Rearrange code and cleanup

- Fix: Incorrect setting of incremental edits for lsp and more

.clangd

@@ -0,0 +1,4 @@
+CompileFlags:
+  Add: [-I/home/syed/main/crib/include, -I/home/syed/main/crib/libs]
+  Remove: []
+  Compiler: clang++

.gitignore

@@ -7,9 +7,11 @@
 
 .vscode
 
-samples/t_*
+samples/tmp*
 
 build
 bin
 
+grammar/.*.scm
+
 __old__

.gitmodules

@@ -130,3 +130,11 @@
 	path = libs/tree-sitter-markdown
 	url = https://github.com/tree-sitter-grammars/tree-sitter-markdown.git
   ignore = dirty
+[submodule "libs/tree-sitter-typescript"]
+	path = libs/tree-sitter-typescript
+  url = https://github.com/tree-sitter/tree-sitter-typescript.git
+  ignore = dirty
+[submodule "libs/tree-sitter-man"]
+	path = libs/tree-sitter-man
+  url = https://github.com/ribru17/tree-sitter-man.git
+  ignore = dirty

Makefile

@@ -13,14 +13,15 @@ CCACHE := ccache
 CXX_DEBUG := $(CCACHE) g++
 CXX_RELEASE := $(CCACHE) clang++
 
-CFLAGS_DEBUG := -std=c++20 -Wall -Wextra -O0 -fno-inline -gsplit-dwarf -g -fsanitize=address -fno-omit-frame-pointer
+CFLAGS_DEBUG := -std=c++20 -Wall -Wextra -O0 -fno-inline -gsplit-dwarf -g -fsanitize=address -fno-omit-frame-pointer -I./include -I./libs
 CFLAGS_RELEASE := -std=c++20 -O3 -march=native -flto=thin \
 	-fno-exceptions -fno-rtti -fstrict-aliasing \
 	-ffast-math -funroll-loops \
 	-fvisibility=hidden \
 	-fomit-frame-pointer -DNDEBUG -s \
 	-mllvm -vectorize-loops \
-	-fno-unwind-tables -fno-asynchronous-unwind-tables
+	-fno-unwind-tables -fno-asynchronous-unwind-tables\
+	-I./include -I./libs
 
 PCH_CFLAGS_DEBUG := $(CFLAGS_DEBUG) -x c++-header
 PCH_CFLAGS_RELEASE := $(CFLAGS_RELEASE) -x c++-header
@@ -34,6 +35,8 @@ TREE_SITTER_LIBS := $(wildcard libs/tree-sitter-*/libtree-sitter*.a)
 
 PHP_LIB := libs/tree-sitter-php/php/libtree-sitter-php.a
 
+TSX_LIB := libs/tree-sitter-typescript/tsx/libtree-sitter-tsx.a
+
 NGINX_OBJ_PARSER := libs/tree-sitter-nginx/build/Release/obj.target/tree_sitter_nginx_binding/src/parser.o
 
 GITIGNORE_OBJ_PARSER := libs/tree-sitter-gitignore/build/Release/obj.target/tree_sitter_ignore_binding/src/parser.o
@@ -41,6 +44,9 @@ GITIGNORE_OBJ_PARSER := libs/tree-sitter-gitignore/build/Release/obj.target/tree
 FISH_OBJ_PARSER := libs/tree-sitter-fish/build/Release/obj.target/tree_sitter_fish_binding/src/parser.o
 FISH_OBJ_SCANNER := libs/tree-sitter-fish/build/Release/obj.target/tree_sitter_fish_binding/src/scanner.o
 
+MAN_OBJ_PARSER := libs/tree-sitter-man/build/Release/obj.target/tree_sitter_man_binding/src/parser.o
+MAN_OBJ_SCANNER := libs/tree-sitter-man/build/Release/obj.target/tree_sitter_man_binding/src/scanner.o
+
 MD_OBJ_PARSER := libs/tree-sitter-markdown/build/Release/obj.target/tree_sitter_markdown_binding/tree-sitter-markdown/src/parser.o
 MD_OBJ_SCANNER := libs/tree-sitter-markdown/build/Release/obj.target/tree_sitter_markdown_binding/tree-sitter-markdown/src/scanner.o
 
@@ -52,17 +58,20 @@ LIBS := \
 	libs/tree-sitter/libtree-sitter.a \
 	$(TREE_SITTER_LIBS) \
 	$(PHP_LIB) \
+	$(TSX_LIB) \
 	$(NGINX_OBJ_PARSER) \
 	$(GITIGNORE_OBJ_PARSER) \
 	$(FISH_OBJ_PARSER) \
 	$(FISH_OBJ_SCANNER) \
+	$(MAN_OBJ_PARSER) \
+	$(MAN_OBJ_SCANNER) \
 	$(MD_OBJ_PARSER) \
 	$(MD_OBJ_SCANNER) \
 	$(MD_I_OBJ_PARSER) \
 	$(MD_I_OBJ_SCANNER) \
 	-lpcre2-8 -lmagic
 
-SRC := $(wildcard $(SRC_DIR)/*.cc)
+SRC := $(wildcard $(SRC_DIR)/**/*.cc) $(wildcard $(SRC_DIR)/*.cc)
 OBJ_DEBUG := $(patsubst $(SRC_DIR)/%.cc,$(OBJ_DIR)/debug/%.o,$(SRC))
 OBJ_RELEASE := $(patsubst $(SRC_DIR)/%.cc,$(OBJ_DIR)/release/%.o,$(SRC))
 

README.md

@@ -6,21 +6,28 @@ A TUI IDE.
 
 # TODO
 
-- [ ] Get lsp warnings byte offsets/lengths and render them as background color.
+- [ ] Locking knot while getting hex is not good idea . make knot true knot so i can copy it super fast with refcount on a pool of nodes.
+    - also edits are somehow still leaking . as in they are not applying properly for very quick edits.
+    - Look into ts_collect_spans and edit_insert (on large files)
+- [ ] Add strikethrough support
+- [ ] Add status bar & RUNNER mode
+- [ ] Fix indentation logic
+- [ ] Fix bug where closing immediately while lsp is loading hangs and then segfaults.
+- [ ] For `"insertTextFormat": 2` in `clangd` and similar use only the last word in the signature when replacing
+- [ ] Keep a list of words in the current buffer. (for auto completion) (maybe?)
+- [ ] Add ecma to js and make tsx
 - [ ] Add support for LSP & autocomplete / snippets.
     - First research
         - `textDocument/documentHighlight` - for highlighting stuff (probably tree-sitter is enough)
         - `textDocument/selectionRange` //
         - `textDocument/completion` - Obviously
         - `textDocument/onTypeFormatting` - seems promising for auto formatting (indentation etc)
-        - `textDocument/inlayHint` & `textDocument/inlineHint` & `textDocument/codeLens`
         - `textDocument/formatting` & `textDocument/rangeFormatting`
         - `textDocument/semanticTokens/*` (probably tree-sitter is enough)
         - `textDocument/linkedEditingRange` - probably useful
         - `textDocument/foldingRange` - i will never use this for folding but it might be useful for other things.
         - `textDocument/rename` & `textDocument/prepareRename` - probably useful
         - And a lot more (just go through each for `clangd` and then expand to say `solargraph`).
-    - Make incremental edits apply. // make a bool field in LSP qhich says if it supports incremental and based on it apply edits
     - Make a universal plug for lsp. So focus more on making a general purpose solid communication interface. Instead of something specific.
         - With a 4ish pass system. (more like each returned value from the lsp is used in 4 ways)
             1. One for stuff like jump to x position. or rename symbol x to y. (stuff that explicitly requires user request to do something)
@@ -28,9 +35,6 @@ A TUI IDE.
             2. One for stuff that only affects highlighting and styles . like symbol highlighting etc.
             3. One for Warnings/errors and inlay hints etc. (stuff that adds virtual text to the editor)
             4. One for fromatting and stuff like that. (stuff that edits the buffer text)
-- [ ] Make tree sitter spans truly incremental - or atleast make them pos based and not byte so minor changes only shifts inline
-    - And make inner trees incremental too
-- [ ] Use LSP to add inlay hints in order to test virtual text. then make an iterator over screen that mimics the renderer for scrolling functions.
 - [ ] Add codeium/copilot support for auto-completion (uses the VAI virtual text) as a test phase.
 - [ ] Add a whitespace highlighter (nerd font). for spaces and tabs at start/end of line. not as virtual but instead at render time.
 - [ ] Once renderer is proven to work well (i.e. redo this commit) merge `experimental` branch into `main`. commit `43f443e` on `experimental`.
@@ -42,9 +46,8 @@ A TUI IDE.
 - [ ] Add alt + click to set multiple cursors.
 - [ ] Add search / replace along with search / virtual cursors are searched pos.
 - [ ] Add support for undo/redo.
-- [ ] Add `.scm` files for all the supported languages. (2/14) Done.
 - [ ] Add splash screen / minigame jumping.
-- [ ] Normalize / validate unicode on file open.
+- [ ] Normalize / validate unicode on file open. so use utf8 purely and fix other types of files
 - [ ] Add git stuff.
 - [ ] Add SQL support. (viewer and basic editor)
 - [ ] Add color picker/palette for hex or other css colors.
@@ -55,5 +58,9 @@ A TUI IDE.
 - [ ] Add the highlight of block edges when cursor is on a bracket (or in). (prolly from lsp)
 - [ ] Add this thing where selection double click on a bracket selects whole block.
     - (only on the first time) and sets mode to `WORD`.
+- [ ] Redo cpp/c/h scm file . also pretty much all of them do manually
+- [ ] Try making `lua-typed` and man pages `tree-sitter` grammar.
 - [ ] Redo folding system and its relation to move_line_* functions. (Currently its a mess)
 - [ ] Make whole thing event driven and not clock driven.
+
+- [ ] Fix in kutu.rb such that windows arent focused on hover . they are only when they are true windows and not just popups . also popus are focused even without hover when they open.

grammar/c.scm

@@ -221,21 +221,6 @@
   _ @type.builtin
   type: _?)
 
-;; #C4B5FF #000000 0 0 0 2
-(type_definition declarator: (type_identifier) @name) @definition.type
-
-;; #C4B5FF #000000 0 0 0 2
-(enum_specifier name: (type_identifier) @name) @definition.type
-
-;; #C4B5FF #000000 0 0 0 2
-(class_specifier name: (type_identifier) @name) @definition.class
-
-;; #C4B5FF #000000 0 0 0 2
-(struct_specifier name: (type_identifier) @name body:(_)) @definition.class
-
-;; #C4B5FF #000000 0 0 0 2
-(declaration type: (union_specifier name: (type_identifier) @name)) @definition.class
-
 ;; #9AD4FF #000000 0 0 0 2
 (namespace_identifier) @module
 

grammar/cpp.scm

@@ -221,21 +221,6 @@
   _ @type.builtin
   type: _?)
 
-;; #C4B5FF #000000 0 0 0 2
-(type_definition declarator: (type_identifier) @name) @definition.type
-
-;; #C4B5FF #000000 0 0 0 2
-(enum_specifier name: (type_identifier) @name) @definition.type
-
-;; #C4B5FF #000000 0 0 0 2
-(class_specifier name: (type_identifier) @name) @definition.class
-
-;; #C4B5FF #000000 0 0 0 2
-(struct_specifier name: (type_identifier) @name body:(_)) @definition.class
-
-;; #C4B5FF #000000 0 0 0 2
-(declaration type: (union_specifier name: (type_identifier) @name)) @definition.class
-
 ;; #9AD4FF #000000 0 0 0 2
 (namespace_identifier) @module
 

grammar/h.scm

@@ -221,21 +221,6 @@
   _ @type.builtin
   type: _?)
 
-;; #C4B5FF #000000 0 0 0 2
-(type_definition declarator: (type_identifier) @name) @definition.type
-
-;; #C4B5FF #000000 0 0 0 2
-(enum_specifier name: (type_identifier) @name) @definition.type
-
-;; #C4B5FF #000000 0 0 0 2
-(class_specifier name: (type_identifier) @name) @definition.class
-
-;; #C4B5FF #000000 0 0 0 2
-(struct_specifier name: (type_identifier) @name body:(_)) @definition.class
-
-;; #C4B5FF #000000 0 0 0 2
-(declaration type: (union_specifier name: (type_identifier) @name)) @definition.class
-
 ;; #9AD4FF #000000 0 0 0 2
 (namespace_identifier) @module
 

grammar/hover.scm

@@ -0,0 +1,334 @@
+;; #82AAFF #000000 1 0 1 4
+(setext_heading
+  (paragraph) @markup.heading.1
+  (setext_h1_underline) @markup.heading.1)
+
+;; #82AAFF #000000 1 0 1 4
+(setext_heading
+  (paragraph) @markup.heading.2
+  (setext_h2_underline) @markup.heading.2)
+
+(atx_heading
+  (atx_h1_marker)) @markup.heading.1
+
+(atx_heading
+  (atx_h2_marker)) @markup.heading.2
+
+;; #82AAFF #000000 1 0 0 4
+(atx_heading
+  (atx_h3_marker)) @markup.heading.3
+
+;; #82AAFF #000000 1 0 0 4
+(atx_heading
+  (atx_h4_marker)) @markup.heading.4
+
+;; #82AAFF #000000 1 0 0 4
+(atx_heading
+  (atx_h5_marker)) @markup.heading.5
+
+;; #82AAFF #000000 1 0 0 4
+(atx_heading
+  (atx_h6_marker)) @markup.heading.6
+
+;; #82AAFF #000000 0 0 0 4
+(info_string) @label
+
+;; #FF6347 #000000 0 0 0 4
+(pipe_table_header
+  (pipe_table_cell) @markup.heading)
+
+;; #FF8F40 #000000 0 0 0 4
+(pipe_table_header
+  "|" @punctuation.special)
+
+(pipe_table_row
+  "|" @punctuation.special)
+
+(pipe_table_delimiter_row
+  "|" @punctuation.special)
+
+(pipe_table_delimiter_cell) @punctuation.special
+
+;; #AAD94C #000000 0 0 0 2
+(indented_code_block) @markup.raw.block
+
+(fenced_code_block) @markup.raw.block
+
+(fenced_code_block
+  (fenced_code_block_delimiter) @markup.raw.block)
+
+(fenced_code_block
+  (info_string
+    (language) @label))
+
+;; #7dcfff #000000 0 0 1 6
+(link_destination) @markup.link.url
+
+;; #7dcfff #000000 0 0 1 6
+[
+  (link_title)
+  (link_label)
+] @markup.link.label
+
+;; #FF8F40 #000000 0 0 0 4
+((link_label)
+  .
+  ":" @punctuation.delimiter)
+
+;; #9ADE7A #000000 0 0 0 4
+[
+  (list_marker_plus)
+  (list_marker_minus)
+  (list_marker_star)
+  (list_marker_dot)
+  (list_marker_parenthesis)
+] @markup.list
+
+(thematic_break) @punctuation.special
+
+;; #FF8F40 #000000 0 0 0 4
+(task_list_marker_unchecked) @markup.list.unchecked
+
+;; #AAD94C #000000 0 0 0 4
+(task_list_marker_checked) @markup.list.checked
+
+[
+  (plus_metadata)
+  (minus_metadata)
+] @keyword.directive
+
+[
+  (block_continuation)
+  (block_quote_marker)
+] @punctuation.special
+
+;; #AAD94C #000000 0 0 0 6
+(backslash_escape) @string.escape
+
+(fenced_code_block
+  (info_string
+    (language) @injection.language (#match? @injection.language "^ruby$"))
+;; !ruby
+  (code_fence_content) @injection.ruby)
+
+(fenced_code_block
+  (info_string
+    (language) @injection.language (#match? @injection.language "^bash$"))
+;; !bash
+  (code_fence_content) @injection.bash)
+
+(fenced_code_block
+  (info_string
+    (language) @injection.language (#match? @injection.language "^cpp$"))
+;; !cpp
+  (code_fence_content) @injection.cpp)
+
+(fenced_code_block
+  (info_string
+    (language) @injection.language (#match? @injection.language "^objective-cpp$"))
+;; !cpp
+  (code_fence_content) @injection.cpp)
+
+(fenced_code_block
+  (info_string
+    (language) @injection.language (#match? @injection.language "^h$"))
+;; !h
+  (code_fence_content) @injection.h)
+
+(fenced_code_block
+  (info_string
+    (language) @injection.language (#match? @injection.language "^c$"))
+;; !c
+  (code_fence_content) @injection.h)
+
+(fenced_code_block
+  (info_string
+    (language) @injection.language (#match? @injection.language "^css$"))
+;; !css
+  (code_fence_content) @injection.css)
+
+(fenced_code_block
+  (info_string
+    (language) @injection.language (#match? @injection.language "^fish$"))
+;; !fish
+  (code_fence_content) @injection.fish)
+
+(fenced_code_block
+  (info_string
+    (language) @injection.language (#match? @injection.language "^go$"))
+;; !go
+  (code_fence_content) @injection.go)
+
+(fenced_code_block
+  (info_string
+    (language) @injection.language (#match? @injection.language "^haskell$"))
+;; !haskell
+  (code_fence_content) @injection.haskell)
+
+(fenced_code_block
+  (info_string
+    (language) @injection.language (#match? @injection.language "^html$"))
+;; !html
+  (code_fence_content) @injection.html)
+
+(fenced_code_block
+  (info_string
+    (language) @injection.language (#match? @injection.language "^javascript$"))
+;; !javascript
+  (code_fence_content) @injection.javascript)
+
+(fenced_code_block
+  (info_string
+    (language) @injection.language (#match? @injection.language "^json$"))
+;; !json
+  (code_fence_content) @injection.json)
+
+(fenced_code_block
+  (info_string
+    (language) @injection.language (#match? @injection.language "^lua$"))
+; - lua format in hover boxes is typed making it unparsable as normal lua
+; - TODO: add a lua grammar with typing or remove this injection
+  (code_fence_content) @injection.lua)
+
+(fenced_code_block
+  (info_string
+    (language) @injection.language (#match? @injection.language "^regex$"))
+;; !regex
+  (code_fence_content) @injection.regex)
+
+(fenced_code_block
+  (info_string
+    (language) @injection.language (#match? @injection.language "^query$"))
+;; !query
+  (code_fence_content) @injection.query)
+
+(fenced_code_block
+  (info_string
+    (language) @injection.language (#match? @injection.language "^markdown$"))
+;; !markdown
+  (code_fence_content) @injection.markdown)
+
+(fenced_code_block
+  (info_string
+    (language) @injection.language (#match? @injection.language "^markdown_inline$"))
+;; !markdown_inline
+  (code_fence_content) @injection.markdown_inline)
+
+(fenced_code_block
+  (info_string
+    (language) @injection.language (#match? @injection.language "^erb$"))
+;; !embedded_template
+  (code_fence_content) @injection.embedded_template)
+
+(fenced_code_block
+  (info_string
+    (language) @injection.language (#match? @injection.language "^python$"))
+;; !python
+  (code_fence_content) @injection.python)
+
+(fenced_code_block
+  (info_string
+    (language) @injection.language (#match? @injection.language "^php$"))
+;; !php
+  (code_fence_content) @injection.php)
+
+(fenced_code_block
+  (info_string
+    (language) @injection.language (#match? @injection.language "^rust$"))
+;; !rust
+  (code_fence_content) @injection.rust)
+
+(fenced_code_block
+  (info_string
+    (language) @injection.language (#match? @injection.language "^sql$"))
+;; !sql
+  (code_fence_content) @injection.sql)
+
+(fenced_code_block
+  (info_string
+    (language) @injection.language (#match? @injection.language "^gitattributes$"))
+;; !gitattributes
+  (code_fence_content) @injection.gitattributes)
+
+(fenced_code_block
+  (info_string
+    (language) @injection.language (#match? @injection.language "^gitignore$"))
+;; !gitignore
+  (code_fence_content) @injection.gitignore)
+
+(fenced_code_block
+  (info_string
+    (language) @injection.language (#match? @injection.language "^gdscript$"))
+;; !gdscript
+  (code_fence_content) @injection.gdscript)
+
+(fenced_code_block
+  (info_string
+    (language) @injection.language (#match? @injection.language "^make$"))
+;; !make
+  (code_fence_content) @injection.make)
+
+(fenced_code_block
+  (info_string
+    (language) @injection.language (#match? @injection.language "^diff$"))
+;; !diff
+  (code_fence_content) @injection.diff)
+
+(fenced_code_block
+  (info_string
+    (language) @injection.language (#match? @injection.language "^ini$"))
+;; !ini
+  (code_fence_content) @injection.ini)
+
+(fenced_code_block
+  (info_string
+    (language) @injection.language (#match? @injection.language "^nginx$"))
+;; !nginx
+  (code_fence_content) @injection.nginx)
+
+(fenced_code_block
+  (info_string
+    (language) @injection.language (#match? @injection.language "^toml$"))
+;; !toml
+  (code_fence_content) @injection.toml)
+
+(fenced_code_block
+  (info_string
+    (language) @injection.language (#match? @injection.language "^yaml$"))
+;; !yaml
+  (code_fence_content) @injection.yaml)
+
+(fenced_code_block
+  (info_string
+    (language) @injection.language (#match? @injection.language "^gomod$"))
+;; !gomod
+  (code_fence_content) @injection.gomod)
+
+(fenced_code_block
+  (info_string
+    (language) @injection.language (#match? @injection.language "^man$"))
+;; !man
+  (code_fence_content) @injection.man)
+
+(fenced_code_block
+  (info_string
+    (language) @injection.language (#match? @injection.language "^cabal$"))
+;; !cabal
+  (code_fence_content) @injection.cabal)
+
+;; !html
+(html_block) @injection.html
+
+;; !yaml
+(minus_metadata) @injection.yaml
+
+;; !toml
+(plus_metadata) @injection.toml
+
+;; !markdown_inline
+(paragraph) @inline
+
+(pipe_table_row
+  (pipe_table_cell) @inline)
+
+(block_quote ((paragraph) @inline))

grammar/javascript.scm

@@ -14,7 +14,7 @@
  (#match? @variable.builtin
   "^(arguments|console|window|document|globalThis|process|module|exports)$"))
 
-;; #59C2FF #000000 0 0 0 3
+;; #59C2FF #000000 0 0 0 1
 ((identifier) @constructor
  (#match? @constructor "^[A-Z][a-zA-Z0-9]*$"))
 
@@ -84,12 +84,10 @@
   key: (property_identifier) @name
   value: [(arrow_function) (function_expression)]) @definition.function
 
-;; #59C2FF #000000 0 0 0 2
-(
-  (call_expression
+;; #59C2FF #000000 0 0 0 0
+((call_expression
     function: (identifier) @name) @reference.call
-  (#not-match? @name "^(require)$")
-)
+  (#not-match? @name "^(require)$"))
 
 ;; #7dcfff #000000 0 0 0 2
 (new_expression
@@ -299,18 +297,18 @@
 ; JSX
 ; ============================================================
 
-;; #59C2FF #000000 0 0 0 2
-(jsx_opening_element (identifier) @tag (#match? @tag "^[a-z][^.]*$"))
-(jsx_closing_element (identifier) @tag (#match? @tag "^[a-z][^.]*$"))
-(jsx_self_closing_element (identifier) @tag (#match? @tag "^[a-z][^.]*$"))
+;; #59C2FF #000000 0 0 0 4
+(jsx_opening_element (identifier) @tag2)
+(jsx_closing_element (identifier) @tag2)
+(jsx_self_closing_element (identifier) @tag2)
 
-;; #F07178 #000000 0 0 0 1
-(jsx_attribute (property_identifier) @attribute)
+;; #F07178 #000000 0 0 0 3
+(jsx_attribute (property_identifier) @attribute2)
 
-;; #BFBDB6 #000000 0 0 0 1
-(jsx_opening_element (["<" ">"]) @punctuation.bracket)
-(jsx_closing_element (["</" ">"]) @punctuation.bracket)
-(jsx_self_closing_element (["<" "/>"]) @punctuation.bracket)
+;; #BFBDB6 #000000 0 0 0 3
+(jsx_opening_element (["<" ">"]) @punctuation.bracket2)
+(jsx_closing_element (["</" ">"]) @punctuation.bracket2)
+(jsx_self_closing_element (["<" "/>"]) @punctuation.bracket2)
 
 ; Injections
 

grammar/jsonc.scm

@@ -0,0 +1,22 @@
+;; #D2A6FF #000000 0 0 0 2
+(pair
+  key: (_) @string.special.key)
+
+;; #AAD94C #000000 0 0 0 1
+(string) @string
+
+;; #7dcfff #000000 0 0 0 2
+(number) @number
+
+;; #F07178 #000000 0 0 0 1
+[
+  (null)
+  (true)
+  (false)
+] @constant.builtin
+
+;; #7dcfff #000000 0 0 0 2
+(escape_sequence) @escape
+
+;; #99ADBF #000000 0 1 0 1
+(comment) @comment

grammar/man.scm

@@ -0,0 +1,23 @@
+;; #82AAFF #000000 1 0 1 2
+(title) @markup.heading.1
+
+;; #ccefc9 #000000 0 0 0 0
+(section_title) @markup.heading.2
+
+;; #FF8F40 #000000 1 0 0 2
+(subsection_title) @markup.heading.3
+
+;; #AAD94C #000000 0 0 0 3
+(option) @variable.parameter
+
+;; #FFD700 #000000 1 0 0 3
+(reference) @markup.link.label
+
+;; #C792EA #000000 0 0 0 3
+(footer) @markup.heading
+
+(section_heading
+  (section_title) @_title
+;; #FFD700 #000000 1 0 0 1
+  (block) @injection.content
+  (#match? @_title "SYNOPSIS"))

grammar/markdown.scm

@@ -297,6 +297,12 @@
 ;; !gomod
   (code_fence_content) @injection.gomod)
 
+(fenced_code_block
+  (info_string
+    (language) @injection.language (#match? @injection.language "^man$"))
+;; !man
+  (code_fence_content) @injection.man)
+
 (fenced_code_block
   (info_string
     (language) @injection.language (#match? @injection.language "^cabal$"))

grammar/ruby.scm

@@ -293,7 +293,7 @@
 (pair
   ":" @punctuation.delimiter)
 
-;; #BFBDB6 #000000 0 0 0 1
+;; #BFBDB6 #000000 0 0 0 3
 [
   "("
   ")"
@@ -507,3 +507,9 @@
   (heredoc_content) @cabal_injection
   ((heredoc_end) @lang
     (#match? @lang "CABAL")))
+
+(heredoc_body
+;; !man
+  (heredoc_content) @man_injection
+  ((heredoc_end) @lang
+    (#match? @lang "MAN")))

grammar/typescript.scm

@@ -0,0 +1,316 @@
+; ============================================================
+; Identifiers
+; ============================================================
+
+;; #FFFFFF #000000 0 0 0 1
+(identifier) @variable
+
+;; #D2A6FF #000000 0 0 0 2
+((identifier) @constant
+ (#match? @constant "^[A-Z_][A-Z0-9_]*$"))
+
+;; #F07178 #000000 0 0 0 3
+((identifier) @variable.builtin
+ (#match? @variable.builtin
+  "^(arguments|console|window|document|globalThis|process|module|exports)$"))
+
+;; #59C2FF #000000 0 0 0 1
+((identifier) @constructor
+ (#match? @constructor "^[A-Z][a-zA-Z0-9]*$"))
+
+; ============================================================
+; Properties
+; ============================================================
+
+;; #F07178 #000000 0 0 0 1
+(property_identifier) @property
+
+; ============================================================
+; Functions
+; ============================================================
+
+;; #FFB454 #000000 0 0 0 3
+(function_declaration
+  name: (identifier) @function)
+
+(function_expression
+  name: (identifier) @function)
+
+;; #FFB454 #000000 0 0 0 2
+(method_definition
+  name: (property_identifier) @function.method)
+
+(variable_declarator
+  name: (identifier) @function
+  value: [(function_expression) (arrow_function)])
+
+(assignment_expression
+  left: (identifier) @function
+  right: [(function_expression) (arrow_function)])
+
+(pair
+  key: (property_identifier) @function.method
+  value: [(function_expression) (arrow_function)])
+
+; ------------------------------------------------------------
+; Function calls
+; ------------------------------------------------------------
+
+;; #FFB454 #000000 0 0 0 2
+(call_expression
+  function: (identifier) @function.call)
+
+;; #FFB454 #000000 0 0 0 2
+(call_expression
+  function: (member_expression
+    property: (property_identifier) @function.method))
+
+; ============================================================
+; Highlighted definitions & references
+; ============================================================
+
+;; #FFB454 #000000 0 0 0 3
+(assignment_expression
+  left: [
+    (identifier) @name
+    (member_expression
+      property: (property_identifier) @name)
+  ]
+  right: [(arrow_function) (function_expression)]
+) @definition.function
+
+;; #FFB454 #000000 0 0 0 3
+(pair
+  key: (property_identifier) @name
+  value: [(arrow_function) (function_expression)]) @definition.function
+
+;; #59C2FF #000000 0 0 0 0
+((call_expression
+    function: (identifier) @name) @reference.call
+  (#not-match? @name "^(require)$"))
+
+;; #7dcfff #000000 0 0 0 2
+(new_expression
+  constructor: (_) @name) @reference.class
+
+;; #D2A6FF #000000 0 0 0 2
+(export_statement value: (assignment_expression left: (identifier) @name right: ([
+ (number)
+ (string)
+ (identifier)
+ (undefined)
+ (null)
+ (new_expression)
+ (binary_expression)
+ (call_expression)
+]))) @definition.constant
+
+; ============================================================
+; Parameters
+; ============================================================
+
+;; #D2A6FF #000000 0 0 0 1
+(formal_parameters
+  [
+    (identifier) @variable.parameter
+    (array_pattern
+      (identifier) @variable.parameter)
+    (object_pattern
+      [
+        (pair_pattern value: (identifier) @variable.parameter)
+        (shorthand_property_identifier_pattern) @variable.parameter
+      ])
+  ])
+
+; ============================================================
+; Keywords (split into semantic groups)
+; ============================================================
+
+;; #FF8F40 #000000 0 0 0 1
+; Declarations
+[
+  "var"
+  "let"
+  "const"
+  "function"
+  "class"
+] @keyword.declaration
+
+;; #FF8F40 #000000 0 0 0 1
+; Control flow
+[
+  "if"
+  "else"
+  "switch"
+  "case"
+  "default"
+  "for"
+  "while"
+  "do"
+  "break"
+  "continue"
+  "return"
+  "throw"
+  "try"
+  "catch"
+  "finally"
+  "extends"
+] @keyword.control
+
+;; #FF8F40 #000000 0 0 0 1
+; Imports / exports
+[
+  "import"
+  "export"
+  "from"
+  "as"
+] @keyword.import
+
+;; #F29668 #000000 0 0 0 1
+; Operators-as-keywords
+[
+  "in"
+  "instanceof"
+  "new"
+  "delete"
+  "typeof"
+  "void"
+  "await"
+  "yield"
+] @keyword.operator
+
+;; #FF8F40 #000000 0 0 0 1
+; Modifiers
+[
+  "async"
+  "static"
+  "get"
+  "set"
+] @keyword.modifier
+
+; ============================================================
+; Literals
+; ============================================================
+
+;; #F07178 #000000 0 0 0 1
+(this) @variable.builtin
+(super) @variable.builtin
+
+;; #D2A6FF #000000 0 0 0 4
+[
+  (true)
+  (false)
+  (null)
+  (undefined)
+] @constant.builtin
+
+;; #D2A6FF #000000 0 0 0 2
+(number) @number
+
+;; #D2A6FF #000000 0 1 0 2
+((string) @use_strict
+  (#match? @use_strict "^['\"]use strict['\"]$"))
+
+;; #AAD94C #000000 0 0 0 0
+(string) @string
+
+;; #AAD94C #000000 0 0 0 0
+(template_string) @string.special
+
+;; #99ADBF #000000 0 1 0 1
+(comment) @comment
+
+; ============================================================
+; Operators & punctuation
+; ============================================================
+
+;; #F29668 #000000 0 1 0 1
+[
+  "+"
+  "-"
+  "*"
+  "/"
+  "%"
+  "**"
+  "++"
+  "--"
+  "=="
+  "!="
+  "==="
+  "!=="
+  "<"
+  "<="
+  ">"
+  ">="
+  "&&"
+  "||"
+  "??"
+  "!"
+  "~"
+  "&"
+  "|"
+  "^"
+  "<<"
+  ">>"
+  ">>>"
+  "="
+  "+="
+  "-="
+  "*="
+  "/="
+  "%="
+  "<<="
+  ">>="
+  ">>>="
+  "&="
+  "|="
+  "^="
+  "&&="
+  "||="
+  "??="
+  "=>"
+] @operator
+
+;; #BFBDB6 #000000 0 0 0 1
+[
+  "."
+  ","
+  ";"
+] @punctuation.delimiter
+
+;; #BFBDB6 #000000 0 0 0 1
+[
+  "("
+  ")"
+  "["
+  "]"
+  "{"
+  "}"
+] @punctuation.bracket
+
+;; #7dcfff #000000 0 0 0 2
+(template_substitution
+  "${" @punctuation.special
+  "}" @punctuation.special)
+
+; ============================================================
+; JSX
+; ============================================================
+
+;; #59C2FF #000000 0 0 0 4
+(jsx_opening_element (identifier) @tag2)
+(jsx_closing_element (identifier) @tag2)
+(jsx_self_closing_element (identifier) @tag2)
+
+;; #F07178 #000000 0 0 0 3
+(jsx_attribute (property_identifier) @attribute2)
+
+;; #BFBDB6 #000000 0 0 0 3
+(jsx_opening_element (["<" ">"]) @punctuation.bracket2)
+(jsx_closing_element (["</" ">"]) @punctuation.bracket2)
+(jsx_self_closing_element (["<" "/>"]) @punctuation.bracket2)
+
+; Injections
+
+;; !regex
+(regex) @string.regex

include/boxes/diagnostics.h

@@ -0,0 +1,20 @@
+#ifndef BOXES_DIAGNOSTICS_H
+#define BOXES_DIAGNOSTICS_H
+
+#include "editor/decl.h"
+#include "io/ui.h"
+#include "pch.h"
+#include "utils/utils.h"
+
+struct DiagnosticBox {
+  std::vector<VWarn> warnings;
+  std::vector<ScreenCell> cells;
+  uint32_t box_width;
+  uint32_t box_height;
+
+  void clear();
+  void render_first();
+  void render(Coord pos);
+};
+
+#endif

include/boxes/hover.h

@@ -0,0 +1,26 @@
+#ifndef BOXES_HOVER_H
+#define BOXES_HOVER_H
+
+#include "editor/decl.h"
+#include "io/ui.h"
+#include "pch.h"
+#include "ts/decl.h"
+#include "utils/utils.h"
+
+struct HoverBox {
+  std::string text;
+  std::atomic<bool> is_markup;
+  uint32_t scroll_;
+  std::vector<ScreenCell> cells;
+  uint32_t box_width;
+  uint32_t box_height;
+  std::vector<Highlight> highlights;
+  std::vector<Span> hover_spans;
+
+  void clear();
+  void scroll(int32_t number);
+  void render_first(bool scroll = false);
+  void render(Coord pos);
+};
+
+#endif

include/config.h

@@ -0,0 +1,291 @@
+#ifndef CONFIG_H
+#define CONFIG_H
+
+#include "lsp/lsp.h"
+#include "pch.h"
+#include "ts/decl.h"
+
+static const std::unordered_map<uint8_t, LSP> kLsps = {
+    {1,
+     {"clangd",
+      {
+          "clangd",
+          "--background-index",
+          "--clang-tidy",
+          "--completion-style=detailed",
+          "--header-insertion=never",
+          "--pch-storage=memory",
+          "--limit-results=50",
+          "--log=error",
+          nullptr,
+      }}},
+    {2,
+     {"ruby-lsp",
+      {
+          "ruby-lsp",
+          nullptr,
+      }}},
+    {3,
+     {"solargraph",
+      {
+          "solargraph",
+          "stdio",
+          nullptr,
+      }}},
+    {4,
+     {"bash-language-server",
+      {
+          "bash-language-server",
+          "start",
+          nullptr,
+      }}},
+    {5,
+     {"vscode-css-language-server",
+      {
+          "vscode-css-language-server",
+          "--stdio",
+          nullptr,
+      }}},
+    {6,
+     {"vscode-json-language-server",
+      {
+          "vscode-json-language-server",
+          "--stdio",
+          nullptr,
+      }}},
+    {7,
+     {"fish-lsp",
+      {
+          "fish-lsp",
+          "start",
+          nullptr,
+      }}},
+    {8,
+     {"gopls",
+      {
+          "gopls",
+          "serve",
+          nullptr,
+      }}},
+    {9,
+     {"haskell-language-server",
+      {
+          "haskell-language-server",
+          "lsp",
+          nullptr,
+      }}},
+    {10,
+     {"emmet-ls",
+      {
+          "emmet-ls",
+          "--stdio",
+          nullptr,
+      }}},
+    {11,
+     {"typescript-language-server",
+      {
+          "typescript-language-server",
+          "--stdio",
+          nullptr,
+      }}},
+    {12,
+     {"lua-language-server",
+      {
+          "lua-language-server",
+          nullptr,
+      }}},
+    {13,
+     {"pyright-langserver",
+      {
+          "pyright-langserver",
+          "--stdio",
+          nullptr,
+      }}},
+    {14,
+     {"rust-analyzer",
+      {
+          "rust-analyzer",
+          nullptr,
+      }}},
+    {15,
+     {"intelephense",
+      {
+          "intelephense",
+          "--stdio",
+          nullptr,
+      }}},
+    {16,
+     {"marksman",
+      {
+          "marksman",
+          "server",
+          nullptr,
+      }}},
+    {17,
+     {"nginx-language-server",
+      {
+          "nginx-language-server",
+          nullptr,
+      }}},
+    {18,
+     {"taplo",
+      {
+          "taplo",
+          "lsp",
+          "stdio",
+          nullptr,
+      }}},
+    {19,
+     {"yaml-language-server",
+      {
+          "yaml-language-server",
+          "--stdio",
+          nullptr,
+      }}},
+    {20,
+     {"sqls",
+      {
+          "sqls",
+          "serve",
+          nullptr,
+      }}},
+    {21,
+     {"make-language-server",
+      {
+          "make-language-server",
+          nullptr,
+      }}},
+    {22,
+     {"sql-language-server",
+      {
+          "sql-language-server",
+          "up",
+          "--method",
+          "stdio",
+          nullptr,
+      }}},
+};
+
+static const std::unordered_map<std::string, Language> kLanguages = {
+    {"bash", {"bash", LANG(bash), 4}},
+    {"c", {"c", LANG(cpp), 1}},
+    {"cpp", {"cpp", LANG(cpp), 1}},
+    {"h", {"h", LANG(cpp), 1}},
+    {"css", {"css", LANG(css), 5}},
+    {"fish", {"fish", LANG(fish), 7}},
+    {"go", {"go", LANG(go), 8}},
+    {"gomod", {"gomod", LANG(gomod), 8}},
+    {"haskell", {"haskell", LANG(haskell), 9}},
+    {"html", {"html", LANG(html), 10}},
+    {"javascript", {"javascript", LANG(javascript), 11}},
+    {"typescript", {"typescript", LANG(tsx), 11}},
+    {"json", {"json", LANG(json), 6}},
+    {"jsonc", {"jsonc", LANG(json), 6}},
+    {"erb", {"erb", LANG(embedded_template), 10}},
+    {"ruby", {"ruby", LANG(ruby), 3}},
+    {"lua", {"lua", LANG(lua), 12}},
+    {"python", {"python", LANG(python), 13}},
+    {"rust", {"rust", LANG(rust), 14}},
+    {"php", {"php", LANG(php), 15}},
+    {"markdown", {"markdown", LANG(markdown), 16}},
+    {"markdown_inline", {"markdown_inline", LANG(markdown_inline), 16}},
+    {"nginx", {"nginx", LANG(nginx), 17}},
+    {"toml", {"toml", LANG(toml), 18}},
+    {"yaml", {"yaml", LANG(yaml), 19}},
+    {"sql", {"sql", LANG(sql), 20}}, // Can use `22` for more accuracy but need
+                                     // config to connect to database
+    {"make", {"make", LANG(make), 21}},
+    {"gdscript", {"gdscript", LANG(gdscript)}}, // TODO: connect to godot
+    {"man", {"man", LANG(man)}},
+    {"diff", {"diff", LANG(diff)}},
+    {"gitattributes", {"gitattributes", LANG(gitattributes)}},
+    {"gitignore", {"gitignore", LANG(gitignore)}},
+    {"query", {"query", LANG(query)}},
+    {"regex", {"regex", LANG(regex)}},
+    {"ini", {"ini", LANG(ini)}},
+};
+
+static const std::unordered_map<std::string, std::string> kExtToLang = {
+    {"sh", "bash"},
+    {"bash", "bash"},
+    {"c", "c"},
+    {"cpp", "cpp"},
+    {"cxx", "cpp"},
+    {"cc", "cpp"},
+    {"hpp", "h"},
+    {"hh", "h"},
+    {"hxx", "h"},
+    {"h", "h"},
+    {"css", "css"},
+    {"fish", "fish"},
+    {"go", "go"},
+    {"hs", "haskell"},
+    {"html", "html"},
+    {"htm", "html"},
+    {"js", "javascript"},
+    {"jsx", "javascript"},
+    {"ts", "typescript"},
+    {"tsx", "typescript"},
+    {"json", "json"},
+    {"jsonc", "jsonc"},
+    {"lua", "lua"},
+    {"make", "make"},
+    {"mk", "make"},
+    {"makefile", "make"},
+    {"man", "man"},
+    {"py", "python"},
+    {"rb", "ruby"},
+    {"rs", "rust"},
+    {"diff", "diff"},
+    {"patch", "diff"},
+    {"erb", "erb"},
+    {"gd", "gdscript"},
+    {"gitattributes", "gitattributes"},
+    {"gitignore", "gitignore"},
+    {"mod", "gomod"},
+    {"ini", "ini"},
+    {"gitmodules", "ini"},
+    {"md", "markdown"},
+    {"markdown", "markdown"},
+    {"conf", "nginx"},
+    {"php", "php"},
+    {"scm", "query"},
+    {"regex", "regex"},
+    {"sql", "sql"},
+    {"toml", "toml"},
+    {"yaml", "yaml"},
+    {"yml", "yaml"},
+};
+
+static const std::unordered_map<std::string, std::string> kMimeToLang = {
+    {"text/x-c", "c"},
+    {"text/x-c++", "cpp"},
+    {"text/x-shellscript", "bash"},
+    {"application/json", "json"},
+    {"text/javascript", "javascript"},
+    {"text/html", "html"},
+    {"text/css", "css"},
+    {"text/x-python", "python"},
+    {"text/x-ruby", "ruby"},
+    {"text/x-go", "go"},
+    {"text/x-haskell", "haskell"},
+    {"text/x-rust", "rust"},
+    {"text/x-lua", "lua"},
+    {"text/x-diff", "diff"},
+    {"text/x-gdscript", "gdscript"},
+    {"text/x-gitattributes", "gitattributes"},
+    {"text/x-gitignore", "gitignore"},
+    {"text/x-gomod", "gomod"},
+    {"text/x-ini", "ini"},
+    {"text/markdown", "markdown"},
+    {"text/x-nginx-conf", "nginx"},
+    {"application/x-php", "php"},
+    {"text/x-tree-sitter-query", "query"},
+    {"text/x-regex", "regex"},
+    {"text/x-sql", "sql"},
+    {"text/x-toml", "toml"},
+    {"text/x-yaml", "yaml"},
+    {"text/x-man", "man"},
+};
+
+#endif

include/editor.h

@@ -1,269 +0,0 @@
-#ifndef EDITOR_H
-#define EDITOR_H
-
-#include "./knot.h"
-#include "./pch.h"
-#include "./ui.h"
-#include "./utils.h"
-#include "ts_def.h"
-#include <cstdint>
-#include <shared_mutex>
-
-#define CHAR 0
-#define WORD 1
-#define LINE 2
-
-#define EXTRA_META 4
-#define INDENT_WIDTH 2
-
-struct Highlight {
-  uint32_t fg;
-  uint32_t bg;
-  uint32_t flags;
-  uint8_t priority;
-};
-
-struct Span {
-  uint32_t start;
-  uint32_t end;
-  Highlight *hl;
-
-  bool operator<(const Span &other) const { return start < other.start; }
-};
-
-struct Spans {
-  std::vector<Span> spans;
-  Queue<std::pair<uint32_t, int64_t>> edits;
-  bool mid_parse = false;
-  std::shared_mutex mtx;
-};
-
-struct Fold {
-  uint32_t start;
-  uint32_t end;
-
-  bool contains(uint32_t line) const { return line >= start && line <= end; }
-  bool operator<(const Fold &other) const { return start < other.start; }
-};
-
-struct SpanCursor {
-  Spans &spans;
-  size_t index = 0;
-  std::vector<Span *> active;
-  std::shared_lock<std::shared_mutex> lock;
-
-  SpanCursor(Spans &s) : spans(s) {}
-  Highlight *get_highlight(uint32_t byte_offset) {
-    for (int i = (int)active.size() - 1; i >= 0; i--)
-      if (active[i]->end <= byte_offset)
-        active.erase(active.begin() + i);
-    while (index < spans.spans.size() &&
-           spans.spans[index].start <= byte_offset) {
-      if (spans.spans[index].end > byte_offset)
-        active.push_back(const_cast<Span *>(&spans.spans[index]));
-      index++;
-    }
-    Highlight *best = nullptr;
-    int max_prio = -1;
-    for (auto *s : active)
-      if (s->hl->priority > max_prio) {
-        max_prio = s->hl->priority;
-        best = s->hl;
-      }
-    return best;
-  }
-  void sync(uint32_t byte_offset) {
-    lock = std::shared_lock(spans.mtx);
-    active.clear();
-    size_t left = 0, right = spans.spans.size();
-    while (left < right) {
-      size_t mid = (left + right) / 2;
-      if (spans.spans[mid].start <= byte_offset)
-        left = mid + 1;
-      else
-        right = mid;
-    }
-    index = left;
-    while (left > 0) {
-      left--;
-      if (spans.spans[left].end > byte_offset)
-        active.push_back(const_cast<Span *>(&spans.spans[left]));
-      else if (byte_offset - spans.spans[left].end > 1000)
-        break;
-    }
-  }
-};
-
-struct VHint {
-  Coord pos;
-  std::string hint;
-
-  bool operator<(const VHint &other) const { return pos < other.pos; }
-};
-
-struct VWarn {
-  uint32_t line;
-  std::string text;
-  int8_t type; // For hl
-
-  bool operator<(const VWarn &other) const { return line < other.line; }
-};
-
-struct VAI {
-  Coord pos;
-  char *text;
-  uint32_t len;
-  uint32_t lines; // number of \n in text for speed .. the ai part will not
-                  // line wrap but multiline ones need to have its own lines
-                  // after the first one
-};
-
-struct TSSetBase {
-  std::string lang;
-  TSParser *parser;
-  std::string query_file;
-  TSQuery *query;
-  std::map<uint16_t, Highlight> query_map;
-  std::map<uint16_t, Language> injection_map;
-  const TSLanguage *language;
-};
-
-struct TSSet : TSSetBase {
-  std::vector<TSRange> ranges;
-};
-
-struct TSSetMain : TSSetBase {
-  TSTree *tree;
-  std::unordered_map<std::string, TSSet> injections;
-};
-
-struct Editor {
-  std::string filename;
-  std::string uri;
-  Knot *root;
-  std::shared_mutex knot_mtx;
-  Coord cursor;
-  uint32_t cursor_preffered;
-  Coord selection;
-  bool selection_active;
-  int selection_type;
-  Coord position;
-  Coord size;
-  Coord scroll;
-  TSSetMain ts;
-  Queue<TSInputEdit> edit_queue;
-  std::vector<Fold> folds;
-  Spans spans;
-  Spans def_spans;
-  uint32_t hooks[94];
-  bool jumper_set;
-  std::shared_mutex v_mtx;
-  std::vector<VHint> hints;
-  std::vector<VWarn> warnings;
-  VAI ai;
-  std::shared_mutex lsp_mtx;
-  struct LSPInstance *lsp;
-  int lsp_version = 1;
-};
-
-inline const Fold *fold_for_line(const std::vector<Fold> &folds,
-                                 uint32_t line) {
-  auto it = std::lower_bound(
-      folds.begin(), folds.end(), line,
-      [](const Fold &fold, uint32_t value) { return fold.start < value; });
-  if (it != folds.end() && it->start == line)
-    return &(*it);
-  if (it != folds.begin()) {
-    --it;
-    if (it->contains(line))
-      return &(*it);
-  }
-  return nullptr;
-}
-
-inline Fold *fold_for_line(std::vector<Fold> &folds, uint32_t line) {
-  const auto *fold =
-      fold_for_line(static_cast<const std::vector<Fold> &>(folds), line);
-  return const_cast<Fold *>(fold);
-}
-
-inline bool line_is_fold_start(const std::vector<Fold> &folds, uint32_t line) {
-  const Fold *fold = fold_for_line(folds, line);
-  return fold && fold->start == line;
-}
-
-inline bool line_is_folded(const std::vector<Fold> &folds, uint32_t line) {
-  return fold_for_line(folds, line) != nullptr;
-}
-
-inline uint32_t next_unfolded_row(const Editor *editor, uint32_t row) {
-  uint32_t limit = editor && editor->root ? editor->root->line_count : 0;
-  while (row < limit) {
-    const Fold *fold = fold_for_line(editor->folds, row);
-    if (!fold)
-      return row;
-    row = fold->end + 1;
-  }
-  return limit;
-}
-
-inline uint32_t prev_unfolded_row(const Editor *editor, uint32_t row) {
-  while (row > 0) {
-    const Fold *fold = fold_for_line(editor->folds, row);
-    if (!fold)
-      return row;
-    if (fold->start == 0)
-      return 0;
-    row = fold->start - 1;
-  }
-  return 0;
-}
-
-void apply_edit(std::vector<Span> &spans, uint32_t x, int64_t y);
-void apply_hook_insertion(Editor *editor, uint32_t line, uint32_t rows);
-void apply_hook_deletion(Editor *editor, uint32_t removal_start,
-                         uint32_t removal_end);
-Editor *new_editor(const char *filename_arg, Coord position, Coord size);
-void save_file(Editor *editor);
-void free_editor(Editor *editor);
-void render_editor(Editor *editor);
-void fold(Editor *editor, uint32_t start_line, uint32_t end_line);
-void cursor_up(Editor *editor, uint32_t number);
-void cursor_down(Editor *editor, uint32_t number);
-Coord move_left(Editor *editor, Coord cursor, uint32_t number);
-Coord move_right(Editor *editor, Coord cursor, uint32_t number);
-Coord move_left_pure(Editor *editor, Coord cursor, uint32_t number);
-Coord move_right_pure(Editor *editor, Coord cursor, uint32_t number);
-void cursor_left(Editor *editor, uint32_t number);
-void cursor_right(Editor *editor, uint32_t number);
-void scroll_up(Editor *editor, int32_t number);
-void scroll_down(Editor *editor, uint32_t number);
-void ensure_cursor(Editor *editor);
-void indent_line(Editor *editor, uint32_t row);
-void dedent_line(Editor *editor, uint32_t row);
-void ensure_scroll(Editor *editor);
-void handle_editor_event(Editor *editor, KeyEvent event);
-void edit_erase(Editor *editor, Coord pos, int64_t len);
-void edit_insert(Editor *editor, Coord pos, char *data, uint32_t len);
-Coord editor_hit_test(Editor *editor, uint32_t x, uint32_t y);
-char *get_selection(Editor *editor, uint32_t *out_len, Coord *out_start);
-void editor_worker(Editor *editor);
-void move_line_down(Editor *editor);
-void move_line_up(Editor *editor);
-void word_boundaries(Editor *editor, Coord coord, uint32_t *prev_col,
-                     uint32_t *next_col, uint32_t *prev_clusters,
-                     uint32_t *next_clusters);
-void word_boundaries_exclusive(Editor *editor, Coord coord, uint32_t *prev_col,
-                               uint32_t *next_col);
-std::vector<Fold>::iterator find_fold_iter(Editor *editor, uint32_t line);
-bool add_fold(Editor *editor, uint32_t start, uint32_t end);
-bool remove_fold(Editor *editor, uint32_t line);
-void apply_line_insertion(Editor *editor, uint32_t line, uint32_t rows);
-void apply_line_deletion(Editor *editor, uint32_t removal_start,
-                         uint32_t removal_end);
-uint32_t leading_indent(const char *line, uint32_t len);
-uint32_t get_indent(Editor *editor, Coord cursor);
-bool closing_after_cursor(const char *line, uint32_t len, uint32_t col);
-void editor_lsp_handle(Editor *editor, json msg);
-
-#endif

include/editor/decl.h

@@ -0,0 +1,45 @@
+#ifndef EDITOR_DECL_H
+#define EDITOR_DECL_H
+
+#include "utils/utils.h"
+
+struct Fold {
+  uint32_t start;
+  uint32_t end;
+
+  bool contains(uint32_t line) const { return line >= start && line <= end; }
+  bool operator<(const Fold &other) const { return start < other.start; }
+};
+
+struct Span {
+  uint32_t start;
+  uint32_t end;
+  Highlight *hl;
+
+  bool operator<(const Span &other) const { return start < other.start; }
+};
+
+struct VWarn {
+  uint32_t line;
+  std::string text;
+  std::string text_full;
+  std::string source;
+  std::string code;
+  std::vector<std::string> see_also;
+  int8_t type;
+  uint32_t start;
+  uint32_t end{UINT32_MAX};
+
+  bool operator<(const VWarn &other) const { return line < other.line; }
+};
+
+struct VAI {
+  Coord pos;
+  char *text;
+  uint32_t len;
+  uint32_t lines; // number of \n in text for speed .. the ai part will not
+                  // line wrap but multiline ones need to have its own lines
+                  // after the first one
+};
+
+#endif

include/editor/editor.h

@@ -0,0 +1,107 @@
+#ifndef EDITOR_H
+#define EDITOR_H
+
+#include "boxes/diagnostics.h"
+#include "boxes/hover.h"
+#include "editor/spans.h"
+#include "io/knot.h"
+#include "io/ui.h"
+#include "ts/decl.h"
+#include "utils/utils.h"
+
+#define CHAR 0
+#define WORD 1
+#define LINE 2
+
+#define EXTRA_META 4
+#define INDENT_WIDTH 2
+
+struct Editor {
+  std::string filename;
+  std::string uri;
+  Knot *root;
+  std::shared_mutex knot_mtx;
+  Coord cursor;
+  uint32_t cursor_preffered;
+  Coord selection;
+  bool selection_active;
+  int selection_type;
+  Coord position;
+  Coord size;
+  Coord scroll;
+  TSSetMain ts;
+  Queue<TSInputEdit> edit_queue;
+  std::vector<Fold> folds;
+  Spans spans;
+  // TODO: Split into 2 groups to have their own mutex's . one for word hl and
+  //       one for hex colors
+  Spans def_spans;
+  uint32_t hooks[94];
+  bool jumper_set;
+  std::shared_mutex v_mtx;
+  std::vector<VWarn> warnings;
+  bool warnings_dirty;
+  VAI ai;
+  std::shared_mutex lsp_mtx;
+  std::shared_ptr<struct LSPInstance> lsp;
+  bool hover_active;
+  HoverBox hover;
+  bool diagnostics_active;
+  DiagnosticBox diagnostics;
+  int lsp_version = 1;
+};
+
+Editor *new_editor(const char *filename_arg, Coord position, Coord size);
+void save_file(Editor *editor);
+void free_editor(Editor *editor);
+void render_editor(Editor *editor);
+void fold(Editor *editor, uint32_t start_line, uint32_t end_line);
+void cursor_up(Editor *editor, uint32_t number);
+void cursor_down(Editor *editor, uint32_t number);
+Coord move_left(Editor *editor, Coord cursor, uint32_t number);
+Coord move_right(Editor *editor, Coord cursor, uint32_t number);
+Coord move_left_pure(Editor *editor, Coord cursor, uint32_t number);
+Coord move_right_pure(Editor *editor, Coord cursor, uint32_t number);
+void cursor_left(Editor *editor, uint32_t number);
+void cursor_right(Editor *editor, uint32_t number);
+void scroll_up(Editor *editor, int32_t number);
+void scroll_down(Editor *editor, uint32_t number);
+void ensure_cursor(Editor *editor);
+void indent_line(Editor *editor, uint32_t row);
+void dedent_line(Editor *editor, uint32_t row);
+void ensure_scroll(Editor *editor);
+void handle_editor_event(Editor *editor, KeyEvent event);
+void edit_erase(Editor *editor, Coord pos, int64_t len);
+void edit_insert(Editor *editor, Coord pos, char *data, uint32_t len);
+Coord editor_hit_test(Editor *editor, uint32_t x, uint32_t y);
+char *get_selection(Editor *editor, uint32_t *out_len, Coord *out_start);
+void editor_worker(Editor *editor);
+void move_line_down(Editor *editor);
+void move_line_up(Editor *editor);
+void word_boundaries(Editor *editor, Coord coord, uint32_t *prev_col,
+                     uint32_t *next_col, uint32_t *prev_clusters,
+                     uint32_t *next_clusters);
+void word_boundaries_exclusive(Editor *editor, Coord coord, uint32_t *prev_col,
+                               uint32_t *next_col);
+std::vector<Fold>::iterator find_fold_iter(Editor *editor, uint32_t line);
+bool add_fold(Editor *editor, uint32_t start, uint32_t end);
+bool remove_fold(Editor *editor, uint32_t line);
+uint32_t leading_indent(const char *line, uint32_t len);
+uint32_t get_indent(Editor *editor, Coord cursor);
+bool closing_after_cursor(const char *line, uint32_t len, uint32_t col);
+void editor_lsp_handle(Editor *editor, json msg);
+
+inline void apply_hook_insertion(Editor *editor, uint32_t line, uint32_t rows) {
+  for (auto &hook : editor->hooks)
+    if (hook > line)
+      hook += rows;
+}
+
+inline void apply_hook_deletion(Editor *editor, uint32_t removal_start,
+                                uint32_t removal_end) {
+  for (auto &hook : editor->hooks)
+    if (hook > removal_start)
+      hook -= removal_end - removal_start + 1;
+}
+
+#endif

include/editor/folds.h

@@ -0,0 +1,150 @@
+#ifndef EDITOR_FOLDS_H
+#define EDITOR_FOLDS_H
+
+#include "editor/editor.h"
+
+inline std::vector<Fold>::iterator find_fold_iter(Editor *editor,
+                                                  uint32_t line) {
+  auto &folds = editor->folds;
+  auto it = std::lower_bound(
+      folds.begin(), folds.end(), line,
+      [](const Fold &fold, uint32_t value) { return fold.start < value; });
+  if (it != folds.end() && it->start == line)
+    return it;
+  if (it != folds.begin()) {
+    --it;
+    if (it->contains(line))
+      return it;
+  }
+  return folds.end();
+}
+
+inline bool add_fold(Editor *editor, uint32_t start, uint32_t end) {
+  if (!editor || !editor->root)
+    return false;
+  if (start > end)
+    std::swap(start, end);
+  if (start >= editor->root->line_count)
+    return false;
+  end = std::min(end, editor->root->line_count - 1);
+  if (start == end)
+    return false;
+  Fold new_fold{start, end};
+  auto &folds = editor->folds;
+  auto it = std::lower_bound(
+      folds.begin(), folds.end(), new_fold.start,
+      [](const Fold &fold, uint32_t value) { return fold.start < value; });
+  if (it != folds.begin()) {
+    auto prev = std::prev(it);
+    if (prev->end + 1 >= new_fold.start) {
+      new_fold.start = std::min(new_fold.start, prev->start);
+      new_fold.end = std::max(new_fold.end, prev->end);
+      it = folds.erase(prev);
+    }
+  }
+  while (it != folds.end() && it->start <= new_fold.end + 1) {
+    new_fold.end = std::max(new_fold.end, it->end);
+    it = folds.erase(it);
+  }
+  folds.insert(it, new_fold);
+  return true;
+}
+
+inline bool remove_fold(Editor *editor, uint32_t line) {
+  auto it = find_fold_iter(editor, line);
+  if (it == editor->folds.end())
+    return false;
+  editor->folds.erase(it);
+  return true;
+}
+
+inline void apply_line_insertion(Editor *editor, uint32_t line, uint32_t rows) {
+  for (auto it = editor->folds.begin(); it != editor->folds.end();) {
+    if (line <= it->start) {
+      it->start += rows;
+      it->end += rows;
+      ++it;
+    } else if (line <= it->end) {
+      it = editor->folds.erase(it);
+    } else {
+      ++it;
+    }
+  }
+}
+
+inline void apply_line_deletion(Editor *editor, uint32_t removal_start,
+                                uint32_t removal_end) {
+  if (removal_start > removal_end)
+    return;
+  uint32_t rows_removed = removal_end - removal_start + 1;
+  std::vector<Fold> updated;
+  updated.reserve(editor->folds.size());
+  for (auto fold : editor->folds) {
+    if (removal_end < fold.start) {
+      fold.start -= rows_removed;
+      fold.end -= rows_removed;
+      updated.push_back(fold);
+      continue;
+    }
+    if (removal_start > fold.end) {
+      updated.push_back(fold);
+      continue;
+    }
+  }
+  editor->folds.swap(updated);
+}
+
+inline const Fold *fold_for_line(const std::vector<Fold> &folds,
+                                 uint32_t line) {
+  auto it = std::lower_bound(
+      folds.begin(), folds.end(), line,
+      [](const Fold &fold, uint32_t value) { return fold.start < value; });
+  if (it != folds.end() && it->start == line)
+    return &(*it);
+  if (it != folds.begin()) {
+    --it;
+    if (it->contains(line))
+      return &(*it);
+  }
+  return nullptr;
+}
+
+inline Fold *fold_for_line(std::vector<Fold> &folds, uint32_t line) {
+  const auto *fold =
+      fold_for_line(static_cast<const std::vector<Fold> &>(folds), line);
+  return const_cast<Fold *>(fold);
+}
+
+inline bool line_is_fold_start(const std::vector<Fold> &folds, uint32_t line) {
+  const Fold *fold = fold_for_line(folds, line);
+  return fold && fold->start == line;
+}
+
+inline bool line_is_folded(const std::vector<Fold> &folds, uint32_t line) {
+  return fold_for_line(folds, line) != nullptr;
+}
+
+inline uint32_t next_unfolded_row(const Editor *editor, uint32_t row) {
+  uint32_t limit = editor && editor->root ? editor->root->line_count : 0;
+  while (row < limit) {
+    const Fold *fold = fold_for_line(editor->folds, row);
+    if (!fold)
+      return row;
+    row = fold->end + 1;
+  }
+  return limit;
+}
+
+inline uint32_t prev_unfolded_row(const Editor *editor, uint32_t row) {
+  while (row > 0) {
+    const Fold *fold = fold_for_line(editor->folds, row);
+    if (!fold)
+      return row;
+    if (fold->start == 0)
+      return 0;
+    row = fold->start - 1;
+  }
+  return 0;
+}
+
+#endif

include/editor/spans.h

@@ -0,0 +1,85 @@
+#ifndef EDITOR_SPANS_H
+#define EDITOR_SPANS_H
+
+#include "editor/decl.h"
+#include "utils/utils.h"
+
+struct Spans {
+  std::vector<Span> spans;
+  Queue<std::pair<uint32_t, int64_t>> edits;
+  bool mid_parse = false;
+  std::shared_mutex mtx;
+};
+
+struct SpanCursor {
+  Spans &spans;
+  size_t index = 0;
+  std::vector<Span *> active;
+  std::shared_lock<std::shared_mutex> lock;
+
+  SpanCursor(Spans &s) : spans(s) {}
+  Highlight *get_highlight(uint32_t byte_offset) {
+    for (int i = (int)active.size() - 1; i >= 0; i--)
+      if (active[i]->end <= byte_offset)
+        active.erase(active.begin() + i);
+    while (index < spans.spans.size() &&
+           spans.spans[index].start <= byte_offset) {
+      if (spans.spans[index].end > byte_offset)
+        active.push_back(const_cast<Span *>(&spans.spans[index]));
+      index++;
+    }
+    Highlight *best = nullptr;
+    int max_prio = -1;
+    for (auto *s : active)
+      if (s->hl->priority > max_prio) {
+        max_prio = s->hl->priority;
+        best = s->hl;
+      }
+    return best;
+  }
+  void sync(uint32_t byte_offset) {
+    lock = std::shared_lock(spans.mtx);
+    active.clear();
+    size_t left = 0, right = spans.spans.size();
+    while (left < right) {
+      size_t mid = (left + right) / 2;
+      if (spans.spans[mid].start <= byte_offset)
+        left = mid + 1;
+      else
+        right = mid;
+    }
+    index = left;
+    while (left > 0) {
+      left--;
+      if (spans.spans[left].end > byte_offset)
+        active.push_back(const_cast<Span *>(&spans.spans[left]));
+      else if (byte_offset - spans.spans[left].end > 1000)
+        break;
+    }
+  }
+};
+
+inline void apply_edit(std::vector<Span> &spans, uint32_t x, int64_t y) {
+  Span key{.start = x, .end = 0, .hl = nullptr};
+  auto it = std::lower_bound(
+      spans.begin(), spans.end(), key,
+      [](const Span &a, const Span &b) { return a.start < b.start; });
+  size_t idx = std::distance(spans.begin(), it);
+  while (idx > 0 && spans.at(idx - 1).end >= x)
+    --idx;
+  for (size_t i = idx; i < spans.size();) {
+    Span &s = spans.at(i);
+    if (s.start < x && s.end >= x) {
+      s.end += y;
+    } else if (s.start > x) {
+      s.start += y;
+      s.end += y;
+    }
+    if (s.end <= s.start)
+      spans.erase(spans.begin() + i);
+    else
+      ++i;
+  }
+}
+
+#endif

include/io/knot.h

@@ -1,13 +1,11 @@
 #ifndef ROPE_H
 #define ROPE_H
 
-#include "./pch.h"
-#include "./utils.h"
+#include "pch.h"
+#include "utils/utils.h"
 
 #define MIN_CHUNK_SIZE 64       // 64 Bytes
 #define MAX_CHUNK_SIZE 1024 * 8 // 8192 Bytes (8 KiB)
-#define MAX(a, b) ((a) > (b) ? (a) : (b))
-#define MIN(a, b) ((a) < (b) ? (a) : (b))
 #define DEPTH(n) ((n) ? (n)->depth : 0)
 
 // Rope node definition

include/io/ui.h

@@ -1,8 +1,8 @@
 #ifndef UI_H
 #define UI_H
 
-#include "./pch.h"
-#include "./utils.h"
+#include "pch.h"
+#include "utils/utils.h"
 
 #define KEY_CHAR 0
 #define KEY_SPECIAL 1
@@ -48,14 +48,16 @@ enum CellFlags : uint8_t {
   CF_NONE = 0,
   CF_ITALIC = 1 << 0,
   CF_BOLD = 1 << 1,
-  CF_UNDERLINE = 1 << 2,
+  CF_UNDERLINE = 1 << 2
 };
 
 struct ScreenCell {
   std::string utf8 = std::string("");
+  uint8_t width = 1;
   uint32_t fg = 0;
   uint32_t bg = 0;
   uint8_t flags = CF_NONE;
+  uint32_t ul_color = 0;
 };
 
 struct KeyEvent {
@@ -86,6 +88,10 @@ void update(uint32_t row, uint32_t col, std::string utf8, uint32_t fg,
             uint32_t bg, uint8_t flags);
 void update(uint32_t row, uint32_t col, const char *utf8, uint32_t fg,
             uint32_t bg, uint8_t flags);
+void update(uint32_t row, uint32_t col, std::string utf8, uint32_t fg,
+            uint32_t bg, uint8_t flags, uint32_t ul_color);
+void update(uint32_t row, uint32_t col, const char *utf8, uint32_t fg,
+            uint32_t bg, uint8_t flags, uint32_t ul_color);
 void set_cursor(int row, int col, int type, bool show_cursor_param);
 void render();
 Coord get_size();

include/lsp.h

@@ -1,55 +0,0 @@
-#ifndef LSP_H
-#define LSP_H
-
-#include "./editor.h"
-#include "./pch.h"
-#include "utils.h"
-
-struct LSP {
-  const char *command;
-  std::vector<const char *> args;
-};
-
-struct LSPPending {
-  std::string method;
-  Editor *editor = nullptr;
-
-  std::function<void(Editor *, std::string, json)> callback;
-};
-
-struct LSPOpenRequest {
-  Language language;
-  Editor *editor;
-};
-
-struct LSPInstance {
-  std::shared_mutex mtx;
-  const LSP *lsp;
-  std::string root_dir;
-  int pid{-1};
-  int stdin_fd{-1};
-  int stdout_fd{-1};
-  bool initialized = false;
-  uint32_t last_id = 0;
-  Queue<json> inbox;
-  Queue<json> outbox;
-  std::unordered_map<uint32_t, LSPPending *> pending;
-  std::vector<Editor *> editors;
-};
-
-extern std::shared_mutex active_lsps_mtx;
-extern std::unordered_map<uint8_t, LSPInstance *> active_lsps;
-
-void lsp_worker();
-void lsp_handle(LSPInstance *lsp, json message);
-
-LSPInstance *get_or_init_lsp(uint8_t lsp_id);
-void close_lsp(uint8_t lsp_id);
-
-void request_add_to_lsp(Language language, Editor *editor);
-void add_to_lsp(Language language, Editor *editor);
-void remove_from_lsp(Editor *editor);
-
-void lsp_send(LSPInstance *lsp, json message, LSPPending *pending);
-
-#endif

include/lsp/lsp.h

@@ -0,0 +1,81 @@
+#ifndef LSP_H
+#define LSP_H
+
+#include "editor/editor.h"
+#include "pch.h"
+#include "utils/utils.h"
+
+struct LSP {
+  const char *command;
+  std::vector<const char *> args;
+};
+
+struct LSPPending {
+  std::string method;
+  Editor *editor = nullptr;
+
+  std::function<void(Editor *, std::string, json)> callback;
+};
+
+struct LSPOpenRequest {
+  Language language;
+  Editor *editor;
+};
+
+struct LSPInstance {
+  std::shared_mutex mtx;
+  const LSP *lsp;
+  std::string root_dir;
+  int pid{-1};
+  int stdin_fd{-1};
+  int stdout_fd{-1};
+  std::atomic<bool> initialized = false;
+  std::atomic<bool> exited = false;
+  bool incremental_sync = false;
+  bool allow_hover = false;
+  bool allow_completion = false;
+  std::string trigger_chars;
+  uint32_t last_id = 0;
+  Queue<json> inbox;
+  Queue<json> outbox;
+  Queue<std::pair<Language, Editor *>> open_queue;
+  std::unordered_map<uint32_t, LSPPending *> pending;
+  std::vector<Editor *> editors;
+};
+
+extern std::shared_mutex active_lsps_mtx;
+extern std::unordered_map<uint8_t, std::shared_ptr<LSPInstance>> active_lsps;
+extern Queue<LSPOpenRequest> lsp_open_queue;
+
+static json client_capabilities = {
+    {"textDocument",
+     {{"publishDiagnostics", {{"relatedInformation", true}}},
+      {"hover", {{"contentFormat", {"markdown", "plaintext"}}}},
+      {"completion",
+       {{"completionItem",
+         {{"snippetSupport", true},
+          {"documentationFormat", {"markdown", "plaintext"}},
+          {"resolveSupport", {{"properties", {"documentation", "detail"}}}},
+          {"insertReplaceSupport", true},
+          {"labelDetailsSupport", true},
+          {"insertTextModeSupport", {{"valueSet", {1}}}}}},
+        {"completionItemKind", {{"valueSet", {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}}}},
+        {"contextSupport", true},
+        {"insertTextMode", 1}}}}}};
+
+void lsp_send(std::shared_ptr<LSPInstance> lsp, json message,
+              LSPPending *pending);
+void lsp_worker();
+
+std::shared_ptr<LSPInstance> get_or_init_lsp(uint8_t lsp_id);
+void clean_lsp(std::shared_ptr<LSPInstance> lsp, uint8_t lsp_id);
+void close_lsp(uint8_t lsp_id);
+
+void open_editor(std::shared_ptr<LSPInstance> lsp,
+                 std::pair<Language, Editor *> entry);
+void request_add_to_lsp(Language language, Editor *editor);
+void add_to_lsp(Language language, Editor *editor);
+void remove_from_lsp(Editor *editor);
+void lsp_handle(std::shared_ptr<LSPInstance> lsp, json message);
+
+#endif

include/main.h

@@ -1,7 +1,7 @@
 #ifndef MAIN_H
 #define MAIN_H
 
-#include "./pch.h"
+#include "pch.h"
 
 #define NORMAL 0
 #define INSERT 1

include/maps.h

@@ -1,139 +0,0 @@
-#ifndef MAPS_H
-#define MAPS_H
-
-#include "./lsp.h"
-#include "./pch.h"
-#include "./ts_def.h"
-
-static const std::unordered_map<uint8_t, LSP> kLsps = {
-    {1,
-     {"clangd",
-      {
-          "clangd",
-          "--background-index",
-          "--clang-tidy",
-          "--completion-style=detailed",
-          "--header-insertion=never",
-          "--pch-storage=memory",
-          "--limit-results=50",
-          "--log=error",
-          nullptr,
-      }}},
-};
-
-static const std::unordered_map<std::string, Language> kLanguages = {
-    {"bash", {"bash", LANG(bash)}},
-    {"c", {"c", LANG(cpp), 1}},
-    {"cpp", {"cpp", LANG(cpp), 1}},
-    {"h", {"h", LANG(cpp), 1}},
-    {"css", {"css", LANG(css)}},
-    {"fish", {"fish", LANG(fish)}},
-    {"go", {"go", LANG(go)}},
-    {"haskell", {"haskell", LANG(haskell)}},
-    {"html", {"html", LANG(html)}},
-    {"javascript", {"javascript", LANG(javascript)}},
-    {"json", {"json", LANG(json)}},
-    {"lua", {"lua", LANG(lua)}},
-    {"make", {"make", LANG(make)}},
-    {"python", {"python", LANG(python)}},
-    {"ruby", {"ruby", LANG(ruby)}},
-    {"rust", {"rust", LANG(rust)}},
-    {"diff", {"diff", LANG(diff)}},
-    {"embedded_template", {"embedded_template", LANG(embedded_template)}},
-    {"gdscript", {"gdscript", LANG(gdscript)}},
-    {"gitattributes", {"gitattributes", LANG(gitattributes)}},
-    {"gitignore", {"gitignore", LANG(gitignore)}},
-    {"gomod", {"gomod", LANG(gomod)}},
-    {"ini", {"ini", LANG(ini)}},
-    {"markdown", {"markdown", LANG(markdown)}},
-    {"markdown_inline", {"markdown_inline", LANG(markdown_inline)}},
-    {"nginx", {"nginx", LANG(nginx)}},
-    {"php", {"php", LANG(php)}},
-    {"query", {"query", LANG(query)}},
-    {"regex", {"regex", LANG(regex)}},
-    {"sql", {"sql", LANG(sql)}},
-    {"toml", {"toml", LANG(toml)}},
-    {"yaml", {"yaml", LANG(yaml)}},
-};
-
-static const std::unordered_map<std::string, std::string> kExtToLang = {
-    {"sh", "bash"},
-    {"bash", "bash"},
-    {"c", "c"},
-    {"cpp", "cpp"},
-    {"cxx", "cpp"},
-    {"cc", "cpp"},
-    {"hpp", "h"},
-    {"hh", "h"},
-    {"hxx", "h"},
-    {"h", "h"},
-    {"css", "css"},
-    {"fish", "fish"},
-    {"go", "go"},
-    {"hs", "haskell"},
-    {"html", "html"},
-    {"htm", "html"},
-    {"js", "javascript"},
-    {"jsx", "javascript"},
-    {"json", "json"},
-    {"jsonc", "json"},
-    {"lua", "lua"},
-    {"mk", "make"},
-    {"makefile", "make"},
-    {"py", "python"},
-    {"rb", "ruby"},
-    {"rs", "rust"},
-    {"diff", "diff"},
-    {"patch", "diff"},
-    {"erb", "embedded_template"},
-    {"etlua", "embedded_template"},
-    {"gd", "gdscript"},
-    {"gitattributes", "gitattributes"},
-    {"gitignore", "gitignore"},
-    {"mod", "gomod"},
-    {"ini", "ini"},
-    {"gitmodules", "ini"},
-    {"md", "markdown"},
-    {"markdown", "markdown"},
-    {"conf", "nginx"},
-    {"php", "php"},
-    {"scm", "query"},
-    {"regex", "regex"},
-    {"sql", "sql"},
-    {"toml", "toml"},
-    {"yaml", "yaml"},
-    {"yml", "yaml"},
-};
-
-static const std::unordered_map<std::string, std::string> kMimeToLang = {
-    {"text/x-c", "c"},
-    {"text/x-c++", "cpp"},
-    {"text/x-shellscript", "bash"},
-    {"application/json", "json"},
-    {"text/javascript", "javascript"},
-    {"text/html", "html"},
-    {"text/css", "css"},
-    {"text/x-python", "python"},
-    {"text/x-ruby", "ruby"},
-    {"text/x-go", "go"},
-    {"text/x-haskell", "haskell"},
-    {"text/x-rust", "rust"},
-    {"text/x-lua", "lua"},
-    {"text/x-diff", "diff"},
-    {"text/x-embedded-template", "embedded_template"},
-    {"text/x-gdscript", "gdscript"},
-    {"text/x-gitattributes", "gitattributes"},
-    {"text/x-gitignore", "gitignore"},
-    {"text/x-gomod", "gomod"},
-    {"text/x-ini", "ini"},
-    {"text/markdown", "markdown"},
-    {"text/x-nginx-conf", "nginx"},
-    {"application/x-php", "php"},
-    {"text/x-tree-sitter-query", "query"},
-    {"text/x-regex", "regex"},
-    {"text/x-sql", "sql"},
-    {"text/x-toml", "toml"},
-    {"text/x-yaml", "yaml"},
-};
-
-#endif

include/pch.h

@@ -4,17 +4,23 @@
 #define PCRE2_CODE_UNIT_WIDTH 8
 #define PCRE_WORKSPACE_SIZE 512
 
-#include "../libs/tree-sitter/lib/include/tree_sitter/api.h"
+extern "C" {
+#include "libgrapheme/grapheme.h"
+#include "unicode_width/unicode_width.h"
+}
+#include "tree-sitter/lib/include/tree_sitter/api.h"
 #include <algorithm>
 #include <atomic>
 #include <cctype>
 #include <chrono>
+#include <cmath>
 #include <cstdarg>
 #include <cstdint>
 #include <cstdio>
 #include <cstdlib>
 #include <cstring>
 #include <deque>
+#include <fcntl.h>
 #include <filesystem>
 #include <fstream>
 #include <functional>
@@ -27,9 +33,13 @@
 #include <pcre2.h>
 #include <queue>
 #include <shared_mutex>
+#include <signal.h>
 #include <string.h>
 #include <string>
 #include <sys/ioctl.h>
+#include <sys/poll.h>
+#include <sys/types.h>
+#include <sys/wait.h>
 #include <termios.h>
 #include <thread>
 #include <unistd.h>

include/ts/decl.h

@@ -1,7 +1,7 @@
-#ifndef TS_DEF_H
-#define TS_DEF_H
+#ifndef TS_DECL_H
+#define TS_DECL_H
 
-#include "./pch.h"
+#include "pch.h"
 
 #define LANG(name) tree_sitter_##name
 #define TS_DEF(name) extern "C" const TSLanguage *LANG(name)()
@@ -12,6 +12,32 @@ struct Language {
   uint8_t lsp_id = 0;
 };
 
+struct Highlight {
+  uint32_t fg;
+  uint32_t bg;
+  uint32_t flags;
+  uint8_t priority;
+};
+
+struct TSSetBase {
+  std::string lang;
+  TSParser *parser;
+  std::string query_file;
+  TSQuery *query;
+  TSTree *tree;
+  std::map<uint16_t, Highlight> query_map;
+  std::map<uint16_t, Language> injection_map;
+  const TSLanguage *language;
+};
+
+struct TSSet : TSSetBase {
+  std::vector<TSRange> ranges;
+};
+
+struct TSSetMain : TSSetBase {
+  std::unordered_map<std::string, TSSet> injections;
+};
+
 TS_DEF(ruby);
 TS_DEF(bash);
 TS_DEF(cpp);
@@ -21,6 +47,8 @@ TS_DEF(go);
 TS_DEF(haskell);
 TS_DEF(html);
 TS_DEF(javascript);
+TS_DEF(tsx);
+TS_DEF(man);
 TS_DEF(json);
 TS_DEF(lua);
 TS_DEF(regex);

include/ts/ts.h

@@ -1,9 +1,9 @@
 #ifndef TS_H
 #define TS_H
 
-#include "./editor.h"
-#include "./pch.h"
-#include "./utils.h"
+#include "editor/editor.h"
+#include "pch.h"
+#include "utils/utils.h"
 
 #define HEX(s) (static_cast<uint32_t>(std::stoul(s, nullptr, 16)))
 
@@ -11,6 +11,8 @@ extern std::unordered_map<std::string, pcre2_code *> regex_cache;
 
 TSQuery *load_query(const char *query_path, TSSetBase *set);
 void ts_collect_spans(Editor *editor);
+bool ts_predicate(TSQuery *query, const TSQueryMatch &match,
+                  std::function<std::string(const TSNode *)> subject_fn);
 void clear_regex_cache();
 
 #endif

include/utils/utils.h

@@ -1,8 +1,8 @@
 #ifndef UTILS_H
 #define UTILS_H
 
-#include "./pch.h"
-#include "./ts_def.h"
+#include "pch.h"
+#include "ts/decl.h"
 
 template <typename T> struct Queue {
   std::queue<T> q;
@@ -12,8 +12,9 @@ template <typename T> struct Queue {
     std::lock_guard<std::mutex> lock(m);
     q.push(val);
   }
-  T front() {
-    std::lock_guard<std::mutex> lock(m);
+  std::optional<T> front() {
+    if (q.empty())
+      return std::nullopt;
     return q.front();
   }
   bool pop(T &val) {
@@ -52,22 +53,47 @@ struct Coord {
   bool operator>=(const Coord &other) const { return !(*this < other); }
 };
 
-std::string path_abs(const std::string &path_str);
-std::string path_to_file_uri(const std::string &path_str);
+struct Match {
+  size_t start;
+  size_t end;
+  std::string text;
+};
+
+#define MAX(a, b) ((a) > (b) ? (a) : (b))
+#define MIN(a, b) ((a) < (b) ? (a) : (b))
+
+std::string clean_text(const std::string &input);
+std::string percent_encode(const std::string &s);
+std::string percent_decode(const std::string &s);
+uint32_t count_clusters(const char *line, size_t len, size_t from, size_t to);
+std::string trim(const std::string &s);
+
 int display_width(const char *str, size_t len);
 uint32_t get_visual_col_from_bytes(const char *line, uint32_t len,
                                    uint32_t byte_limit);
 uint32_t get_bytes_from_visual_col(const char *line, uint32_t len,
                                    uint32_t target_visual_col);
+int utf8_byte_offset_to_utf16(const char *s, size_t byte_pos);
+
 void log(const char *fmt, ...);
+
+std::string path_abs(const std::string &path_str);
+std::string path_to_file_uri(const std::string &path_str);
 std::string get_exe_dir();
 char *load_file(const char *path, uint32_t *out_len);
 char *detect_file_type(const char *filename);
-int utf8_byte_offset_to_utf16(const char *s, size_t byte_pos);
 Language language_for_file(const char *filename);
+
 void copy_to_clipboard(const char *text, size_t len);
 char *get_from_clipboard(uint32_t *out_len);
-uint32_t count_clusters(const char *line, size_t len, size_t from, size_t to);
+
+template <typename T>
+inline T *safe_get(std::map<uint16_t, T> &m, uint16_t key) {
+  auto it = m.find(key);
+  if (it == m.end())
+    return nullptr;
+  return &it->second;
+}
 
 template <typename Func, typename... Args>
 auto throttle(std::chrono::milliseconds min_duration, Func &&func,

samples/lua.lua

@@ -13,7 +13,7 @@ local name = "Lua"
 print(self)
 
 -- Functions
-function greet(user)
+local function greet(user)
 	print("Hello, " .. user)
 end
 

samples/markdown.md

@@ -31,9 +31,10 @@ This is a paragraph with **bold text**, *italic text*, ~~strikethrough~~, and `i
 
 `Inline code` example and a fenced code block:
 
-```python
-def hello_world():
-    print("Hello, world!")
+```lua
+local s2 = [[Long
+multi-line
+string]]
 ```
 
 ![Image](https://example.com/image.jpg)

samples/ruby.rb

@@ -12,7 +12,7 @@ end
 # Emoji-heavy strings
 emojis = "👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏👋🌏"
 
-# Mixed-width CJK blocks
+# Mixed-width CJK block
 cjk_samples = [
   "漢字テスト",
   "測試中文字串",
@@ -173,6 +173,11 @@ end
 # Method definition
 def greet_person(name)
   puts "#{Utilities.random_greeting}, #{name}!"
+  if (name == "harry")
+    return true
+  else
+    return "s"
+  end
 end
 
 # Calling methods

samples/yaml.yaml

@@ -0,0 +1,66 @@
+# ============================================================
+# Basic types
+# ============================================================
+
+title: "Example YAML Configuration"
+enabled: true
+count: 42
+pi: 3.14159
+empty: ""
+
+# ============================================================
+# Arrays / Lists
+# ============================================================
+fruits:
+  - apple
+  - banana
+  - cherry
+
+numbers:
+  - 1
+  - 2
+  - 3
+  - 4
+  - 5
+
+matrix:
+  - [1, 2]
+  - [3, 4]
+
+# ============================================================
+# Nested objects / maps
+# ============================================================
+owner:
+  name: Alice
+  dob: 1979-05-27T07:32:00Z
+
+database:
+  server: 192.168.1.1
+  ports:
+    - 8001
+    - 8001
+    - 8002
+  connection_max: 5000
+  enabled: true
+
+servers:
+  alpha:
+    ip: 10.0.0.1
+    dc: east
+  beta:
+    ip: 10.0.0.2
+    dc: west
+
+# ============================================================
+# Multiline string
+# ============================================================
+description: |
+  This is a YAML file
+  used for testing syntax highlighting.
+  It supports multiple lines.
+
+# ============================================================
+# Special characters
+# ============================================================
+regex_pattern: "^[A-Za-z0-9_]+$"
+path: "C:\\Users\\Alice\\Documents"

src/boxes/diagnostics.cc

@@ -0,0 +1,150 @@
+#include "boxes/diagnostics.h"
+
+void DiagnosticBox::clear() {
+  warnings.clear();
+  cells.clear();
+  box_width = 0;
+  box_height = 0;
+}
+
+void DiagnosticBox::render_first() {
+  if (warnings.empty())
+    return;
+  uint32_t longest_line = 8 + warnings[0].source.length();
+  for (auto &warn : warnings) {
+    longest_line = MAX(longest_line, (uint32_t)warn.text.length() + 7);
+    longest_line = MAX(longest_line, (uint32_t)warn.code.length() + 4);
+    for (auto &see_also : warn.see_also)
+      longest_line = MAX(longest_line, (uint32_t)see_also.length() + 4);
+  }
+  uint32_t content_width = MIN(longest_line, 150u);
+  box_width = content_width + 2;
+  cells.assign(box_width * 25, {" ", 0, 0, 0, 0, 0});
+  auto set = [&](uint32_t r, uint32_t c, const char *text, uint32_t fg,
+                 uint32_t bg, uint8_t flags) {
+    cells[r * box_width + c] = {std::string(text), 0, fg, bg, flags, 0};
+  };
+  uint32_t base_bg = 0;
+  uint32_t border_fg = 0x82AAFF;
+  uint32_t r = 0;
+  if (warnings[0].source != "") {
+    std::string src_txt = "Source: ";
+    for (uint32_t i = 0; i < src_txt.length() && i < content_width; i++)
+      set(1, i + 1, (char[2]){src_txt[i], 0}, 0x3EAAFF, base_bg, 0);
+    for (uint32_t i = 0; i < warnings[0].source.length() && i < content_width;
+         i++)
+      set(1, i + 1 + src_txt.length(), (char[2]){warnings[0].source[i], 0},
+          0xffffff, base_bg, 0);
+    r++;
+  }
+  int idx = 1;
+  for (auto &warn : warnings) {
+    char buf[4];
+    std::snprintf(buf, sizeof(buf), "%2d", idx % 100);
+    std::string line_txt = std::string(buf) + ". ";
+    for (uint32_t i = 0; i < line_txt.length(); i++)
+      set(r + 1, i + 1, (char[2]){line_txt[i], 0}, 0xffffff, base_bg, 0);
+    if (r >= 23)
+      break;
+    const char *err_sym = "";
+    uint32_t c_sym = 0xAAAAAA;
+    switch (warn.type) {
+    case 1:
+      err_sym = "";
+      c_sym = 0xFF0000;
+      break;
+    case 2:
+      err_sym = "";
+      c_sym = 0xFFFF00;
+      break;
+    case 3:
+      err_sym = "";
+      c_sym = 0xFF00FF;
+      break;
+    case 4:
+      err_sym = "";
+      c_sym = 0xAAAAAA;
+      break;
+    }
+    std::string text = warn.text_full + " " + err_sym;
+    uint32_t i = 0;
+    while (i < text.length() && r < 23) {
+      uint32_t c = 4;
+      while (c < content_width && i < text.length()) {
+        if (text[i] == '\n') {
+          while (i < text.length() && text[i] == '\n')
+            i++;
+          break;
+        }
+        uint32_t cluster_len = grapheme_next_character_break_utf8(
+            text.c_str() + i, text.length() - i);
+        std::string cluster = text.substr(i, cluster_len);
+        int width = display_width(cluster.c_str(), cluster_len);
+        if (c + width > content_width)
+          break;
+        set(r + 1, c + 1, cluster.c_str(), c_sym, base_bg, 0);
+        c += width;
+        i += cluster_len;
+        for (int w = 1; w < width; w++)
+          set(r + 1, c - w + 1, "\x1b", c_sym, base_bg, 0);
+      }
+      r++;
+    }
+    if (r >= 23)
+      break;
+    if (warn.code != "") {
+      for (uint32_t i = 0; i < warn.code.length() && i + 5 < content_width; i++)
+        set(r + 1, i + 5, (char[2]){warn.code[i], 0}, 0x81cdc6, base_bg, 0);
+      r++;
+    }
+    if (r >= 23)
+      break;
+    for (std::string &see_also : warn.see_also) {
+      uint32_t fg = 0xB55EFF;
+      uint8_t colon_count = 0;
+      for (uint32_t i = 0; i < see_also.length() && i + 5 < content_width;
+           i++) {
+        set(r + 1, i + 5, (char[2]){see_also[i], 0}, fg, base_bg, 0);
+        if (see_also[i] == ':')
+          colon_count++;
+        if (colon_count == 2)
+          fg = 0xFFFFFF;
+      }
+      r++;
+      if (r >= 23)
+        break;
+    };
+    idx++;
+  }
+  box_height = 2 + r;
+  set(0, 0, "┌", border_fg, base_bg, 0);
+  for (uint32_t i = 1; i < box_width - 1; i++)
+    set(0, i, "─", border_fg, base_bg, 0);
+  set(0, box_width - 1, "┐", border_fg, base_bg, 0);
+  for (uint32_t r = 1; r < box_height - 1; r++) {
+    set(r, 0, "│", border_fg, base_bg, 0);
+    set(r, box_width - 1, "│", border_fg, base_bg, 0);
+  }
+  set(box_height - 1, 0, "└", border_fg, base_bg, 0);
+  for (uint32_t i = 1; i < box_width - 1; i++)
+    set(box_height - 1, i, "─", border_fg, base_bg, 0);
+  set(box_height - 1, box_width - 1, "┘", border_fg, base_bg, 0);
+  cells.resize(box_width * box_height);
+}
+
+void DiagnosticBox::render(Coord pos) {
+  int32_t start_row = (int32_t)pos.row - (int32_t)box_height;
+  if (start_row < 0)
+    start_row = pos.row + 1;
+  int32_t start_col = pos.col;
+  if (start_col + box_width > cols) {
+    start_col = cols - box_width;
+    if (start_col < 0)
+      start_col = 0;
+  }
+  for (uint32_t r = 0; r < box_height; r++)
+    for (uint32_t c = 0; c < box_width; c++)
+      update(start_row + r, start_col + c, cells[r * box_width + c].utf8,
+             cells[r * box_width + c].fg, cells[r * box_width + c].bg,
+             cells[r * box_width + c].flags);
+}

src/boxes/hover.cc

@@ -0,0 +1,220 @@
+#include "boxes/hover.h"
+#include "ts/ts.h"
+
+void HoverBox::clear() {
+  text = "";
+  scroll_ = 0;
+  is_markup = false;
+  box_width = 0;
+  box_height = 0;
+  cells.clear();
+  highlights.clear();
+  hover_spans.clear();
+}
+
+void HoverBox::scroll(int32_t number) {
+  if (text.empty() || number == 0)
+    return;
+  uint32_t line_count = 0;
+  for (uint32_t i = 0; i < text.length(); i++)
+    if (text[i] == '\n')
+      line_count++;
+  scroll_ = MAX((int32_t)scroll_ + number, 0);
+  if (scroll_ > line_count)
+    scroll_ = line_count;
+  render_first(true);
+}
+
+void HoverBox::render_first(bool scroll) {
+  if (!scroll) {
+    std::vector<Span> base_spans;
+    std::vector<Span> injected_spans;
+    TSSetBase ts = TSSetBase{};
+    if (is_markup) {
+      highlights.reserve(1024);
+      base_spans.reserve(1024);
+      injected_spans.reserve(1024);
+      hover_spans.reserve(1024);
+      std::string query_path = get_exe_dir() + "/../grammar/hover.scm";
+      ts.language = LANG(markdown)();
+      ts.query = load_query(query_path.c_str(), &ts);
+      ts.parser = ts_parser_new();
+      ts_parser_set_language(ts.parser, ts.language);
+      ts.tree = ts_parser_parse_string(ts.parser, nullptr, text.c_str(),
+                                       text.length());
+      TSQueryCursor *cursor = ts_query_cursor_new();
+      ts_query_cursor_exec(cursor, ts.query, ts_tree_root_node(ts.tree));
+      TSQueryMatch match;
+      while (ts_query_cursor_next_match(cursor, &match)) {
+        auto subject_fn = [&](const TSNode *node) -> std::string {
+          uint32_t start = ts_node_start_byte(*node);
+          uint32_t end = ts_node_end_byte(*node);
+          return text.substr(start, end - start);
+        };
+        if (!ts_predicate(ts.query, match, subject_fn))
+          continue;
+        for (uint32_t i = 0; i < match.capture_count; i++) {
+          TSQueryCapture cap = match.captures[i];
+          uint32_t start = ts_node_start_byte(cap.node);
+          uint32_t end = ts_node_end_byte(cap.node);
+          if (Language *inj_lang = safe_get(ts.injection_map, cap.index)) {
+            TSSetBase inj_ts = TSSetBase{};
+            inj_ts.language = inj_lang->fn();
+            inj_ts.query_file =
+                get_exe_dir() + "/../grammar/" + inj_lang->name + ".scm";
+            inj_ts.query = load_query(inj_ts.query_file.c_str(), &inj_ts);
+            inj_ts.parser = ts_parser_new();
+            ts_parser_set_language(inj_ts.parser, inj_ts.language);
+            TSPoint start_p = ts_node_start_point(cap.node);
+            TSPoint end_p = ts_node_end_point(cap.node);
+            std::vector<TSRange> ranges = {{start_p, end_p, start, end}};
+            ts_parser_set_included_ranges(inj_ts.parser, ranges.data(), 1);
+            inj_ts.tree = ts_parser_parse_string(inj_ts.parser, nullptr,
+                                                 text.c_str(), text.length());
+            TSQueryCursor *inj_cursor = ts_query_cursor_new();
+            ts_query_cursor_exec(inj_cursor, inj_ts.query,
+                                 ts_tree_root_node(inj_ts.tree));
+            TSQueryMatch inj_match;
+            while (ts_query_cursor_next_match(inj_cursor, &inj_match)) {
+              auto subject_fn = [&](const TSNode *node) -> std::string {
+                uint32_t start = ts_node_start_byte(*node);
+                uint32_t end = ts_node_end_byte(*node);
+                return text.substr(start, end - start);
+              };
+              if (!ts_predicate(inj_ts.query, inj_match, subject_fn))
+                continue;
+              for (uint32_t i = 0; i < inj_match.capture_count; i++) {
+                TSQueryCapture inj_cap = inj_match.captures[i];
+                uint32_t start = ts_node_start_byte(inj_cap.node);
+                uint32_t end = ts_node_end_byte(inj_cap.node);
+                if (Highlight *hl = safe_get(inj_ts.query_map, inj_cap.index)) {
+                  highlights.push_back(*hl);
+                  Highlight *hl_f = &highlights.back();
+                  injected_spans.push_back({start, end, hl_f});
+                }
+              }
+            }
+            ts_query_cursor_delete(inj_cursor);
+            ts_tree_delete(inj_ts.tree);
+            ts_parser_delete(inj_ts.parser);
+            ts_query_delete(inj_ts.query);
+            continue;
+          }
+          if (Highlight *hl = safe_get(ts.query_map, cap.index)) {
+            highlights.push_back(*hl);
+            Highlight *hl_f = &highlights.back();
+            base_spans.push_back({start, end, hl_f});
+          }
+        }
+      }
+      ts_query_cursor_delete(cursor);
+      ts_query_delete(ts.query);
+      ts_tree_delete(ts.tree);
+      ts_parser_delete(ts.parser);
+    }
+    for (const auto &inj : injected_spans) {
+      base_spans.erase(std::remove_if(base_spans.begin(), base_spans.end(),
+                                      [&](const Span &base) {
+                                        return !(base.end <= inj.start ||
+                                                 base.start >= inj.end);
+                                      }),
+                       base_spans.end());
+    }
+    hover_spans.insert(hover_spans.end(), base_spans.begin(), base_spans.end());
+    hover_spans.insert(hover_spans.end(), injected_spans.begin(),
+                       injected_spans.end());
+    std::sort(hover_spans.begin(), hover_spans.end());
+  }
+  uint32_t longest_line = 0;
+  uint32_t current_width = 0;
+  for (size_t j = 0; j < text.length(); j++) {
+    if (text[j] == '\n') {
+      longest_line = std::max(longest_line, current_width);
+      current_width = 0;
+    } else {
+      current_width += 1;
+    }
+  }
+  // HACK: the 1 is added so the longest line doesnt wrap which should be fixed
+  //       in the loop instead as it was never meant to wrap in the first place
+  longest_line = MAX(longest_line, current_width) + 1;
+  uint32_t content_width = MIN(longest_line, 130u);
+  box_width = content_width + 2;
+  size_t i = 0;
+  size_t lines_skipped = 0;
+  while (i < text.length() && lines_skipped < scroll_) {
+    if (text[i] == '\n')
+      lines_skipped++;
+    i++;
+  }
+  Spans spans{};
+  spans.spans = hover_spans;
+  uint32_t border_fg = 0x82AAFF;
+  uint32_t base_bg = 0;
+  SpanCursor span_cursor(spans);
+  span_cursor.sync(i);
+  cells.assign(box_width * 26, ScreenCell{" ", 0, 0, 0, 0, 0});
+  auto set = [&](uint32_t r, uint32_t c, const char *text, uint32_t fg,
+                 uint32_t bg, uint8_t flags) {
+    cells[r * box_width + c] = {std::string(text), 0, fg, bg, flags, 0};
+  };
+  uint32_t r = 0;
+  while (i < text.length() && r < 24) {
+    uint32_t c = 0;
+    while (c < content_width && i < text.length()) {
+      if (text[i] == '\n') {
+        while (i < text.length() && text[i] == '\n')
+          i++;
+        break;
+      }
+      uint32_t cluster_len = grapheme_next_character_break_utf8(
+          text.c_str() + i, text.length() - i);
+      std::string cluster = text.substr(i, cluster_len);
+      int width = display_width(cluster.c_str(), cluster_len);
+      if (c + width > content_width)
+        break;
+      Highlight *hl = span_cursor.get_highlight(i);
+      uint32_t fg = hl ? hl->fg : 0xFFFFFF;
+      uint32_t bg = hl ? hl->bg : 0;
+      uint32_t flags = hl ? hl->flags : 0;
+      set(r + 1, c + 1, cluster.c_str(), fg, bg | base_bg, flags);
+      c += width;
+      i += cluster_len;
+      for (int w = 1; w < width; w++)
+        set(r + 1, c - w + 1, "\x1b", 0xFFFFFF, base_bg, 0);
+    }
+    r++;
+  }
+  if (!scroll)
+    box_height = r + 2;
+  set(0, 0, "┌", border_fg, base_bg, 0);
+  for (uint32_t i = 1; i < box_width - 1; i++)
+    set(0, i, "─", border_fg, base_bg, 0);
+  set(0, box_width - 1, "┐", border_fg, base_bg, 0);
+  for (uint32_t r = 1; r < box_height - 1; r++) {
+    set(r, 0, "│", border_fg, base_bg, 0);
+    set(r, box_width - 1, "│", border_fg, base_bg, 0);
+  }
+  set(box_height - 1, 0, "└", border_fg, base_bg, 0);
+  for (uint32_t i = 1; i < box_width - 1; i++)
+    set(box_height - 1, i, "─", border_fg, base_bg, 0);
+  set(box_height - 1, box_width - 1, "┘", border_fg, base_bg, 0);
+  cells.resize(box_width * box_height);
+}
+
+void HoverBox::render(Coord pos) {
+  int32_t start_row = (int32_t)pos.row - (int32_t)box_height;
+  if (start_row < 0)
+    start_row = pos.row + 1;
+  int32_t start_col = pos.col;
+  if (start_col + box_width > cols) {
+    start_col = cols - box_width;
+    if (start_col < 0)
+      start_col = 0;
+  }
+  for (uint32_t r = 0; r < box_height; r++)
+    for (uint32_t c = 0; c < box_width; c++)
+      update(start_row + r, start_col + c, cells[r * box_width + c].utf8,
+             cells[r * box_width + c].fg, cells[r * box_width + c].bg,
+             cells[r * box_width + c].flags);
+}

src/editor/adjustment.cc

@@ -1,241 +1,5 @@
-extern "C" {
-#include "../libs/libgrapheme/grapheme.h"
-}
-#include "../include/editor.h"
-#include "../include/utils.h"
-#include <cmath>
-
-void scroll_up(Editor *editor, int32_t number) {
-  if (!editor || number == 0)
-    return;
-  uint32_t numlen =
-      EXTRA_META + static_cast<int>(std::log10(editor->root->line_count + 1));
-  uint32_t render_width = editor->size.col - numlen;
-  uint32_t line_index = editor->scroll.row;
-  LineIterator *it = begin_l_iter(editor->root, line_index);
-  if (!it)
-    return;
-  uint32_t len;
-  char *line = next_line(it, &len);
-  if (!line) {
-    free(it->buffer);
-    free(it);
-    return;
-  }
-  if (len > 0 && line[len - 1] == '\n')
-    len--;
-  uint32_t current_byte_offset = 0;
-  uint32_t col = 0;
-  std::vector<uint32_t> segment_starts;
-  segment_starts.reserve(16);
-  if (current_byte_offset < editor->scroll.col)
-    segment_starts.push_back(0);
-  while (current_byte_offset < editor->scroll.col &&
-         current_byte_offset < len) {
-    uint32_t cluster_len = grapheme_next_character_break_utf8(
-        line + current_byte_offset, len - current_byte_offset);
-    int width = display_width(line + current_byte_offset, cluster_len);
-    if (col + width > render_width) {
-      segment_starts.push_back(current_byte_offset);
-      col = 0;
-    }
-    current_byte_offset += cluster_len;
-    col += width;
-  }
-  for (auto it_seg = segment_starts.rbegin(); it_seg != segment_starts.rend();
-       ++it_seg) {
-    if (--number == 0) {
-      editor->scroll = {line_index, *it_seg};
-      free(it->buffer);
-      free(it);
-      return;
-    }
-  }
-  line = prev_line(it, &len);
-  if (!line) {
-    editor->scroll = {0, 0};
-    free(it->buffer);
-    free(it);
-    return;
-  }
-  do {
-    line_index--;
-    line = prev_line(it, &len);
-    if (!line) {
-      editor->scroll = {0, 0};
-      free(it->buffer);
-      free(it);
-      return;
-    }
-    const Fold *fold = fold_for_line(editor->folds, line_index);
-    if (fold) {
-      while (line && line_index > fold->start) {
-        free(line);
-        line = prev_line(it, &len);
-        line_index--;
-        if (!line) {
-          editor->scroll = {0, 0};
-          free(it->buffer);
-          free(it);
-          return;
-        }
-      }
-      if (--number == 0) {
-        editor->scroll = {fold->start, 0};
-        free(it->buffer);
-        free(it);
-        return;
-      }
-      if (fold->start == 0) {
-        editor->scroll = {0, 0};
-        free(it->buffer);
-        free(it);
-        return;
-      }
-      line_index = fold->start - 1;
-      line = prev_line(it, &len);
-      if (!line) {
-        editor->scroll = {0, 0};
-        free(it->buffer);
-        free(it);
-        return;
-      }
-      continue;
-    }
-    if (len > 0 && line[len - 1] == '\n')
-      len--;
-    current_byte_offset = 0;
-    col = 0;
-    std::vector<uint32_t> segment_starts;
-    segment_starts.reserve(16);
-    segment_starts.push_back(0);
-    while (current_byte_offset < len) {
-      uint32_t cluster_len = grapheme_next_character_break_utf8(
-          line + current_byte_offset, len - current_byte_offset);
-      int width = display_width(line + current_byte_offset, cluster_len);
-      if (col + width > render_width) {
-        segment_starts.push_back(current_byte_offset);
-        col = 0;
-      }
-      current_byte_offset += cluster_len;
-      col += width;
-    }
-    for (auto it_seg = segment_starts.rbegin(); it_seg != segment_starts.rend();
-         ++it_seg) {
-      if (--number == 0) {
-        editor->scroll = {line_index, *it_seg};
-        free(it->buffer);
-        free(it);
-        return;
-      }
-    }
-  } while (number > 0);
-  free(it->buffer);
-  free(it);
-}
-
-void scroll_down(Editor *editor, uint32_t number) {
-  if (!editor || number == 0)
-    return;
-  uint32_t numlen =
-      EXTRA_META + static_cast<int>(std::log10(editor->root->line_count + 1));
-  uint32_t render_width = editor->size.col - numlen;
-  uint32_t line_index = editor->scroll.row;
-  LineIterator *it = begin_l_iter(editor->root, line_index);
-  if (!it)
-    return;
-  const uint32_t max_visual_lines = editor->size.row;
-  Coord *scroll_queue = (Coord *)malloc(sizeof(Coord) * max_visual_lines);
-  uint32_t q_head = 0;
-  uint32_t q_size = 0;
-  uint32_t visual_seen = 0;
-  bool first_visual_line = true;
-  while (true) {
-    const Fold *fold = fold_for_line(editor->folds, line_index);
-    if (fold) {
-      Coord fold_coord = {fold->start, 0};
-      if (q_size < max_visual_lines) {
-        scroll_queue[(q_head + q_size) % max_visual_lines] = fold_coord;
-        q_size++;
-      } else {
-        scroll_queue[q_head] = fold_coord;
-        q_head = (q_head + 1) % max_visual_lines;
-      }
-      visual_seen++;
-      if (visual_seen >= number + max_visual_lines) {
-        editor->scroll = scroll_queue[q_head];
-        break;
-      }
-      uint32_t skip_until = fold->end;
-      while (line_index <= skip_until) {
-        char *line = next_line(it, nullptr);
-        if (!line) {
-          free(scroll_queue);
-          free(it->buffer);
-          free(it);
-          return;
-        }
-        line_index++;
-      }
-      continue;
-    }
-    uint32_t line_len;
-    char *line = next_line(it, &line_len);
-    if (!line)
-      break;
-    if (line_len && line[line_len - 1] == '\n')
-      line_len--;
-    uint32_t current_byte_offset = 0;
-    if (first_visual_line) {
-      current_byte_offset += editor->scroll.col;
-      first_visual_line = false;
-    }
-    while (current_byte_offset < line_len ||
-           (line_len == 0 && current_byte_offset == 0)) {
-      Coord coord = {line_index, current_byte_offset};
-      if (q_size < max_visual_lines) {
-        scroll_queue[(q_head + q_size) % max_visual_lines] = coord;
-        q_size++;
-      } else {
-        scroll_queue[q_head] = coord;
-        q_head = (q_head + 1) % max_visual_lines;
-      }
-      visual_seen++;
-      if (visual_seen >= number + max_visual_lines) {
-        editor->scroll = scroll_queue[q_head];
-        free(scroll_queue);
-        free(it->buffer);
-        free(it);
-        return;
-      }
-      uint32_t col = 0;
-      uint32_t local_render_offset = 0;
-      uint32_t left = line_len - current_byte_offset;
-      while (left > 0 && col < render_width) {
-        uint32_t cluster_len = grapheme_next_character_break_utf8(
-            line + current_byte_offset + local_render_offset, left);
-        int width = display_width(
-            line + current_byte_offset + local_render_offset, cluster_len);
-        if (col + width > render_width)
-          break;
-        local_render_offset += cluster_len;
-        left -= cluster_len;
-        col += width;
-      }
-      current_byte_offset += local_render_offset;
-      if (line_len == 0)
-        break;
-    }
-    line_index++;
-  }
-  if (q_size > 0) {
-    uint32_t advance = (q_size > number) ? number : (q_size - 1);
-    editor->scroll = scroll_queue[(q_head + advance) % max_visual_lines];
-  }
-  free(it->buffer);
-  free(it);
-  free(scroll_queue);
-}
+#include "editor/editor.h"
+#include "editor/folds.h"
 
 void ensure_cursor(Editor *editor) {
   std::shared_lock knot_lock(editor->knot_mtx);

src/editor/boundaries.cc

@@ -0,0 +1,115 @@
+#include "editor/editor.h"
+
+uint32_t scan_left(const char *line, uint32_t len, uint32_t off) {
+  if (off > len)
+    off = len;
+  uint32_t i = off;
+  while (i > 0) {
+    unsigned char c = (unsigned char)line[i - 1];
+    if ((c & 0x80) != 0)
+      break;
+    if (!((c == '_') || (c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') ||
+          (c >= 'a' && c <= 'z')))
+      break;
+    --i;
+  }
+  return i;
+}
+
+uint32_t scan_right(const char *line, uint32_t len, uint32_t off) {
+  if (off > len)
+    off = len;
+  uint32_t i = off;
+  while (i < len) {
+    unsigned char c = (unsigned char)line[i];
+    if ((c & 0x80) != 0)
+      break;
+    if (!((c == '_') || (c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') ||
+          (c >= 'a' && c <= 'z')))
+      break;
+    ++i;
+  }
+  return i;
+}
+
+void word_boundaries_exclusive(Editor *editor, Coord coord, uint32_t *prev_col,
+                               uint32_t *next_col) {
+  if (!editor)
+    return;
+  std::shared_lock lock(editor->knot_mtx);
+  LineIterator *it = begin_l_iter(editor->root, coord.row);
+  if (!it)
+    return;
+  uint32_t line_len;
+  char *line = next_line(it, &line_len);
+  if (!line)
+    return;
+  if (line_len && line[line_len - 1] == '\n')
+    line_len--;
+  uint32_t col = coord.col;
+  if (col > line_len)
+    col = line_len;
+  uint32_t left = scan_left(line, line_len, col);
+  uint32_t right = scan_right(line, line_len, col);
+  if (prev_col)
+    *prev_col = left;
+  if (next_col)
+    *next_col = right;
+  free(it->buffer);
+  free(it);
+}
+
+uint32_t word_jump_right(const char *line, size_t len, uint32_t pos) {
+  if (pos >= len)
+    return len;
+  size_t next = grapheme_next_word_break_utf8(line + pos, len - pos);
+  return static_cast<uint32_t>(pos + next);
+}
+
+uint32_t word_jump_left(const char *line, size_t len, uint32_t col) {
+  if (col == 0)
+    return 0;
+  size_t pos = 0;
+  size_t last = 0;
+  size_t cursor = col;
+  while (pos < len) {
+    size_t next = pos + grapheme_next_word_break_utf8(line + pos, len - pos);
+    if (next >= cursor)
+      break;
+    last = next;
+    pos = next;
+  }
+  return static_cast<uint32_t>(last);
+}
+
+void word_boundaries(Editor *editor, Coord coord, uint32_t *prev_col,
+                     uint32_t *next_col, uint32_t *prev_clusters,
+                     uint32_t *next_clusters) {
+  if (!editor)
+    return;
+  std::shared_lock lock(editor->knot_mtx);
+  LineIterator *it = begin_l_iter(editor->root, coord.row);
+  if (!it)
+    return;
+  uint32_t line_len;
+  char *line = next_line(it, &line_len);
+  if (!line)
+    return;
+  if (line_len && line[line_len - 1] == '\n')
+    line_len--;
+  size_t col = coord.col;
+  if (col > line_len)
+    col = line_len;
+  size_t left = word_jump_left(line, line_len, col);
+  size_t right = word_jump_right(line, line_len, col);
+  if (prev_col)
+    *prev_col = static_cast<uint32_t>(left);
+  if (next_col)
+    *next_col = static_cast<uint32_t>(right);
+  if (prev_clusters)
+    *prev_clusters = count_clusters(line, line_len, left, col);
+  if (next_clusters)
+    *next_clusters = count_clusters(line, line_len, col, right);
+  free(it->buffer);
+  free(it);
+}

src/editor/click.cc

@@ -0,0 +1,97 @@
+#include "editor/editor.h"
+#include "editor/folds.h"
+#include "main.h"
+
+Coord editor_hit_test(Editor *editor, uint32_t x, uint32_t y) {
+  if (mode == INSERT)
+    x++;
+  uint32_t numlen =
+      EXTRA_META + static_cast<int>(std::log10(editor->root->line_count + 1));
+  bool is_gutter_click = (x < numlen);
+  uint32_t render_width = editor->size.col - numlen;
+  x = MAX(x, numlen) - numlen;
+  uint32_t target_visual_row = y;
+  uint32_t visual_row = 0;
+  uint32_t line_index = editor->scroll.row;
+  uint32_t last_line_index = editor->scroll.row;
+  uint32_t last_col = editor->scroll.col;
+  bool first_visual_line = true;
+  std::shared_lock knot_lock(editor->knot_mtx);
+  LineIterator *it = begin_l_iter(editor->root, line_index);
+  if (!it)
+    return editor->scroll;
+  while (visual_row <= target_visual_row) {
+    const Fold *fold = fold_for_line(editor->folds, line_index);
+    if (fold) {
+      if (visual_row == target_visual_row) {
+        free(it->buffer);
+        free(it);
+        if (is_gutter_click) {
+          remove_fold(editor, fold->start);
+          return {UINT32_MAX, UINT32_MAX};
+        }
+        return {fold->start > 0 ? fold->start - 1 : 0, 0};
+      }
+      visual_row++;
+      while (line_index <= fold->end) {
+        char *l = next_line(it, nullptr);
+        if (!l)
+          break;
+        line_index++;
+      }
+      last_line_index = fold->end;
+      last_col = 0;
+      continue;
+    }
+    uint32_t line_len;
+    char *line = next_line(it, &line_len);
+    if (!line)
+      break;
+    if (line_len && line[line_len - 1] == '\n')
+      line_len--;
+    last_line_index = line_index;
+    last_col = line_len;
+    uint32_t offset = first_visual_line ? editor->scroll.col : 0;
+    first_visual_line = false;
+    while (offset < line_len || (line_len == 0 && offset == 0)) {
+      uint32_t col = 0;
+      uint32_t advance = 0;
+      uint32_t left = line_len - offset;
+      uint32_t last_good_offset = offset;
+      while (left > 0 && col < render_width) {
+        uint32_t g =
+            grapheme_next_character_break_utf8(line + offset + advance, left);
+        int w = display_width(line + offset + advance, g);
+        if (col + w > render_width)
+          break;
+        if (visual_row == target_visual_row && x < col + w) {
+          free(it->buffer);
+          free(it);
+          return {line_index, offset + advance};
+        }
+        advance += g;
+        last_good_offset = offset + advance;
+        left -= g;
+        col += w;
+      }
+      last_col = last_good_offset;
+      if (visual_row == target_visual_row) {
+        free(it->buffer);
+        free(it);
+        return {line_index, last_good_offset};
+      }
+      visual_row++;
+      if (visual_row > target_visual_row)
+        break;
+      if (advance == 0)
+        break;
+      offset += advance;
+      if (line_len == 0)
+        break;
+    }
+    line_index++;
+  }
+  free(it->buffer);
+  free(it);
+  return {last_line_index, last_col};
+}

src/editor/cursor.cc

@@ -1,8 +1,6 @@
-extern "C" {
-#include "../libs/libgrapheme/grapheme.h"
-}
-#include "../include/editor.h"
-#include "../include/utils.h"
+#include "editor/editor.h"
+#include "editor/folds.h"
+#include "utils/utils.h"
 
 Coord move_right_pure(Editor *editor, Coord cursor, uint32_t number) {
   Coord result = cursor;

src/editor/edit.cc

@@ -0,0 +1,274 @@
+#include "editor/editor.h"
+#include "editor/folds.h"
+#include "lsp/lsp.h"
+
+void edit_erase(Editor *editor, Coord pos, int64_t len) {
+  if (len == 0)
+    return;
+  if (len < 0) {
+    std::shared_lock lock_1(editor->knot_mtx);
+    uint32_t cursor_original =
+        line_to_byte(editor->root, editor->cursor.row, nullptr) +
+        editor->cursor.col;
+    TSPoint old_point = {pos.row, pos.col};
+    uint32_t byte_pos = line_to_byte(editor->root, pos.row, nullptr) + pos.col;
+    Coord point = move_left_pure(editor, pos, -len);
+    json lsp_range;
+    bool do_lsp = (editor->lsp != nullptr);
+    if (do_lsp) {
+      LineIterator *it = begin_l_iter(editor->root, point.row);
+      char *line = next_line(it, nullptr);
+      int utf16_start = 0;
+      if (line)
+        utf16_start = utf8_byte_offset_to_utf16(line, point.col);
+      free(it->buffer);
+      free(it);
+      it = begin_l_iter(editor->root, pos.row);
+      line = next_line(it, nullptr);
+      int utf16_end = 0;
+      if (line)
+        utf16_end = utf8_byte_offset_to_utf16(line, pos.col);
+      free(it->buffer);
+      free(it);
+      lsp_range = {{"start", {{"line", point.row}, {"character", utf16_start}}},
+                   {"end", {{"line", pos.row}, {"character", utf16_end}}}};
+    }
+    uint32_t start = line_to_byte(editor->root, point.row, nullptr) + point.col;
+    if (cursor_original > start && cursor_original <= byte_pos) {
+      editor->cursor = point;
+      editor->cursor_preffered = UINT32_MAX;
+    } else if (cursor_original > byte_pos) {
+      uint32_t cursor_new = cursor_original - (byte_pos - start);
+      uint32_t new_col;
+      uint32_t new_row = byte_to_line(editor->root, cursor_new, &new_col);
+      editor->cursor = {new_row, new_col};
+      editor->cursor_preffered = UINT32_MAX;
+    }
+    lock_1.unlock();
+    uint32_t start_row = point.row;
+    uint32_t end_row = pos.row;
+    apply_line_deletion(editor, start_row + 1, end_row);
+    apply_hook_deletion(editor, start_row + 1, end_row);
+    std::unique_lock lock_2(editor->knot_mtx);
+    editor->root = erase(editor->root, start, byte_pos - start);
+    lock_2.unlock();
+    if (editor->ts.tree) {
+      TSInputEdit edit = {
+          .start_byte = start,
+          .old_end_byte = byte_pos,
+          .new_end_byte = start,
+          .start_point = {point.row, point.col},
+          .old_end_point = old_point,
+          .new_end_point = {point.row, point.col},
+      };
+      editor->edit_queue.push(edit);
+    }
+    if (do_lsp) {
+      if (editor->lsp->incremental_sync) {
+        json message = {
+            {"jsonrpc", "2.0"},
+            {"method", "textDocument/didChange"},
+            {"params",
+             {{"textDocument",
+               {{"uri", editor->uri}, {"version", ++editor->lsp_version}}},
+              {"contentChanges",
+               json::array({{{"range", lsp_range}, {"text", ""}}})}}}};
+        lsp_send(editor->lsp, message, nullptr);
+      } else {
+        char *buf = read(editor->root, 0, editor->root->char_count);
+        std::string text(buf);
+        free(buf);
+        json message = {
+            {"jsonrpc", "2.0"},
+            {"method", "textDocument/didChange"},
+            {"params",
+             {{"textDocument",
+               {{"uri", editor->uri}, {"version", ++editor->lsp_version}}},
+              {"contentChanges", json::array({{{"text", text}}})}}}};
+        lsp_send(editor->lsp, message, nullptr);
+      }
+    }
+    std::unique_lock lock_3(editor->spans.mtx);
+    apply_edit(editor->spans.spans, start, start - byte_pos);
+    if (editor->spans.mid_parse)
+      editor->spans.edits.push({start, start - byte_pos});
+    std::unique_lock lock_4(editor->def_spans.mtx);
+    apply_edit(editor->def_spans.spans, byte_pos, start - byte_pos);
+  } else {
+    std::shared_lock lock_1(editor->knot_mtx);
+    uint32_t cursor_original =
+        line_to_byte(editor->root, editor->cursor.row, nullptr) +
+        editor->cursor.col;
+    TSPoint old_point = {pos.row, pos.col};
+    uint32_t byte_pos = line_to_byte(editor->root, pos.row, nullptr) + pos.col;
+    Coord point = move_right_pure(editor, pos, len);
+    json lsp_range;
+    bool do_lsp = (editor->lsp != nullptr);
+    if (do_lsp) {
+      LineIterator *it = begin_l_iter(editor->root, pos.row);
+      char *line = next_line(it, nullptr);
+      int utf16_start = 0;
+      if (line)
+        utf16_start = utf8_byte_offset_to_utf16(line, pos.col);
+      free(it->buffer);
+      free(it);
+      it = begin_l_iter(editor->root, point.row);
+      line = next_line(it, nullptr);
+      int utf16_end = 0;
+      if (line)
+        utf16_end = utf8_byte_offset_to_utf16(line, point.col);
+      free(it->buffer);
+      free(it);
+      lsp_range = {{"start", {{"line", pos.row}, {"character", utf16_start}}},
+                   {"end", {{"line", point.row}, {"character", utf16_end}}}};
+    }
+    uint32_t end = line_to_byte(editor->root, point.row, nullptr) + point.col;
+    if (cursor_original > byte_pos && cursor_original <= end) {
+      editor->cursor = pos;
+      editor->cursor_preffered = UINT32_MAX;
+    } else if (cursor_original > end) {
+      uint32_t cursor_new = cursor_original - (end - byte_pos);
+      uint32_t new_col;
+      uint32_t new_row = byte_to_line(editor->root, cursor_new, &new_col);
+      editor->cursor = {new_row, new_col};
+      editor->cursor_preffered = UINT32_MAX;
+    }
+    lock_1.unlock();
+    uint32_t start_row = pos.row;
+    uint32_t end_row = point.row;
+    apply_line_deletion(editor, start_row + 1, end_row);
+    apply_hook_deletion(editor, start_row + 1, end_row);
+    std::unique_lock lock_2(editor->knot_mtx);
+    editor->root = erase(editor->root, byte_pos, end - byte_pos);
+    lock_2.unlock();
+    if (editor->ts.tree) {
+      TSInputEdit edit = {
+          .start_byte = byte_pos,
+          .old_end_byte = end,
+          .new_end_byte = byte_pos,
+          .start_point = old_point,
+          .old_end_point = {point.row, point.col},
+          .new_end_point = old_point,
+      };
+      editor->edit_queue.push(edit);
+    }
+    if (do_lsp) {
+      if (editor->lsp->incremental_sync) {
+        json message = {
+            {"jsonrpc", "2.0"},
+            {"method", "textDocument/didChange"},
+            {"params",
+             {{"textDocument",
+               {{"uri", editor->uri}, {"version", ++editor->lsp_version}}},
+              {"contentChanges",
+               json::array({{{"range", lsp_range}, {"text", ""}}})}}}};
+        lsp_send(editor->lsp, message, nullptr);
+      } else {
+        char *buf = read(editor->root, 0, editor->root->char_count);
+        std::string text(buf);
+        free(buf);
+        json message = {
+            {"jsonrpc", "2.0"},
+            {"method", "textDocument/didChange"},
+            {"params",
+             {{"textDocument",
+               {{"uri", editor->uri}, {"version", ++editor->lsp_version}}},
+              {"contentChanges", json::array({{{"text", text}}})}}}};
+        lsp_send(editor->lsp, message, nullptr);
+      }
+    }
+    std::unique_lock lock_3(editor->spans.mtx);
+    apply_edit(editor->spans.spans, byte_pos, byte_pos - end);
+    if (editor->spans.mid_parse)
+      editor->spans.edits.push({byte_pos, byte_pos - end});
+    std::unique_lock lock_4(editor->def_spans.mtx);
+    apply_edit(editor->def_spans.spans, byte_pos, byte_pos - end);
+  }
+}
+
+void edit_insert(Editor *editor, Coord pos, char *data, uint32_t len) {
+  std::shared_lock lock_1(editor->knot_mtx);
+  uint32_t cursor_original =
+      line_to_byte(editor->root, editor->cursor.row, nullptr) +
+      editor->cursor.col;
+  uint32_t byte_pos = line_to_byte(editor->root, pos.row, nullptr) + pos.col;
+  TSPoint start_point = {pos.row, pos.col};
+  if (cursor_original > byte_pos) {
+    uint32_t cursor_new = cursor_original + len;
+    uint32_t new_col;
+    uint32_t new_row = byte_to_line(editor->root, cursor_new, &new_col);
+    editor->cursor = {new_row, new_col};
+  }
+  lock_1.unlock();
+  std::unique_lock lock_2(editor->knot_mtx);
+  editor->root = insert(editor->root, byte_pos, data, len);
+  lock_2.unlock();
+  uint32_t cols = 0;
+  uint32_t rows = 0;
+  for (uint32_t i = 0; i < len; i++) {
+    if (data[i] == '\n') {
+      rows++;
+      cols = 0;
+    } else {
+      cols++;
+    }
+  }
+  apply_line_insertion(editor, pos.row, rows);
+  apply_hook_insertion(editor, pos.row, rows);
+  if (editor->ts.tree) {
+    TSInputEdit edit = {
+        .start_byte = byte_pos,
+        .old_end_byte = byte_pos,
+        .new_end_byte = byte_pos + len,
+        .start_point = start_point,
+        .old_end_point = start_point,
+        .new_end_point = {start_point.row + rows,
+                          (rows == 0) ? (start_point.column + cols) : cols},
+    };
+    editor->edit_queue.push(edit);
+  }
+  if (editor->lsp) {
+    if (editor->lsp->incremental_sync) {
+      lock_1.lock();
+      LineIterator *it = begin_l_iter(editor->root, pos.row);
+      char *line = next_line(it, nullptr);
+      int utf16_col = 0;
+      if (line)
+        utf16_col = utf8_byte_offset_to_utf16(line, pos.col);
+      free(it->buffer);
+      free(it);
+      lock_1.unlock();
+      json message = {
+          {"jsonrpc", "2.0"},
+          {"method", "textDocument/didChange"},
+          {"params",
+           {{"textDocument",
+             {{"uri", editor->uri}, {"version", ++editor->lsp_version}}},
+            {"contentChanges",
+             json::array(
+                 {{{"range",
+                    {{"start", {{"line", pos.row}, {"character", utf16_col}}},
+                     {"end", {{"line", pos.row}, {"character", utf16_col}}}}},
+                   {"text", std::string(data, len)}}})}}}};
+      lsp_send(editor->lsp, message, nullptr);
+    } else {
+      char *buf = read(editor->root, 0, editor->root->char_count);
+      std::string text(buf);
+      free(buf);
+      json message = {
+          {"jsonrpc", "2.0"},
+          {"method", "textDocument/didChange"},
+          {"params",
+           {{"textDocument",
+             {{"uri", editor->uri}, {"version", ++editor->lsp_version}}},
+            {"contentChanges", json::array({{{"text", text}}})}}}};
+      lsp_send(editor->lsp, message, nullptr);
+    }
+  }
+  std::unique_lock lock_3(editor->spans.mtx);
+  apply_edit(editor->spans.spans, byte_pos, len);
+  if (editor->spans.mid_parse)
+    editor->spans.edits.push({byte_pos, len});
+  std::unique_lock lock_4(editor->def_spans.mtx);
+  apply_edit(editor->def_spans.spans, byte_pos, len);
+}

src/editor/editor.cc

@@ -0,0 +1,79 @@
+#include "editor/editor.h"
+#include "lsp/lsp.h"
+#include "utils/utils.h"
+
+Editor *new_editor(const char *filename_arg, Coord position, Coord size) {
+  Editor *editor = new Editor();
+  if (!editor)
+    return nullptr;
+  uint32_t len = 0;
+  std::string filename = path_abs(filename_arg);
+  char *str = load_file(filename.c_str(), &len);
+  if (!str) {
+    free_editor(editor);
+    return nullptr;
+  }
+  editor->filename = filename;
+  editor->uri = path_to_file_uri(filename);
+  editor->position = position;
+  editor->size = size;
+  editor->cursor_preffered = UINT32_MAX;
+  if (len == 0) {
+    free(str);
+    str = (char *)malloc(1);
+    *str = '\n';
+    len = 1;
+  }
+  editor->root = load(str, len, optimal_chunk_size(len));
+  free(str);
+  Language language = language_for_file(filename.c_str());
+  if (language.name != "unknown" && len <= (1024 * 128)) {
+    editor->ts.parser = ts_parser_new();
+    editor->ts.language = language.fn();
+    ts_parser_set_language(editor->ts.parser, editor->ts.language);
+    editor->ts.query_file =
+        get_exe_dir() + "/../grammar/" + language.name + ".scm";
+    request_add_to_lsp(language, editor);
+  }
+  return editor;
+}
+
+void free_tsset(TSSetMain *set) {
+  if (set->parser)
+    ts_parser_delete(set->parser);
+  if (set->tree)
+    ts_tree_delete(set->tree);
+  if (set->query)
+    ts_query_delete(set->query);
+  for (auto &inj : set->injections) {
+    if (inj.second.parser)
+      ts_parser_delete(inj.second.parser);
+    if (inj.second.query)
+      ts_query_delete(inj.second.query);
+    if (inj.second.tree)
+      ts_tree_delete(inj.second.tree);
+  }
+}
+
+void free_editor(Editor *editor) {
+  remove_from_lsp(editor);
+  free_tsset(&editor->ts);
+  free_rope(editor->root);
+  delete editor;
+}
+
+void save_file(Editor *editor) {
+  if (!editor || !editor->root)
+    return;
+  char *str = read(editor->root, 0, editor->root->char_count);
+  if (!str)
+    return;
+  std::ofstream out(editor->filename);
+  out.write(str, editor->root->char_count);
+  free(str);
+  json msg = {{"jsonrpc", "2.0"},
+              {"method", "textDocument/didSave"},
+              {"params", {{"textDocument", {{"uri", editor->uri}}}}}};
+  if (editor->lsp)
+    lsp_send(editor->lsp, msg, nullptr);
+}

src/editor/events.cc

@@ -1,14 +1,16 @@
-#include "../include/editor.h"
-#include "../include/main.h"
-#include "../include/ts.h"
-#include <cstdint>
-#include <sys/ioctl.h>
+#include "editor/editor.h"
+#include "editor/folds.h"
+#include "lsp/lsp.h"
+#include "main.h"
 
 void handle_editor_event(Editor *editor, KeyEvent event) {
   static std::chrono::steady_clock::time_point last_click_time =
       std::chrono::steady_clock::now();
   static uint32_t click_count = 0;
   static Coord last_click_pos = {UINT32_MAX, UINT32_MAX};
+  Coord start = editor->cursor;
+  if (editor->hover_active)
+    editor->hover_active = false;
   if (event.key_type == KEY_MOUSE) {
     auto now = std::chrono::steady_clock::now();
     auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(
@@ -205,7 +207,6 @@ void handle_editor_event(Editor *editor, KeyEvent event) {
   switch (mode) {
   case NORMAL:
     if (event.key_type == KEY_CHAR && event.len == 1) {
-      Coord start = editor->cursor;
       switch (event.c[0]) {
       case 'u':
         if (editor->root->line_count > 0) {
@@ -230,6 +231,71 @@ void handle_editor_event(Editor *editor, KeyEvent event) {
           editor->selection_type = LINE;
         }
         break;
+      case CTRL('h'):
+        editor->hover.scroll(-1);
+        editor->hover_active = true;
+        break;
+      case CTRL('l'):
+        editor->hover.scroll(1);
+        editor->hover_active = true;
+        break;
+      case 'h':
+        if (editor->lsp && editor->lsp->allow_hover) {
+          LineIterator *it = begin_l_iter(editor->root, editor->cursor.row);
+          char *line = next_line(it, nullptr);
+          if (!line) {
+            free(it->buffer);
+            free(it);
+            break;
+          }
+          uint32_t col = utf8_byte_offset_to_utf16(line, editor->cursor.col);
+          free(it->buffer);
+          free(it);
+          json hover_request = {
+              {"jsonrpc", "2.0"},
+              {"method", "textDocument/hover"},
+              {"params",
+               {{"textDocument", {{"uri", editor->uri}}},
+                {"position",
+                 {{"line", editor->cursor.row}, {"character", col}}}}}};
+          LSPPending *pending = new LSPPending();
+          pending->editor = editor;
+          pending->method = "textDocument/hover";
+          pending->callback = [](Editor *editor, std::string, json hover) {
+            if (hover.contains("result") && !hover["result"].is_null()) {
+              auto &contents = hover["result"]["contents"];
+              std::string hover_text = "";
+              bool is_markup = false;
+              if (contents.is_object()) {
+                hover_text += contents["value"].get<std::string>();
+                is_markup = (contents["kind"].get<std::string>() == "markdown");
+              } else if (contents.is_array()) {
+                for (auto &block : contents) {
+                  if (block.is_string()) {
+                    hover_text += block.get<std::string>() + "\n";
+                  } else if (block.is_object() && block.contains("language") &&
+                             block.contains("value")) {
+                    std::string lang = block["language"].get<std::string>();
+                    std::string val = block["value"].get<std::string>();
+                    is_markup = true;
+                    hover_text += "```" + lang + "\n" + val + "\n```\n";
+                  }
+                }
+              } else if (contents.is_string()) {
+                hover_text += contents.get<std::string>();
+              }
+              if (!hover_text.empty()) {
+                editor->hover.clear();
+                editor->hover.text = clean_text(hover_text);
+                editor->hover.is_markup = is_markup;
+                editor->hover.render_first();
+                editor->hover_active = true;
+              }
+            }
+          };
+          lsp_send(editor->lsp, hover_request, pending);
+        }
+        break;
       case 'a':
         mode = INSERT;
         cursor_right(editor, 1);
@@ -298,6 +364,45 @@ void handle_editor_event(Editor *editor, KeyEvent event) {
       case CTRL('s'):
         save_file(editor);
         break;
+      case CTRL(' '):
+        if (editor->lsp) {
+          json msg = {
+              {"jsonrpc", "2.0"},
+              {"method", "textDocument/completion"},
+              {
+                  "params",
+                  {
+                      {
+                          "textDocument",
+                          {
+                              {"uri", editor->uri},
+                          },
+                      },
+                      {
+                          "position",
+                          {
+                              {"line", editor->cursor.row},
+                              {"character", editor->cursor.col},
+                          },
+                      },
+                      {
+                          "context",
+                          {
+                              {"triggerKind", 1},
+                          },
+                      },
+                  },
+              },
+          };
+          LSPPending *pending = new LSPPending();
+          pending->editor = editor;
+          pending->method = "textDocument/completion";
+          pending->callback = [](Editor *editor, std::string, json completion) {
+            log("%s\n", completion.dump().c_str());
+          };
+          lsp_send(editor->lsp, msg, pending);
+        }
+        break;
       case 'p':
         uint32_t len;
         char *text = get_from_clipboard(&len);
@@ -558,65 +663,3 @@ void handle_editor_event(Editor *editor, KeyEvent event) {
   if ((event.key_type == KEY_CHAR || event.key_type == KEY_PASTE) && event.c)
     free(event.c);
 }
-
-static Highlight HL_UNDERLINE = {0, 0, 1 << 2, 100};
-
-void editor_worker(Editor *editor) {
-  if (!editor || !editor->root)
-    return;
-  if (editor->root->char_count > (1024 * 200))
-    return;
-  if (editor->ts.query_file != "" && !editor->ts.query)
-    editor->ts.query = load_query(editor->ts.query_file.c_str(), &editor->ts);
-  if (editor->ts.parser && editor->ts.query)
-    ts_collect_spans(editor);
-  uint32_t prev_col, next_col;
-  word_boundaries_exclusive(editor, editor->cursor, &prev_col, &next_col);
-  if (next_col - prev_col > 0 && next_col - prev_col < 256 - 4) {
-    std::shared_lock lock(editor->knot_mtx);
-    uint32_t offset = line_to_byte(editor->root, editor->cursor.row, nullptr);
-    char *word = read(editor->root, offset + prev_col, next_col - prev_col);
-    if (word) {
-      char buf[256];
-      snprintf(buf, sizeof(buf), "\\b%s\\b", word);
-      std::vector<std::pair<size_t, size_t>> results =
-          search_rope(editor->root, buf);
-      std::unique_lock lock(editor->def_spans.mtx);
-      editor->def_spans.spans.clear();
-      for (const auto &match : results) {
-        Span s;
-        s.start = match.first;
-        s.end = match.first + match.second;
-        s.hl = &HL_UNDERLINE;
-        editor->def_spans.spans.push_back(s);
-      }
-      std::sort(editor->def_spans.spans.begin(), editor->def_spans.spans.end());
-      lock.unlock();
-      free(word);
-    }
-  } else {
-    std::unique_lock lock(editor->def_spans.mtx);
-    editor->def_spans.spans.clear();
-    lock.unlock();
-  }
-}
-
-void editor_lsp_handle(Editor *editor, json msg) {
-  if (msg.contains("method") &&
-      msg["method"] == "textDocument/publishDiagnostics") {
-    std::unique_lock lock(editor->v_mtx);
-    editor->warnings.clear();
-    json diagnostics = msg["params"]["diagnostics"];
-    for (size_t i = 0; i < diagnostics.size(); i++) {
-      json d = diagnostics[i];
-      VWarn w;
-      w.line = d["range"]["start"]["line"];
-      std::string text = d["message"].get<std::string>();
-      auto pos = text.find('\n');
-      w.text = (pos == std::string::npos) ? text : text.substr(0, pos);
-      w.type = d["severity"].get<int>();
-      editor->warnings.push_back(w);
-    }
-    std::sort(editor->warnings.begin(), editor->warnings.end());
-  }
-}

src/editor/indents.cc

@@ -1,4 +1,4 @@
-#include "../include/editor.h"
+#include "editor/editor.h"
 
 uint32_t leading_indent(const char *line, uint32_t len) {
   uint32_t indent = 0;

src/editor/lsp.cc

@@ -0,0 +1,63 @@
+#include "editor/editor.h"
+
+void editor_lsp_handle(Editor *editor, json msg) {
+  if (msg.contains("method") &&
+      msg["method"] == "textDocument/publishDiagnostics") {
+    std::unique_lock lock(editor->v_mtx);
+    editor->warnings.clear();
+    json diagnostics = msg["params"]["diagnostics"];
+    for (size_t i = 0; i < diagnostics.size(); i++) {
+      json d = diagnostics[i];
+      VWarn w;
+      // HACK: convert back to utf-8 but as this is only visually affecting it
+      //       is not worth getting the line string from the rope.
+      w.line = d["range"]["start"]["line"];
+      w.start = d["range"]["start"]["character"];
+      uint32_t end = d["range"]["end"]["character"];
+      if (d["range"]["end"]["line"] == w.line)
+        w.end = end;
+      std::string text = trim(d["message"].get<std::string>());
+      w.text_full = text;
+      auto pos = text.find('\n');
+      w.text = (pos == std::string::npos) ? text : text.substr(0, pos);
+      if (d.contains("source"))
+        w.source = d["source"].get<std::string>();
+      if (d.contains("code")) {
+        w.code = "[";
+        if (d["code"].is_string())
+          w.code += d["code"].get<std::string>() + "] ";
+        else if (d["code"].is_number())
+          w.code += std::to_string(d["code"].get<int>()) + "] ";
+        else
+          w.code.clear();
+        if (d.contains("codeDescription") &&
+            d["codeDescription"].contains("href"))
+          w.code += d["codeDescription"]["href"].get<std::string>();
+      }
+      if (d.contains("relatedInformation")) {
+        json related = d["relatedInformation"];
+        for (size_t j = 0; j < related.size(); j++) {
+          json rel = related[j];
+          std::string message = rel["message"].get<std::string>();
+          auto pos = message.find('\n');
+          message =
+              (pos == std::string::npos) ? message : message.substr(0, pos);
+          std::string uri =
+              percent_decode(rel["location"]["uri"].get<std::string>());
+          auto pos2 = uri.find_last_of('/');
+          if (pos2 != std::string::npos)
+            uri = uri.substr(pos2 + 1);
+          std::string row = std::to_string(
+              rel["location"]["range"]["start"]["line"].get<int>());
+          w.see_also.push_back(uri + ":" + row + ": " + message);
+        }
+      }
+      w.type = 1;
+      if (d.contains("severity"))
+        w.type = d["severity"].get<int>();
+      editor->warnings.push_back(w);
+    }
+    std::sort(editor->warnings.begin(), editor->warnings.end());
+    editor->warnings_dirty = true;
+  }
+}

src/editor/move_line.cc

@@ -0,0 +1,121 @@
+#include "editor/editor.h"
+#include "editor/folds.h"
+#include "main.h"
+
+void move_line_up(Editor *editor) {
+  if (!editor || !editor->root || editor->cursor.row == 0)
+    return;
+  if (mode == NORMAL || mode == INSERT) {
+    uint32_t line_len, line_cluster_len;
+    std::shared_lock lock(editor->knot_mtx);
+    LineIterator *it = begin_l_iter(editor->root, editor->cursor.row);
+    char *line = next_line(it, &line_len);
+    if (!line) {
+      lock.unlock();
+      return;
+    }
+    if (line_len > 0 && line[line_len - 1] == '\n')
+      line_len--;
+    line_cluster_len = count_clusters(line, line_len, 0, line_len);
+    uint32_t target_row = prev_unfolded_row(editor, editor->cursor.row - 1);
+    uint32_t up_by = editor->cursor.row - target_row;
+    if (up_by > 1)
+      up_by--;
+    lock.unlock();
+    Coord cursor = editor->cursor;
+    edit_erase(editor, {cursor.row, 0}, line_cluster_len);
+    edit_erase(editor, {cursor.row, 0}, -1);
+    edit_insert(editor, {cursor.row - up_by, 0}, (char *)"\n", 1);
+    edit_insert(editor, {cursor.row - up_by, 0}, line, line_len);
+    free(it->buffer);
+    free(it);
+    editor->cursor = {cursor.row - up_by, cursor.col};
+  } else if (mode == SELECT) {
+    uint32_t start_row = MIN(editor->cursor.row, editor->selection.row);
+    uint32_t end_row = MAX(editor->cursor.row, editor->selection.row);
+    uint32_t start_byte = line_to_byte(editor->root, start_row, nullptr);
+    uint32_t end_byte = line_to_byte(editor->root, end_row + 1, nullptr);
+    char *selected_text = read(editor->root, start_byte, end_byte - start_byte);
+    if (!selected_text)
+      return;
+    uint32_t selected_len = count_clusters(selected_text, end_byte - start_byte,
+                                           0, end_byte - start_byte);
+    Coord cursor = editor->cursor;
+    Coord selection = editor->selection;
+    edit_erase(editor, {start_row, 0}, selected_len);
+    edit_insert(editor, {start_row - 1, 0}, selected_text,
+                end_byte - start_byte);
+    free(selected_text);
+    editor->cursor = {cursor.row - 1, cursor.col};
+    editor->selection = {selection.row - 1, selection.col};
+  }
+}
+
+void move_line_down(Editor *editor) {
+  if (!editor || !editor->root)
+    return;
+  if (mode == NORMAL || mode == INSERT) {
+    if (editor->cursor.row >= editor->root->line_count - 1)
+      return;
+    uint32_t line_len, line_cluster_len;
+    std::shared_lock lock(editor->knot_mtx);
+    LineIterator *it = begin_l_iter(editor->root, editor->cursor.row);
+    char *line = next_line(it, &line_len);
+    if (!line) {
+      lock.unlock();
+      return;
+    }
+    if (line_len && line[line_len - 1] == '\n')
+      line_len--;
+    line_cluster_len = count_clusters(line, line_len, 0, line_len);
+    uint32_t target_row = next_unfolded_row(editor, editor->cursor.row + 1);
+    if (target_row >= editor->root->line_count) {
+      free(line);
+      lock.unlock();
+      return;
+    }
+    uint32_t down_by = target_row - editor->cursor.row;
+    if (down_by > 1)
+      down_by--;
+    uint32_t ln;
+    line_to_byte(editor->root, editor->cursor.row + down_by - 1, &ln);
+    lock.unlock();
+    Coord cursor = editor->cursor;
+    edit_erase(editor, {cursor.row, 0}, line_cluster_len);
+    edit_erase(editor, {cursor.row, 0}, -1);
+    edit_insert(editor, {cursor.row + down_by, 0}, (char *)"\n", 1);
+    edit_insert(editor, {cursor.row + down_by, 0}, line, line_len);
+    free(it->buffer);
+    free(it);
+    editor->cursor = {cursor.row + down_by, cursor.col};
+  } else if (mode == SELECT) {
+    if (editor->cursor.row >= editor->root->line_count - 1 ||
+        editor->selection.row >= editor->root->line_count - 1)
+      return;
+    std::shared_lock lock(editor->knot_mtx);
+    uint32_t start_row = MIN(editor->cursor.row, editor->selection.row);
+    uint32_t end_row = MAX(editor->cursor.row, editor->selection.row);
+    uint32_t target_row = next_unfolded_row(editor, end_row + 1);
+    if (target_row >= editor->root->line_count)
+      return;
+    uint32_t down_by = target_row - end_row;
+    if (down_by > 1)
+      down_by--;
+    uint32_t start_byte = line_to_byte(editor->root, start_row, nullptr);
+    uint32_t end_byte = line_to_byte(editor->root, end_row + 1, nullptr);
+    char *selected_text = read(editor->root, start_byte, end_byte - start_byte);
+    lock.unlock();
+    if (!selected_text)
+      return;
+    uint32_t selected_len = count_clusters(selected_text, end_byte - start_byte,
+                                           0, end_byte - start_byte);
+    Coord cursor = editor->cursor;
+    Coord selection = editor->selection;
+    edit_erase(editor, {start_row, 0}, selected_len);
+    edit_insert(editor, {start_row + down_by, 0}, selected_text,
+                end_byte - start_byte);
+    free(selected_text);
+    editor->cursor = {cursor.row + down_by, cursor.col};
+    editor->selection = {selection.row + down_by, selection.col};
+  }
+}

src/editor/renderer.cc

@@ -1,73 +1,6 @@
-extern "C" {
-#include "../libs/libgrapheme/grapheme.h"
-}
-#include "../include/editor.h"
-#include "../include/lsp.h"
-#include "../include/main.h"
-#include "../include/utils.h"
-
-Editor *new_editor(const char *filename_arg, Coord position, Coord size) {
-  Editor *editor = new Editor();
-  if (!editor)
-    return nullptr;
-  uint32_t len = 0;
-  std::string filename = path_abs(filename_arg);
-  char *str = load_file(filename.c_str(), &len);
-  if (!str) {
-    free_editor(editor);
-    return nullptr;
-  }
-  editor->filename = filename;
-  editor->uri = path_to_file_uri(filename);
-  editor->position = position;
-  editor->size = size;
-  editor->cursor_preffered = UINT32_MAX;
-  editor->root = load(str, len, optimal_chunk_size(len));
-  free(str);
-  Language language = language_for_file(filename.c_str());
-  if (language.name != "unknown" && len <= (1024 * 128)) {
-    editor->ts.parser = ts_parser_new();
-    editor->ts.language = language.fn();
-    ts_parser_set_language(editor->ts.parser, editor->ts.language);
-    editor->ts.query_file =
-        get_exe_dir() + "/../grammar/" + language.name + ".scm";
-    request_add_to_lsp(language, editor);
-  }
-  return editor;
-}
-
-void free_tsset(TSSetMain *set) {
-  if (set->parser)
-    ts_parser_delete(set->parser);
-  if (set->tree)
-    ts_tree_delete(set->tree);
-  if (set->query)
-    ts_query_delete(set->query);
-  for (auto &inj : set->injections) {
-    if (inj.second.parser)
-      ts_parser_delete(inj.second.parser);
-    if (inj.second.query)
-      ts_query_delete(inj.second.query);
-  }
-}
-
-void free_editor(Editor *editor) {
-  remove_from_lsp(editor);
-  free_tsset(&editor->ts);
-  free_rope(editor->root);
-  delete editor;
-}
-
-void save_file(Editor *editor) {
-  if (!editor || !editor->root)
-    return;
-  char *str = read(editor->root, 0, editor->root->char_count);
-  if (!str)
-    return;
-  std::ofstream out(editor->filename);
-  out.write(str, editor->root->char_count);
-  free(str);
-}
+#include "editor/editor.h"
+#include "editor/folds.h"
+#include "main.h"
 
 void render_editor(Editor *editor) {
   uint32_t sel_start = 0, sel_end = 0;
@@ -83,6 +16,7 @@ void render_editor(Editor *editor) {
   auto hook_it = v.begin();
   while (hook_it != v.end() && hook_it->first <= editor->scroll.row)
     ++hook_it;
+  std::unique_lock warn_lock(editor->v_mtx);
   auto warn_it = editor->warnings.begin();
   while (warn_it != editor->warnings.end() &&
          warn_it->line < editor->scroll.row)
@@ -147,7 +81,6 @@ void render_editor(Editor *editor) {
   uint32_t global_byte_offset = line_to_byte(editor->root, line_index, nullptr);
   span_cursor.sync(global_byte_offset);
   def_span_cursor.sync(global_byte_offset);
-  std::shared_lock v_lock(editor->v_mtx);
   while (rendered_rows < editor->size.row) {
     const Fold *fold = fold_for_line(editor->folds, line_index);
     if (fold) {
@@ -193,6 +126,14 @@ void render_editor(Editor *editor) {
       break;
     if (line_len > 0 && line[line_len - 1] == '\n')
       line_len--;
+    uint32_t content_end = line_len;
+    while (content_end > 0 &&
+           (line[content_end - 1] == ' ' || line[content_end - 1] == '\t'))
+      content_end--;
+    uint32_t content_start = 0;
+    while (content_start < line_len &&
+           (line[content_start] == ' ' || line[content_start] == '\t'))
+      content_start++;
     std::vector<VWarn> line_warnings;
     while (warn_it != editor->warnings.end() && warn_it->line == line_index) {
       line_warnings.push_back(*warn_it);
@@ -253,6 +194,30 @@ void render_editor(Editor *editor) {
         if (editor->selection_active && absolute_byte_pos >= sel_start &&
             absolute_byte_pos < sel_end)
           bg = 0x555555;
+        uint32_t u_color = 0;
+        for (const auto &w : line_warnings) {
+          if (w.start <= current_byte_offset + local_render_offset &&
+              current_byte_offset + local_render_offset < w.end) {
+            switch (w.type) {
+            case 1:
+              u_color = 0xff0000;
+              fl |= CF_UNDERLINE;
+              break;
+            case 2:
+              u_color = 0xffff00;
+              fl |= CF_UNDERLINE;
+              break;
+            case 3:
+              u_color = 0xff00ff;
+              fl |= CF_UNDERLINE;
+              break;
+            case 4:
+              u_color = 0xA0A0A0;
+              fl |= CF_UNDERLINE;
+              break;
+            }
+          }
+        }
         uint32_t cluster_len = grapheme_next_character_break_utf8(
             line + current_byte_offset + local_render_offset, line_left);
         std::string cluster(line + current_byte_offset + local_render_offset,
@@ -260,8 +225,19 @@ void render_editor(Editor *editor) {
         int width = display_width(cluster.c_str(), cluster_len);
         if (col + width > render_width)
           break;
-        update(editor->position.row + rendered_rows, render_x + col,
-               cluster.c_str(), fg, bg | color, fl);
+        if (current_byte_offset + local_render_offset >= content_start &&
+            current_byte_offset + local_render_offset < content_end) {
+          update(editor->position.row + rendered_rows, render_x + col,
+                 cluster.c_str(), fg, bg | color, fl, u_color);
+        } else {
+          if (cluster[0] == ' ') {
+            update(editor->position.row + rendered_rows, render_x + col, "·",
+                   0x282828, bg | color, fl, u_color);
+          } else {
+            update(editor->position.row + rendered_rows, render_x + col, "->  ",
+                   0x282828, bg | color, (fl & ~CF_BOLD) | CF_ITALIC, u_color);
+          }
+        }
         local_render_offset += cluster_len;
         line_left -= cluster_len;
         col += width;
@@ -351,7 +327,11 @@ void render_editor(Editor *editor) {
                  cluster.c_str(), fg_color, color, 0);
           col += width;
           warn_idx += cluster_len;
+          while (width-- > 1)
+            update(editor->position.row + rendered_rows, render_x + col - width,
+                   "\x1b", fg_color, color, 0);
         }
+        line_warnings.clear();
       }
       while (col < render_width) {
         update(editor->position.row + rendered_rows, render_x + col, " ", 0,
@@ -393,6 +373,81 @@ void render_editor(Editor *editor) {
                0x555555 | color, 0);
         col++;
       }
+      if (!line_warnings.empty()) {
+        VWarn warn = line_warnings.front();
+        update(editor->position.row + rendered_rows, render_x + col, " ", 0,
+               color, 0);
+        col++;
+        for (size_t i = 0; i < line_warnings.size(); i++) {
+          if (line_warnings[i].type < warn.type)
+            warn = line_warnings[i];
+          std::string err_sym = " ";
+          uint32_t fg_color = 0;
+          switch (line_warnings[i].type) {
+          case 1:
+            err_sym = "";
+            fg_color = 0xFF0000;
+            goto final2;
+          case 2:
+            err_sym = "";
+            fg_color = 0xFFFF00;
+            goto final2;
+          case 3:
+            err_sym = "";
+            fg_color = 0xFF00FF;
+            goto final2;
+          case 4:
+            err_sym = "";
+            fg_color = 0xAAAAAA;
+            goto final2;
+          final2:
+            if (col < render_width) {
+              update(editor->position.row + rendered_rows, render_x + col,
+                     err_sym, fg_color, color, 0);
+              col++;
+              update(editor->position.row + rendered_rows, render_x + col, " ",
+                     fg_color, color, 0);
+              col++;
+            }
+          }
+        }
+        if (col < render_width) {
+          update(editor->position.row + rendered_rows, render_x + col, " ", 0,
+                 0 | color, 0);
+          col++;
+        }
+        size_t warn_idx = 0;
+        uint32_t fg_color = 0;
+        switch (warn.type) {
+        case 1:
+          fg_color = 0xFF0000;
+          break;
+        case 2:
+          fg_color = 0xFFFF00;
+          break;
+        case 3:
+          fg_color = 0xFF00FF;
+          break;
+        case 4:
+          fg_color = 0xAAAAAA;
+          break;
+        }
+        while (col < render_width && warn_idx < warn.text.length()) {
+          uint32_t cluster_len = grapheme_next_character_break_utf8(
+              warn.text.c_str() + warn_idx, warn.text.length() - warn_idx);
+          std::string cluster = warn.text.substr(warn_idx, cluster_len);
+          int width = display_width(cluster.c_str(), cluster_len);
+          if (col + width > render_width)
+            break;
+          update(editor->position.row + rendered_rows, render_x + col,
+                 cluster.c_str(), fg_color, color, 0);
+          col += width;
+          warn_idx += cluster_len;
+          while (width-- > 1)
+            update(editor->position.row + rendered_rows, render_x + col - width,
+                   "\x1b", fg_color, color, 0);
+        }
+      }
       while (col < render_width) {
         update(editor->position.row + rendered_rows, render_x + col, " ", 0,
                0 | color, 0);
@@ -418,6 +473,10 @@ void render_editor(Editor *editor) {
       break;
     }
     set_cursor(cursor.row, cursor.col, type, true);
+    if (editor->hover_active)
+      editor->hover.render(cursor);
+    else if (editor->diagnostics_active)
+      editor->diagnostics.render(cursor);
   }
   while (rendered_rows < editor->size.row) {
     for (uint32_t col = 0; col < editor->size.col; col++)

src/editor/scroll.cc

@@ -0,0 +1,234 @@
+#include "editor/editor.h"
+#include "editor/folds.h"
+
+void scroll_up(Editor *editor, int32_t number) {
+  if (!editor || number == 0)
+    return;
+  uint32_t numlen =
+      EXTRA_META + static_cast<int>(std::log10(editor->root->line_count + 1));
+  uint32_t render_width = editor->size.col - numlen;
+  uint32_t line_index = editor->scroll.row;
+  LineIterator *it = begin_l_iter(editor->root, line_index);
+  if (!it)
+    return;
+  uint32_t len;
+  char *line = next_line(it, &len);
+  if (!line) {
+    free(it->buffer);
+    free(it);
+    return;
+  }
+  if (len > 0 && line[len - 1] == '\n')
+    len--;
+  uint32_t current_byte_offset = 0;
+  uint32_t col = 0;
+  std::vector<uint32_t> segment_starts;
+  segment_starts.reserve(16);
+  if (current_byte_offset < editor->scroll.col)
+    segment_starts.push_back(0);
+  while (current_byte_offset < editor->scroll.col &&
+         current_byte_offset < len) {
+    uint32_t cluster_len = grapheme_next_character_break_utf8(
+        line + current_byte_offset, len - current_byte_offset);
+    int width = display_width(line + current_byte_offset, cluster_len);
+    if (col + width > render_width) {
+      segment_starts.push_back(current_byte_offset);
+      col = 0;
+    }
+    current_byte_offset += cluster_len;
+    col += width;
+  }
+  for (auto it_seg = segment_starts.rbegin(); it_seg != segment_starts.rend();
+       ++it_seg) {
+    if (--number == 0) {
+      editor->scroll = {line_index, *it_seg};
+      free(it->buffer);
+      free(it);
+      return;
+    }
+  }
+  line = prev_line(it, &len);
+  if (!line) {
+    editor->scroll = {0, 0};
+    free(it->buffer);
+    free(it);
+    return;
+  }
+  do {
+    line_index--;
+    line = prev_line(it, &len);
+    if (!line) {
+      editor->scroll = {0, 0};
+      free(it->buffer);
+      free(it);
+      return;
+    }
+    const Fold *fold = fold_for_line(editor->folds, line_index);
+    if (fold) {
+      while (line && line_index > fold->start) {
+        free(line);
+        line = prev_line(it, &len);
+        line_index--;
+        if (!line) {
+          editor->scroll = {0, 0};
+          free(it->buffer);
+          free(it);
+          return;
+        }
+      }
+      if (--number == 0) {
+        editor->scroll = {fold->start, 0};
+        free(it->buffer);
+        free(it);
+        return;
+      }
+      if (fold->start == 0) {
+        editor->scroll = {0, 0};
+        free(it->buffer);
+        free(it);
+        return;
+      }
+      line_index = fold->start - 1;
+      line = prev_line(it, &len);
+      if (!line) {
+        editor->scroll = {0, 0};
+        free(it->buffer);
+        free(it);
+        return;
+      }
+      continue;
+    }
+    if (len > 0 && line[len - 1] == '\n')
+      len--;
+    current_byte_offset = 0;
+    col = 0;
+    std::vector<uint32_t> segment_starts;
+    segment_starts.reserve(16);
+    segment_starts.push_back(0);
+    while (current_byte_offset < len) {
+      uint32_t cluster_len = grapheme_next_character_break_utf8(
+          line + current_byte_offset, len - current_byte_offset);
+      int width = display_width(line + current_byte_offset, cluster_len);
+      if (col + width > render_width) {
+        segment_starts.push_back(current_byte_offset);
+        col = 0;
+      }
+      current_byte_offset += cluster_len;
+      col += width;
+    }
+    for (auto it_seg = segment_starts.rbegin(); it_seg != segment_starts.rend();
+         ++it_seg) {
+      if (--number == 0) {
+        editor->scroll = {line_index, *it_seg};
+        free(it->buffer);
+        free(it);
+        return;
+      }
+    }
+  } while (number > 0);
+  free(it->buffer);
+  free(it);
+}
+
+void scroll_down(Editor *editor, uint32_t number) {
+  if (!editor || number == 0)
+    return;
+  uint32_t numlen =
+      EXTRA_META + static_cast<int>(std::log10(editor->root->line_count + 1));
+  uint32_t render_width = editor->size.col - numlen;
+  uint32_t line_index = editor->scroll.row;
+  LineIterator *it = begin_l_iter(editor->root, line_index);
+  if (!it)
+    return;
+  const uint32_t max_visual_lines = editor->size.row;
+  Coord *scroll_queue = (Coord *)malloc(sizeof(Coord) * max_visual_lines);
+  uint32_t q_head = 0;
+  uint32_t q_size = 0;
+  uint32_t visual_seen = 0;
+  bool first_visual_line = true;
+  while (true) {
+    const Fold *fold = fold_for_line(editor->folds, line_index);
+    if (fold) {
+      Coord fold_coord = {fold->start, 0};
+      if (q_size < max_visual_lines) {
+        scroll_queue[(q_head + q_size) % max_visual_lines] = fold_coord;
+        q_size++;
+      } else {
+        scroll_queue[q_head] = fold_coord;
+        q_head = (q_head + 1) % max_visual_lines;
+      }
+      visual_seen++;
+      if (visual_seen >= number + max_visual_lines) {
+        editor->scroll = scroll_queue[q_head];
+        break;
+      }
+      uint32_t skip_until = fold->end;
+      while (line_index <= skip_until) {
+        char *line = next_line(it, nullptr);
+        if (!line) {
+          free(scroll_queue);
+          free(it->buffer);
+          free(it);
+          return;
+        }
+        line_index++;
+      }
+      continue;
+    }
+    uint32_t line_len;
+    char *line = next_line(it, &line_len);
+    if (!line)
+      break;
+    if (line_len && line[line_len - 1] == '\n')
+      line_len--;
+    uint32_t current_byte_offset = 0;
+    if (first_visual_line) {
+      current_byte_offset += editor->scroll.col;
+      first_visual_line = false;
+    }
+    while (current_byte_offset < line_len ||
+           (line_len == 0 && current_byte_offset == 0)) {
+      Coord coord = {line_index, current_byte_offset};
+      if (q_size < max_visual_lines) {
+        scroll_queue[(q_head + q_size) % max_visual_lines] = coord;
+        q_size++;
+      } else {
+        scroll_queue[q_head] = coord;
+        q_head = (q_head + 1) % max_visual_lines;
+      }
+      visual_seen++;
+      if (visual_seen >= number + max_visual_lines) {
+        editor->scroll = scroll_queue[q_head];
+        free(scroll_queue);
+        free(it->buffer);
+        free(it);
+        return;
+      }
+      uint32_t col = 0;
+      uint32_t local_render_offset = 0;
+      uint32_t left = line_len - current_byte_offset;
+      while (left > 0 && col < render_width) {
+        uint32_t cluster_len = grapheme_next_character_break_utf8(
+            line + current_byte_offset + local_render_offset, left);
+        int width = display_width(
+            line + current_byte_offset + local_render_offset, cluster_len);
+        if (col + width > render_width)
+          break;
+        local_render_offset += cluster_len;
+        left -= cluster_len;
+        col += width;
+      }
+      current_byte_offset += local_render_offset;
+      if (line_len == 0)
+        break;
+    }
+    line_index++;
+  }
+  if (q_size > 0) {
+    uint32_t advance = (q_size > number) ? number : (q_size - 1);
+    editor->scroll = scroll_queue[(q_head + advance) % max_visual_lines];
+  }
+  free(it->buffer);
+  free(it);
+  free(scroll_queue);
+}

src/editor/selection.cc

@@ -0,0 +1,58 @@
+#include "editor/editor.h"
+
+char *get_selection(Editor *editor, uint32_t *out_len, Coord *out_start) {
+  std::shared_lock lock(editor->knot_mtx);
+  Coord start, end;
+  if (editor->cursor >= editor->selection) {
+    uint32_t prev_col, next_col;
+    switch (editor->selection_type) {
+    case CHAR:
+      start = editor->selection;
+      end = move_right(editor, editor->cursor, 1);
+      break;
+    case WORD:
+      word_boundaries(editor, editor->selection, &prev_col, &next_col, nullptr,
+                      nullptr);
+      start = {editor->selection.row, prev_col};
+      end = editor->cursor;
+      break;
+    case LINE:
+      start = {editor->selection.row, 0};
+      end = editor->cursor;
+      break;
+    }
+  } else {
+    start = editor->cursor;
+    uint32_t prev_col, next_col, line_len;
+    switch (editor->selection_type) {
+    case CHAR:
+      end = move_right(editor, editor->selection, 1);
+      break;
+    case WORD:
+      word_boundaries(editor, editor->selection, &prev_col, &next_col, nullptr,
+                      nullptr);
+      end = {editor->selection.row, next_col};
+      break;
+    case LINE:
+      LineIterator *it = begin_l_iter(editor->root, editor->selection.row);
+      char *line = next_line(it, &line_len);
+      if (!line)
+        return nullptr;
+      if (line_len > 0 && line[line_len - 1] == '\n')
+        line_len--;
+      end = {editor->selection.row, line_len};
+      free(it->buffer);
+      free(it);
+      break;
+    }
+  }
+  if (out_start)
+    *out_start = start;
+  uint32_t start_byte =
+      line_to_byte(editor->root, start.row, nullptr) + start.col;
+  uint32_t end_byte = line_to_byte(editor->root, end.row, nullptr) + end.col;
+  char *text = read(editor->root, start_byte, end_byte - start_byte);
+  if (out_len)
+    *out_len = end_byte - start_byte;
+  return text;
+}

src/editor/worker.cc

@@ -0,0 +1,90 @@
+#include "editor/editor.h"
+#include "ts/ts.h"
+
+static Highlight HL_UNDERLINE = {0, 0, CF_UNDERLINE, UINT8_MAX - 1};
+
+void hover_diagnostic(Editor *editor) {
+  std::shared_lock lock(editor->v_mtx);
+  static uint32_t last_line = UINT32_MAX;
+  if (last_line == editor->cursor.row && !editor->warnings_dirty)
+    return;
+  VWarn dummy;
+  dummy.line = editor->cursor.row;
+  editor->warnings_dirty = false;
+  last_line = editor->cursor.row;
+  auto first =
+      std::lower_bound(editor->warnings.begin(), editor->warnings.end(), dummy);
+  auto last =
+      std::upper_bound(editor->warnings.begin(), editor->warnings.end(), dummy);
+  std::vector<VWarn> warnings_at_line(first, last);
+  if (warnings_at_line.size() == 0) {
+    editor->diagnostics_active = false;
+    return;
+  }
+  editor->diagnostics.clear();
+  editor->diagnostics.warnings.swap(warnings_at_line);
+  editor->diagnostics.render_first();
+  editor->diagnostics_active = true;
+}
+
+void editor_worker(Editor *editor) {
+  if (!editor || !editor->root)
+    return;
+  if (editor->root->char_count > (1024 * 200))
+    return;
+  if (editor->ts.query_file != "" && !editor->ts.query)
+    editor->ts.query = load_query(editor->ts.query_file.c_str(), &editor->ts);
+  if (editor->ts.parser && editor->ts.query)
+    ts_collect_spans(editor);
+  uint32_t prev_col, next_col;
+  word_boundaries_exclusive(editor, editor->cursor, &prev_col, &next_col);
+  std::unique_lock lock(editor->def_spans.mtx);
+  editor->def_spans.spans.clear();
+  if (next_col - prev_col > 0 && next_col - prev_col < 256 - 4) {
+    std::shared_lock lockk(editor->knot_mtx);
+    uint32_t offset = line_to_byte(editor->root, editor->cursor.row, nullptr);
+    char *word = read(editor->root, offset + prev_col, next_col - prev_col);
+    lockk.unlock();
+    if (word) {
+      char buf[256];
+      snprintf(buf, sizeof(buf), "\\b%s\\b", word);
+      std::vector<std::pair<size_t, size_t>> results =
+          search_rope(editor->root, buf);
+      for (const auto &match : results) {
+        Span s;
+        s.start = match.first;
+        s.end = match.first + match.second;
+        s.hl = &HL_UNDERLINE;
+        editor->def_spans.spans.push_back(s);
+      }
+      free(word);
+    }
+  }
+  uint8_t top = 0;
+  static Highlight *hl_s = (Highlight *)calloc(200, sizeof(Highlight));
+  if (!hl_s)
+    exit(ENOMEM);
+  std::shared_lock lockk(editor->knot_mtx);
+  std::vector<std::pair<size_t, size_t>> results =
+      search_rope(editor->root, "(0x|#)[0-9a-fA-F]{6}");
+  for (int i = 0; i < results.size() && top < 200; i++) {
+    Span s;
+    s.start = results[i].first;
+    s.end = results[i].first + results[i].second;
+    char *buf = read(editor->root, s.start, s.end - s.start);
+    int x = buf[0] == '#' ? 1 : 2;
+    uint32_t bg = HEX(buf + x);
+    free(buf);
+    uint8_t r = bg >> 16, g = (bg >> 8) & 0xFF, b = bg & 0xFF;
+    double luminance = 0.299 * r + 0.587 * g + 0.114 * b;
+    uint32_t fg = (luminance > 128) ? 0x010101 : 0xFEFEFE;
+    hl_s[top] = {fg, bg, CF_BOLD, UINT8_MAX};
+    s.hl = &hl_s[top];
+    editor->def_spans.spans.push_back(s);
+    top++;
+  }
+  std::sort(editor->def_spans.spans.begin(), editor->def_spans.spans.end());
+  lock.unlock();
+  lockk.unlock();
+  hover_diagnostic(editor);
+}

src/editor_ctrl.cc

@@ -1,745 +0,0 @@
-extern "C" {
-#include "../libs/libgrapheme/grapheme.h"
-}
-#include "../include/editor.h"
-#include "../include/lsp.h"
-#include "../include/main.h"
-#include "../include/utils.h"
-
-uint32_t scan_left(const char *line, uint32_t len, uint32_t off) {
-  if (off > len)
-    off = len;
-  uint32_t i = off;
-  while (i > 0) {
-    unsigned char c = (unsigned char)line[i - 1];
-    if ((c & 0x80) != 0)
-      break;
-    if (!((c == '_') || (c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') ||
-          (c >= 'a' && c <= 'z')))
-      break;
-    --i;
-  }
-  return i;
-}
-
-uint32_t scan_right(const char *line, uint32_t len, uint32_t off) {
-  if (off > len)
-    off = len;
-  uint32_t i = off;
-  while (i < len) {
-    unsigned char c = (unsigned char)line[i];
-    if ((c & 0x80) != 0)
-      break;
-    if (!((c == '_') || (c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') ||
-          (c >= 'a' && c <= 'z')))
-      break;
-    ++i;
-  }
-  return i;
-}
-
-void word_boundaries_exclusive(Editor *editor, Coord coord, uint32_t *prev_col,
-                               uint32_t *next_col) {
-  if (!editor)
-    return;
-  std::shared_lock lock(editor->knot_mtx);
-  LineIterator *it = begin_l_iter(editor->root, coord.row);
-  if (!it)
-    return;
-  uint32_t line_len;
-  char *line = next_line(it, &line_len);
-  if (!line)
-    return;
-  if (line_len && line[line_len - 1] == '\n')
-    line_len--;
-  uint32_t col = coord.col;
-  if (col > line_len)
-    col = line_len;
-  uint32_t left = scan_left(line, line_len, col);
-  uint32_t right = scan_right(line, line_len, col);
-  if (prev_col)
-    *prev_col = left;
-  if (next_col)
-    *next_col = right;
-  free(it->buffer);
-  free(it);
-}
-
-uint32_t word_jump_right(const char *line, size_t len, uint32_t pos) {
-  if (pos >= len)
-    return len;
-  size_t next = grapheme_next_word_break_utf8(line + pos, len - pos);
-  return static_cast<uint32_t>(pos + next);
-}
-
-uint32_t word_jump_left(const char *line, size_t len, uint32_t col) {
-  if (col == 0)
-    return 0;
-  size_t pos = 0;
-  size_t last = 0;
-  size_t cursor = col;
-  while (pos < len) {
-    size_t next = pos + grapheme_next_word_break_utf8(line + pos, len - pos);
-    if (next >= cursor)
-      break;
-    last = next;
-    pos = next;
-  }
-  return static_cast<uint32_t>(last);
-}
-
-void word_boundaries(Editor *editor, Coord coord, uint32_t *prev_col,
-                     uint32_t *next_col, uint32_t *prev_clusters,
-                     uint32_t *next_clusters) {
-  if (!editor)
-    return;
-  std::shared_lock lock(editor->knot_mtx);
-  LineIterator *it = begin_l_iter(editor->root, coord.row);
-  if (!it)
-    return;
-  uint32_t line_len;
-  char *line = next_line(it, &line_len);
-  if (!line)
-    return;
-  if (line_len && line[line_len - 1] == '\n')
-    line_len--;
-  size_t col = coord.col;
-  if (col > line_len)
-    col = line_len;
-  size_t left = word_jump_left(line, line_len, col);
-  size_t right = word_jump_right(line, line_len, col);
-  if (prev_col)
-    *prev_col = static_cast<uint32_t>(left);
-  if (next_col)
-    *next_col = static_cast<uint32_t>(right);
-  if (prev_clusters)
-    *prev_clusters = count_clusters(line, line_len, left, col);
-  if (next_clusters)
-    *next_clusters = count_clusters(line, line_len, col, right);
-  free(it->buffer);
-  free(it);
-}
-
-Coord editor_hit_test(Editor *editor, uint32_t x, uint32_t y) {
-  if (mode == INSERT)
-    x++;
-  uint32_t numlen =
-      EXTRA_META + static_cast<int>(std::log10(editor->root->line_count + 1));
-  bool is_gutter_click = (x < numlen);
-  uint32_t render_width = editor->size.col - numlen;
-  x = MAX(x, numlen) - numlen;
-  uint32_t target_visual_row = y;
-  uint32_t visual_row = 0;
-  uint32_t line_index = editor->scroll.row;
-  uint32_t last_line_index = editor->scroll.row;
-  uint32_t last_col = editor->scroll.col;
-  bool first_visual_line = true;
-  std::shared_lock knot_lock(editor->knot_mtx);
-  LineIterator *it = begin_l_iter(editor->root, line_index);
-  if (!it)
-    return editor->scroll;
-  while (visual_row <= target_visual_row) {
-    const Fold *fold = fold_for_line(editor->folds, line_index);
-    if (fold) {
-      if (visual_row == target_visual_row) {
-        free(it->buffer);
-        free(it);
-        if (is_gutter_click) {
-          remove_fold(editor, fold->start);
-          return {UINT32_MAX, UINT32_MAX};
-        }
-        return {fold->start > 0 ? fold->start - 1 : 0, 0};
-      }
-      visual_row++;
-      while (line_index <= fold->end) {
-        char *l = next_line(it, nullptr);
-        if (!l)
-          break;
-        line_index++;
-      }
-      last_line_index = fold->end;
-      last_col = 0;
-      continue;
-    }
-    uint32_t line_len;
-    char *line = next_line(it, &line_len);
-    if (!line)
-      break;
-    if (line_len && line[line_len - 1] == '\n')
-      line_len--;
-    last_line_index = line_index;
-    last_col = line_len;
-    uint32_t offset = first_visual_line ? editor->scroll.col : 0;
-    first_visual_line = false;
-    while (offset < line_len || (line_len == 0 && offset == 0)) {
-      uint32_t col = 0;
-      uint32_t advance = 0;
-      uint32_t left = line_len - offset;
-      uint32_t last_good_offset = offset;
-      while (left > 0 && col < render_width) {
-        uint32_t g =
-            grapheme_next_character_break_utf8(line + offset + advance, left);
-        int w = display_width(line + offset + advance, g);
-        if (col + w > render_width)
-          break;
-        if (visual_row == target_visual_row && x < col + w) {
-          free(it->buffer);
-          free(it);
-          return {line_index, offset + advance};
-        }
-        advance += g;
-        last_good_offset = offset + advance;
-        left -= g;
-        col += w;
-      }
-      last_col = last_good_offset;
-      if (visual_row == target_visual_row) {
-        free(it->buffer);
-        free(it);
-        return {line_index, last_good_offset};
-      }
-      visual_row++;
-      if (visual_row > target_visual_row)
-        break;
-      if (advance == 0)
-        break;
-      offset += advance;
-      if (line_len == 0)
-        break;
-    }
-    line_index++;
-  }
-  free(it->buffer);
-  free(it);
-  return {last_line_index, last_col};
-}
-
-void move_line_up(Editor *editor) {
-  if (!editor || !editor->root || editor->cursor.row == 0)
-    return;
-  if (mode == NORMAL || mode == INSERT) {
-    uint32_t line_len, line_cluster_len;
-    std::shared_lock lock(editor->knot_mtx);
-    LineIterator *it = begin_l_iter(editor->root, editor->cursor.row);
-    char *line = next_line(it, &line_len);
-    if (!line) {
-      lock.unlock();
-      return;
-    }
-    if (line_len > 0 && line[line_len - 1] == '\n')
-      line_len--;
-    line_cluster_len = count_clusters(line, line_len, 0, line_len);
-    uint32_t target_row = prev_unfolded_row(editor, editor->cursor.row - 1);
-    uint32_t up_by = editor->cursor.row - target_row;
-    if (up_by > 1)
-      up_by--;
-    lock.unlock();
-    Coord cursor = editor->cursor;
-    edit_erase(editor, {cursor.row, 0}, line_cluster_len);
-    edit_erase(editor, {cursor.row, 0}, -1);
-    edit_insert(editor, {cursor.row - up_by, 0}, (char *)"\n", 1);
-    edit_insert(editor, {cursor.row - up_by, 0}, line, line_len);
-    free(it->buffer);
-    free(it);
-    editor->cursor = {cursor.row - up_by, cursor.col};
-  } else if (mode == SELECT) {
-    uint32_t start_row = MIN(editor->cursor.row, editor->selection.row);
-    uint32_t end_row = MAX(editor->cursor.row, editor->selection.row);
-    uint32_t start_byte = line_to_byte(editor->root, start_row, nullptr);
-    uint32_t end_byte = line_to_byte(editor->root, end_row + 1, nullptr);
-    char *selected_text = read(editor->root, start_byte, end_byte - start_byte);
-    if (!selected_text)
-      return;
-    uint32_t selected_len = count_clusters(selected_text, end_byte - start_byte,
-                                           0, end_byte - start_byte);
-    Coord cursor = editor->cursor;
-    Coord selection = editor->selection;
-    edit_erase(editor, {start_row, 0}, selected_len);
-    edit_insert(editor, {start_row - 1, 0}, selected_text,
-                end_byte - start_byte);
-    free(selected_text);
-    editor->cursor = {cursor.row - 1, cursor.col};
-    editor->selection = {selection.row - 1, selection.col};
-  }
-}
-
-void move_line_down(Editor *editor) {
-  if (!editor || !editor->root)
-    return;
-  if (mode == NORMAL || mode == INSERT) {
-    if (editor->cursor.row >= editor->root->line_count - 1)
-      return;
-    uint32_t line_len, line_cluster_len;
-    std::shared_lock lock(editor->knot_mtx);
-    LineIterator *it = begin_l_iter(editor->root, editor->cursor.row);
-    char *line = next_line(it, &line_len);
-    if (!line) {
-      lock.unlock();
-      return;
-    }
-    if (line_len && line[line_len - 1] == '\n')
-      line_len--;
-    line_cluster_len = count_clusters(line, line_len, 0, line_len);
-    uint32_t target_row = next_unfolded_row(editor, editor->cursor.row + 1);
-    if (target_row >= editor->root->line_count) {
-      free(line);
-      lock.unlock();
-      return;
-    }
-    uint32_t down_by = target_row - editor->cursor.row;
-    if (down_by > 1)
-      down_by--;
-    uint32_t ln;
-    line_to_byte(editor->root, editor->cursor.row + down_by - 1, &ln);
-    lock.unlock();
-    Coord cursor = editor->cursor;
-    edit_erase(editor, {cursor.row, 0}, line_cluster_len);
-    edit_erase(editor, {cursor.row, 0}, -1);
-    edit_insert(editor, {cursor.row + down_by, 0}, (char *)"\n", 1);
-    edit_insert(editor, {cursor.row + down_by, 0}, line, line_len);
-    free(it->buffer);
-    free(it);
-    editor->cursor = {cursor.row + down_by, cursor.col};
-  } else if (mode == SELECT) {
-    if (editor->cursor.row >= editor->root->line_count - 1 ||
-        editor->selection.row >= editor->root->line_count - 1)
-      return;
-    std::shared_lock lock(editor->knot_mtx);
-    uint32_t start_row = MIN(editor->cursor.row, editor->selection.row);
-    uint32_t end_row = MAX(editor->cursor.row, editor->selection.row);
-    uint32_t target_row = next_unfolded_row(editor, end_row + 1);
-    if (target_row >= editor->root->line_count)
-      return;
-    uint32_t down_by = target_row - end_row;
-    if (down_by > 1)
-      down_by--;
-    uint32_t start_byte = line_to_byte(editor->root, start_row, nullptr);
-    uint32_t end_byte = line_to_byte(editor->root, end_row + 1, nullptr);
-    char *selected_text = read(editor->root, start_byte, end_byte - start_byte);
-    lock.unlock();
-    if (!selected_text)
-      return;
-    uint32_t selected_len = count_clusters(selected_text, end_byte - start_byte,
-                                           0, end_byte - start_byte);
-    Coord cursor = editor->cursor;
-    Coord selection = editor->selection;
-    edit_erase(editor, {start_row, 0}, selected_len);
-    edit_insert(editor, {start_row + down_by, 0}, selected_text,
-                end_byte - start_byte);
-    free(selected_text);
-    editor->cursor = {cursor.row + down_by, cursor.col};
-    editor->selection = {selection.row + down_by, selection.col};
-  }
-}
-
-void edit_erase(Editor *editor, Coord pos, int64_t len) {
-  if (len == 0)
-    return;
-  if (len < 0) {
-    std::shared_lock lock_1(editor->knot_mtx);
-    uint32_t cursor_original =
-        line_to_byte(editor->root, editor->cursor.row, nullptr) +
-        editor->cursor.col;
-    TSPoint old_point = {pos.row, pos.col};
-    uint32_t byte_pos = line_to_byte(editor->root, pos.row, nullptr) + pos.col;
-    Coord point = move_left_pure(editor, pos, -len);
-    json lsp_range;
-    bool do_lsp = (editor->lsp != nullptr);
-    if (do_lsp) {
-      LineIterator *it = begin_l_iter(editor->root, point.row);
-      char *line = next_line(it, nullptr);
-      int utf16_start = 0;
-      if (line)
-        utf16_start = utf8_byte_offset_to_utf16(line, point.col);
-      free(it->buffer);
-      free(it);
-      it = begin_l_iter(editor->root, pos.row);
-      line = next_line(it, nullptr);
-      int utf16_end = 0;
-      if (line)
-        utf16_end = utf8_byte_offset_to_utf16(line, pos.col);
-      free(it->buffer);
-      free(it);
-      lsp_range = {{"start", {{"line", point.row}, {"character", utf16_start}}},
-                   {"end", {{"line", pos.row}, {"character", utf16_end}}}};
-    }
-    uint32_t start = line_to_byte(editor->root, point.row, nullptr) + point.col;
-    if (cursor_original > start && cursor_original <= byte_pos) {
-      editor->cursor = point;
-      editor->cursor_preffered = UINT32_MAX;
-    } else if (cursor_original > byte_pos) {
-      uint32_t cursor_new = cursor_original - (byte_pos - start);
-      uint32_t new_col;
-      uint32_t new_row = byte_to_line(editor->root, cursor_new, &new_col);
-      editor->cursor = {new_row, new_col};
-      editor->cursor_preffered = UINT32_MAX;
-    }
-    lock_1.unlock();
-    uint32_t start_row = point.row;
-    uint32_t end_row = pos.row;
-    apply_line_deletion(editor, start_row + 1, end_row);
-    apply_hook_deletion(editor, start_row + 1, end_row);
-    std::unique_lock lock_2(editor->knot_mtx);
-    editor->root = erase(editor->root, start, byte_pos - start);
-    lock_2.unlock();
-    if (editor->ts.tree) {
-      TSInputEdit edit = {
-          .start_byte = start,
-          .old_end_byte = byte_pos,
-          .new_end_byte = start,
-          .start_point = {point.row, point.col},
-          .old_end_point = old_point,
-          .new_end_point = {point.row, point.col},
-      };
-      editor->edit_queue.push(edit);
-    }
-    if (do_lsp) {
-      json message = {
-          {"jsonrpc", "2.0"},
-          {"method", "textDocument/didChange"},
-          {"params",
-           {{"textDocument",
-             {{"uri", editor->uri}, {"version", ++editor->lsp_version}}},
-            {"contentChanges",
-             json::array({{{"range", lsp_range}, {"text", ""}}})}}}};
-      lsp_send(editor->lsp, message, nullptr);
-    }
-    std::unique_lock lock_3(editor->spans.mtx);
-    apply_edit(editor->spans.spans, start, start - byte_pos);
-    if (editor->spans.mid_parse)
-      editor->spans.edits.push({start, start - byte_pos});
-    std::unique_lock lock_4(editor->def_spans.mtx);
-    apply_edit(editor->def_spans.spans, byte_pos, start - byte_pos);
-  } else {
-    std::shared_lock lock_1(editor->knot_mtx);
-    uint32_t cursor_original =
-        line_to_byte(editor->root, editor->cursor.row, nullptr) +
-        editor->cursor.col;
-    TSPoint old_point = {pos.row, pos.col};
-    uint32_t byte_pos = line_to_byte(editor->root, pos.row, nullptr) + pos.col;
-    Coord point = move_right_pure(editor, pos, len);
-    json lsp_range;
-    bool do_lsp = (editor->lsp != nullptr);
-    if (do_lsp) {
-      LineIterator *it = begin_l_iter(editor->root, pos.row);
-      char *line = next_line(it, nullptr);
-      int utf16_start = 0;
-      if (line)
-        utf16_start = utf8_byte_offset_to_utf16(line, pos.col);
-      free(it->buffer);
-      free(it);
-      it = begin_l_iter(editor->root, point.row);
-      line = next_line(it, nullptr);
-      int utf16_end = 0;
-      if (line)
-        utf16_end = utf8_byte_offset_to_utf16(line, point.col);
-      free(it->buffer);
-      free(it);
-      lsp_range = {{"start", {{"line", pos.row}, {"character", utf16_start}}},
-                   {"end", {{"line", point.row}, {"character", utf16_end}}}};
-    }
-    uint32_t end = line_to_byte(editor->root, point.row, nullptr) + point.col;
-    if (cursor_original > byte_pos && cursor_original <= end) {
-      editor->cursor = pos;
-      editor->cursor_preffered = UINT32_MAX;
-    } else if (cursor_original > end) {
-      uint32_t cursor_new = cursor_original - (end - byte_pos);
-      uint32_t new_col;
-      uint32_t new_row = byte_to_line(editor->root, cursor_new, &new_col);
-      editor->cursor = {new_row, new_col};
-      editor->cursor_preffered = UINT32_MAX;
-    }
-    lock_1.unlock();
-    uint32_t start_row = pos.row;
-    uint32_t end_row = point.row;
-    apply_line_deletion(editor, start_row + 1, end_row);
-    apply_hook_deletion(editor, start_row + 1, end_row);
-    std::unique_lock lock_2(editor->knot_mtx);
-    editor->root = erase(editor->root, byte_pos, end - byte_pos);
-    lock_2.unlock();
-    if (editor->ts.tree) {
-      TSInputEdit edit = {
-          .start_byte = byte_pos,
-          .old_end_byte = end,
-          .new_end_byte = byte_pos,
-          .start_point = old_point,
-          .old_end_point = {point.row, point.col},
-          .new_end_point = old_point,
-      };
-      editor->edit_queue.push(edit);
-    }
-    if (do_lsp) {
-      json message = {
-          {"jsonrpc", "2.0"},
-          {"method", "textDocument/didChange"},
-          {"params",
-           {{"textDocument",
-             {{"uri", editor->uri}, {"version", ++editor->lsp_version}}},
-            {"contentChanges",
-             json::array({{{"range", lsp_range}, {"text", ""}}})}}}};
-      lsp_send(editor->lsp, message, nullptr);
-    }
-    std::unique_lock lock_3(editor->spans.mtx);
-    apply_edit(editor->spans.spans, byte_pos, byte_pos - end);
-    if (editor->spans.mid_parse)
-      editor->spans.edits.push({byte_pos, byte_pos - end});
-    std::unique_lock lock_4(editor->def_spans.mtx);
-    apply_edit(editor->def_spans.spans, byte_pos, byte_pos - end);
-  }
-}
-
-void edit_insert(Editor *editor, Coord pos, char *data, uint32_t len) {
-  std::shared_lock lock_1(editor->knot_mtx);
-  uint32_t cursor_original =
-      line_to_byte(editor->root, editor->cursor.row, nullptr) +
-      editor->cursor.col;
-  uint32_t byte_pos = line_to_byte(editor->root, pos.row, nullptr) + pos.col;
-  TSPoint start_point = {pos.row, pos.col};
-  if (cursor_original > byte_pos) {
-    uint32_t cursor_new = cursor_original + len;
-    uint32_t new_col;
-    uint32_t new_row = byte_to_line(editor->root, cursor_new, &new_col);
-    editor->cursor = {new_row, new_col};
-  }
-  lock_1.unlock();
-  std::unique_lock lock_2(editor->knot_mtx);
-  editor->root = insert(editor->root, byte_pos, data, len);
-  lock_2.unlock();
-  uint32_t cols = 0;
-  uint32_t rows = 0;
-  for (uint32_t i = 0; i < len; i++) {
-    if (data[i] == '\n') {
-      rows++;
-      cols = 0;
-    } else {
-      cols++;
-    }
-  }
-  apply_line_insertion(editor, pos.row, rows);
-  apply_hook_insertion(editor, pos.row, rows);
-  if (editor->ts.tree) {
-    TSInputEdit edit = {
-        .start_byte = byte_pos,
-        .old_end_byte = byte_pos,
-        .new_end_byte = byte_pos + len,
-        .start_point = start_point,
-        .old_end_point = start_point,
-        .new_end_point = {start_point.row + rows,
-                          (rows == 0) ? (start_point.column + cols) : cols},
-    };
-    editor->edit_queue.push(edit);
-  }
-  if (editor->lsp) {
-    lock_1.lock();
-    LineIterator *it = begin_l_iter(editor->root, pos.row);
-    char *line = next_line(it, nullptr);
-    int utf16_col = 0;
-    if (line)
-      utf16_col = utf8_byte_offset_to_utf16(line, pos.col);
-    free(it->buffer);
-    free(it);
-    lock_1.unlock();
-    json message = {
-        {"jsonrpc", "2.0"},
-        {"method", "textDocument/didChange"},
-        {"params",
-         {{"textDocument",
-           {{"uri", editor->uri}, {"version", ++editor->lsp_version}}},
-          {"contentChanges",
-           json::array(
-               {{{"range",
-                  {{"start", {{"line", pos.row}, {"character", utf16_col}}},
-                   {"end", {{"line", pos.row}, {"character", utf16_col}}}}},
-                 {"text", std::string(data, len)}}})}}}};
-    lsp_send(editor->lsp, message, nullptr);
-  }
-  std::unique_lock lock_3(editor->spans.mtx);
-  apply_edit(editor->spans.spans, byte_pos, len);
-  if (editor->spans.mid_parse)
-    editor->spans.edits.push({byte_pos, len});
-  std::unique_lock lock_4(editor->def_spans.mtx);
-  apply_edit(editor->def_spans.spans, byte_pos, len);
-}
-
-char *get_selection(Editor *editor, uint32_t *out_len, Coord *out_start) {
-  std::shared_lock lock(editor->knot_mtx);
-  Coord start, end;
-  if (editor->cursor >= editor->selection) {
-    uint32_t prev_col, next_col;
-    switch (editor->selection_type) {
-    case CHAR:
-      start = editor->selection;
-      end = move_right(editor, editor->cursor, 1);
-      break;
-    case WORD:
-      word_boundaries(editor, editor->selection, &prev_col, &next_col, nullptr,
-                      nullptr);
-      start = {editor->selection.row, prev_col};
-      end = editor->cursor;
-      break;
-    case LINE:
-      start = {editor->selection.row, 0};
-      end = editor->cursor;
-      break;
-    }
-  } else {
-    start = editor->cursor;
-    uint32_t prev_col, next_col, line_len;
-    switch (editor->selection_type) {
-    case CHAR:
-      end = move_right(editor, editor->selection, 1);
-      break;
-    case WORD:
-      word_boundaries(editor, editor->selection, &prev_col, &next_col, nullptr,
-                      nullptr);
-      end = {editor->selection.row, next_col};
-      break;
-    case LINE:
-      LineIterator *it = begin_l_iter(editor->root, editor->selection.row);
-      char *line = next_line(it, &line_len);
-      if (!line)
-        return nullptr;
-      if (line_len > 0 && line[line_len - 1] == '\n')
-        line_len--;
-      end = {editor->selection.row, line_len};
-      free(it->buffer);
-      free(it);
-      break;
-    }
-  }
-  if (out_start)
-    *out_start = start;
-  uint32_t start_byte =
-      line_to_byte(editor->root, start.row, nullptr) + start.col;
-  uint32_t end_byte = line_to_byte(editor->root, end.row, nullptr) + end.col;
-  char *text = read(editor->root, start_byte, end_byte - start_byte);
-  if (out_len)
-    *out_len = end_byte - start_byte;
-  return text;
-}
-
-void apply_edit(std::vector<Span> &spans, uint32_t x, int64_t y) {
-  Span key{.start = x, .end = 0, .hl = nullptr};
-  auto it = std::lower_bound(
-      spans.begin(), spans.end(), key,
-      [](const Span &a, const Span &b) { return a.start < b.start; });
-  size_t idx = std::distance(spans.begin(), it);
-  while (idx > 0 && spans.at(idx - 1).end >= x)
-    --idx;
-  for (size_t i = idx; i < spans.size();) {
-    Span &s = spans.at(i);
-    if (s.start < x && s.end >= x) {
-      s.end += y;
-    } else if (s.start > x) {
-      s.start += y;
-      s.end += y;
-    }
-    if (s.end <= s.start)
-      spans.erase(spans.begin() + i);
-    else
-      ++i;
-  }
-}
-
-std::vector<Fold>::iterator find_fold_iter(Editor *editor, uint32_t line) {
-  auto &folds = editor->folds;
-  auto it = std::lower_bound(
-      folds.begin(), folds.end(), line,
-      [](const Fold &fold, uint32_t value) { return fold.start < value; });
-  if (it != folds.end() && it->start == line)
-    return it;
-  if (it != folds.begin()) {
-    --it;
-    if (it->contains(line))
-      return it;
-  }
-  return folds.end();
-}
-
-bool add_fold(Editor *editor, uint32_t start, uint32_t end) {
-  if (!editor || !editor->root)
-    return false;
-  if (start > end)
-    std::swap(start, end);
-  if (start >= editor->root->line_count)
-    return false;
-  end = std::min(end, editor->root->line_count - 1);
-  if (start == end)
-    return false;
-  Fold new_fold{start, end};
-  auto &folds = editor->folds;
-  auto it = std::lower_bound(
-      folds.begin(), folds.end(), new_fold.start,
-      [](const Fold &fold, uint32_t value) { return fold.start < value; });
-  if (it != folds.begin()) {
-    auto prev = std::prev(it);
-    if (prev->end + 1 >= new_fold.start) {
-      new_fold.start = std::min(new_fold.start, prev->start);
-      new_fold.end = std::max(new_fold.end, prev->end);
-      it = folds.erase(prev);
-    }
-  }
-  while (it != folds.end() && it->start <= new_fold.end + 1) {
-    new_fold.end = std::max(new_fold.end, it->end);
-    it = folds.erase(it);
-  }
-  folds.insert(it, new_fold);
-  return true;
-}
-
-bool remove_fold(Editor *editor, uint32_t line) {
-  auto it = find_fold_iter(editor, line);
-  if (it == editor->folds.end())
-    return false;
-  editor->folds.erase(it);
-  return true;
-}
-
-void apply_line_insertion(Editor *editor, uint32_t line, uint32_t rows) {
-  for (auto it = editor->folds.begin(); it != editor->folds.end();) {
-    if (line <= it->start) {
-      it->start += rows;
-      it->end += rows;
-      ++it;
-    } else if (line <= it->end) {
-      it = editor->folds.erase(it);
-    } else {
-      ++it;
-    }
-  }
-}
-
-void apply_line_deletion(Editor *editor, uint32_t removal_start,
-                         uint32_t removal_end) {
-  if (removal_start > removal_end)
-    return;
-  uint32_t rows_removed = removal_end - removal_start + 1;
-  std::vector<Fold> updated;
-  updated.reserve(editor->folds.size());
-  for (auto fold : editor->folds) {
-    if (removal_end < fold.start) {
-      fold.start -= rows_removed;
-      fold.end -= rows_removed;
-      updated.push_back(fold);
-      continue;
-    }
-    if (removal_start > fold.end) {
-      updated.push_back(fold);
-      continue;
-    }
-  }
-  editor->folds.swap(updated);
-}
-
-void apply_hook_insertion(Editor *editor, uint32_t line, uint32_t rows) {
-  for (auto &hook : editor->hooks)
-    if (hook > line)
-      hook += rows;
-}
-
-void apply_hook_deletion(Editor *editor, uint32_t removal_start,
-                         uint32_t removal_end) {
-  for (auto &hook : editor->hooks)
-    if (hook > removal_start)
-      hook -= removal_end - removal_start + 1;
-}

src/io/input.cc

@@ -1,12 +1,4 @@
-extern "C" {
-#include "../libs/libgrapheme/grapheme.h"
-}
-#include "../include/ui.h"
-#include <cstdint>
-#include <cstdio>
-#include <cstdlib>
-#include <cstring>
-#include <unistd.h>
+#include "io/ui.h"
 
 static Queue<char> input_queue;
 

src/io/knot.cc

@@ -1,12 +1,4 @@
-#include "../include/knot.h"
-#include <assert.h>
-#include <cmath>
-#include <cstdint>
-#include <cstdlib>
-#include <cstring>
-#include <pcre2.h>
-#include <stdio.h>
-#include <string.h>
+#include "io/knot.h"
 
 static void update(Knot *n) {
   if (!n)

src/io/renderer.cc

@@ -1,4 +1,4 @@
-#include "../include/ui.h"
+#include "io/ui.h"
 
 uint32_t rows, cols;
 bool show_cursor = 0;
@@ -55,9 +55,13 @@ void update(uint32_t row, uint32_t col, std::string utf8, uint32_t fg,
   uint32_t idx = row * cols + col;
   std::lock_guard<std::mutex> lock(screen_mutex);
   screen[idx].utf8 = utf8 != "" ? utf8 : "";
+  if (utf8 == "")
+    return;
+  screen[idx].width = display_width(utf8.c_str(), utf8.size());
   screen[idx].fg = fg;
   screen[idx].bg = bg;
   screen[idx].flags = flags;
+  screen[idx].ul_color = 0;
 }
 
 void update(uint32_t row, uint32_t col, const char *utf8, uint32_t fg,
@@ -67,15 +71,56 @@ void update(uint32_t row, uint32_t col, const char *utf8, uint32_t fg,
   uint32_t idx = row * cols + col;
   std::lock_guard<std::mutex> lock(screen_mutex);
   screen[idx].utf8 = utf8 ? utf8 : "";
+  if (utf8 == nullptr)
+    return;
+  screen[idx].width = display_width(utf8, strlen(utf8));
   screen[idx].fg = fg;
   screen[idx].bg = bg;
   screen[idx].flags = flags;
+  screen[idx].ul_color = 0;
+}
+
+void update(uint32_t row, uint32_t col, std::string utf8, uint32_t fg,
+            uint32_t bg, uint8_t flags, uint32_t ul_color) {
+  if (row >= rows || col >= cols)
+    return;
+  uint32_t idx = row * cols + col;
+  std::lock_guard<std::mutex> lock(screen_mutex);
+  screen[idx].utf8 = utf8 != "" ? utf8 : "";
+  if (utf8 == "")
+    return;
+  screen[idx].width = display_width(utf8.c_str(), utf8.size());
+  screen[idx].fg = fg;
+  screen[idx].bg = bg;
+  screen[idx].flags = flags;
+  screen[idx].ul_color = ul_color;
+}
+
+void update(uint32_t row, uint32_t col, const char *utf8, uint32_t fg,
+            uint32_t bg, uint8_t flags, uint32_t ul_color) {
+  if (row >= rows || col >= cols)
+    return;
+  uint32_t idx = row * cols + col;
+  std::lock_guard<std::mutex> lock(screen_mutex);
+  screen[idx].utf8 = utf8 ? utf8 : "";
+  if (utf8 == nullptr)
+    return;
+  screen[idx].width = display_width(utf8, strlen(utf8));
+  screen[idx].fg = fg;
+  screen[idx].bg = bg;
+  screen[idx].flags = flags;
+  screen[idx].ul_color = ul_color;
+}
+
+inline bool is_empty_cell(const ScreenCell &c) {
+  return c.utf8.empty() || c.utf8 == " " || c.utf8 == "\x1b";
 }
 
 void render() {
   static bool first_render = true;
   uint32_t current_fg = 0;
   uint32_t current_bg = 0;
+  uint32_t current_ul_color = 0;
   bool current_italic = false;
   bool current_bold = false;
   bool current_underline = false;
@@ -94,15 +139,25 @@ void render() {
       uint32_t idx = row * cols + col;
       ScreenCell &old_cell = old_screen[idx];
       ScreenCell &new_cell = screen[idx];
-      bool content_changed = old_cell.utf8 != new_cell.utf8;
-      bool style_changed =
-          (old_cell.fg != new_cell.fg) || (old_cell.bg != new_cell.bg) ||
-          ((old_cell.flags & CF_ITALIC) != (new_cell.flags & CF_ITALIC)) ||
-          ((old_cell.flags & CF_BOLD) != (new_cell.flags & CF_BOLD)) ||
-          ((old_cell.flags & CF_UNDERLINE) != (new_cell.flags & CF_UNDERLINE));
-      if (content_changed || style_changed) {
-        if (first_change_col == -1)
+      bool content_changed =
+          old_cell.utf8 != new_cell.utf8 || old_cell.fg != new_cell.fg ||
+          old_cell.bg != new_cell.bg || old_cell.flags != new_cell.flags ||
+          old_cell.ul_color != new_cell.ul_color;
+      if (content_changed) {
+        if (first_change_col == -1) {
           first_change_col = col;
+          if (first_change_col > 0) {
+            for (int back = 1; back <= 3 && first_change_col - back >= 0;
+                 ++back) {
+              ScreenCell &prev_cell =
+                  screen[row * cols + (first_change_col - back)];
+              if (prev_cell.width > 1) {
+                first_change_col -= back;
+                break;
+              }
+            }
+          }
+        }
         last_change_col = col;
       }
     }
@@ -115,6 +170,20 @@ void render() {
       int idx = row * cols + col;
       ScreenCell &old_cell = old_screen[idx];
       ScreenCell &new_cell = screen[idx];
+      int width = new_cell.width > 0 ? new_cell.width : 1;
+      bool overlap = false;
+      if (width > 1) {
+        for (int i = 1; i < width; ++i) {
+          int next_col = col + i;
+          if (next_col >= cols)
+            break;
+          const ScreenCell &next = screen[row * cols + next_col];
+          if (!is_empty_cell(next)) {
+            overlap = true;
+            break;
+          }
+        }
+      }
       if (current_fg != new_cell.fg) {
         if (new_cell.fg) {
           char fb[64];
@@ -150,24 +219,43 @@ void render() {
         current_bold = bold;
       }
       bool underline = (new_cell.flags & CF_UNDERLINE) != 0;
+      if (underline) {
+        if (new_cell.ul_color != current_ul_color) {
+          if (new_cell.ul_color) {
+            char ubuf[64];
+            snprintf(ubuf, sizeof(ubuf), "\x1b[58;2;%d;%d;%dm",
+                     (new_cell.ul_color >> 16) & 0xFF,
+                     (new_cell.ul_color >> 8) & 0xFF,
+                     (new_cell.ul_color >> 0) & 0xFF);
+            out.append(ubuf);
+          } else {
+            out += "\x1b[59m";
+          }
+          current_ul_color = new_cell.ul_color;
+        }
+      }
       if (underline != current_underline) {
         out += underline ? "\x1b[4m" : "\x1b[24m";
         current_underline = underline;
       }
-      if (!new_cell.utf8.empty()) {
-        if (new_cell.utf8[0] == '\t')
-          out.append("    ");
-        else if (new_cell.utf8[0] == '\x1b')
-          out.append("");
-        else
-          out.append(new_cell.utf8);
+      if (width > 1 && overlap) {
+        for (int i = 1; i < width; ++i)
+          out.push_back(' ');
       } else {
-        out.append(1, ' ');
+        if (!new_cell.utf8.empty()) {
+          if (new_cell.utf8[0] == '\t')
+            out.append("    ");
+          else if (new_cell.utf8[0] != '\x1b')
+            out.append(new_cell.utf8);
+        } else {
+          out.push_back(' ');
+        }
       }
       old_cell.utf8 = new_cell.utf8;
       old_cell.fg = new_cell.fg;
       old_cell.bg = new_cell.bg;
       old_cell.flags = new_cell.flags;
+      old_cell.width = new_cell.width;
     }
   }
   out += "\x1b[0m";

src/lsp.cc

@@ -1,326 +0,0 @@
-#include "../include/lsp.h"
-#include "../include/maps.h"
-#include <fcntl.h>
-#include <signal.h>
-#include <sys/poll.h>
-#include <sys/types.h>
-#include <sys/wait.h>
-#include <unistd.h>
-
-std::shared_mutex active_lsps_mtx;
-std::unordered_map<uint8_t, LSPInstance *> active_lsps;
-
-Queue<LSPOpenRequest> lsp_open_queue;
-
-static bool init_lsp(LSPInstance *lsp) {
-  int in_pipe[2];
-  int out_pipe[2];
-  if (pipe(in_pipe) == -1 || pipe(out_pipe) == -1) {
-    perror("pipe");
-    return false;
-  }
-  pid_t pid = fork();
-  if (pid == -1) {
-    perror("fork");
-    return false;
-  }
-  if (pid == 0) {
-    dup2(in_pipe[0], STDIN_FILENO);
-    dup2(out_pipe[1], STDOUT_FILENO);
-    int devnull = open("/dev/null", O_WRONLY);
-    if (devnull >= 0) {
-      dup2(devnull, STDERR_FILENO);
-      close(devnull);
-    }
-    close(in_pipe[1]);
-    close(out_pipe[0]);
-    execvp(lsp->lsp->command, (char *const *)(lsp->lsp->args.data()));
-    perror("execvp");
-    return false;
-  }
-  lsp->pid = pid;
-  lsp->stdin_fd = in_pipe[1];
-  lsp->stdout_fd = out_pipe[0];
-  close(in_pipe[0]);
-  close(out_pipe[1]);
-  return true;
-}
-
-LSPInstance *get_or_init_lsp(uint8_t lsp_id) {
-  std::unique_lock lock(active_lsps_mtx);
-  auto it = active_lsps.find(lsp_id);
-  if (it == active_lsps.end()) {
-    auto map_it = kLsps.find(lsp_id);
-    if (map_it == kLsps.end())
-      return nullptr;
-    LSPInstance *lsp = new LSPInstance();
-    lsp->lsp = &map_it->second;
-    if (!init_lsp(lsp)) {
-      delete lsp;
-      return nullptr;
-    }
-    LSPPending *pending = new LSPPending();
-    pending->method = "initialize";
-    pending->editor = nullptr;
-    pending->callback = [lsp](Editor *, std::string, json) {
-      lsp->initialized = true;
-      json initialized = {{"jsonrpc", "2.0"},
-                          {"method", "initialized"},
-                          {"params", json::object()}};
-      lsp_send(lsp, initialized, nullptr);
-    };
-    json init_message = {
-        {"jsonrpc", "2.0"},
-        {"method", "initialize"},
-        {"params",
-         {{"processId", getpid()},
-          {"rootUri", "file://" + std::filesystem::current_path().string()},
-          {"capabilities", json::object()}}}};
-    lsp_send(lsp, init_message, pending);
-    active_lsps[lsp_id] = lsp;
-    return lsp;
-  }
-  return it->second;
-}
-
-void lsp_send(LSPInstance *lsp, json message, LSPPending *pending) {
-  if (!lsp || lsp->stdin_fd == -1)
-    return;
-  std::unique_lock lock(lsp->mtx);
-  if (pending) {
-    message["id"] = lsp->last_id++;
-    uint32_t id = message["id"].get<uint32_t>();
-    lsp->pending[id] = pending;
-  }
-  lsp->outbox.push(message);
-}
-
-void close_lsp(uint8_t lsp_id) {
-  std::shared_lock active_lsps_lock(active_lsps_mtx);
-  auto it = active_lsps.find(lsp_id);
-  if (it == active_lsps.end())
-    return;
-  LSPInstance *lsp = it->second;
-  active_lsps_lock.unlock();
-  LSPPending *shutdown_pending = new LSPPending();
-  shutdown_pending->method = "shutdown";
-  shutdown_pending->callback = [lsp, lsp_id](Editor *, std::string, json) {
-    json exit = {{"jsonrpc", "2.0"}, {"method", "exit"}};
-    lsp_send(lsp, exit, nullptr);
-  };
-  json shutdown = {{"jsonrpc", "2.0"}, {"method", "shutdown"}};
-  lsp_send(lsp, shutdown, shutdown_pending);
-  std::thread t([lsp, lsp_id] {
-    std::this_thread::sleep_for(100ms);
-    std::unique_lock active_lsps_lock(active_lsps_mtx);
-    std::unique_lock lock(lsp->mtx);
-    if (kill(lsp->pid, 0) == 0)
-      kill(lsp->pid, SIGKILL);
-    waitpid(lsp->pid, nullptr, 0);
-    close(lsp->stdin_fd);
-    close(lsp->stdout_fd);
-    while (!lsp->outbox.empty())
-      lsp->outbox.pop();
-    while (!lsp->inbox.empty())
-      lsp->inbox.pop();
-    for (auto &kv : lsp->pending)
-      delete kv.second;
-    delete lsp;
-    active_lsps.erase(lsp_id);
-  });
-  t.detach();
-}
-
-static std::optional<json> read_lsp_message(int fd) {
-  std::string header;
-  char c;
-  while (true) {
-    ssize_t n = read(fd, &c, 1);
-    if (n <= 0)
-      return std::nullopt;
-    header.push_back(c);
-    if (header.size() >= 4 && header.substr(header.size() - 4) == "\r\n\r\n")
-      break;
-  }
-  size_t pos = header.find("Content-Length:");
-  if (pos == std::string::npos)
-    return std::nullopt;
-  pos += strlen("Content-Length:");
-  while (pos < header.size() && std::isspace(header[pos]))
-    pos++;
-  size_t end = pos;
-  while (end < header.size() && std::isdigit(header[end]))
-    end++;
-  size_t len = std::stoul(header.substr(pos, end - pos));
-  std::string body(len, '\0');
-  size_t got = 0;
-  while (got < len) {
-    ssize_t n = read(fd, &body[got], len - got);
-    if (n <= 0)
-      return std::nullopt;
-    got += n;
-  }
-  return json::parse(body);
-}
-
-static Editor *editor_for_uri(LSPInstance *lsp, std::string uri) {
-  if (uri.empty())
-    return nullptr;
-  for (auto &editor : lsp->editors)
-    if (editor->uri == uri)
-      return editor;
-  return nullptr;
-}
-
-void lsp_worker() {
-  LSPOpenRequest request;
-  while (lsp_open_queue.pop(request))
-    add_to_lsp(request.language, request.editor);
-  std::shared_lock active_lsps_lock(active_lsps_mtx);
-  for (auto &kv : active_lsps) {
-    LSPInstance *lsp = kv.second;
-    std::unique_lock lock(lsp->mtx);
-    while (!lsp->outbox.empty()) {
-      json message;
-      message = lsp->outbox.front();
-      if (!lsp->initialized) {
-        std::string m = message.value("method", "");
-        if (m != "initialize")
-          break;
-      }
-      lsp->outbox.pop(message);
-      std::string payload = message.dump();
-      std::string header =
-          "Content-Length: " + std::to_string(payload.size()) + "\r\n\r\n";
-      std::string out = header + payload;
-      const char *ptr = out.data();
-      size_t remaining = out.size();
-      while (remaining > 0) {
-        ssize_t written = write(lsp->stdin_fd, ptr, remaining);
-        if (written == 0)
-          break;
-        else if (written == -1) {
-          if (errno == EINTR)
-            continue;
-          perror("write");
-          break;
-        } else {
-          ptr += written;
-          remaining -= written;
-        }
-      }
-    }
-    pollfd pfd{lsp->stdout_fd, POLLIN, 0};
-    while (poll(&pfd, 1, 0) > 0) {
-      auto msg = read_lsp_message(lsp->stdout_fd);
-      if (!msg)
-        break;
-      if (msg->contains("id")) {
-        uint32_t id = msg->at("id").get<uint32_t>();
-        auto it = lsp->pending.find(id);
-        if (it != lsp->pending.end()) {
-          LSPPending *pend = it->second;
-          lock.unlock();
-          if (pend->callback)
-            pend->callback(pend->editor, pend->method, *msg);
-          delete pend;
-          lock.lock();
-          lsp->pending.erase(it);
-        }
-      } else if (msg->contains("method")) {
-        std::string uri;
-        if (msg->contains("params")) {
-          auto &p = (*msg)["params"];
-          if (p.contains("textDocument") && p["textDocument"].contains("uri"))
-            uri = p["textDocument"]["uri"].get<std::string>();
-          else if (p.contains("uri"))
-            uri = p["uri"].get<std::string>();
-        }
-        Editor *ed = editor_for_uri(lsp, uri);
-        lock.unlock();
-        if (ed)
-          editor_lsp_handle(ed, *msg);
-        else
-          lsp_handle(lsp, *msg);
-        lock.lock();
-      }
-    }
-  }
-}
-
-void request_add_to_lsp(Language language, Editor *editor) {
-  lsp_open_queue.push({language, editor});
-}
-
-void add_to_lsp(Language language, Editor *editor) {
-  LSPInstance *lsp = get_or_init_lsp(language.lsp_id);
-  if (!lsp)
-    return;
-  std::unique_lock lock(lsp->mtx);
-  if (editor->lsp == lsp)
-    return;
-  lsp->editors.push_back(editor);
-  lock.unlock();
-  std::unique_lock lock2(editor->lsp_mtx);
-  editor->lsp = lsp;
-  lock2.unlock();
-  std::unique_lock lock3(editor->knot_mtx);
-  char *buf = read(editor->root, 0, editor->root->char_count);
-  std::string text(buf);
-  free(buf);
-  json message = {{"jsonrpc", "2.0"},
-                  {"method", "textDocument/didOpen"},
-                  {"params",
-                   {{"textDocument",
-                     {{"uri", editor->uri},
-                      {"languageId", language.name},
-                      {"version", 1},
-                      {"text", text}}}}}};
-  lock3.unlock();
-  lsp_send(lsp, message, nullptr);
-}
-
-static uint8_t find_lsp_id(LSPInstance *needle) {
-  for (const auto &[id, lsp] : active_lsps)
-    if (lsp == needle)
-      return id;
-  return 0;
-}
-
-void remove_from_lsp(Editor *editor) {
-  auto lsp = editor->lsp;
-  if (!lsp)
-    return;
-  std::unique_lock lock1(lsp->mtx);
-  lsp->editors.erase(
-      std::remove(lsp->editors.begin(), lsp->editors.end(), editor),
-      lsp->editors.end());
-  lock1.unlock();
-  std::unique_lock lock2(editor->lsp_mtx);
-  editor->lsp = nullptr;
-  lock2.unlock();
-  json message = {{"jsonrpc", "2.0"},
-                  {"method", "textDocument/didClose"},
-                  {"params", {{"textDocument", {{"uri", editor->uri}}}}}};
-  lsp_send(lsp, message, nullptr);
-  uint8_t lsp_id = find_lsp_id(lsp);
-  if (lsp_id && lsp->editors.empty())
-    close_lsp(lsp_id);
-}
-
-void lsp_handle(LSPInstance *, json message) {
-  std::string method = message.value("method", "");
-  if (method == "window/showMessage") {
-    if (message.contains("params")) {
-      auto &p = message["params"];
-      if (p.contains("message"))
-        log("%s\n", p["message"].get<std::string>().c_str());
-    }
-  } else if (method == "window/logMessage") {
-    if (message.contains("params")) {
-      auto &p = message["params"];
-      if (p.contains("message"))
-        log("%s\n", p["message"].get<std::string>().c_str());
-    }
-  }
-}

src/lsp/handlers.cc

@@ -0,0 +1,87 @@
+#include "lsp/lsp.h"
+
+Queue<LSPOpenRequest> lsp_open_queue;
+
+void request_add_to_lsp(Language language, Editor *editor) {
+  lsp_open_queue.push({language, editor});
+}
+
+void add_to_lsp(Language language, Editor *editor) {
+  std::shared_ptr<LSPInstance> lsp = get_or_init_lsp(language.lsp_id);
+  if (!lsp)
+    return;
+  std::unique_lock lock(lsp->mtx);
+  if (editor->lsp == lsp)
+    return;
+  lsp->editors.push_back(editor);
+  lsp->open_queue.push({language, editor});
+  lock.unlock();
+}
+
+void open_editor(std::shared_ptr<LSPInstance> lsp,
+                 std::pair<Language, Editor *> entry) {
+  Language language = entry.first;
+  Editor *editor = entry.second;
+  if (editor->lsp == lsp)
+    return;
+  editor->lsp = lsp;
+  std::unique_lock lock3(editor->knot_mtx);
+  char *buf = read(editor->root, 0, editor->root->char_count);
+  std::string text(buf);
+  free(buf);
+  json message = {{"jsonrpc", "2.0"},
+                  {"method", "textDocument/didOpen"},
+                  {"params",
+                   {{"textDocument",
+                     {{"uri", editor->uri},
+                      {"languageId", language.name},
+                      {"version", 1},
+                      {"text", text}}}}}};
+  lock3.unlock();
+  lsp_send(lsp, message, nullptr);
+}
+
+static uint8_t find_lsp_id(std::shared_ptr<LSPInstance> needle) {
+  for (const auto &[id, lsp] : active_lsps)
+    if (lsp == needle)
+      return id;
+  return 0;
+}
+
+void remove_from_lsp(Editor *editor) {
+  auto lsp = editor->lsp;
+  if (!lsp)
+    return;
+  std::unique_lock lock1(lsp->mtx);
+  lsp->editors.erase(
+      std::remove(lsp->editors.begin(), lsp->editors.end(), editor),
+      lsp->editors.end());
+  lock1.unlock();
+  std::unique_lock lock2(editor->lsp_mtx);
+  editor->lsp = nullptr;
+  lock2.unlock();
+  json message = {{"jsonrpc", "2.0"},
+                  {"method", "textDocument/didClose"},
+                  {"params", {{"textDocument", {{"uri", editor->uri}}}}}};
+  lsp_send(lsp, message, nullptr);
+  uint8_t lsp_id = find_lsp_id(lsp);
+  if (lsp_id && lsp->editors.empty())
+    close_lsp(lsp_id);
+}
+
+void lsp_handle(std::shared_ptr<LSPInstance>, json message) {
+  std::string method = message.value("method", "");
+  if (method == "window/showMessage") {
+    if (message.contains("params")) {
+      auto &p = message["params"];
+      if (p.contains("message"))
+        log("%s\n", p["message"].get<std::string>().c_str());
+    }
+  } else if (method == "window/logMessage") {
+    if (message.contains("params")) {
+      auto &p = message["params"];
+      if (p.contains("message"))
+        log("%s\n", p["message"].get<std::string>().c_str());
+    }
+  }
+}

src/lsp/process.cc

@@ -0,0 +1,150 @@
+#include "config.h"
+#include "lsp/lsp.h"
+
+static bool init_lsp(std::shared_ptr<LSPInstance> lsp) {
+  int in_pipe[2];
+  int out_pipe[2];
+  if (pipe(in_pipe) == -1 || pipe(out_pipe) == -1) {
+    perror("pipe");
+    return false;
+  }
+  pid_t pid = fork();
+  if (pid == -1) {
+    perror("fork");
+    return false;
+  }
+  if (pid == 0) {
+    dup2(in_pipe[0], STDIN_FILENO);
+    dup2(out_pipe[1], STDOUT_FILENO);
+#ifdef __clang__
+    int devnull = open("/dev/null", O_WRONLY);
+    if (devnull >= 0) {
+      dup2(devnull, STDERR_FILENO);
+      close(devnull);
+    }
+#else
+    int log = open("/tmp/lsp.log", O_WRONLY | O_CREAT | O_TRUNC, 0644);
+    if (log >= 0) {
+      dup2(log, STDERR_FILENO);
+      close(log);
+    }
+#endif
+    close(in_pipe[0]);
+    close(in_pipe[1]);
+    close(out_pipe[0]);
+    close(out_pipe[1]);
+    execvp(lsp->lsp->command, (char *const *)(lsp->lsp->args.data()));
+    perror("execvp");
+    _exit(127);
+  }
+  lsp->pid = pid;
+  lsp->stdin_fd = in_pipe[1];
+  lsp->stdout_fd = out_pipe[0];
+  close(in_pipe[0]);
+  close(out_pipe[1]);
+  return true;
+}
+
+std::shared_ptr<LSPInstance> get_or_init_lsp(uint8_t lsp_id) {
+  std::unique_lock lock(active_lsps_mtx);
+  auto it = active_lsps.find(lsp_id);
+  if (it == active_lsps.end()) {
+    auto map_it = kLsps.find(lsp_id);
+    if (map_it == kLsps.end())
+      return nullptr;
+    std::shared_ptr<LSPInstance> lsp = std::make_shared<LSPInstance>();
+    lsp->lsp = &map_it->second;
+    if (!init_lsp(lsp))
+      return nullptr;
+    LSPPending *pending = new LSPPending();
+    pending->method = "initialize";
+    pending->editor = nullptr;
+    pending->callback = [lsp](Editor *, std::string, json msg) {
+      if (msg.contains("result") && msg["result"].contains("capabilities")) {
+        auto &caps = msg["result"]["capabilities"];
+        if (caps.contains("textDocumentSync")) {
+          auto &sync = caps["textDocumentSync"];
+          if (sync.is_number()) {
+            int change_type = sync.get<int>();
+            lsp->incremental_sync = (change_type == 2);
+          } else if (sync.is_object() && sync.contains("change")) {
+            int change_type = sync["change"].get<int>();
+            lsp->incremental_sync = (change_type == 2);
+          }
+        }
+        if (caps.contains("hoverProvider"))
+          lsp->allow_hover = caps["hoverProvider"].get<bool>();
+        else
+          lsp->allow_hover = false;
+        if (caps.contains("completionProvider"))
+          lsp->allow_completion = true;
+      }
+      lsp->initialized = true;
+      json initialized = {{"jsonrpc", "2.0"},
+                          {"method", "initialized"},
+                          {"params", json::object()}};
+      lsp_send(lsp, initialized, nullptr);
+    };
+    json init_message = {
+        {"jsonrpc", "2.0"},
+        {"method", "initialize"},
+        {"params",
+         {{"processId", getpid()},
+          {"rootUri", "file://" + percent_encode(path_abs("."))},
+          {"capabilities", client_capabilities}}}};
+    lsp_send(lsp, init_message, pending);
+    active_lsps[lsp_id] = lsp;
+    return lsp;
+  }
+  return it->second;
+}
+
+void close_lsp(uint8_t lsp_id) {
+  std::shared_lock active_lsps_lock(active_lsps_mtx);
+  auto it = active_lsps.find(lsp_id);
+  if (it == active_lsps.end())
+    return;
+  std::shared_ptr<LSPInstance> lsp = it->second;
+  active_lsps_lock.unlock();
+  lsp->exited = true;
+  lsp->initialized = false;
+  LSPPending *shutdown_pending = new LSPPending();
+  shutdown_pending->method = "shutdown";
+  shutdown_pending->callback = [lsp, lsp_id](Editor *, std::string, json) {
+    json exit = {{"jsonrpc", "2.0"}, {"method", "exit"}};
+    lsp_send(lsp, exit, nullptr);
+  };
+  json shutdown = {{"jsonrpc", "2.0"}, {"method", "shutdown"}};
+  lsp_send(lsp, shutdown, shutdown_pending);
+  std::thread t([lsp, lsp_id] {
+    std::this_thread::sleep_for(100ms);
+    std::unique_lock active_lsps_lock(active_lsps_mtx);
+    std::unique_lock lock(lsp->mtx);
+    if (lsp->pid != -1 && kill(lsp->pid, 0) == 0)
+      kill(lsp->pid, SIGKILL);
+    waitpid(lsp->pid, nullptr, 0);
+    close(lsp->stdin_fd);
+    close(lsp->stdout_fd);
+    for (auto &kv : lsp->pending)
+      delete kv.second;
+    for (auto &editor : lsp->editors) {
+      std::unique_lock editor_lock(editor->lsp_mtx);
+      editor->lsp = nullptr;
+    }
+    active_lsps.erase(lsp_id);
+  });
+  t.detach();
+}
+
+void clean_lsp(std::shared_ptr<LSPInstance> lsp, uint8_t lsp_id) {
+  for (auto &kv : lsp->pending)
+    delete kv.second;
+  lsp->pid = -1;
+  close(lsp->stdin_fd);
+  close(lsp->stdout_fd);
+  for (auto &editor : lsp->editors) {
+    std::unique_lock editor_lock(editor->lsp_mtx);
+    editor->lsp = nullptr;
+  }
+  active_lsps.erase(lsp_id);
+}

src/lsp/workers.cc

@@ -0,0 +1,170 @@
+#include "lsp/lsp.h"
+
+std::shared_mutex active_lsps_mtx;
+std::unordered_map<uint8_t, std::shared_ptr<LSPInstance>> active_lsps;
+
+void lsp_send(std::shared_ptr<LSPInstance> lsp, json message,
+              LSPPending *pending) {
+  if (!lsp || lsp->stdin_fd == -1)
+    return;
+  std::unique_lock lock(lsp->mtx);
+  if (pending) {
+    message["id"] = lsp->last_id++;
+    uint32_t id = message["id"].get<uint32_t>();
+    lsp->pending[id] = pending;
+  }
+  lsp->outbox.push(message);
+}
+
+static std::optional<json> read_lsp_message(int fd) {
+  std::string header;
+  char c;
+  while (true) {
+    ssize_t n = read(fd, &c, 1);
+    if (n <= 0)
+      return std::nullopt;
+    header.push_back(c);
+    if (header.size() >= 4 && header.substr(header.size() - 4) == "\r\n\r\n")
+      break;
+  }
+  size_t pos = header.find("Content-Length:");
+  if (pos == std::string::npos)
+    return std::nullopt;
+  pos += strlen("Content-Length:");
+  while (pos < header.size() && std::isspace(header[pos]))
+    pos++;
+  size_t end = pos;
+  while (end < header.size() && std::isdigit(header[end]))
+    end++;
+  size_t len = std::stoul(header.substr(pos, end - pos));
+  std::string body(len, '\0');
+  size_t got = 0;
+  while (got < len) {
+    ssize_t n = read(fd, &body[got], len - got);
+    if (n <= 0)
+      return std::nullopt;
+    got += n;
+  }
+  return json::parse(body);
+}
+
+static Editor *editor_for_uri(std::shared_ptr<LSPInstance> lsp,
+                              std::string uri) {
+  if (uri.empty())
+    return nullptr;
+  for (auto &editor : lsp->editors)
+    if (editor->uri == uri)
+      return editor;
+  return nullptr;
+}
+
+void lsp_worker() {
+  LSPOpenRequest request;
+  while (lsp_open_queue.pop(request))
+    add_to_lsp(request.language, request.editor);
+  std::unique_lock active_lsps_lock(active_lsps_mtx);
+  for (auto &kv : active_lsps) {
+    std::shared_ptr<LSPInstance> lsp = kv.second;
+    std::unique_lock lock(lsp->mtx);
+    int status;
+    pid_t res = waitpid(lsp->pid, &status, WNOHANG);
+    if (res == lsp->pid) {
+      clean_lsp(lsp, kv.first);
+      return;
+    }
+    if (lsp->initialized) {
+      std::pair<Language, Editor *> request;
+      while (lsp->open_queue.pop(request)) {
+        lock.unlock();
+        open_editor(lsp, request);
+        lock.lock();
+      }
+    }
+    while (!lsp->outbox.empty()) {
+      json message = lsp->outbox.front();
+      std::string m = message.value("method", "");
+      if (lsp->exited) {
+        if (m != "exit" && m != "shutdown") {
+          lsp->outbox.pop(message);
+          continue;
+        }
+      }
+      if (!lsp->initialized) {
+        if (m != "initialize" && m != "exit" && m != "shutdown")
+          break;
+      }
+      lsp->outbox.pop(message);
+      std::string payload = message.dump();
+      std::string header =
+          "Content-Length: " + std::to_string(payload.size()) + "\r\n\r\n";
+      std::string out = header + payload;
+      const char *ptr = out.data();
+      size_t remaining = out.size();
+      while (remaining > 0) {
+        int status;
+        pid_t res = waitpid(lsp->pid, &status, WNOHANG);
+        if (res == lsp->pid) {
+          clean_lsp(lsp, kv.first);
+          return;
+        }
+        ssize_t written = write(lsp->stdin_fd, ptr, remaining);
+        if (written == 0)
+          break;
+        else if (written == -1) {
+          if (errno == EINTR)
+            continue;
+          perror("write");
+          clean_lsp(lsp, kv.first);
+          return;
+        } else {
+          ptr += written;
+          remaining -= written;
+        }
+      }
+    }
+    pollfd pfd{lsp->stdout_fd, POLLIN | POLLHUP | POLLERR, 0};
+    int r = poll(&pfd, 1, 0);
+    if (r > 0 && pfd.revents & (POLLHUP | POLLERR)) {
+      clean_lsp(lsp, kv.first);
+      return;
+    }
+    while ((r = poll(&pfd, 1, 0) > 0)) {
+      if (r > 0 && pfd.revents & (POLLHUP | POLLERR)) {
+        clean_lsp(lsp, kv.first);
+        return;
+      }
+      auto msg = read_lsp_message(lsp->stdout_fd);
+      if (!msg)
+        break;
+      if (msg->contains("id")) {
+        uint32_t id = msg->at("id").get<uint32_t>();
+        auto it = lsp->pending.find(id);
+        if (it != lsp->pending.end()) {
+          LSPPending *pend = it->second;
+          lock.unlock();
+          if (pend->callback)
+            pend->callback(pend->editor, pend->method, *msg);
+          delete pend;
+          lock.lock();
+          lsp->pending.erase(it);
+        }
+      } else if (msg->contains("method")) {
+        std::string uri;
+        if (msg->contains("params")) {
+          auto &p = (*msg)["params"];
+          if (p.contains("textDocument") && p["textDocument"].contains("uri"))
+            uri = p["textDocument"]["uri"].get<std::string>();
+          else if (p.contains("uri"))
+            uri = p["uri"].get<std::string>();
+        }
+        Editor *ed = editor_for_uri(lsp, uri);
+        lock.unlock();
+        if (ed)
+          editor_lsp_handle(ed, *msg);
+        else
+          lsp_handle(lsp, *msg);
+        lock.lock();
+      }
+    }
+  }
+}

src/main.cc

@@ -1,12 +1,8 @@
-#include "../include/main.h"
-#include "../include/editor.h"
-#include "../include/lsp.h"
-#include "../include/ts.h"
-#include "../include/ui.h"
-#include <atomic>
-#include <cstdint>
-#include <sys/ioctl.h>
-#include <thread>
+#include "main.h"
+#include "editor/editor.h"
+#include "io/ui.h"
+#include "lsp/lsp.h"
+#include "ts/ts.h"
 
 std::atomic<bool> running{true};
 Queue<KeyEvent> event_queue;

src/ts.cc

@@ -1,340 +0,0 @@
-#include "../include/ts.h"
-#include "../include/editor.h"
-#include "../include/knot.h"
-#include "../include/maps.h"
-#include <algorithm>
-#include <cstdint>
-#include <fstream>
-#include <string>
-#include <unordered_map>
-
-std::unordered_map<std::string, pcre2_code *> regex_cache;
-
-void clear_regex_cache() {
-  for (auto &kv : regex_cache)
-    pcre2_code_free(kv.second);
-  regex_cache.clear();
-}
-
-pcre2_code *get_re(const std::string &pattern) {
-  auto it = regex_cache.find(pattern);
-  if (it != regex_cache.end())
-    return it->second;
-  int errornum;
-  PCRE2_SIZE erroffset;
-  pcre2_code *re =
-      pcre2_compile((PCRE2_SPTR)pattern.c_str(), PCRE2_ZERO_TERMINATED, 0,
-                    &errornum, &erroffset, nullptr);
-  regex_cache[pattern] = re;
-  return re;
-}
-
-TSQuery *load_query(const char *query_path, TSSetBase *set) {
-  const TSLanguage *lang = set->language;
-  std::ifstream file(query_path, std::ios::in | std::ios::binary);
-  if (!file.is_open())
-    return nullptr;
-  std::string highlight_query((std::istreambuf_iterator<char>(file)),
-                              std::istreambuf_iterator<char>());
-  int errornumber = 0;
-  PCRE2_SIZE erroroffset = 0;
-  pcre2_code *re = pcre2_compile(
-      (PCRE2_SPTR) R"((@[A-Za-z0-9_.]+)|(;; \#[0-9a-fA-F]{6} \#[0-9a-fA-F]{6} [01] [01] [01] \d+)|(;; !(\w+)))",
-      PCRE2_ZERO_TERMINATED, 0, &errornumber, &erroroffset, nullptr);
-  if (!re)
-    return nullptr;
-  pcre2_match_data *match_data =
-      pcre2_match_data_create_from_pattern(re, nullptr);
-  std::map<std::string, int> capture_name_cache;
-  Highlight *c_hl = nullptr;
-  Language c_lang = {"unknown", nullptr, 0};
-  int i = 0;
-  PCRE2_SIZE offset = 0;
-  PCRE2_SIZE subject_length = highlight_query.size();
-  while (offset < subject_length) {
-    int rc = pcre2_match(re, (PCRE2_SPTR)highlight_query.c_str(),
-                         subject_length, offset, 0, match_data, nullptr);
-    if (rc <= 0)
-      break;
-    PCRE2_SIZE *ovector = pcre2_get_ovector_pointer(match_data);
-    std::string mct =
-        highlight_query.substr(ovector[0], ovector[1] - ovector[0]);
-    if (!mct.empty() && mct[0] == '@') {
-      std::string capture_name = mct;
-      if (!capture_name_cache.count(capture_name)) {
-        if (c_hl) {
-          set->query_map[i] = *c_hl;
-          delete c_hl;
-          c_hl = nullptr;
-        }
-        if (c_lang.fn != nullptr) {
-          set->injection_map[i] = c_lang;
-          c_lang = {"unknown", nullptr, 0};
-        }
-        capture_name_cache[capture_name] = i;
-        i++;
-      }
-    } else if (mct.substr(0, 4) == ";; #") {
-      if (c_hl)
-        delete c_hl;
-      c_hl = new Highlight();
-      c_hl->fg = HEX(mct.substr(4, 6));
-      c_hl->bg = HEX(mct.substr(12, 6));
-      int bold = std::stoi(mct.substr(19, 1));
-      int italic = std::stoi(mct.substr(21, 1));
-      int underline = std::stoi(mct.substr(23, 1));
-      c_hl->priority = std::stoi(mct.substr(25));
-      c_hl->flags = (bold ? CF_BOLD : 0) | (italic ? CF_ITALIC : 0) |
-                    (underline ? CF_UNDERLINE : 0);
-    } else if (mct.substr(0, 4) == ";; !") {
-      auto it = kLanguages.find(mct.substr(4));
-      if (it != kLanguages.end())
-        c_lang = it->second;
-      else
-        c_lang = {"unknown", nullptr, 0};
-    }
-    offset = ovector[1];
-  }
-  if (c_hl)
-    delete c_hl;
-  pcre2_match_data_free(match_data);
-  pcre2_code_free(re);
-  uint32_t error_offset = 0;
-  TSQueryError error_type = (TSQueryError)0;
-  TSQuery *q = ts_query_new(lang, highlight_query.c_str(),
-                            (uint32_t)highlight_query.length(), &error_offset,
-                            &error_type);
-  if (!q)
-    log("Failed to create TSQuery at offset %u, error type %d", error_offset,
-        (int)error_type);
-  return q;
-}
-
-static inline const TSNode *find_capture_node(const TSQueryMatch &match,
-                                              uint32_t capture_id) {
-  for (uint32_t i = 0; i < match.capture_count; i++)
-    if (match.captures[i].index == capture_id)
-      return &match.captures[i].node;
-  return nullptr;
-}
-
-static inline std::string node_text(const TSNode &node, Knot *source) {
-  uint32_t start = ts_node_start_byte(node);
-  uint32_t end = ts_node_end_byte(node);
-  char *text = read(source, start, end - start);
-  std::string final = std::string(text, end - start);
-  free(text);
-  return final;
-}
-
-static inline bool ts_predicate(TSQuery *query, const TSQueryMatch &match,
-                                Knot *source) {
-  uint32_t step_count;
-  const TSQueryPredicateStep *steps =
-      ts_query_predicates_for_pattern(query, match.pattern_index, &step_count);
-  if (!steps || step_count != 4)
-    return true;
-  if (source->char_count >= (1024 * 64))
-    return false;
-  std::string command;
-  std::string regex_txt;
-  uint32_t subject_id = 0;
-  for (uint32_t i = 0; i < step_count; i++) {
-    const TSQueryPredicateStep *step = &steps[i];
-    if (step->type == TSQueryPredicateStepTypeDone)
-      break;
-    switch (step->type) {
-    case TSQueryPredicateStepTypeString: {
-      uint32_t length = 0;
-      const char *s =
-          ts_query_string_value_for_id(query, step->value_id, &length);
-      if (i == 0)
-        command.assign(s, length);
-      else
-        regex_txt.assign(s, length);
-      break;
-    }
-    case TSQueryPredicateStepTypeCapture: {
-      subject_id = step->value_id;
-      break;
-    }
-    case TSQueryPredicateStepTypeDone:
-      break;
-    }
-  }
-  const TSNode *node = find_capture_node(match, subject_id);
-  std::string subject = node_text(*node, source);
-  pcre2_code *re = get_re(regex_txt);
-  pcre2_match_data *md = pcre2_match_data_create_from_pattern(re, nullptr);
-  int rc = pcre2_match(re, (PCRE2_SPTR)subject.c_str(), subject.size(), 0, 0,
-                       md, nullptr);
-  pcre2_match_data_free(md);
-  bool ok = (rc >= 0);
-  return (command == "match?" ? ok : !ok);
-}
-
-const char *read_ts(void *payload, uint32_t byte_index, TSPoint,
-                    uint32_t *bytes_read) {
-  Editor *editor = (Editor *)payload;
-  if (byte_index >= editor->root->char_count) {
-    *bytes_read = 0;
-    return "";
-  }
-  return leaf_from_offset(editor->root, byte_index, bytes_read);
-}
-
-template <typename T>
-static inline T *safe_get(std::map<uint16_t, T> &m, uint16_t key) {
-  auto it = m.find(key);
-  if (it == m.end())
-    return nullptr;
-  return &it->second;
-}
-
-void ts_collect_spans(Editor *editor) {
-  static int parse_counter = 0;
-  if (!editor->ts.parser || !editor->root || !editor->ts.query)
-    return;
-  const bool injections_enabled = editor->root->char_count < (1024 * 32);
-  for (auto &inj : editor->ts.injections)
-    inj.second.ranges.clear();
-  TSInput tsinput{
-      .payload = editor,
-      .read = read_ts,
-      .encoding = TSInputEncodingUTF8,
-      .decode = nullptr,
-  };
-  TSTree *tree = nullptr;
-  TSTree *copy = nullptr;
-  std::unique_lock knot_mtx(editor->knot_mtx);
-  if (editor->ts.tree)
-    copy = ts_tree_copy(editor->ts.tree);
-  knot_mtx.unlock();
-  std::vector<TSInputEdit> edits;
-  TSInputEdit edit;
-  if (copy)
-    while (editor->edit_queue.pop(edit)) {
-      edits.push_back(edit);
-      ts_tree_edit(copy, &edits.back());
-    }
-  if (copy && edits.empty() && parse_counter < 64) {
-    parse_counter++;
-    ts_tree_delete(copy);
-    return;
-  }
-  parse_counter = 0;
-  editor->spans.mid_parse = true;
-  std::shared_lock lock(editor->knot_mtx);
-  tree = ts_parser_parse(editor->ts.parser, copy, tsinput);
-  lock.unlock();
-  if (copy)
-    ts_tree_delete(copy);
-  if (editor->ts.tree)
-    ts_tree_delete(editor->ts.tree);
-  editor->ts.tree = tree;
-  copy = ts_tree_copy(tree);
-  std::vector<Span> new_spans;
-  new_spans.reserve(4096);
-  struct PendingRanges {
-    std::vector<TSRange> ranges;
-    TSSet *tsset = nullptr;
-  };
-  struct WorkItem {
-    TSSetBase *tsset;
-    TSTree *tree;
-    int depth;
-    TSSet *as_injection;
-  };
-  const int kMaxInjectionDepth = 4;
-  std::vector<WorkItem> work;
-  work.push_back(
-      {reinterpret_cast<TSSetBase *>(&editor->ts), copy, 0, nullptr});
-  auto overlaps = [](const Span &s, const TSRange &r) {
-    return !(s.end <= r.start_byte || s.start >= r.end_byte);
-  };
-  auto remove_overlapping_spans = [&](const std::vector<TSRange> &ranges) {
-    if (ranges.empty())
-      return;
-    new_spans.erase(
-        std::remove_if(new_spans.begin(), new_spans.end(),
-                       [&](const Span &sp) {
-                         return std::any_of(
-                             ranges.begin(), ranges.end(),
-                             [&](const TSRange &r) { return overlaps(sp, r); });
-                       }),
-        new_spans.end());
-  };
-  while (!work.empty()) {
-    WorkItem item = work.back();
-    work.pop_back();
-    TSQuery *q = item.tsset->query;
-    if (!q) {
-      ts_tree_delete(item.tree);
-      continue;
-    }
-    TSQueryCursor *cursor = ts_query_cursor_new();
-    ts_query_cursor_exec(cursor, q, ts_tree_root_node(item.tree));
-    std::unordered_map<std::string, PendingRanges> pending_injections;
-    TSQueryMatch match;
-    while (ts_query_cursor_next_match(cursor, &match)) {
-      if (!ts_predicate(q, match, editor->root))
-        continue;
-      for (uint32_t i = 0; i < match.capture_count; i++) {
-        TSQueryCapture cap = match.captures[i];
-        uint32_t start = ts_node_start_byte(cap.node);
-        uint32_t end = ts_node_end_byte(cap.node);
-        if (Highlight *hl = safe_get(item.tsset->query_map, cap.index))
-          new_spans.push_back({start, end, hl});
-        if (!injections_enabled)
-          continue;
-        if (Language *inj_lang =
-                safe_get(item.tsset->injection_map, cap.index)) {
-          auto &pending = pending_injections[inj_lang->name];
-          TSSet &tsset =
-              editor->ts.injections.try_emplace(inj_lang->name).first->second;
-          if (!tsset.parser) {
-            tsset.lang = inj_lang->name;
-            tsset.parser = ts_parser_new();
-            ts_parser_set_language(tsset.parser, inj_lang->fn());
-            tsset.language = inj_lang->fn();
-            tsset.query_file =
-                get_exe_dir() + "/../grammar/" + inj_lang->name + ".scm";
-            tsset.query = load_query(tsset.query_file.c_str(), &tsset);
-          }
-          pending.tsset = &tsset;
-          pending.ranges.push_back(TSRange{
-              ts_node_start_point(cap.node),
-              ts_node_end_point(cap.node),
-              start,
-              end,
-          });
-        }
-      }
-    }
-    ts_query_cursor_delete(cursor);
-    if (injections_enabled && item.depth < kMaxInjectionDepth) {
-      for (auto &[lang_name, pending] : pending_injections) {
-        TSSet *tsset = pending.tsset;
-        if (!tsset || pending.ranges.empty() || !tsset->parser || !tsset->query)
-          continue;
-        tsset->ranges = std::move(pending.ranges);
-        remove_overlapping_spans(tsset->ranges);
-        ts_parser_set_included_ranges(tsset->parser, tsset->ranges.data(),
-                                      tsset->ranges.size());
-        lock.lock();
-        TSTree *inj_tree = ts_parser_parse(tsset->parser, nullptr, tsinput);
-        lock.unlock();
-        work.push_back({reinterpret_cast<TSSetBase *>(tsset), inj_tree,
-                        item.depth + 1, tsset});
-      }
-    }
-    ts_tree_delete(item.tree);
-  }
-  std::pair<uint32_t, int64_t> span_edit;
-  while (editor->spans.edits.pop(span_edit))
-    apply_edit(new_spans, span_edit.first, span_edit.second);
-  std::sort(new_spans.begin(), new_spans.end());
-  std::unique_lock span_mtx(editor->spans.mtx);
-  editor->spans.mid_parse = false;
-  editor->spans.spans.swap(new_spans);
-}

src/ts/ts.cc

@@ -0,0 +1,179 @@
+#include "ts/ts.h"
+#include "editor/editor.h"
+#include "io/knot.h"
+
+const char *read_ts(void *payload, uint32_t byte_index, TSPoint,
+                    uint32_t *bytes_read) {
+  Editor *editor = (Editor *)payload;
+  if (byte_index >= editor->root->char_count) {
+    *bytes_read = 0;
+    return "";
+  }
+  return leaf_from_offset(editor->root, byte_index, bytes_read);
+}
+
+void ts_collect_spans(Editor *editor) {
+  static int parse_counter = 0;
+  if (!editor->ts.parser || !editor->root || !editor->ts.query)
+    return;
+  const bool injections_enabled = editor->root->char_count < (1024 * 32);
+  for (auto &inj : editor->ts.injections)
+    inj.second.ranges.clear();
+  TSInput tsinput{
+      .payload = editor,
+      .read = read_ts,
+      .encoding = TSInputEncodingUTF8,
+      .decode = nullptr,
+  };
+  std::vector<TSInputEdit> edits;
+  TSInputEdit edit;
+  if (!editor->edit_queue.empty()) {
+    while (editor->edit_queue.pop(edit))
+      edits.push_back(edit);
+    if (editor->ts.tree) {
+      for (auto &e : edits)
+        ts_tree_edit(editor->ts.tree, &e);
+    }
+    for (auto &inj : editor->ts.injections) {
+      if (inj.second.tree) {
+        for (auto &e : edits) {
+          TSInputEdit inj_edit = e;
+          for (auto &r : inj.second.ranges) {
+            if (e.start_byte >= r.start_byte && e.start_byte <= r.end_byte) {
+              inj_edit.start_byte -= r.start_byte;
+              inj_edit.old_end_byte -= r.start_byte;
+              inj_edit.new_end_byte -= r.start_byte;
+            }
+          }
+          ts_tree_edit(inj.second.tree, &inj_edit);
+        }
+      }
+    }
+  } else if (editor->ts.tree && parse_counter < 64) {
+    parse_counter++;
+    return;
+  }
+  parse_counter = 0;
+  editor->spans.mid_parse = true;
+  std::shared_lock lock(editor->knot_mtx);
+  TSTree *tree = ts_parser_parse(editor->ts.parser, editor->ts.tree, tsinput);
+  if (!tree)
+    return;
+  if (editor->ts.tree)
+    ts_tree_delete(editor->ts.tree);
+  editor->ts.tree = tree;
+  lock.unlock();
+  std::vector<Span> new_spans;
+  new_spans.reserve(4096);
+  struct PendingRanges {
+    std::vector<TSRange> ranges;
+    TSSet *tsset = nullptr;
+  };
+  struct WorkItem {
+    TSSetBase *tsset;
+    TSTree *tree;
+    int depth;
+  };
+  const int kMaxInjectionDepth = 4;
+  std::vector<WorkItem> work;
+  work.push_back(
+      {reinterpret_cast<TSSetBase *>(&editor->ts), editor->ts.tree, 0});
+  auto overlaps = [](const Span &s, const TSRange &r) {
+    return !(s.end <= r.start_byte || s.start >= r.end_byte);
+  };
+  auto remove_overlapping_spans = [&](const std::vector<TSRange> &ranges) {
+    if (ranges.empty())
+      return;
+    new_spans.erase(
+        std::remove_if(new_spans.begin(), new_spans.end(),
+                       [&](const Span &sp) {
+                         return std::any_of(
+                             ranges.begin(), ranges.end(),
+                             [&](const TSRange &r) { return overlaps(sp, r); });
+                       }),
+        new_spans.end());
+  };
+  while (!work.empty()) {
+    WorkItem item = work.back();
+    work.pop_back();
+    TSQuery *q = item.tsset->query;
+    if (!q)
+      continue;
+    TSQueryCursor *cursor = ts_query_cursor_new();
+    ts_query_cursor_exec(cursor, q, ts_tree_root_node(item.tsset->tree));
+    std::unordered_map<std::string, PendingRanges> pending_injections;
+    TSQueryMatch match;
+    while (ts_query_cursor_next_match(cursor, &match)) {
+      auto subject_fn = [&](const TSNode *node) -> std::string {
+        uint32_t start = ts_node_start_byte(*node);
+        uint32_t end = ts_node_end_byte(*node);
+        char *text = read(editor->root, start, end - start);
+        std::string final = std::string(text, end - start);
+        free(text);
+        return final;
+      };
+      if (!ts_predicate(q, match, subject_fn))
+        continue;
+      for (uint32_t i = 0; i < match.capture_count; i++) {
+        TSQueryCapture cap = match.captures[i];
+        uint32_t start = ts_node_start_byte(cap.node);
+        uint32_t end = ts_node_end_byte(cap.node);
+        if (Highlight *hl = safe_get(item.tsset->query_map, cap.index))
+          new_spans.push_back({start, end, hl});
+        if (!injections_enabled)
+          continue;
+        if (Language *inj_lang =
+                safe_get(item.tsset->injection_map, cap.index)) {
+          auto &pending = pending_injections[inj_lang->name];
+          TSSet &tsset =
+              editor->ts.injections.try_emplace(inj_lang->name).first->second;
+          if (!tsset.parser) {
+            tsset.lang = inj_lang->name;
+            tsset.parser = ts_parser_new();
+            ts_parser_set_language(tsset.parser, inj_lang->fn());
+            tsset.language = inj_lang->fn();
+            tsset.query_file =
+                get_exe_dir() + "/../grammar/" + inj_lang->name + ".scm";
+            tsset.query = load_query(tsset.query_file.c_str(), &tsset);
+          }
+          pending.tsset = &tsset;
+          pending.ranges.push_back(TSRange{
+              ts_node_start_point(cap.node),
+              ts_node_end_point(cap.node),
+              start,
+              end,
+          });
+        }
+      }
+    }
+    ts_query_cursor_delete(cursor);
+    if (injections_enabled && item.depth < kMaxInjectionDepth) {
+      for (auto &[lang_name, pending] : pending_injections) {
+        TSSet *tsset = pending.tsset;
+        if (!tsset || pending.ranges.empty() || !tsset->parser || !tsset->query)
+          continue;
+        tsset->ranges = std::move(pending.ranges);
+        remove_overlapping_spans(tsset->ranges);
+        ts_parser_set_included_ranges(tsset->parser, tsset->ranges.data(),
+                                      tsset->ranges.size());
+        lock.lock();
+        TSTree *tree = ts_parser_parse(tsset->parser, tsset->tree, tsinput);
+        if (!tree)
+          continue;
+        if (tsset->tree)
+          ts_tree_delete(tsset->tree);
+        tsset->tree = tree;
+        lock.unlock();
+        work.push_back({reinterpret_cast<TSSetBase *>(tsset), tsset->tree,
+                        item.depth + 1});
+      }
+    }
+  }
+  std::pair<uint32_t, int64_t> span_edit;
+  while (editor->spans.edits.pop(span_edit))
+    apply_edit(new_spans, span_edit.first, span_edit.second);
+  std::sort(new_spans.begin(), new_spans.end());
+  std::unique_lock span_mtx(editor->spans.mtx);
+  editor->spans.mid_parse = false;
+  editor->spans.spans.swap(new_spans);
+}

src/ts/utils.cc

@@ -0,0 +1,156 @@
+#include "config.h"
+#include "ts/ts.h"
+
+std::unordered_map<std::string, pcre2_code *> regex_cache;
+
+static inline const TSNode *find_capture_node(const TSQueryMatch &match,
+                                              uint32_t capture_id) {
+  for (uint32_t i = 0; i < match.capture_count; i++)
+    if (match.captures[i].index == capture_id)
+      return &match.captures[i].node;
+  return nullptr;
+}
+
+void clear_regex_cache() {
+  for (auto &kv : regex_cache)
+    pcre2_code_free(kv.second);
+  regex_cache.clear();
+}
+
+pcre2_code *get_re(const std::string &pattern) {
+  auto it = regex_cache.find(pattern);
+  if (it != regex_cache.end())
+    return it->second;
+  int errornum;
+  PCRE2_SIZE erroffset;
+  pcre2_code *re =
+      pcre2_compile((PCRE2_SPTR)pattern.c_str(), PCRE2_ZERO_TERMINATED, 0,
+                    &errornum, &erroffset, nullptr);
+  regex_cache[pattern] = re;
+  return re;
+}
+
+TSQuery *load_query(const char *query_path, TSSetBase *set) {
+  const TSLanguage *lang = set->language;
+  std::ifstream file(query_path, std::ios::in | std::ios::binary);
+  if (!file.is_open())
+    return nullptr;
+  std::string highlight_query((std::istreambuf_iterator<char>(file)),
+                              std::istreambuf_iterator<char>());
+  int errornumber = 0;
+  PCRE2_SIZE erroroffset = 0;
+  pcre2_code *re = pcre2_compile(
+      (PCRE2_SPTR) R"((@[A-Za-z0-9_.]+)|(;; \#[0-9a-fA-F]{6} \#[0-9a-fA-F]{6} [01] [01] [01] \d+)|(;; !(\w+)))",
+      PCRE2_ZERO_TERMINATED, 0, &errornumber, &erroroffset, nullptr);
+  if (!re)
+    return nullptr;
+  pcre2_match_data *match_data =
+      pcre2_match_data_create_from_pattern(re, nullptr);
+  std::map<std::string, int> capture_name_cache;
+  Highlight *c_hl = nullptr;
+  Language c_lang = {"unknown", nullptr, 0};
+  int i = 0;
+  PCRE2_SIZE offset = 0;
+  PCRE2_SIZE subject_length = highlight_query.size();
+  while (offset < subject_length) {
+    int rc = pcre2_match(re, (PCRE2_SPTR)highlight_query.c_str(),
+                         subject_length, offset, 0, match_data, nullptr);
+    if (rc <= 0)
+      break;
+    PCRE2_SIZE *ovector = pcre2_get_ovector_pointer(match_data);
+    std::string mct =
+        highlight_query.substr(ovector[0], ovector[1] - ovector[0]);
+    if (!mct.empty() && mct[0] == '@') {
+      std::string capture_name = mct;
+      if (!capture_name_cache.count(capture_name)) {
+        if (c_hl) {
+          set->query_map[i] = *c_hl;
+          delete c_hl;
+          c_hl = nullptr;
+        }
+        if (c_lang.fn != nullptr) {
+          set->injection_map[i] = c_lang;
+          c_lang = {"unknown", nullptr, 0};
+        }
+        capture_name_cache[capture_name] = i;
+        i++;
+      }
+    } else if (mct.substr(0, 4) == ";; #") {
+      if (c_hl)
+        delete c_hl;
+      c_hl = new Highlight();
+      c_hl->fg = HEX(mct.substr(4, 6));
+      c_hl->bg = HEX(mct.substr(12, 6));
+      int bold = std::stoi(mct.substr(19, 1));
+      int italic = std::stoi(mct.substr(21, 1));
+      int underline = std::stoi(mct.substr(23, 1));
+      c_hl->priority = std::stoi(mct.substr(25));
+      c_hl->flags = (bold ? CF_BOLD : 0) | (italic ? CF_ITALIC : 0) |
+                    (underline ? CF_UNDERLINE : 0);
+    } else if (mct.substr(0, 4) == ";; !") {
+      auto it = kLanguages.find(mct.substr(4));
+      if (it != kLanguages.end())
+        c_lang = it->second;
+      else
+        c_lang = {"unknown", nullptr, 0};
+    }
+    offset = ovector[1];
+  }
+  if (c_hl)
+    delete c_hl;
+  pcre2_match_data_free(match_data);
+  pcre2_code_free(re);
+  uint32_t error_offset = 0;
+  TSQueryError error_type = (TSQueryError)0;
+  TSQuery *q = ts_query_new(lang, highlight_query.c_str(),
+                            (uint32_t)highlight_query.length(), &error_offset,
+                            &error_type);
+  if (!q)
+    log("Failed to create TSQuery at offset %u, error type %d", error_offset,
+        (int)error_type);
+  return q;
+}
+
+bool ts_predicate(TSQuery *query, const TSQueryMatch &match,
+                  std::function<std::string(const TSNode *)> subject_fn) {
+  uint32_t step_count;
+  const TSQueryPredicateStep *steps =
+      ts_query_predicates_for_pattern(query, match.pattern_index, &step_count);
+  if (!steps || step_count != 4)
+    return true;
+  std::string command;
+  std::string regex_txt;
+  uint32_t subject_id = 0;
+  for (uint32_t i = 0; i < step_count; i++) {
+    const TSQueryPredicateStep *step = &steps[i];
+    if (step->type == TSQueryPredicateStepTypeDone)
+      break;
+    switch (step->type) {
+    case TSQueryPredicateStepTypeString: {
+      uint32_t length = 0;
+      const char *s =
+          ts_query_string_value_for_id(query, step->value_id, &length);
+      if (i == 0)
+        command.assign(s, length);
+      else
+        regex_txt.assign(s, length);
+      break;
+    }
+    case TSQueryPredicateStepTypeCapture: {
+      subject_id = step->value_id;
+      break;
+    }
+    case TSQueryPredicateStepTypeDone:
+      break;
+    }
+  }
+  const TSNode *node = find_capture_node(match, subject_id);
+  pcre2_code *re = get_re(regex_txt);
+  std::string subject = subject_fn(node);
+  pcre2_match_data *md = pcre2_match_data_create_from_pattern(re, nullptr);
+  int rc = pcre2_match(re, (PCRE2_SPTR)subject.c_str(), subject.size(), 0, 0,
+                       md, nullptr);
+  pcre2_match_data_free(md);
+  bool ok = (rc >= 0);
+  return (command == "match?" ? ok : !ok);
+}

src/utils/system.cc

@@ -1,24 +1,16 @@
-extern "C" {
-#include "../libs/libgrapheme/grapheme.h"
-#include "../libs/unicode_width/unicode_width.h"
-}
-#include "../include/maps.h"
-#include "../include/utils.h"
+#include "config.h"
+#include "utils/utils.h"
 
-static std::string percent_encode(const std::string &s) {
-  static const char *hex = "0123456789ABCDEF";
-  std::string out;
-  for (unsigned char c : s) {
-    if (std::isalnum(c) || c == '-' || c == '_' || c == '.' || c == '~' ||
-        c == '/') {
-      out.push_back(c);
-    } else {
-      out.push_back('%');
-      out.push_back(hex[c >> 4]);
-      out.push_back(hex[c & 0xF]);
-    }
-  }
-  return out;
+void log(const char *fmt, ...) {
+  FILE *fp = fopen("/tmp/log.txt", "a");
+  if (!fp)
+    return;
+  va_list args;
+  va_start(args, fmt);
+  vfprintf(fp, fmt, args);
+  va_end(args);
+  fputc('\n', fp);
+  fclose(fp);
 }
 
 std::string path_abs(const std::string &path_str) {
@@ -31,92 +23,6 @@ std::string path_to_file_uri(const std::string &path_str) {
   return "file://" + percent_encode(path_abs(path_str));
 }
 
-uint64_t fnv1a_64(const char *s, size_t len) {
-  uint64_t hash = 1469598103934665603ull;
-  for (size_t i = 0; i < len; ++i) {
-    hash ^= (uint8_t)s[i];
-    hash *= 1099511628211ull;
-  }
-  return hash;
-}
-
-char *get_from_clipboard(uint32_t *out_len) {
-  FILE *pipe = popen("xclip -selection clipboard -o", "r");
-  if (!pipe) {
-    *out_len = 0;
-    return nullptr;
-  }
-  size_t capacity = 4096;
-  size_t length = 0;
-  char *buffer = (char *)malloc(capacity);
-  if (!buffer) {
-    pclose(pipe);
-    *out_len = 0;
-    return nullptr;
-  }
-  size_t n;
-  while ((n = fread(buffer + length, 1, capacity - length, pipe)) > 0) {
-    length += n;
-    if (length == capacity) {
-      capacity *= 2;
-      char *tmp = (char *)realloc(buffer, capacity);
-      if (!tmp) {
-        free(buffer);
-        pclose(pipe);
-        *out_len = 0;
-        return nullptr;
-      }
-      buffer = tmp;
-    }
-  }
-  pclose(pipe);
-  char *result = (char *)realloc(buffer, length + 1);
-  if (result) {
-    result[length] = '\0';
-    buffer = result;
-  } else {
-    buffer[length] = '\0';
-  }
-  *out_len = length;
-  return buffer;
-}
-
-void copy_to_clipboard(const char *text, size_t len) {
-  FILE *pipe = popen("xclip -selection clipboard", "w");
-  if (!pipe)
-    return;
-  fwrite(text, sizeof(char), len, pipe);
-  pclose(pipe);
-}
-
-int display_width(const char *str, size_t len) {
-  if (!str || !*str)
-    return 0;
-  if (str[0] == '\t')
-    return 4;
-  unicode_width_state_t state;
-  unicode_width_init(&state);
-  int width = 0;
-  for (size_t j = 0; j < len; j++) {
-    unsigned char c = str[j];
-    if (c < 128) {
-      int char_width = unicode_width_process(&state, c);
-      if (char_width > 0)
-        width += char_width;
-    } else {
-      uint_least32_t cp;
-      size_t bytes = grapheme_decode_utf8(str + j, strlen(str) - j, &cp);
-      if (bytes > 1) {
-        int char_width = unicode_width_process(&state, cp);
-        if (char_width > 0)
-          width += char_width;
-        j += bytes - 1;
-      }
-    }
-  }
-  return width;
-}
-
 std::string get_exe_dir() {
   char exe_path[PATH_MAX];
   ssize_t count = readlink("/proc/self/exe", exe_path, PATH_MAX);
@@ -127,76 +33,6 @@ std::string get_exe_dir() {
   return path.substr(0, path.find_last_of('/'));
 }
 
-uint32_t get_visual_col_from_bytes(const char *line, uint32_t len,
-                                   uint32_t byte_limit) {
-  if (!line)
-    return 0;
-  uint32_t visual_col = 0;
-  uint32_t current_byte = 0;
-  if (len > 0 && line[len - 1] == '\n')
-    len--;
-  while (current_byte < byte_limit && current_byte < len) {
-    uint32_t inc = grapheme_next_character_break_utf8(line + current_byte,
-                                                      len - current_byte);
-    if (current_byte + inc > byte_limit)
-      break;
-    int w = display_width(line + current_byte, inc);
-    if (w < 0)
-      w = 0;
-    visual_col += (uint32_t)w;
-    current_byte += inc;
-  }
-  return visual_col;
-}
-
-uint32_t get_bytes_from_visual_col(const char *line, uint32_t len,
-                                   uint32_t target_visual_col) {
-  if (!line)
-    return 0;
-  uint32_t current_byte = 0;
-  uint32_t visual_col = 0;
-  if (len > 0 && line[len - 1] == '\n')
-    len--;
-  while (current_byte < len && visual_col < target_visual_col) {
-    uint32_t inc = grapheme_next_character_break_utf8(line + current_byte,
-                                                      len - current_byte);
-    int w = display_width(line + current_byte, inc);
-    if (w < 0)
-      w = 0;
-    if (visual_col + (uint32_t)w > target_visual_col)
-      return current_byte;
-    visual_col += (uint32_t)w;
-    current_byte += inc;
-  }
-  return current_byte;
-}
-
-uint32_t count_clusters(const char *line, size_t len, size_t from, size_t to) {
-  uint32_t count = 0;
-  size_t pos = from;
-  while (pos < to && pos < len) {
-    size_t next =
-        pos + grapheme_next_character_break_utf8(line + pos, len - pos);
-    if (next > to)
-      break;
-    pos = next;
-    count++;
-  }
-  return count;
-}
-
-void log(const char *fmt, ...) {
-  FILE *fp = fopen("/tmp/log.txt", "a");
-  if (!fp)
-    return;
-  va_list args;
-  va_start(args, fmt);
-  vfprintf(fp, fmt, args);
-  va_end(args);
-  fputc('\n', fp);
-  fclose(fp);
-}
-
 char *load_file(const char *path, uint32_t *out_len) {
   std::ifstream file(path, std::ios::in | std::ios::binary | std::ios::ate);
   if (!file.is_open())
@@ -269,24 +105,51 @@ Language language_for_file(const char *filename) {
   return {"unknown", nullptr};
 }
 
-int utf8_byte_offset_to_utf16(const char *s, size_t byte_pos) {
-  int utf16_units = 0;
-  size_t i = 0;
-  while (i < byte_pos) {
-    unsigned char c = s[i];
-    if ((c & 0x80) == 0x00) {
-      i += 1;
-      utf16_units += 1;
-    } else if ((c & 0xE0) == 0xC0) {
-      i += 2;
-      utf16_units += 1;
-    } else if ((c & 0xF0) == 0xE0) {
-      i += 3;
-      utf16_units += 1;
-    } else {
-      i += 4;
-      utf16_units += 2;
+char *get_from_clipboard(uint32_t *out_len) {
+  FILE *pipe = popen("xclip -selection clipboard -o", "r");
+  if (!pipe) {
+    *out_len = 0;
+    return nullptr;
+  }
+  size_t capacity = 4096;
+  size_t length = 0;
+  char *buffer = (char *)malloc(capacity);
+  if (!buffer) {
+    pclose(pipe);
+    *out_len = 0;
+    return nullptr;
+  }
+  size_t n;
+  while ((n = fread(buffer + length, 1, capacity - length, pipe)) > 0) {
+    length += n;
+    if (length == capacity) {
+      capacity *= 2;
+      char *tmp = (char *)realloc(buffer, capacity);
+      if (!tmp) {
+        free(buffer);
+        pclose(pipe);
+        *out_len = 0;
+        return nullptr;
+      }
+      buffer = tmp;
     }
   }
-  return utf16_units;
+  pclose(pipe);
+  char *result = (char *)realloc(buffer, length + 1);
+  if (result) {
+    result[length] = '\0';
+    buffer = result;
+  } else {
+    buffer[length] = '\0';
+  }
+  *out_len = length;
+  return buffer;
+}
+
+void copy_to_clipboard(const char *text, size_t len) {
+  FILE *pipe = popen("xclip -selection clipboard", "w");
+  if (!pipe)
+    return;
+  fwrite(text, sizeof(char), len, pipe);
+  pclose(pipe);
 }

src/utils/text.cc

@@ -0,0 +1,105 @@
+#include "utils/utils.h"
+
+std::string percent_decode(const std::string &s) {
+  std::string out;
+  out.reserve(s.size());
+  for (size_t i = 0; i < s.size(); ++i) {
+    if (s[i] == '%' && i + 2 < s.size() && std::isxdigit(s[i + 1]) &&
+        std::isxdigit(s[i + 2])) {
+      auto hex = [](char c) -> int {
+        if ('0' <= c && c <= '9')
+          return c - '0';
+        if ('a' <= c && c <= 'f')
+          return c - 'a' + 10;
+        if ('A' <= c && c <= 'F')
+          return c - 'A' + 10;
+        return 0;
+      };
+      char decoded = (hex(s[i + 1]) << 4) | hex(s[i + 2]);
+      out.push_back(decoded);
+      i += 2;
+    } else {
+      out.push_back(s[i]);
+    }
+  }
+  return out;
+}
+
+std::string percent_encode(const std::string &s) {
+  static const char *hex = "0123456789ABCDEF";
+  std::string out;
+  out.reserve(s.size() * 3);
+  for (unsigned char c : s) {
+    if (std::isalnum(c) || c == '-' || c == '_' || c == '.' || c == '~' ||
+        c == '/') {
+      out.push_back(c);
+    } else {
+      out.push_back('%');
+      out.push_back(hex[c >> 4]);
+      out.push_back(hex[c & 0xF]);
+    }
+  }
+  return out;
+}
+
+std::string trim(const std::string &s) {
+  size_t start = s.find_first_not_of(" \t\n\r");
+  if (start == std::string::npos)
+    return "";
+  size_t end = s.find_last_not_of(" \t\n\r");
+  return s.substr(start, end - start + 1);
+}
+
+std::string clean_text(const std::string &input) {
+  std::string result = input;
+  static const std::unordered_map<std::string, std::string> entities = {
+      {"&nbsp;", " "}, {"&lt;", "<"},    {"&gt;", ">"},
+      {"&amp;", "&"},  {"&quot;", "\""}, {"&apos;", "'"}};
+  for (const auto &e : entities) {
+    size_t pos = 0;
+    while ((pos = result.find(e.first, pos)) != std::string::npos) {
+      result.replace(pos, e.first.length(), e.second);
+      pos += e.second.length();
+    }
+  }
+  int errorcode;
+  PCRE2_SIZE erroroffset;
+  pcre2_code *re =
+      pcre2_compile((PCRE2_SPTR) "(\n\\s*)+", PCRE2_ZERO_TERMINATED, 0,
+                    &errorcode, &erroroffset, nullptr);
+  if (!re)
+    return result;
+  pcre2_match_data *match_data =
+      pcre2_match_data_create_from_pattern(re, nullptr);
+  PCRE2_SIZE offset = 0;
+  std::string clean;
+  while (offset < result.size()) {
+    int rc = pcre2_match(re, (PCRE2_SPTR)result.c_str(), result.size(), offset,
+                         0, match_data, nullptr);
+    if (rc < 0) {
+      clean += result.substr(offset);
+      break;
+    }
+    PCRE2_SIZE *ovector = pcre2_get_ovector_pointer(match_data);
+    clean += result.substr(offset, ovector[0] - offset) + "\n";
+    offset = ovector[1];
+  }
+  pcre2_match_data_free(match_data);
+  pcre2_code_free(re);
+  std::string final_str;
+  size_t start = 0;
+  while (start < clean.size()) {
+    size_t end = clean.find('\n', start);
+    if (end == std::string::npos)
+      end = clean.size();
+    std::string line = clean.substr(start, end - start);
+    size_t first = line.find_first_not_of(" \t\r");
+    size_t last = line.find_last_not_of(" \t\r");
+    if (first != std::string::npos)
+      final_str += line.substr(first, last - first + 1) + "\n";
+    start = end + 1;
+  }
+  if (!final_str.empty() && final_str.back() == '\n')
+    final_str.pop_back();
+  return final_str;
+}

src/utils/unicode.cc

@@ -0,0 +1,109 @@
+#include "utils/utils.h"
+
+int display_width(const char *str, size_t len) {
+  if (!str || !*str)
+    return 0;
+  if (str[0] == '\t')
+    return 4;
+  unicode_width_state_t state;
+  unicode_width_init(&state);
+  int width = 0;
+  for (size_t j = 0; j < len; j++) {
+    unsigned char c = str[j];
+    if (c < 128) {
+      int char_width = unicode_width_process(&state, c);
+      if (char_width > 0)
+        width += char_width;
+    } else {
+      uint_least32_t cp;
+      size_t bytes = grapheme_decode_utf8(str + j, strlen(str) - j, &cp);
+      if (bytes > 1) {
+        int char_width = unicode_width_process(&state, cp);
+        if (char_width > 0)
+          width += char_width;
+        j += bytes - 1;
+      }
+    }
+  }
+  return width;
+}
+
+uint32_t get_visual_col_from_bytes(const char *line, uint32_t len,
+                                   uint32_t byte_limit) {
+  if (!line)
+    return 0;
+  uint32_t visual_col = 0;
+  uint32_t current_byte = 0;
+  if (len > 0 && line[len - 1] == '\n')
+    len--;
+  while (current_byte < byte_limit && current_byte < len) {
+    uint32_t inc = grapheme_next_character_break_utf8(line + current_byte,
+                                                      len - current_byte);
+    if (current_byte + inc > byte_limit)
+      break;
+    int w = display_width(line + current_byte, inc);
+    if (w < 0)
+      w = 0;
+    visual_col += (uint32_t)w;
+    current_byte += inc;
+  }
+  return visual_col;
+}
+
+uint32_t get_bytes_from_visual_col(const char *line, uint32_t len,
+                                   uint32_t target_visual_col) {
+  if (!line)
+    return 0;
+  uint32_t current_byte = 0;
+  uint32_t visual_col = 0;
+  if (len > 0 && line[len - 1] == '\n')
+    len--;
+  while (current_byte < len && visual_col < target_visual_col) {
+    uint32_t inc = grapheme_next_character_break_utf8(line + current_byte,
+                                                      len - current_byte);
+    int w = display_width(line + current_byte, inc);
+    if (w < 0)
+      w = 0;
+    if (visual_col + (uint32_t)w > target_visual_col)
+      return current_byte;
+    visual_col += (uint32_t)w;
+    current_byte += inc;
+  }
+  return current_byte;
+}
+
+uint32_t count_clusters(const char *line, size_t len, size_t from, size_t to) {
+  uint32_t count = 0;
+  size_t pos = from;
+  while (pos < to && pos < len) {
+    size_t next =
+        pos + grapheme_next_character_break_utf8(line + pos, len - pos);
+    if (next > to)
+      break;
+    pos = next;
+    count++;
+  }
+  return count;
+}
+
+int utf8_byte_offset_to_utf16(const char *s, size_t byte_pos) {
+  int utf16_units = 0;
+  size_t i = 0;
+  while (i < byte_pos) {
+    unsigned char c = s[i];
+    if ((c & 0x80) == 0x00) {
+      i += 1;
+      utf16_units += 1;
+    } else if ((c & 0xE0) == 0xC0) {
+      i += 2;
+      utf16_units += 1;
+    } else if ((c & 0xF0) == 0xE0) {
+      i += 3;
+      utf16_units += 1;
+    } else {
+      i += 4;
+      utf16_units += 2;
+    }
+  }
+  return utf16_units;
+}