Add tree-sitter injections support & cleanup

.gitmodules

@@ -66,3 +66,6 @@
 [submodule "libs/tree-sitter-fish"]
 	path = libs/tree-sitter-fish
 	url = https://github.com/ram02z/tree-sitter-fish
+[submodule "libs/tree-sitter-rust"]
+	path = libs/tree-sitter-rust
+	url = https://github.com/tree-sitter/tree-sitter-rust.git

Makefile

@@ -1,10 +1,14 @@
 SRC_DIR := src
 BIN_DIR := bin
 OBJ_DIR := build
+INCLUDE_DIR := include
 
 TARGET_DEBUG := $(BIN_DIR)/crib-dbg
 TARGET_RELEASE := $(BIN_DIR)/crib
 
+PCH_DEBUG := $(OBJ_DIR)/debug/pch.h.gch
+PCH_RELEASE := $(OBJ_DIR)/release/pch.h.gch
+
 CCACHE := ccache
 CXX_DEBUG := $(CCACHE) g++
 CXX_RELEASE := $(CCACHE) clang++
@@ -18,6 +22,9 @@ CFLAGS_RELEASE := -std=c++20 -O3 -march=native -flto=thin \
 	-mllvm -vectorize-loops \
 	-fno-unwind-tables -fno-asynchronous-unwind-tables
 
+PCH_CFLAGS_DEBUG := $(CFLAGS_DEBUG) -x c++-header
+PCH_CFLAGS_RELEASE := $(CFLAGS_RELEASE) -x c++-header
+
 UNICODE_SRC := $(wildcard libs/unicode_width/*.c)
 
 UNICODE_OBJ_DEBUG := $(patsubst libs/unicode_width/%.c,$(OBJ_DIR)/debug/unicode_width/%.o,$(UNICODE_SRC))
@@ -50,21 +57,29 @@ test: $(TARGET_DEBUG)
 
 release: $(TARGET_RELEASE)
 
-$(TARGET_DEBUG): $(OBJ_DEBUG) $(UNICODE_OBJ_DEBUG)
+$(PCH_DEBUG): $(INCLUDE_DIR)/pch.h
-	mkdir -p $(BIN_DIR)
-	$(CXX_DEBUG) $(CFLAGS_DEBUG) -o $@ $^ $(LIBS)
-
-$(TARGET_RELEASE): $(OBJ_RELEASE) $(UNICODE_OBJ_RELEASE)
-	mkdir -p $(BIN_DIR)
-	$(CXX_RELEASE) $(CFLAGS_RELEASE) -o $@ $^ $(LIBS)
-
-$(OBJ_DIR)/debug/%.o: $(SRC_DIR)/%.cc
 	mkdir -p $(dir $@)
-	$(CXX_DEBUG) $(CFLAGS_DEBUG) -MMD -MP -c $< -o $@
+	$(CXX_DEBUG) $(PCH_CFLAGS_DEBUG) -o $@ $<
 
-$(OBJ_DIR)/release/%.o: $(SRC_DIR)/%.cc
+$(PCH_RELEASE): $(INCLUDE_DIR)/pch.h
 	mkdir -p $(dir $@)
-	$(CXX_RELEASE) $(CFLAGS_RELEASE) -MMD -MP -c $< -o $@
+	$(CXX_RELEASE) $(PCH_CFLAGS_RELEASE) -o $@ $<
+
+$(TARGET_DEBUG): $(PCH_DEBUG) $(OBJ_DEBUG) $(UNICODE_OBJ_DEBUG)
+	mkdir -p $(BIN_DIR)
+	$(CXX_DEBUG) $(CFLAGS_DEBUG) -o $@ $(OBJ_DEBUG) $(UNICODE_OBJ_DEBUG) $(LIBS)
+
+$(TARGET_RELEASE): $(PCH_RELEASE) $(OBJ_RELEASE) $(UNICODE_OBJ_RELEASE)
+	mkdir -p $(BIN_DIR)
+	$(CXX_RELEASE) $(CFLAGS_RELEASE) -o $@ $(OBJ_RELEASE) $(UNICODE_OBJ_RELEASE) $(LIBS)
+
+$(OBJ_DIR)/debug/%.o: $(SRC_DIR)/%.cc $(PCH_DEBUG)
+	mkdir -p $(dir $@)
+	$(CXX_DEBUG) $(CFLAGS_DEBUG) -include $(INCLUDE_DIR)/pch.h -MMD -MP -c $< -o $@
+
+$(OBJ_DIR)/release/%.o: $(SRC_DIR)/%.cc $(PCH_RELEASE)
+	mkdir -p $(dir $@)
+	$(CXX_RELEASE) $(CFLAGS_RELEASE) -include $(INCLUDE_DIR)/pch.h -MMD -MP -c $< -o $@
 
 $(OBJ_DIR)/debug/unicode_width/%.o: libs/unicode_width/%.c
 	mkdir -p $(dir $@)

README.md

@@ -6,8 +6,6 @@ A TUI IDE.
 
 # TODO
 
-- [ ] Add a virtual text support (text that is rendered in the editor but not in the actual text)
-    - Add a whitespace highlighter (nerd font). for spaces and tabs at start/end of line. as a test.
 - [ ] Add support for LSP & autocomplete / snippets.
     - First research
         - `textDocument/documentHighlight` - for highlighting stuff (probably tree-sitter is enough)
@@ -21,6 +19,7 @@ A TUI IDE.
         - `textDocument/foldingRange` - i will never use this for folding but it might be useful for other things.
         - `textDocument/rename` & `textDocument/prepareRename` - probably useful
         - And a lot more (just go through each for `clangd` and then expand to say `solargraph`).
+    - Make incremental edits apply.
     - Make a universal plug for lsp. So focus more on making a general purpose solid communication interface. Instead of something specific.
         - With a 4ish pass system. (more like each returned value from the lsp is used in 4 ways)
             1. One for stuff like jump to x position. or rename symbol x to y. (stuff that explicitly requires user request to do something)
@@ -28,6 +27,10 @@ A TUI IDE.
             2. One for stuff that only affects highlighting and styles . like symbol highlighting etc.
             3. One for Warnings/errors and inlay hints etc. (stuff that adds virtual text to the editor)
             4. One for fromatting and stuff like that. (stuff that edits the buffer text)
+- [ ] Use LSP to add inlay hints in order to test virtual text. then make an iterator over screen that mimics the renderer for scrolling functions.
+- [ ] Add codeium/copilot support for auto-completion (uses the VAI virtual text) as a test phase.
+- [ ] Add a whitespace highlighter (nerd font). for spaces and tabs at start/end of line. not as virtual but instead at render time.
+- [ ] Once renderer is proven to work well (i.e. redo this commit) merge `experimental` branch into `main`. commit `43f443e` on `experimental`.
 - [ ] Add snippets from wherever i get them. (like luasnip or vsnip)
 - [ ] Add this thing where select at end of screen scrolls down. (and vice versa)
     - Can be acheived by updating `main.cc` to send drag events to the selected editor instead of just under cursor.
@@ -38,7 +41,6 @@ A TUI IDE.
 - [ ] Add support for undo/redo.
 - [ ] Add `.scm` files for all the supported languages. (2/14) Done.
 - [ ] Add splash screen / minigame jumping.
-- [ ] Add codeium/copilot support.
 - [ ] Normalize / validate unicode on file open.
 - [ ] Add git stuff.
 - [ ] Add SQL support. (viewer and basic editor)
@@ -51,3 +53,4 @@ A TUI IDE.
 - [ ] Add this thing where selection double click on a bracket selects whole block.
     - (only on the first time) and sets mode to `WORD`.
 - [ ] Redo folding system and its relation to move_line_* functions. (Currently its a mess)
+- [ ] Make whole thing event driven and not clock driven.

grammar/cpp.scm

@@ -0,0 +1,613 @@
+;; #ffffff #000000 0 0 0 1
+((identifier) @variable
+  (#set! priority 95))
+
+(preproc_def
+  (preproc_arg) @variable)
+
+;; #fbb152 #000000 0 0 0 1
+[
+  "default"
+  "goto"
+  "asm"
+  "__asm__"
+] @keyword
+
+[
+  "enum"
+  "struct"
+  "union"
+  "typedef"
+] @keyword.type
+
+[
+  "sizeof"
+  "offsetof"
+] @keyword.operator
+
+(alignof_expression
+  .
+  _ @keyword.operator)
+
+;; #fbb152 #000000 0 0 0 1
+"return" @keyword.return
+
+;; #fbb152 #000000 0 0 0 1
+[
+  "while"
+  "for"
+  "do"
+  "continue"
+  "break"
+] @keyword.repeat
+
+;; #fbb152 #000000 0 0 0 1
+[
+  "if"
+  "else"
+  "case"
+  "switch"
+] @keyword.conditional
+
+;; #fbb152 #000000 0 0 0 1
+[
+  "#if"
+  "#ifdef"
+  "#ifndef"
+  "#else"
+  "#elif"
+  "#endif"
+  "#elifdef"
+  "#elifndef"
+  (preproc_directive)
+] @keyword.directive
+
+;; #fbb152 #000000 0 0 0 1
+"#define" @keyword.directive.define
+
+;; #fbb152 #000000 0 0 0 1
+"#include" @keyword.import
+
+;; #bd9ae6 #000000 0 0 0 1
+[
+  ";"
+  ":"
+  ","
+  "."
+  "::"
+] @punctuation.delimiter
+
+;; #e6a24c #000000 0 0 0 2
+"..." @punctuation.special
+
+;; #bd9ae6 #000000 0 0 0 1
+[
+  "("
+  ")"
+  "["
+  "]"
+  "{"
+  "}"
+] @punctuation.bracket
+
+;; #ffffff #000000 0 0 0 1
+[
+  "="
+  "-"
+  "*"
+  "/"
+  "+"
+  "%"
+  "~"
+  "|"
+  "&"
+  "^"
+  "<<"
+  ">>"
+  "->"
+  "<"
+  "<="
+  ">="
+  ">"
+  "=="
+  "!="
+  "!"
+  "&&"
+  "||"
+  "-="
+  "+="
+  "*="
+  "/="
+  "%="
+  "|="
+  "&="
+  "^="
+  ">>="
+  "<<="
+  "--"
+  "++"
+] @operator
+
+;; #ffffff #000000 0 0 0 1
+(comma_expression
+  "," @operator)
+
+;; #51eeba #000000 0 0 0 1
+[
+  (true)
+  (false)
+] @boolean
+
+;; #fbb152 #000000 0 0 0 1
+(conditional_expression
+  [
+    "?"
+    ":"
+  ] @keyword.conditional.ternary)
+
+;; #aad84c #000000 0 0 0 1
+(string_literal) @string
+
+(system_lib_string) @string
+
+;; #e6a24c #000000 0 0 0 2
+(escape_sequence) @string.escape
+
+;; #ebda8c #000000 0 0 0 1
+(null) @constant.builtin
+
+;; #ebda8c #000000 0 0 0 1
+(number_literal) @number
+
+;; #ebda8c #000000 0 0 0 1
+(char_literal) @character
+
+;; #aad84c #000000 0 0 0 1
+(preproc_defined) @function.macro
+
+;; #ffffff #000000 0 0 0 1
+((field_expression
+  (field_identifier) @property) @_parent)
+
+(field_designator) @property
+
+((field_identifier) @property)
+
+;; #fbb152 #000000 0 0 0 1
+(statement_identifier) @label
+
+(declaration
+  type: (type_identifier) @_type
+  declarator: (identifier) @label
+  (#match? @_type "^__label__$"))
+
+;; #aad84c #000000 0 0 0 1
+[
+  (type_identifier)
+  (type_descriptor)
+] @type
+
+;; #fbb152 #000000 0 0 0 1
+(storage_class_specifier) @keyword.modifier
+
+[
+  (type_qualifier)
+  (gnu_asm_qualifier)
+  "__extension__"
+] @keyword.modifier
+
+;; #fbb152 #000000 0 0 0 1
+(linkage_specification
+  "extern" @keyword.modifier)
+
+;; #aad84c #000000 0 0 0 1
+(type_definition
+  declarator: (type_identifier) @type.definition)
+
+;; #aad84c #000000 0 0 0 1
+(primitive_type) @type.builtin
+
+(sized_type_specifier
+  _ @type.builtin
+  type: _?)
+
+;; #ebda8c #000000 0 0 0 1
+((identifier) @constant
+  (#match? @constant "^[A-Z][A-Z0-9_]+$"))
+
+(preproc_def
+  (preproc_arg) @constant
+  (#match? @constant "^[A-Z][A-Z0-9_]+$"))
+
+(enumerator
+  name: (identifier) @constant)
+
+(case_statement
+  value: (identifier) @constant)
+
+;; #ebda8c #000000 0 0 0 1
+((identifier) @constant.builtin
+  (#match? @constant.builtin "^(stderr|stdin|stdout|__FILE__|__LINE__|__DATE__|__TIME__|__STDC__|__STDC_VERSION__|__STDC_HOSTED__|__cplusplus|__OBJC__|__ASSEMBLER__|__BASE_FILE__|__FILE_NAME__|__INCLUDE_LEVEL__|__TIMESTAMP__|__clang__|__clang_major__|__clang_minor__|__clang_patchlevel__|__clang_version__|__clang_literal_encoding__|__clang_wide_literal_encoding__|__FUNCTION__|__func__|__PRETTY_FUNCTION__|__VA_ARGS__|__VA_OPT__)$"))
+
+(preproc_def
+  (preproc_arg) @constant.builtin
+  (#match? @constant.builtin "^(stderr|stdin|stdout|__FILE__|__LINE__|__DATE__|__TIME__|__STDC__|__STDC_VERSION__|__STDC_HOSTED__|__cplusplus|__OBJC__|__ASSEMBLER__|__BASE_FILE__|__FILE_NAME__|__INCLUDE_LEVEL__|__TIMESTAMP__|__clang__|__clang_major__|__clang_minor__|__clang_patchlevel__|__clang_version__|__clang_literal_encoding__|__clang_wide_literal_encoding__|__FUNCTION__|__func__|__PRETTY_FUNCTION__|__VA_ARGS__|__VA_OPT__)$"))
+
+;; #ffffff #000000 0 0 0 1
+(attribute_specifier
+  (argument_list
+    (identifier) @variable.builtin))
+
+(attribute_specifier
+  (argument_list
+    (call_expression
+      function: (identifier) @variable.builtin)))
+
+;; #aad84c #000000 0 0 0 1
+((call_expression
+  function: (identifier) @function.builtin)
+  (#match? @function.builtin "^__builtin_"))
+
+((call_expression
+  function: (identifier) @function.builtin))
+
+;; #ebda8c #000000 0 0 0 1
+(preproc_def
+  name: (_) @constant.macro)
+
+(preproc_call
+  directive: (preproc_directive) @_u
+  argument: (_) @constant.macro
+  (#match? @_u "^#undef$"))
+
+(preproc_ifdef
+  name: (identifier) @constant.macro)
+
+(preproc_elifdef
+  name: (identifier) @constant.macro)
+
+(preproc_defined
+  (identifier) @constant.macro)
+
+;; #aad84c #000000 0 0 0 3
+(call_expression
+  function: (identifier) @function.call)
+
+(call_expression
+  function: (field_expression
+    field: (field_identifier) @function.call))
+
+;; #aad84c #000000 0 0 0 3
+(function_declarator
+  declarator: (identifier) @function)
+
+(function_declarator
+  declarator: (parenthesized_declarator
+    (pointer_declarator
+      declarator: (field_identifier) @function)))
+
+;; #aad84c #000000 0 0 0 3
+(preproc_function_def
+  name: (identifier) @function.macro)
+
+;; #AAAAAA #000000 0 1 0 1
+(comment) @comment @spell
+
+;; #AAAAAA #000000 0 1 0 1
+((comment) @comment.documentation
+  (#match? @comment.documentation "^/[*][*][^*].*[*]/$"))
+
+;; #ffffff #000000 0 0 0 1
+(parameter_declaration
+  declarator: (identifier) @variable.parameter)
+
+(parameter_declaration
+  declarator: (array_declarator) @variable.parameter)
+
+(parameter_declaration
+  declarator: (pointer_declarator) @variable.parameter)
+
+(preproc_params
+  (identifier) @variable.parameter)
+
+;; #fbb152 #000000 0 0 0 1
+[
+  "__attribute__"
+  "__declspec"
+  "__based"
+  "__cdecl"
+  "__clrcall"
+  "__stdcall"
+  "__fastcall"
+  "__thiscall"
+  "__vectorcall"
+  (ms_pointer_modifier)
+  (attribute_declaration)
+] @attribute
+
+;; #ffffff #000000 0 0 0 1
+((identifier) @variable.member
+  (#match? @variable.member "^m_.*$"))
+
+(parameter_declaration
+  declarator: (reference_declarator) @variable.parameter)
+
+(variadic_parameter_declaration
+  declarator: (variadic_declarator
+    (_) @variable.parameter))
+
+(optional_parameter_declaration
+  declarator: (_) @variable.parameter)
+
+;; #ffffff #000000 0 0 0 1
+((field_expression
+  (field_identifier) @function.method) @_parent)
+
+(field_declaration
+  (field_identifier) @variable.member)
+
+(field_initializer
+  (field_identifier) @property)
+
+(function_declarator
+  declarator: (field_identifier) @function.method)
+
+;; #aad84c #000000 0 0 0 3
+(concept_definition
+  name: (identifier) @type.definition)
+
+(alias_declaration
+  name: (type_identifier) @type.definition)
+
+;; #aad84c #000000 0 0 0 1
+(auto) @type.builtin
+
+;; #aad84c #000000 0 0 0 1
+(namespace_identifier) @module
+
+;; #aad84c #000000 0 0 0 1
+((namespace_identifier) @type
+  (#match? @type "^[A-Z]"))
+
+;; #ebda8c #000000 0 0 0 1
+(case_statement
+  value: (qualified_identifier
+    (identifier) @constant))
+
+;; #fbb152 #000000 0 0 0 1
+(using_declaration
+  .
+  "using"
+  .
+  "namespace"
+  .
+  [
+    (qualified_identifier)
+    (identifier)
+  ] @module)
+
+;; #aad84c #000000 0 0 0 3
+(destructor_name
+  (identifier) @function.method)
+
+;; #aad84c #000000 0 0 0 3
+(function_declarator
+  (qualified_identifier
+    (identifier) @function))
+
+(function_declarator
+  (qualified_identifier
+    (qualified_identifier
+      (identifier) @function)))
+
+(function_declarator
+  (qualified_identifier
+    (qualified_identifier
+      (qualified_identifier
+        (identifier) @function))))
+
+((qualified_identifier
+  (qualified_identifier
+    (qualified_identifier
+      (qualified_identifier
+        (identifier) @function)))) @_parent)
+
+(function_declarator
+  (template_function
+    (identifier) @function))
+
+(operator_name) @function
+
+"operator" @function
+
+;; #aad84c #000000 0 0 0 3
+"static_assert" @function.builtin
+
+;; #aad84c #000000 0 0 0 3
+(call_expression
+  (qualified_identifier
+    (identifier) @function.call))
+
+(call_expression
+  (qualified_identifier
+    (qualified_identifier
+      (identifier) @function.call)))
+
+(call_expression
+  (qualified_identifier
+    (qualified_identifier
+      (qualified_identifier
+        (identifier) @function.call))))
+
+((qualified_identifier
+  (qualified_identifier
+    (qualified_identifier
+      (qualified_identifier
+        (identifier) @function.call)))) @_parent)
+
+(call_expression
+  (template_function
+    (identifier) @function.call))
+
+(call_expression
+  (qualified_identifier
+    (template_function
+      (identifier) @function.call)))
+
+(call_expression
+  (qualified_identifier
+    (qualified_identifier
+      (template_function
+        (identifier) @function.call))))
+
+(call_expression
+  (qualified_identifier
+    (qualified_identifier
+      (qualified_identifier
+        (template_function
+          (identifier) @function.call)))))
+
+((qualified_identifier
+  (qualified_identifier
+    (qualified_identifier
+      (qualified_identifier
+        (template_function
+          (identifier) @function.call))))) @_parent)
+
+(function_declarator
+  (template_method
+    (field_identifier) @function.method))
+
+;; #aad84c #000000 0 0 0 3
+(call_expression
+  (field_expression
+    (field_identifier) @function.method.call))
+
+(call_expression
+  (field_expression
+    (template_method
+      (field_identifier) @function.method.call)))
+
+;; #aad84c #000000 0 0 0 3
+((function_declarator
+  (qualified_identifier
+    (identifier) @constructor))
+  (#match? @constructor "^[A-Z]"))
+
+((call_expression
+  function: (identifier) @constructor)
+  (#match? @constructor "^[A-Z]"))
+
+((call_expression
+  function: (qualified_identifier
+    name: (identifier) @constructor))
+  (#match? @constructor "^[A-Z]"))
+
+((call_expression
+  function: (field_expression
+    field: (field_identifier) @constructor))
+  (#match? @constructor "^[A-Z]"))
+
+((field_initializer
+  (field_identifier) @constructor
+  (argument_list))
+  (#match? @constructor "^[A-Z]"))
+
+;; #ffffff #000000 0 0 0 1
+(this) @variable.builtin
+
+;; #ebda8c #000000 0 0 0 1
+(null
+  "nullptr" @constant.builtin)
+
+;; #51eeba #000000 0 0 0 2
+(true) @boolean_true
+
+;; #ee513a #000000 0 0 0 2
+(false) @boolean_false
+
+;; #aad84c #000000 0 0 0 1
+(raw_string_literal) @string
+
+;; #fbb152 #000000 0 0 0 1
+[
+  "try"
+  "catch"
+  "noexcept"
+  "throw"
+] @keyword.exception
+
+;; #fbb152 #000000 0 0 0 1
+[
+  "decltype"
+  "explicit"
+  "friend"
+  "override"
+  "using"
+  "requires"
+  "constexpr"
+] @keyword
+
+;; #fbb152 #000000 0 0 0 1
+[
+  "class"
+  "namespace"
+  "template"
+  "typename"
+  "concept"
+] @keyword.type
+
+;; #fbb152 #000000 0 0 0 1
+[
+  "co_await"
+  "co_yield"
+  "co_return"
+] @keyword.coroutine
+
+;; #fbb152 #000000 0 0 0 1
+[
+  "public"
+  "private"
+  "protected"
+  "final"
+  "virtual"
+] @keyword.modifier
+
+;; #fbb152 #000000 0 0 0 1
+[
+  "new"
+  "delete"
+  "xor"
+  "bitand"
+  "bitor"
+  "compl"
+  "not"
+  "xor_eq"
+  "and_eq"
+  "or_eq"
+  "not_eq"
+  "and"
+  "or"
+] @keyword.operator
+
+;; #ffffff #000000 0 0 0 1
+"<=>" @operator
+
+;; #bd9ae6 #000000 0 0 0 1
+"::" @punctuation.delimiter
+
+;; #bd9ae6 #000000 0 0 0 1
+(template_argument_list
+  [
+    "<"
+    ">"
+  ] @punctuation.bracket)
+
+(template_parameter_list
+  [
+    "<"
+    ">"
+  ] @punctuation.bracket)
+
+;; #ffffff #000000 0 0 0 1
+(literal_suffix) @operator

grammar/ruby.scm

@@ -1,3 +1,7 @@
+; This is an injection test - it should hight all heredoc content as bash code
+;; !bash - this part should be ignored (anything after the first wordbreak after the `!`)
+(heredoc_content) @ruby_injection
+
 ;; #ffffff #000000 0 0 0 1
 [
   (identifier)

include/editor.h

@@ -1,23 +1,18 @@
 #ifndef EDITOR_H
 #define EDITOR_H
 
-#include "../libs/tree-sitter/lib/include/tree_sitter/api.h"
 #include "./knot.h"
+#include "./pch.h"
 #include "./ui.h"
 #include "./utils.h"
-#include <algorithm>
+#include "ts_def.h"
 #include <cstdint>
-#include <map>
-#include <shared_mutex>
-#include <unordered_map>
-#include <vector>
 
 #define CHAR 0
 #define WORD 1
 #define LINE 2
 
 #define EXTRA_META 4
-
 #define INDENT_WIDTH 2
 
 struct Highlight {
@@ -98,8 +93,54 @@ struct SpanCursor {
   }
 };
 
+struct VHint {
+  Coord pos;
+  char *text; // Can only be a single line with ascii only
+  uint32_t len;
+
+  bool operator<(const VHint &other) const { return pos < other.pos; }
+};
+
+struct VWarn {
+  uint32_t line;
+  char *text; // Can only be a single line
+  uint32_t len;
+  int8_t type; // For hl
+
+  bool operator<(const VWarn &other) const { return line < other.line; }
+};
+
+struct VAI {
+  Coord pos;
+  char *text;
+  uint32_t len;
+  uint32_t lines; // number of \n in text for speed .. the ai part will not
+                  // line wrap but multiline ones need to have its own lines
+                  // after the first one
+};
+
+struct TSSetBase {
+  std::string lang;
+  TSTree *tree;
+  TSParser *parser;
+  std::string query_file;
+  TSQuery *query;
+  std::map<uint16_t, Highlight> query_map;
+  std::map<uint16_t, Language> injection_map;
+  const TSLanguage *language;
+};
+
+struct TSSet : TSSetBase {
+  std::vector<TSRange> ranges;
+};
+
+struct TSSetMain : TSSetBase {
+  std::vector<TSSet> injections;
+};
+
 struct Editor {
-  const char *filename;
+  std::string filename;
+  std::string uri;
   Knot *root;
   std::shared_mutex knot_mtx;
   Coord cursor;
@@ -110,17 +151,18 @@ struct Editor {
   Coord position;
   Coord size;
   Coord scroll;
-  TSTree *tree;
+  TSSetMain ts;
-  TSParser *parser;
-  TSQuery *query;
-  const TSLanguage *language;
   Queue<TSInputEdit> edit_queue;
-  std::vector<Highlight> query_map;
   std::vector<Fold> folds;
   Spans spans;
   Spans def_spans;
   uint32_t hooks[94];
   bool jumper_set;
+  std::vector<VHint> hints;
+  std::vector<VWarn> warnings;
+  VAI ai;
+  std::shared_mutex lsp_mtx;
+  struct LSPInstance *lsp;
 };
 
 inline const Fold *fold_for_line(const std::vector<Fold> &folds,
@@ -188,6 +230,8 @@ void cursor_up(Editor *editor, uint32_t number);
 void cursor_down(Editor *editor, uint32_t number);
 Coord move_left(Editor *editor, Coord cursor, uint32_t number);
 Coord move_right(Editor *editor, Coord cursor, uint32_t number);
+Coord move_left_pure(Editor *editor, Coord cursor, uint32_t number);
+Coord move_right_pure(Editor *editor, Coord cursor, uint32_t number);
 void cursor_left(Editor *editor, uint32_t number);
 void cursor_right(Editor *editor, uint32_t number);
 void scroll_up(Editor *editor, int32_t number);
@@ -218,5 +262,6 @@ void apply_line_deletion(Editor *editor, uint32_t removal_start,
 uint32_t leading_indent(const char *line, uint32_t len);
 uint32_t get_indent(Editor *editor, Coord cursor);
 bool closing_after_cursor(const char *line, uint32_t len, uint32_t col);
+// void editor_lsp_handle(Editor *editor, json msg);
 
 #endif

include/knot.h

@@ -1,9 +1,8 @@
 #ifndef ROPE_H
 #define ROPE_H
 
+#include "./pch.h"
 #include "./utils.h"
-#include <cstdint>
-#include <vector>
 
 #define MIN_CHUNK_SIZE 64       // 64 Bytes
 #define MAX_CHUNK_SIZE 1024 * 8 // 8192 Bytes (8 KiB)

include/lsp.h

@@ -0,0 +1,55 @@
+#ifndef LSP_H
+#define LSP_H
+
+#include "./editor.h"
+#include "./pch.h"
+#include "utils.h"
+
+struct LSP {
+  const char *command;
+  std::vector<const char *> args;
+};
+
+struct LSPPending {
+  std::string method;
+  Editor *editor = nullptr;
+
+  std::function<void(Editor *, std::string, json)> callback;
+};
+
+struct LSPOpenRequest {
+  Language language;
+  Editor *editor;
+};
+
+struct LSPInstance {
+  std::shared_mutex mtx;
+  const LSP *lsp;
+  std::string root_dir;
+  int pid{-1};
+  int stdin_fd{-1};
+  int stdout_fd{-1};
+  bool initialized = false;
+  uint32_t last_id = 0;
+  Queue<json> inbox;
+  Queue<json> outbox;
+  std::unordered_map<uint32_t, LSPPending *> pending;
+  std::vector<Editor *> editors;
+};
+
+extern std::shared_mutex active_lsps_mtx;
+extern std::unordered_map<uint8_t, LSPInstance *> active_lsps;
+
+void lsp_worker();
+void lsp_handle(LSPInstance *lsp, json message);
+
+LSPInstance *get_or_init_lsp(uint8_t lsp_id);
+void close_lsp(uint8_t lsp_id);
+
+void request_add_to_lsp(Language language, Editor *editor);
+void add_to_lsp(Language language, Editor *editor);
+void remove_from_lsp(Editor *editor);
+
+void lsp_send(LSPInstance *lsp, json message, LSPPending *pending);
+
+#endif

include/main.h

@@ -1,8 +1,7 @@
 #ifndef MAIN_H
 #define MAIN_H
 
-#include <atomic>
+#include "./pch.h"
-#include <vector>
 
 #define NORMAL 0
 #define INSERT 1

include/maps.h

@@ -0,0 +1,65 @@
+#ifndef MAPS_H
+#define MAPS_H
+
+#include "./lsp.h"
+#include "./pch.h"
+#include "./ts_def.h"
+#include <unordered_map>
+
+static const std::unordered_map<std::string, Language> kLanguages = {
+    {"bash", {"bash", tree_sitter_bash}},
+    {"c", {"c", tree_sitter_c, 1}},
+    {"cpp", {"cpp", tree_sitter_cpp, 1}},
+    {"h", {"h", tree_sitter_cpp, 1}},
+    {"css", {"css", tree_sitter_css}},
+    {"fish", {"fish", tree_sitter_fish}},
+    {"go", {"go", tree_sitter_go}},
+    {"haskell", {"haskell", tree_sitter_haskell}},
+    {"html", {"html", tree_sitter_html}},
+    {"javascript", {"javascript", tree_sitter_javascript}},
+    {"json", {"json", tree_sitter_json}},
+    {"lua", {"lua", tree_sitter_lua}},
+    {"make", {"make", tree_sitter_make}},
+    {"python", {"python", tree_sitter_python}},
+    {"ruby", {"ruby", tree_sitter_ruby}},
+};
+
+static const std::unordered_map<uint8_t, LSP> kLsps = {
+    {1,
+     {"clangd",
+      {
+          "clangd",
+          "--background-index",
+          "--clang-tidy",
+          "--completion-style=detailed",
+          "--header-insertion=iwyu",
+          "--log=error",
+          nullptr,
+      }}},
+};
+
+static const std::unordered_map<std::string, std::string> kExtToLang = {
+    {"sh", "bash"},       {"bash", "bash"},  {"c", "c"},       {"cpp", "cpp"},
+    {"cxx", "cpp"},       {"cc", "cpp"},     {"hpp", "h"},     {"hh", "h"},
+    {"hxx", "h"},         {"h", "h"},        {"css", "css"},   {"fish", "fish"},
+    {"go", "go"},         {"hs", "haskell"}, {"html", "html"}, {"htm", "html"},
+    {"js", "javascript"}, {"json", "json"},  {"lua", "lua"},   {"mk", "make"},
+    {"makefile", "make"}, {"py", "python"},  {"rb", "ruby"},
+};
+
+static const std::unordered_map<std::string, std::string> kMimeToLang = {
+    {"text/x-c", "c"},
+    {"text/x-c++", "cpp"},
+    {"text/x-shellscript", "bash"},
+    {"application/json", "json"},
+    {"text/javascript", "javascript"},
+    {"text/html", "html"},
+    {"text/css", "css"},
+    {"text/x-python", "python"},
+    {"text/x-ruby", "ruby"},
+    {"text/x-go", "go"},
+    {"text/x-haskell", "haskell"},
+    {"text/x-lua", "lua"},
+};
+
+#endif

include/pch.h

@@ -0,0 +1,42 @@
+#ifndef PCH_H
+#define PCH_H
+
+#define PCRE2_CODE_UNIT_WIDTH 8
+#define PCRE_WORKSPACE_SIZE 512
+
+#include "../libs/tree-sitter/lib/include/tree_sitter/api.h"
+#include <algorithm>
+#include <atomic>
+#include <cctype>
+#include <chrono>
+#include <cstdarg>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <deque>
+#include <filesystem>
+#include <fstream>
+#include <functional>
+#include <limits.h>
+#include <magic.h>
+#include <map>
+#include <mutex>
+#include <nlohmann/json.hpp>
+#include <optional>
+#include <pcre2.h>
+#include <queue>
+#include <shared_mutex>
+#include <string.h>
+#include <string>
+#include <sys/ioctl.h>
+#include <termios.h>
+#include <thread>
+#include <unistd.h>
+#include <unordered_map>
+#include <vector>
+
+using json = nlohmann::json;
+using namespace std::chrono_literals;
+
+#endif

include/ts.h

@@ -2,14 +2,14 @@
 #define TS_H
 
 #include "./editor.h"
+#include "./pch.h"
 #include "./utils.h"
-#include <pcre2.h>
 
 #define HEX(s) (static_cast<uint32_t>(std::stoul(s, nullptr, 16)))
 
 extern std::unordered_map<std::string, pcre2_code *> regex_cache;
 
-TSQuery *load_query(const char *query_path, Editor *editor);
+TSQuery *load_query(const char *query_path, TSSetBase *set);
 void ts_collect_spans(Editor *editor);
 void clear_regex_cache();
 

include/ts_def.h

@@ -1,9 +1,12 @@
-#include "../libs/tree-sitter/lib/include/tree_sitter/api.h"
+#ifndef TS_DEF_H
-#include <string>
+#define TS_DEF_H
+
+#include "./pch.h"
 
 struct Language {
   std::string name;
   const TSLanguage *(*fn)();
+  uint8_t lsp_id = 0;
 };
 
 extern "C" {
@@ -21,4 +24,13 @@ const TSLanguage *tree_sitter_lua();
 const TSLanguage *tree_sitter_make();
 const TSLanguage *tree_sitter_python();
 const TSLanguage *tree_sitter_ruby();
+const TSLanguage *tree_sitter_rust();
+// TO ADD
+// sql
+// wasm
+// conf
+// yaml, toml
+// godot
 }
+
+#endif

include/ui.h

@@ -1,16 +1,8 @@
 #ifndef UI_H
 #define UI_H
 
+#include "./pch.h"
 #include "./utils.h"
-#include <atomic>
-#include <cstdint>
-#include <mutex>
-#include <string.h>
-#include <string>
-#include <sys/ioctl.h>
-#include <termios.h>
-#include <unistd.h>
-#include <vector>
 
 #define KEY_CHAR 0
 #define KEY_SPECIAL 1

include/utils.h

@@ -1,16 +1,8 @@
 #ifndef UTILS_H
 #define UTILS_H
 
+#include "./pch.h"
 #include "./ts_def.h"
-#include <chrono>
-#include <functional>
-#include <mutex>
-#include <queue>
-#include <string>
-#include <thread>
-
-#define PCRE2_CODE_UNIT_WIDTH 8
-#define PCRE_WORKSPACE_SIZE 512
 
 template <typename T> struct Queue {
   std::queue<T> q;
@@ -20,6 +12,10 @@ template <typename T> struct Queue {
     std::lock_guard<std::mutex> lock(m);
     q.push(val);
   }
+  T front() {
+    std::lock_guard<std::mutex> lock(m);
+    return q.front();
+  }
   bool pop(T &val) {
     std::lock_guard<std::mutex> lock(m);
     if (q.empty())
@@ -28,6 +24,10 @@ template <typename T> struct Queue {
     q.pop();
     return true;
   }
+  void pop() {
+    std::lock_guard<std::mutex> lock(m);
+    q.pop();
+  }
   bool empty() {
     std::lock_guard<std::mutex> lock(m);
     return q.empty();
@@ -52,6 +52,7 @@ struct Coord {
   bool operator>=(const Coord &other) const { return !(*this < other); }
 };
 
+std::string path_to_file_uri(const std::string &path_str);
 int display_width(const char *str, size_t len);
 uint32_t get_visual_col_from_bytes(const char *line, uint32_t len,
                                    uint32_t byte_limit);

samples/ruby.rb

@@ -44,9 +44,19 @@ puts "Emoji count: #{emojis.length}"
 
 # Multi-line string with unicode
 multi = <<~EOF
-  これは複数行テキストです。
+  # Function recursion demo
-  Emojis inside heredoc: 🎉🔥✨💀❤️🧡💛💚💙💜🖤🤍🤎
+  factorial() {
-  End of block.
+    local n="$1"
+    if ((n <= 1)); then
+      echo 1
+    else
+      local prev
+      prev=$(factorial $((n - 1)))
+      echo $((n * prev))
+    fi
+  }
+
+  log INFO "factorial(5) = $(factorial 5)"
 EOF
 
 puts multi

src/editor.cc

@@ -2,10 +2,9 @@ extern "C" {
 #include "../libs/libgrapheme/grapheme.h"
 }
 #include "../include/editor.h"
+#include "../include/lsp.h"
 #include "../include/main.h"
-#include "../include/ts.h"
 #include "../include/utils.h"
-#include <cmath>
 
 Editor *new_editor(const char *filename, Coord position, Coord size) {
   Editor *editor = new Editor();
@@ -18,28 +17,44 @@ Editor *new_editor(const char *filename, Coord position, Coord size) {
     return nullptr;
   }
   editor->filename = filename;
+  editor->uri = path_to_file_uri(filename);
   editor->position = position;
   editor->size = size;
   editor->cursor_preffered = UINT32_MAX;
   editor->root = load(str, len, optimal_chunk_size(len));
   free(str);
-  if (len <= (1024 * 128)) {
-    editor->parser = ts_parser_new();
   Language language = language_for_file(filename);
-    editor->language = language.fn();
+  if (language.name != "unknown" && len <= (1024 * 128)) {
-    ts_parser_set_language(editor->parser, editor->language);
+    editor->ts.parser = ts_parser_new();
-    std::string query = get_exe_dir() + "/../grammar/" + language.name + ".scm";
+    editor->ts.language = language.fn();
-    editor->query = load_query(query.c_str(), editor);
+    ts_parser_set_language(editor->ts.parser, editor->ts.language);
+    editor->ts.query_file =
+        get_exe_dir() + "/../grammar/" + language.name + ".scm";
+    request_add_to_lsp(language, editor);
   }
   return editor;
 }
 
+void free_tsset(TSSetMain *set) {
+  if (set->parser)
+    ts_parser_delete(set->parser);
+  if (set->tree)
+    ts_tree_delete(set->tree);
+  if (set->query)
+    ts_query_delete(set->query);
+  for (auto &inj : set->injections) {
+    if (inj.parser)
+      ts_parser_delete(inj.parser);
+    if (inj.tree)
+      ts_tree_delete(inj.tree);
+    if (inj.query)
+      ts_query_delete(inj.query);
+  }
+}
+
 void free_editor(Editor *editor) {
-  ts_parser_delete(editor->parser);
+  remove_from_lsp(editor);
-  if (editor->tree)
+  free_tsset(&editor->ts);
-    ts_tree_delete(editor->tree);
-  if (editor->query)
-    ts_query_delete(editor->query);
   free_rope(editor->root);
   delete editor;
 }
@@ -56,6 +71,8 @@ void render_editor(Editor *editor) {
       v.push_back({editor->hooks[i], '!' + i});
   std::sort(v.begin(), v.end());
   auto hook_it = v.begin();
+  while (hook_it != v.end() && hook_it->first <= editor->scroll.row)
+    ++hook_it;
   std::shared_lock knot_lock(editor->knot_mtx);
   if (editor->selection_active) {
     Coord start, end;
@@ -137,9 +154,10 @@ void render_editor(Editor *editor) {
       for (; i < render_width; i++)
         update(rendered_rows, i + render_x, " ", 0xc6c6c6, 0, 0);
       rendered_rows++;
-
       uint32_t skip_until = fold->end;
       while (line_index <= skip_until) {
+        if (hook_it != v.end() && hook_it->first == line_index + 1)
+          hook_it++;
         uint32_t line_len;
         char *line = next_line(it, &line_len);
         if (!line)
@@ -166,14 +184,10 @@ void render_editor(Editor *editor) {
       if (current_byte_offset == 0 || rendered_rows == 0) {
         const char *hook = nullptr;
         char h[2] = {0, 0};
-        auto it2 = hook_it;
+        if (hook_it != v.end() && hook_it->first == line_index + 1) {
-        for (; it2 != v.end(); ++it2) {
+          h[0] = hook_it->second;
-          if (it2->first == line_index + 1) {
-            h[0] = it2->second;
           hook = h;
-            hook_it = it2;
+          hook_it++;
-            break;
-          }
         }
         update(editor->position.row + rendered_rows, editor->position.col, hook,
                0xAAAAAA, 0, 0);
@@ -258,14 +272,10 @@ void render_editor(Editor *editor) {
       uint32_t color = editor->cursor.row == line_index ? 0x222222 : 0;
       const char *hook = nullptr;
       char h[2] = {0, 0};
-      auto it2 = hook_it;
+      if (hook_it != v.end() && hook_it->first == line_index + 1) {
-      for (; it2 != v.end(); ++it2) {
+        h[0] = hook_it->second;
-        if (it2->first == line_index + 1) {
-          h[0] = it2->second;
         hook = h;
-          hook_it = it2;
+        hook_it++;
-          break;
-        }
       }
       update(editor->position.row + rendered_rows, editor->position.col, hook,
              0xAAAAAA, 0, 0);

src/editor_ctrl.cc

@@ -4,7 +4,6 @@ extern "C" {
 #include "../include/editor.h"
 #include "../include/main.h"
 #include "../include/utils.h"
-#include <cmath>
 
 uint32_t scan_left(const char *line, uint32_t len, uint32_t off) {
   if (off > len)
@@ -214,328 +213,6 @@ Coord editor_hit_test(Editor *editor, uint32_t x, uint32_t y) {
   return {last_line_index, last_col};
 }
 
-Coord move_right_pure(Editor *editor, Coord cursor, uint32_t number) {
-  Coord result = cursor;
-  if (!editor || !editor->root || number == 0)
-    return result;
-  uint32_t row = result.row;
-  uint32_t col = result.col;
-  uint32_t line_len = 0;
-  LineIterator *it = begin_l_iter(editor->root, row);
-  if (!it)
-    return result;
-  char *line = next_line(it, &line_len);
-  if (!line) {
-    free(it->buffer);
-    free(it);
-    return result;
-  }
-  if (line_len > 0 && line[line_len - 1] == '\n')
-    --line_len;
-  while (number > 0) {
-    if (col >= line_len) {
-      uint32_t next_row = row + 1;
-      if (next_row >= editor->root->line_count) {
-        col = line_len;
-        break;
-      }
-      row = next_row;
-      col = 0;
-      line = next_line(it, &line_len);
-      if (!line)
-        break;
-      if (line_len > 0 && line[line_len - 1] == '\n')
-        --line_len;
-    } else {
-      uint32_t inc =
-          grapheme_next_character_break_utf8(line + col, line_len - col);
-      if (inc == 0)
-        break;
-      col += inc;
-    }
-    number--;
-  }
-  free(it->buffer);
-  free(it);
-  result.row = row;
-  result.col = col;
-  return result;
-}
-
-Coord move_left_pure(Editor *editor, Coord cursor, uint32_t number) {
-  Coord result = cursor;
-  if (!editor || !editor->root || number == 0)
-    return result;
-  uint32_t row = result.row;
-  uint32_t col = result.col;
-  uint32_t len = 0;
-  LineIterator *it = begin_l_iter(editor->root, row);
-  char *line = next_line(it, &len);
-  if (!line) {
-    free(it->buffer);
-    free(it);
-    return result;
-  }
-  if (len > 0 && line[len - 1] == '\n')
-    --len;
-  bool iterator_ahead = true;
-  while (number > 0) {
-    if (col == 0) {
-      if (row == 0)
-        break;
-      if (iterator_ahead) {
-        prev_line(it, nullptr);
-        iterator_ahead = false;
-      }
-      line = nullptr;
-      row--;
-      line = prev_line(it, &len);
-      if (!line)
-        break;
-      if (len > 0 && line[len - 1] == '\n')
-        --len;
-      col = len;
-    } else {
-      uint32_t new_col = 0;
-      while (new_col < col) {
-        uint32_t inc =
-            grapheme_next_character_break_utf8(line + new_col, len - new_col);
-        if (new_col + inc >= col)
-          break;
-        new_col += inc;
-      }
-      col = new_col;
-    }
-    number--;
-  }
-  free(it->buffer);
-  free(it);
-  result.row = row;
-  result.col = col;
-  return result;
-}
-
-Coord move_right(Editor *editor, Coord cursor, uint32_t number) {
-  Coord result = cursor;
-  if (!editor || !editor->root || number == 0)
-    return result;
-  uint32_t row = result.row;
-  uint32_t col = result.col;
-  uint32_t line_len = 0;
-  LineIterator *it = begin_l_iter(editor->root, row);
-  if (!it)
-    return result;
-  uint32_t target_row = next_unfolded_row(editor, row);
-  while (row < target_row) {
-    if (!next_line(it, &line_len)) {
-      free(it->buffer);
-      free(it);
-      return result;
-    }
-    ++row;
-  }
-  char *line = next_line(it, &line_len);
-  if (!line) {
-    free(it->buffer);
-    free(it);
-    return result;
-  }
-  if (line_len > 0 && line[line_len - 1] == '\n')
-    --line_len;
-  while (number > 0) {
-    if (col >= line_len) {
-      uint32_t next_row = next_unfolded_row(editor, row + 1);
-      if (next_row >= editor->root->line_count) {
-        col = line_len;
-        break;
-      }
-      while (row < next_row) {
-        line = next_line(it, &line_len);
-        if (!line) {
-          free(it->buffer);
-          free(it);
-          result.row = row;
-          result.col = col;
-          return result;
-        }
-        ++row;
-      }
-      if (line_len > 0 && line[line_len - 1] == '\n')
-        --line_len;
-      col = 0;
-      --number;
-      continue;
-    } else {
-      uint32_t inc =
-          grapheme_next_character_break_utf8(line + col, line_len - col);
-      if (inc == 0)
-        break;
-      col += inc;
-      --number;
-    }
-  }
-  free(it->buffer);
-  free(it);
-  result.row = row;
-  result.col = col;
-  return result;
-}
-
-Coord move_left(Editor *editor, Coord cursor, uint32_t number) {
-  Coord result = cursor;
-  if (!editor || !editor->root || number == 0)
-    return result;
-  uint32_t row = result.row;
-  uint32_t col = result.col;
-  uint32_t len = 0;
-  LineIterator *it = begin_l_iter(editor->root, row);
-  char *line = next_line(it, &len);
-  if (!line) {
-    free(it->buffer);
-    free(it);
-    return result;
-  }
-  if (len > 0 && line[len - 1] == '\n')
-    --len;
-  bool iterator_ahead = true;
-  while (number > 0) {
-    if (col == 0) {
-      if (row == 0)
-        break;
-      if (iterator_ahead) {
-        prev_line(it, nullptr);
-        iterator_ahead = false;
-      }
-      line = nullptr;
-      while (row > 0) {
-        row--;
-        line = prev_line(it, &len);
-        if (!line)
-          break;
-        const Fold *fold = fold_for_line(editor->folds, row);
-        if (fold) {
-          while (line && row > fold->start) {
-            line = prev_line(it, &len);
-            row--;
-          }
-          line = nullptr;
-          continue;
-        }
-        break;
-      }
-      if (!line)
-        break;
-      if (len > 0 && line[len - 1] == '\n')
-        --len;
-      col = len;
-    } else {
-      uint32_t new_col = 0;
-      while (new_col < col) {
-        uint32_t inc =
-            grapheme_next_character_break_utf8(line + new_col, len - new_col);
-        if (new_col + inc >= col)
-          break;
-        new_col += inc;
-      }
-      col = new_col;
-    }
-    number--;
-  }
-  free(it->buffer);
-  free(it);
-  result.row = row;
-  result.col = col;
-  return result;
-}
-
-void cursor_down(Editor *editor, uint32_t number) {
-  if (!editor || !editor->root || number == 0)
-    return;
-  uint32_t len;
-  LineIterator *it = begin_l_iter(editor->root, editor->cursor.row);
-  char *line_content = next_line(it, &len);
-  if (line_content == nullptr)
-    return;
-  if (editor->cursor_preffered == UINT32_MAX)
-    editor->cursor_preffered =
-        get_visual_col_from_bytes(line_content, len, editor->cursor.col);
-  uint32_t visual_col = editor->cursor_preffered;
-  free(it->buffer);
-  free(it);
-  uint32_t target_row = editor->cursor.row;
-  while (number > 0 && target_row < editor->root->line_count - 1) {
-    target_row = next_unfolded_row(editor, target_row + 1);
-    if (target_row >= editor->root->line_count) {
-      target_row = editor->root->line_count - 1;
-      break;
-    }
-    number--;
-  }
-  it = begin_l_iter(editor->root, target_row);
-  line_content = next_line(it, &len);
-  if (!line_content)
-    return;
-  if (len > 0 && line_content[len - 1] == '\n')
-    --len;
-  editor->cursor.row = target_row;
-  editor->cursor.col = get_bytes_from_visual_col(line_content, len, visual_col);
-  free(it->buffer);
-  free(it);
-}
-
-void cursor_up(Editor *editor, uint32_t number) {
-  if (!editor || !editor->root || number == 0 || editor->cursor.row == 0)
-    return;
-  uint32_t len;
-  LineIterator *it = begin_l_iter(editor->root, editor->cursor.row);
-  char *line_content = next_line(it, &len);
-  if (!line_content)
-    return;
-  if (editor->cursor_preffered == UINT32_MAX)
-    editor->cursor_preffered =
-        get_visual_col_from_bytes(line_content, len, editor->cursor.col);
-  uint32_t visual_col = editor->cursor_preffered;
-  free(it->buffer);
-  free(it);
-  uint32_t target_row = editor->cursor.row;
-  while (number > 0 && target_row > 0) {
-    target_row = prev_unfolded_row(editor, target_row - 1);
-    if (target_row == 0) {
-      number--;
-      break;
-    }
-    number--;
-  }
-  it = begin_l_iter(editor->root, target_row);
-  line_content = next_line(it, &len);
-  if (line_content) {
-    if (len > 0 && line_content[len - 1] == '\n')
-      --len;
-    editor->cursor.row = target_row;
-    editor->cursor.col =
-        get_bytes_from_visual_col(line_content, len, visual_col);
-  } else {
-    editor->cursor.row = 0;
-    editor->cursor.col = 0;
-  }
-  free(it->buffer);
-  free(it);
-}
-
-void cursor_right(Editor *editor, uint32_t number) {
-  if (!editor || !editor->root || number == 0)
-    return;
-  editor->cursor = move_right(editor, editor->cursor, number);
-  editor->cursor_preffered = UINT32_MAX;
-}
-
-void cursor_left(Editor *editor, uint32_t number) {
-  if (!editor || !editor->root || number == 0)
-    return;
-  editor->cursor = move_left(editor, editor->cursor, number);
-  editor->cursor_preffered = UINT32_MAX;
-}
-
 void move_line_up(Editor *editor) {
   if (!editor || !editor->root || editor->cursor.row == 0)
     return;
@@ -684,7 +361,7 @@ void edit_erase(Editor *editor, Coord pos, int64_t len) {
     std::unique_lock lock_2(editor->knot_mtx);
     editor->root = erase(editor->root, start, byte_pos - start);
     lock_2.unlock();
-    if (editor->tree) {
+    if (editor->ts.tree) {
       TSInputEdit edit = {
           .start_byte = start,
           .old_end_byte = byte_pos,
@@ -728,7 +405,7 @@ void edit_erase(Editor *editor, Coord pos, int64_t len) {
     std::unique_lock lock_2(editor->knot_mtx);
     editor->root = erase(editor->root, byte_pos, end - byte_pos);
     lock_2.unlock();
-    if (editor->tree) {
+    if (editor->ts.tree) {
       TSInputEdit edit = {
           .start_byte = byte_pos,
           .old_end_byte = end,
@@ -777,7 +454,7 @@ void edit_insert(Editor *editor, Coord pos, char *data, uint32_t len) {
   }
   apply_line_insertion(editor, pos.row, rows);
   apply_hook_insertion(editor, pos.row, rows);
-  if (editor->tree) {
+  if (editor->ts.tree) {
     TSInputEdit edit = {
         .start_byte = byte_pos,
         .old_end_byte = byte_pos,

src/editor_cursor.cc

@@ -0,0 +1,327 @@
+extern "C" {
+#include "../libs/libgrapheme/grapheme.h"
+}
+#include "../include/editor.h"
+#include "../include/utils.h"
+
+Coord move_right_pure(Editor *editor, Coord cursor, uint32_t number) {
+  Coord result = cursor;
+  if (!editor || !editor->root || number == 0)
+    return result;
+  uint32_t row = result.row;
+  uint32_t col = result.col;
+  uint32_t line_len = 0;
+  LineIterator *it = begin_l_iter(editor->root, row);
+  if (!it)
+    return result;
+  char *line = next_line(it, &line_len);
+  if (!line) {
+    free(it->buffer);
+    free(it);
+    return result;
+  }
+  if (line_len > 0 && line[line_len - 1] == '\n')
+    --line_len;
+  while (number > 0) {
+    if (col >= line_len) {
+      uint32_t next_row = row + 1;
+      if (next_row >= editor->root->line_count) {
+        col = line_len;
+        break;
+      }
+      row = next_row;
+      col = 0;
+      line = next_line(it, &line_len);
+      if (!line)
+        break;
+      if (line_len > 0 && line[line_len - 1] == '\n')
+        --line_len;
+    } else {
+      uint32_t inc =
+          grapheme_next_character_break_utf8(line + col, line_len - col);
+      if (inc == 0)
+        break;
+      col += inc;
+    }
+    number--;
+  }
+  free(it->buffer);
+  free(it);
+  result.row = row;
+  result.col = col;
+  return result;
+}
+
+Coord move_left_pure(Editor *editor, Coord cursor, uint32_t number) {
+  Coord result = cursor;
+  if (!editor || !editor->root || number == 0)
+    return result;
+  uint32_t row = result.row;
+  uint32_t col = result.col;
+  uint32_t len = 0;
+  LineIterator *it = begin_l_iter(editor->root, row);
+  char *line = next_line(it, &len);
+  if (!line) {
+    free(it->buffer);
+    free(it);
+    return result;
+  }
+  if (len > 0 && line[len - 1] == '\n')
+    --len;
+  bool iterator_ahead = true;
+  while (number > 0) {
+    if (col == 0) {
+      if (row == 0)
+        break;
+      if (iterator_ahead) {
+        prev_line(it, nullptr);
+        iterator_ahead = false;
+      }
+      line = nullptr;
+      row--;
+      line = prev_line(it, &len);
+      if (!line)
+        break;
+      if (len > 0 && line[len - 1] == '\n')
+        --len;
+      col = len;
+    } else {
+      uint32_t new_col = 0;
+      while (new_col < col) {
+        uint32_t inc =
+            grapheme_next_character_break_utf8(line + new_col, len - new_col);
+        if (new_col + inc >= col)
+          break;
+        new_col += inc;
+      }
+      col = new_col;
+    }
+    number--;
+  }
+  free(it->buffer);
+  free(it);
+  result.row = row;
+  result.col = col;
+  return result;
+}
+
+Coord move_right(Editor *editor, Coord cursor, uint32_t number) {
+  Coord result = cursor;
+  if (!editor || !editor->root || number == 0)
+    return result;
+  uint32_t row = result.row;
+  uint32_t col = result.col;
+  uint32_t line_len = 0;
+  LineIterator *it = begin_l_iter(editor->root, row);
+  if (!it)
+    return result;
+  uint32_t target_row = next_unfolded_row(editor, row);
+  while (row < target_row) {
+    if (!next_line(it, &line_len)) {
+      free(it->buffer);
+      free(it);
+      return result;
+    }
+    ++row;
+  }
+  char *line = next_line(it, &line_len);
+  if (!line) {
+    free(it->buffer);
+    free(it);
+    return result;
+  }
+  if (line_len > 0 && line[line_len - 1] == '\n')
+    --line_len;
+  while (number > 0) {
+    if (col >= line_len) {
+      uint32_t next_row = next_unfolded_row(editor, row + 1);
+      if (next_row >= editor->root->line_count) {
+        col = line_len;
+        break;
+      }
+      while (row < next_row) {
+        line = next_line(it, &line_len);
+        if (!line) {
+          free(it->buffer);
+          free(it);
+          result.row = row;
+          result.col = col;
+          return result;
+        }
+        ++row;
+      }
+      if (line_len > 0 && line[line_len - 1] == '\n')
+        --line_len;
+      col = 0;
+      --number;
+      continue;
+    } else {
+      uint32_t inc =
+          grapheme_next_character_break_utf8(line + col, line_len - col);
+      if (inc == 0)
+        break;
+      col += inc;
+      --number;
+    }
+  }
+  free(it->buffer);
+  free(it);
+  result.row = row;
+  result.col = col;
+  return result;
+}
+
+Coord move_left(Editor *editor, Coord cursor, uint32_t number) {
+  Coord result = cursor;
+  if (!editor || !editor->root || number == 0)
+    return result;
+  uint32_t row = result.row;
+  uint32_t col = result.col;
+  uint32_t len = 0;
+  LineIterator *it = begin_l_iter(editor->root, row);
+  char *line = next_line(it, &len);
+  if (!line) {
+    free(it->buffer);
+    free(it);
+    return result;
+  }
+  if (len > 0 && line[len - 1] == '\n')
+    --len;
+  bool iterator_ahead = true;
+  while (number > 0) {
+    if (col == 0) {
+      if (row == 0)
+        break;
+      if (iterator_ahead) {
+        prev_line(it, nullptr);
+        iterator_ahead = false;
+      }
+      line = nullptr;
+      while (row > 0) {
+        row--;
+        line = prev_line(it, &len);
+        if (!line)
+          break;
+        const Fold *fold = fold_for_line(editor->folds, row);
+        if (fold) {
+          while (line && row > fold->start) {
+            line = prev_line(it, &len);
+            row--;
+          }
+          line = nullptr;
+          continue;
+        }
+        break;
+      }
+      if (!line)
+        break;
+      if (len > 0 && line[len - 1] == '\n')
+        --len;
+      col = len;
+    } else {
+      uint32_t new_col = 0;
+      while (new_col < col) {
+        uint32_t inc =
+            grapheme_next_character_break_utf8(line + new_col, len - new_col);
+        if (new_col + inc >= col)
+          break;
+        new_col += inc;
+      }
+      col = new_col;
+    }
+    number--;
+  }
+  free(it->buffer);
+  free(it);
+  result.row = row;
+  result.col = col;
+  return result;
+}
+
+void cursor_down(Editor *editor, uint32_t number) {
+  if (!editor || !editor->root || number == 0)
+    return;
+  uint32_t len;
+  LineIterator *it = begin_l_iter(editor->root, editor->cursor.row);
+  char *line_content = next_line(it, &len);
+  if (line_content == nullptr)
+    return;
+  if (editor->cursor_preffered == UINT32_MAX)
+    editor->cursor_preffered =
+        get_visual_col_from_bytes(line_content, len, editor->cursor.col);
+  uint32_t visual_col = editor->cursor_preffered;
+  free(it->buffer);
+  free(it);
+  uint32_t target_row = editor->cursor.row;
+  while (number > 0 && target_row < editor->root->line_count - 1) {
+    target_row = next_unfolded_row(editor, target_row + 1);
+    if (target_row >= editor->root->line_count) {
+      target_row = editor->root->line_count - 1;
+      break;
+    }
+    number--;
+  }
+  it = begin_l_iter(editor->root, target_row);
+  line_content = next_line(it, &len);
+  if (!line_content)
+    return;
+  if (len > 0 && line_content[len - 1] == '\n')
+    --len;
+  editor->cursor.row = target_row;
+  editor->cursor.col = get_bytes_from_visual_col(line_content, len, visual_col);
+  free(it->buffer);
+  free(it);
+}
+
+void cursor_up(Editor *editor, uint32_t number) {
+  if (!editor || !editor->root || number == 0 || editor->cursor.row == 0)
+    return;
+  uint32_t len;
+  LineIterator *it = begin_l_iter(editor->root, editor->cursor.row);
+  char *line_content = next_line(it, &len);
+  if (!line_content)
+    return;
+  if (editor->cursor_preffered == UINT32_MAX)
+    editor->cursor_preffered =
+        get_visual_col_from_bytes(line_content, len, editor->cursor.col);
+  uint32_t visual_col = editor->cursor_preffered;
+  free(it->buffer);
+  free(it);
+  uint32_t target_row = editor->cursor.row;
+  while (number > 0 && target_row > 0) {
+    target_row = prev_unfolded_row(editor, target_row - 1);
+    if (target_row == 0) {
+      number--;
+      break;
+    }
+    number--;
+  }
+  it = begin_l_iter(editor->root, target_row);
+  line_content = next_line(it, &len);
+  if (line_content) {
+    if (len > 0 && line_content[len - 1] == '\n')
+      --len;
+    editor->cursor.row = target_row;
+    editor->cursor.col =
+        get_bytes_from_visual_col(line_content, len, visual_col);
+  } else {
+    editor->cursor.row = 0;
+    editor->cursor.col = 0;
+  }
+  free(it->buffer);
+  free(it);
+}
+
+void cursor_right(Editor *editor, uint32_t number) {
+  if (!editor || !editor->root || number == 0)
+    return;
+  editor->cursor = move_right(editor, editor->cursor, number);
+  editor->cursor_preffered = UINT32_MAX;
+}
+
+void cursor_left(Editor *editor, uint32_t number) {
+  if (!editor || !editor->root || number == 0)
+    return;
+  editor->cursor = move_left(editor, editor->cursor, number);
+  editor->cursor_preffered = UINT32_MAX;
+}

src/editor_events.cc

@@ -535,7 +535,11 @@ static Highlight HL_UNDERLINE = {0, 0, 1 << 2, 100};
 void editor_worker(Editor *editor) {
   if (!editor || !editor->root)
     return;
-  if (editor->parser && editor->query)
+  if (editor->root->char_count > (1024 * 200))
+    return;
+  if (editor->ts.query_file != "" && !editor->ts.query)
+    editor->ts.query = load_query(editor->ts.query_file.c_str(), &editor->ts);
+  if (editor->ts.parser && editor->ts.query)
     ts_collect_spans(editor);
   uint32_t prev_col, next_col;
   word_boundaries_exclusive(editor, editor->cursor, &prev_col, &next_col);

src/lsp.cc

@@ -0,0 +1,328 @@
+#include "../include/lsp.h"
+#include "../include/maps.h"
+#include <fcntl.h>
+#include <signal.h>
+#include <sys/poll.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+std::shared_mutex active_lsps_mtx;
+std::unordered_map<uint8_t, LSPInstance *> active_lsps;
+
+Queue<LSPOpenRequest> lsp_open_queue;
+
+static bool init_lsp(LSPInstance *lsp) {
+  log("starting %s\n", lsp->lsp->command);
+  int in_pipe[2];
+  int out_pipe[2];
+  if (pipe(in_pipe) == -1 || pipe(out_pipe) == -1) {
+    perror("pipe");
+    return false;
+  }
+  pid_t pid = fork();
+  if (pid == -1) {
+    perror("fork");
+    return false;
+  }
+  if (pid == 0) {
+    dup2(in_pipe[0], STDIN_FILENO);
+    dup2(out_pipe[1], STDOUT_FILENO);
+    int devnull = open("/dev/null", O_WRONLY);
+    if (devnull >= 0) {
+      dup2(devnull, STDERR_FILENO);
+      close(devnull);
+    }
+    close(in_pipe[1]);
+    close(out_pipe[0]);
+    execvp(lsp->lsp->command, (char *const *)(lsp->lsp->args.data()));
+    perror("execvp");
+    return false;
+  }
+  lsp->pid = pid;
+  lsp->stdin_fd = in_pipe[1];
+  lsp->stdout_fd = out_pipe[0];
+  close(in_pipe[0]);
+  close(out_pipe[1]);
+  return true;
+}
+
+LSPInstance *get_or_init_lsp(uint8_t lsp_id) {
+  std::unique_lock lock(active_lsps_mtx);
+  auto it = active_lsps.find(lsp_id);
+  if (it == active_lsps.end()) {
+    auto map_it = kLsps.find(lsp_id);
+    if (map_it == kLsps.end())
+      return nullptr;
+    LSPInstance *lsp = new LSPInstance();
+    lsp->lsp = &map_it->second;
+    if (!init_lsp(lsp)) {
+      delete lsp;
+      return nullptr;
+    }
+    LSPPending *pending = new LSPPending();
+    pending->method = "initialize";
+    pending->editor = nullptr;
+    pending->callback = [lsp](Editor *, std::string, json) {
+      lsp->initialized = true;
+      json initialized = {{"jsonrpc", "2.0"},
+                          {"method", "initialized"},
+                          {"params", json::object()}};
+      lsp_send(lsp, initialized, nullptr);
+    };
+    json init_message = {
+        {"jsonrpc", "2.0"},
+        {"method", "initialize"},
+        {"params",
+         {{"processId", getpid()},
+          {"rootUri", "file://" + std::filesystem::current_path().string()},
+          {"capabilities", json::object()}}}};
+    lsp_send(lsp, init_message, pending);
+    active_lsps[lsp_id] = lsp;
+    return lsp;
+  }
+  return it->second;
+}
+
+void lsp_send(LSPInstance *lsp, json message, LSPPending *pending) {
+  if (!lsp || lsp->stdin_fd == -1)
+    return;
+  std::unique_lock lock(lsp->mtx);
+  if (pending) {
+    message["id"] = lsp->last_id++;
+    uint32_t id = message["id"].get<uint32_t>();
+    lsp->pending[id] = pending;
+  }
+  lsp->outbox.push(message);
+}
+
+void close_lsp(uint8_t lsp_id) {
+  std::shared_lock active_lsps_lock(active_lsps_mtx);
+  auto it = active_lsps.find(lsp_id);
+  if (it == active_lsps.end())
+    return;
+  LSPInstance *lsp = it->second;
+  active_lsps_lock.unlock();
+  LSPPending *shutdown_pending = new LSPPending();
+  shutdown_pending->method = "shutdown";
+  shutdown_pending->callback = [lsp, lsp_id](Editor *, std::string, json) {
+    json exit = {{"jsonrpc", "2.0"}, {"method", "exit"}};
+    lsp_send(lsp, exit, nullptr);
+  };
+  json shutdown = {{"jsonrpc", "2.0"}, {"method", "shutdown"}};
+  lsp_send(lsp, shutdown, shutdown_pending);
+  std::thread t([lsp, lsp_id] {
+    std::this_thread::sleep_for(100ms);
+    std::unique_lock active_lsps_lock(active_lsps_mtx);
+    std::unique_lock lock(lsp->mtx);
+    if (kill(lsp->pid, 0) == 0)
+      kill(lsp->pid, SIGKILL);
+    waitpid(lsp->pid, nullptr, 0);
+    close(lsp->stdin_fd);
+    close(lsp->stdout_fd);
+    while (!lsp->outbox.empty())
+      lsp->outbox.pop();
+    while (!lsp->inbox.empty())
+      lsp->inbox.pop();
+    for (auto &kv : lsp->pending)
+      delete kv.second;
+    delete lsp;
+    active_lsps.erase(lsp_id);
+  });
+  t.detach();
+}
+
+static std::optional<json> read_lsp_message(int fd) {
+  std::string header;
+  char c;
+  while (true) {
+    ssize_t n = read(fd, &c, 1);
+    if (n <= 0)
+      return std::nullopt;
+    header.push_back(c);
+    if (header.size() >= 4 && header.substr(header.size() - 4) == "\r\n\r\n")
+      break;
+  }
+  size_t pos = header.find("Content-Length:");
+  if (pos == std::string::npos)
+    return std::nullopt;
+  pos += strlen("Content-Length:");
+  while (pos < header.size() && std::isspace(header[pos]))
+    pos++;
+  size_t end = pos;
+  while (end < header.size() && std::isdigit(header[end]))
+    end++;
+  size_t len = std::stoul(header.substr(pos, end - pos));
+  std::string body(len, '\0');
+  size_t got = 0;
+  while (got < len) {
+    ssize_t n = read(fd, &body[got], len - got);
+    if (n <= 0)
+      return std::nullopt;
+    got += n;
+  }
+  return json::parse(body);
+}
+
+static Editor *editor_for_uri(LSPInstance *lsp, std::string uri) {
+  if (uri.empty())
+    return nullptr;
+  for (auto &editor : lsp->editors)
+    if (editor->uri == uri)
+      return editor;
+  return nullptr;
+}
+
+void lsp_worker() {
+  LSPOpenRequest request;
+  while (lsp_open_queue.pop(request))
+    add_to_lsp(request.language, request.editor);
+  std::shared_lock active_lsps_lock(active_lsps_mtx);
+  for (auto &kv : active_lsps) {
+    LSPInstance *lsp = kv.second;
+    std::unique_lock lock(lsp->mtx);
+    while (!lsp->outbox.empty()) {
+      json message;
+      message = lsp->outbox.front();
+      if (!lsp->initialized) {
+        std::string m = message.value("method", "");
+        if (m != "initialize")
+          break;
+      }
+      lsp->outbox.pop(message);
+      std::string payload = message.dump();
+      std::string header =
+          "Content-Length: " + std::to_string(payload.size()) + "\r\n\r\n";
+      std::string out = header + payload;
+      const char *ptr = out.data();
+      size_t remaining = out.size();
+      while (remaining > 0) {
+        ssize_t written = write(lsp->stdin_fd, ptr, remaining);
+        if (written == 0)
+          break;
+        else if (written == -1) {
+          if (errno == EINTR)
+            continue;
+          perror("write");
+          break;
+        } else {
+          ptr += written;
+          remaining -= written;
+        }
+      }
+    }
+    pollfd pfd{lsp->stdout_fd, POLLIN, 0};
+    while (poll(&pfd, 1, 0) > 0) {
+      auto msg = read_lsp_message(lsp->stdout_fd);
+      if (!msg)
+        break;
+      if (msg->contains("id")) {
+        uint32_t id = msg->at("id").get<uint32_t>();
+        auto it = lsp->pending.find(id);
+        if (it != lsp->pending.end()) {
+          LSPPending *pend = it->second;
+          lock.unlock();
+          if (pend->callback)
+            pend->callback(pend->editor, pend->method, *msg);
+          delete pend;
+          lock.lock();
+          lsp->pending.erase(it);
+        }
+      } else if (msg->contains("method")) {
+        std::string uri;
+        if (msg->contains("params")) {
+          auto &p = (*msg)["params"];
+          if (p.contains("textDocument") && p["textDocument"].contains("uri"))
+            uri = p["textDocument"]["uri"].get<std::string>();
+          else if (p.contains("uri"))
+            uri = p["uri"].get<std::string>();
+        }
+        Editor *ed = editor_for_uri(lsp, uri);
+        lock.unlock();
+        if (ed)
+          // editor_lsp_handle(ed, *msg)
+          ;
+        else
+          lsp_handle(lsp, *msg);
+        lock.lock();
+      }
+    }
+  }
+}
+
+void request_add_to_lsp(Language language, Editor *editor) {
+  lsp_open_queue.push({language, editor});
+}
+
+void add_to_lsp(Language language, Editor *editor) {
+  LSPInstance *lsp = get_or_init_lsp(language.lsp_id);
+  if (!lsp)
+    return;
+  std::unique_lock lock(lsp->mtx);
+  if (editor->lsp == lsp)
+    return;
+  lsp->editors.push_back(editor);
+  lock.unlock();
+  std::unique_lock lock2(editor->lsp_mtx);
+  editor->lsp = lsp;
+  lock2.unlock();
+  std::unique_lock lock3(editor->knot_mtx);
+  char *buf = read(editor->root, 0, editor->root->char_count);
+  std::string text(buf);
+  free(buf);
+  json message = {{"jsonrpc", "2.0"},
+                  {"method", "textDocument/didOpen"},
+                  {"params",
+                   {{"textDocument",
+                     {{"uri", editor->uri},
+                      {"languageId", language.name},
+                      {"version", 1},
+                      {"text", text}}}}}};
+  lock3.unlock();
+  lsp_send(lsp, message, nullptr);
+}
+
+static uint8_t find_lsp_id(LSPInstance *needle) {
+  for (const auto &[id, lsp] : active_lsps)
+    if (lsp == needle)
+      return id;
+  return 0;
+}
+
+void remove_from_lsp(Editor *editor) {
+  auto lsp = editor->lsp;
+  if (!lsp)
+    return;
+  std::unique_lock lock1(lsp->mtx);
+  lsp->editors.erase(
+      std::remove(lsp->editors.begin(), lsp->editors.end(), editor),
+      lsp->editors.end());
+  lock1.unlock();
+  std::unique_lock lock2(editor->lsp_mtx);
+  editor->lsp = nullptr;
+  lock2.unlock();
+  json message = {{"jsonrpc", "2.0"},
+                  {"method", "textDocument/didClose"},
+                  {"params", {{"textDocument", {{"uri", editor->uri}}}}}};
+  lsp_send(lsp, message, nullptr);
+  uint8_t lsp_id = find_lsp_id(lsp);
+  if (lsp_id && lsp->editors.empty())
+    close_lsp(lsp_id);
+}
+
+void lsp_handle(LSPInstance *, json message) {
+  std::string method = message.value("method", "");
+  if (method == "window/showMessage") {
+    if (message.contains("params")) {
+      auto &p = message["params"];
+      if (p.contains("message"))
+        log("%s\n", p["message"].get<std::string>().c_str());
+    }
+  } else if (method == "window/logMessage") {
+    if (message.contains("params")) {
+      auto &p = message["params"];
+      if (p.contains("message"))
+        log("%s\n", p["message"].get<std::string>().c_str());
+    }
+  }
+}

src/main.cc

@@ -1,5 +1,6 @@
 #include "../include/main.h"
 #include "../include/editor.h"
+#include "../include/lsp.h"
 #include "../include/ts.h"
 #include "../include/ui.h"
 #include <atomic>
@@ -7,8 +8,6 @@
 #include <sys/ioctl.h>
 #include <thread>
 
-using namespace std::chrono_literals;
-
 std::atomic<bool> running{true};
 Queue<KeyEvent> event_queue;
 std::vector<Editor *> editors;
@@ -18,7 +17,12 @@ uint8_t mode = NORMAL;
 
 void background_worker() {
   while (running)
-    throttle(16ms, editor_worker, editors[current_editor]);
+    throttle(8ms, editor_worker, editors[current_editor]);
+}
+
+void background_lsp() {
+  while (running)
+    throttle(8ms, lsp_worker);
 }
 
 void input_listener() {
@@ -71,6 +75,7 @@ int main(int argc, char *argv[]) {
 
   std::thread input_thread(input_listener);
   std::thread work_thread(background_worker);
+  std::thread lsp_thread(background_lsp);
 
   while (running) {
     KeyEvent event;
@@ -98,9 +103,22 @@ int main(int argc, char *argv[]) {
   if (work_thread.joinable())
     work_thread.join();
 
+  if (lsp_thread.joinable())
+    lsp_thread.join();
+
   end_screen();
 
+  for (auto editor : editors)
     free_editor(editor);
+
+  while (true) {
+    std::unique_lock lk(active_lsps_mtx);
+    if (active_lsps.empty())
+      break;
+    lk.unlock();
+    throttle(16ms, lsp_worker);
+  }
+
   clear_regex_cache();
   return 0;
 }

src/renderer.cc

@@ -1,4 +1,3 @@
-// includes
 #include "../include/ui.h"
 
 uint32_t rows, cols;

src/ts.cc

@@ -1,6 +1,7 @@
 #include "../include/ts.h"
 #include "../include/editor.h"
 #include "../include/knot.h"
+#include "../include/maps.h"
 #include <algorithm>
 #include <cstdint>
 #include <fstream>
@@ -28,8 +29,8 @@ pcre2_code *get_re(const std::string &pattern) {
   return re;
 }
 
-TSQuery *load_query(const char *query_path, Editor *editor) {
+TSQuery *load_query(const char *query_path, TSSetBase *set) {
-  const TSLanguage *lang = editor->language;
+  const TSLanguage *lang = set->language;
   std::ifstream file(query_path, std::ios::in | std::ios::binary);
   if (!file.is_open())
     return nullptr;
@@ -38,7 +39,7 @@ TSQuery *load_query(const char *query_path, Editor *editor) {
   int errornumber = 0;
   PCRE2_SIZE erroroffset = 0;
   pcre2_code *re = pcre2_compile(
-      (PCRE2_SPTR) R"((@[A-Za-z0-9_.]+)|(;; \#[0-9a-fA-F]{6} \#[0-9a-fA-F]{6} [01] [01] [01] \d+))",
+      (PCRE2_SPTR) R"((@[A-Za-z0-9_.]+)|(;; \#[0-9a-fA-F]{6} \#[0-9a-fA-F]{6} [01] [01] [01] \d+)|(;; !(\w+)))",
       PCRE2_ZERO_TERMINATED, 0, &errornumber, &erroroffset, nullptr);
   if (!re)
     return nullptr;
@@ -46,9 +47,8 @@ TSQuery *load_query(const char *query_path, Editor *editor) {
       pcre2_match_data_create_from_pattern(re, nullptr);
   std::map<std::string, int> capture_name_cache;
   Highlight *c_hl = nullptr;
+  Language c_lang = {"unknown", nullptr, 0};
   int i = 0;
-  int limit = 20;
-  editor->query_map.resize(limit);
   PCRE2_SIZE offset = 0;
   PCRE2_SIZE subject_length = highlight_query.size();
   while (offset < subject_length) {
@@ -63,18 +63,18 @@ TSQuery *load_query(const char *query_path, Editor *editor) {
       std::string capture_name = mct;
       if (!capture_name_cache.count(capture_name)) {
         if (c_hl) {
-          if (i >= limit) {
+          set->query_map[i] = *c_hl;
-            limit += 20;
-            editor->query_map.resize(limit);
-          }
-          editor->query_map[i] = *c_hl;
           delete c_hl;
           c_hl = nullptr;
         }
+        if (c_lang.fn != nullptr) {
+          set->injection_map[i] = c_lang;
+          c_lang = {"unknown", nullptr, 0};
+        }
         capture_name_cache[capture_name] = i;
         i++;
       }
-    } else if (mct.size() >= 2 && mct[0] == ';' && mct[1] == ';') {
+    } else if (mct.substr(0, 4) == ";; #") {
       if (c_hl)
         delete c_hl;
       c_hl = new Highlight();
@@ -86,6 +86,10 @@ TSQuery *load_query(const char *query_path, Editor *editor) {
       c_hl->priority = std::stoi(mct.substr(25));
       c_hl->flags = (bold ? CF_BOLD : 0) | (italic ? CF_ITALIC : 0) |
                     (underline ? CF_UNDERLINE : 0);
+    } else if (mct.substr(0, 4) == ";; !") {
+      auto it = kLanguages.find(mct.substr(4));
+      if (it != kLanguages.end())
+        c_lang = it->second;
     }
     offset = ovector[1];
   }
@@ -174,26 +178,32 @@ const char *read_ts(void *payload, uint32_t byte_index, TSPoint,
   return leaf_from_offset(editor->root, byte_index, bytes_read);
 }
 
-static inline Highlight *safe_get(std::vector<Highlight> &vec, size_t index) {
+template <typename T>
-  if (index >= vec.size())
+static inline T *safe_get(std::map<uint16_t, T> &m, uint16_t key) {
+  auto it = m.find(key);
+  if (it == m.end())
     return nullptr;
-  return &vec[index];
+  return &it->second;
 }
 
 void ts_collect_spans(Editor *editor) {
   static int parse_counter = 0;
-  if (!editor->parser || !editor->root || !editor->query)
+  if (!editor->ts.parser || !editor->root || !editor->ts.query)
     return;
-  TSInput tsinput = {
+  const bool injections_enabled = editor->root->char_count < (1024 * 32);
+  for (auto &inj : editor->ts.injections)
+    inj.ranges.clear();
+  TSInput tsinput{
       .payload = editor,
       .read = read_ts,
       .encoding = TSInputEncodingUTF8,
       .decode = nullptr,
   };
-  TSTree *tree, *copy = nullptr;
+  TSTree *tree = nullptr;
+  TSTree *copy = nullptr;
   std::unique_lock knot_mtx(editor->knot_mtx);
-  if (editor->tree)
+  if (editor->ts.tree)
-    copy = ts_tree_copy(editor->tree);
+    copy = ts_tree_copy(editor->ts.tree);
   knot_mtx.unlock();
   std::vector<TSInputEdit> edits;
   TSInputEdit edit;
@@ -201,7 +211,7 @@ void ts_collect_spans(Editor *editor) {
     while (editor->edit_queue.pop(edit)) {
       edits.push_back(edit);
       ts_tree_edit(copy, &edits.back());
-    };
+    }
   if (copy && edits.empty() && parse_counter < 64) {
     parse_counter++;
     ts_tree_delete(copy);
@@ -210,41 +220,129 @@ void ts_collect_spans(Editor *editor) {
   parse_counter = 0;
   editor->spans.mid_parse = true;
   std::shared_lock lock(editor->knot_mtx);
-  tree = ts_parser_parse(editor->parser, copy, tsinput);
+  tree = ts_parser_parse(editor->ts.parser, copy, tsinput);
   lock.unlock();
   if (copy)
     ts_tree_delete(copy);
-  knot_mtx.lock();
+  if (editor->ts.tree)
-  if (editor->tree)
+    ts_tree_delete(editor->ts.tree);
-    ts_tree_delete(editor->tree);
+  editor->ts.tree = tree;
-  editor->tree = tree;
   copy = ts_tree_copy(tree);
-  knot_mtx.unlock();
+  std::unordered_map<std::string, TSSet *> inj_lookup;
+  for (auto &inj : editor->ts.injections)
+    if (inj.lang != "unknown")
+      inj_lookup[inj.lang] = &inj;
   TSQueryCursor *cursor = ts_query_cursor_new();
-  ts_query_cursor_exec(cursor, editor->query, ts_tree_root_node(copy));
+  ts_query_cursor_exec(cursor, editor->ts.query, ts_tree_root_node(copy));
   std::vector<Span> new_spans;
   new_spans.reserve(4096);
+  struct PendingRanges {
+    std::vector<TSRange> ranges;
+    TSSet *tsset = nullptr;
+  };
+  std::unordered_map<std::string, PendingRanges> pending_injections;
   TSQueryMatch match;
   while (ts_query_cursor_next_match(cursor, &match)) {
-    if (!ts_predicate(editor->query, match, editor->root))
+    if (!ts_predicate(editor->ts.query, match, editor->root))
       continue;
     for (uint32_t i = 0; i < match.capture_count; i++) {
       TSQueryCapture cap = match.captures[i];
       uint32_t start = ts_node_start_byte(cap.node);
       uint32_t end = ts_node_end_byte(cap.node);
-      Highlight *hl = safe_get(editor->query_map, cap.index);
+      if (Highlight *hl = safe_get(editor->ts.query_map, cap.index))
-      if (hl)
         new_spans.push_back({start, end, hl});
+      if (!injections_enabled)
+        continue;
+      if (Language *inj_lang = safe_get(editor->ts.injection_map, cap.index)) {
+        auto &pending = pending_injections[inj_lang->name];
+        if (!pending.tsset) {
+          if (auto it = inj_lookup.find(inj_lang->name);
+              it != inj_lookup.end()) {
+            pending.tsset = it->second;
+          } else {
+            TSSet fresh{};
+            fresh.lang = inj_lang->name;
+            fresh.parser = ts_parser_new();
+            ts_parser_set_language(fresh.parser, inj_lang->fn());
+            fresh.language = inj_lang->fn();
+            fresh.query_file =
+                get_exe_dir() + "/../grammar/" + inj_lang->name + ".scm";
+            fresh.query = load_query(fresh.query_file.c_str(), &fresh);
+            editor->ts.injections.push_back(std::move(fresh));
+            pending.tsset = &editor->ts.injections.back();
+            inj_lookup[inj_lang->name] = pending.tsset;
+          }
+        }
+        pending.ranges.push_back(TSRange{
+            ts_node_start_point(cap.node),
+            ts_node_end_point(cap.node),
+            start,
+            end,
+        });
+      }
+    }
+  }
+  auto overlaps = [](const Span &s, const TSRange &r) {
+    return !(s.end <= r.start_byte || s.start >= r.end_byte);
+  };
+  if (injections_enabled) {
+    for (auto &[lang_name, pending] : pending_injections) {
+      TSSet *tsset = pending.tsset;
+      if (!tsset)
+        continue;
+      tsset->ranges = std::move(pending.ranges);
+      if (tsset->ranges.size() > 1)
+        new_spans.erase(std::remove_if(new_spans.begin(), new_spans.end(),
+                                       [&](const Span &sp) {
+                                         return std::any_of(
+                                             tsset->ranges.begin(),
+                                             tsset->ranges.end(),
+                                             [&](const TSRange &r) {
+                                               return overlaps(sp, r);
+                                             });
+                                       }),
+                        new_spans.end());
+    }
+    for (auto &inj : editor->ts.injections) {
+      if (!inj.parser || !inj.query || inj.ranges.size() == 0)
+        continue;
+      ts_parser_set_included_ranges(inj.parser, inj.ranges.data(),
+                                    inj.ranges.size());
+      std::pair<uint32_t, int64_t> span_edit;
+      while (editor->spans.edits.pop(span_edit))
+        apply_edit(new_spans, span_edit.first, span_edit.second);
+      knot_mtx.lock();
+      TSTree *inj_tree = ts_parser_parse(inj.parser, inj.tree, tsinput);
+      knot_mtx.unlock();
+      if (inj.tree)
+        ts_tree_delete(inj.tree);
+      inj.tree = inj_tree;
+      TSTree *inj_copy = ts_tree_copy(inj_tree);
+      TSQueryCursor *inj_cursor = ts_query_cursor_new();
+      ts_query_cursor_exec(inj_cursor, inj.query, ts_tree_root_node(inj_copy));
+      TSQueryMatch inj_match;
+      while (ts_query_cursor_next_match(inj_cursor, &inj_match)) {
+        if (!ts_predicate(inj.query, inj_match, editor->root))
+          continue;
+        for (uint32_t i = 0; i < inj_match.capture_count; i++) {
+          TSQueryCapture cap = inj_match.captures[i];
+          uint32_t start = ts_node_start_byte(cap.node);
+          uint32_t end = ts_node_end_byte(cap.node);
+          if (Highlight *hl = safe_get(inj.query_map, cap.index))
+            new_spans.push_back({start, end, hl});
+        }
+      }
+      ts_query_cursor_delete(inj_cursor);
+      ts_tree_delete(inj_copy);
     }
   }
   ts_query_cursor_delete(cursor);
   ts_tree_delete(copy);
-  std::sort(new_spans.begin(), new_spans.end());
   std::pair<uint32_t, int64_t> span_edit;
   while (editor->spans.edits.pop(span_edit))
     apply_edit(new_spans, span_edit.first, span_edit.second);
+  std::sort(new_spans.begin(), new_spans.end());
   std::unique_lock span_mtx(editor->spans.mtx);
   editor->spans.mid_parse = false;
   editor->spans.spans.swap(new_spans);
-  span_mtx.unlock();
 }

src/utils.cc

@@ -2,20 +2,31 @@ extern "C" {
 #include "../libs/libgrapheme/grapheme.h"
 #include "../libs/unicode_width/unicode_width.h"
 }
+#include "../include/maps.h"
 #include "../include/utils.h"
-#include <algorithm>
+
-#include <cstdarg>
+static std::string percent_encode(const std::string &s) {
-#include <cstdint>
+  static const char *hex = "0123456789ABCDEF";
-#include <cstdio>
+  std::string out;
-#include <cstdlib>
+  for (unsigned char c : s) {
-#include <cstring>
+    if (std::isalnum(c) || c == '-' || c == '_' || c == '.' || c == '~' ||
-#include <fstream>
+        c == '/') {
-#include <limits.h>
+      out.push_back(c);
-#include <magic.h>
+    } else {
-#include <string.h>
+      out.push_back('%');
-#include <string>
+      out.push_back(hex[c >> 4]);
-#include <unistd.h>
+      out.push_back(hex[c & 0xF]);
-#include <unordered_map>
+    }
+  }
+  return out;
+}
+
+std::string path_to_file_uri(const std::string &path_str) {
+  namespace fs = std::filesystem;
+  fs::path p = fs::weakly_canonical(fs::absolute(fs::path(path_str)));
+  std::string generic = p.generic_string();
+  return "file://" + percent_encode(generic);
+}
 
 uint64_t fnv1a_64(const char *s, size_t len) {
   uint64_t hash = 1469598103934665603ull;
@@ -231,61 +242,21 @@ char *detect_file_type(const char *filename) {
   return result;
 }
 
-static const std::unordered_map<std::string, Language> ext_map = {
-    {"sh", {"bash", tree_sitter_bash}},
-    {"bash", {"bash", tree_sitter_bash}},
-    {"c", {"c", tree_sitter_c}},
-    {"cpp", {"cpp", tree_sitter_cpp}},
-    {"cxx", {"cpp", tree_sitter_cpp}},
-    {"cc", {"cpp", tree_sitter_cpp}},
-    {"hpp", {"cpp", tree_sitter_cpp}},
-    {"hh", {"cpp", tree_sitter_cpp}},
-    {"hxx", {"cpp", tree_sitter_cpp}},
-    {"h", {"cpp", tree_sitter_cpp}},
-    {"css", {"css", tree_sitter_css}},
-    {"fish", {"fish", tree_sitter_fish}},
-    {"go", {"go", tree_sitter_go}},
-    {"hs", {"haskell", tree_sitter_haskell}},
-    {"html", {"html", tree_sitter_html}},
-    {"htm", {"html", tree_sitter_html}},
-    {"js", {"javascript", tree_sitter_javascript}},
-    {"json", {"json", tree_sitter_json}},
-    {"lua", {"lua", tree_sitter_lua}},
-    {"mk", {"make", tree_sitter_make}},
-    {"makefile", {"make", tree_sitter_make}},
-    {"py", {"python", tree_sitter_python}},
-    {"rb", {"ruby", tree_sitter_ruby}},
-};
-
-static const std::unordered_map<std::string, Language> mime_map = {
-    {"text/x-c", {"c", tree_sitter_c}},
-    {"text/x-c++", {"cpp", tree_sitter_cpp}},
-    {"text/x-shellscript", {"bash", tree_sitter_bash}},
-    {"application/json", {"json", tree_sitter_json}},
-    {"text/javascript", {"javascript", tree_sitter_javascript}},
-    {"text/html", {"html", tree_sitter_html}},
-    {"text/css", {"css", tree_sitter_css}},
-    {"text/x-python", {"python", tree_sitter_python}},
-    {"text/x-ruby", {"ruby", tree_sitter_ruby}},
-    {"text/x-go", {"go", tree_sitter_go}},
-    {"text/x-haskell", {"haskell", tree_sitter_haskell}},
-    {"text/x-lua", {"lua", tree_sitter_lua}},
-};
-
 Language language_for_file(const char *filename) {
   std::string ext = file_extension(filename);
+  std::string lang_name;
   if (!ext.empty()) {
-    auto it = ext_map.find(ext);
+    auto it = kExtToLang.find(ext);
-    if (it != ext_map.end())
+    if (it != kExtToLang.end())
-      return it->second;
+      return kLanguages.find(it->second)->second;
   }
   char *mime = detect_file_type(filename);
   if (mime) {
     std::string mime_type(mime);
     free(mime);
-    auto it = mime_map.find(mime_type);
+    auto it = kMimeToLang.find(mime_type);
-    if (it != mime_map.end())
+    if (it != kMimeToLang.end())
-      return it->second;
+      return kLanguages.find(it->second)->second;
   }
   return {"unknown", nullptr};
 }