Completions bug fixes

.clangd

@@ -1,4 +1,4 @@
 CompileFlags:
-  Add: [-I/home/syed/main/crib/include, -I/home/syed/main/crib/libs]
+  Add: [-I/home/syed/main/crib/include, -I/home/syed/main/crib/libs, c++20]
   Remove: []
   Compiler: clang++

README.md

@@ -6,12 +6,19 @@ A TUI IDE.
 
 # TODO
 
-- [ ] Finish autocomplete box.
+- [ ] Check why fish is behaving soo off with completions filtering
+- [ ] Normalize completions edits if local filtering is used
+- [ ] Capture ctrl+h,l for scrolling documentation
+- [ ] Documentation fix position and make it async first render
+- [ ] Allow completion list to be scrolled up and down and show only x max at a time
+- [ ] Dont filter case sensitive.
+- [ ] Do not recompute word under cursor if not changed
+- [ ] Finish autocomplete box style functions.
 - [ ] Add status bar & RUNNER mode
 - [ ] Get code context from tree-sitter
 - [ ] Maybe hide boxes in !`normal` mode
 - [ ] expand color regex to match css colors if in css file
-- [ ] Fix indentation logic
+- [ ] Fix indentation logic - tree-sitter indents too if possible
     - Make it work by one getting the identation used in a file by first checking if it has any line with 2 or more spaces then the least one is set to be the indent or if it is tabs then tabs but if there are none then use a table of file type to its indentation or use 2 spaces as default. store this info as `1 = tab` and `2 or more = those many spaces`.
     - Use this when indenting and unindenting. And also when getting the identation of a line.
     - Also indent when going immediately to newline should follow indent of previous line regardless of file default.
@@ -19,25 +26,11 @@ A TUI IDE.
 - [ ] For `"insertTextFormat": 2` in `clangd` and similar use only the last word in the signature when replacing
 - [ ] Keep a list of words in the current buffer. (for auto completion) (maybe?)
 - [ ] Add ecma to js and make tsx
-- [ ] Add support for LSP & autocomplete / snippets.
+- [ ] Switch to like `RapidJSON` ro something more basic but faster than rn
-    - First research
+    - also decrease use of `std::string` so much in ui stuff and lsp and warnings etc.
-        - `textDocument/documentHighlight` - for highlighting stuff (probably tree-sitter is enough)
+- [ ] Add lsp jumping support for goto definition, hover etc.
-        - `textDocument/selectionRange` //
+- [ ] Add lsp rename support for renaming a symbol. (also see what tree-sitter can do here)
-        - `textDocument/completion` - Obviously
+- [ ] Check into more lsp stuff i can add.
-        - `textDocument/onTypeFormatting` - seems promising for auto formatting (indentation etc)
-        - `textDocument/formatting` & `textDocument/rangeFormatting`
-        - `textDocument/semanticTokens/*` (probably tree-sitter is enough)
-        - `textDocument/linkedEditingRange` - probably useful
-        - `textDocument/foldingRange` - i will never use this for folding but it might be useful for other things.
-        - `textDocument/rename` & `textDocument/prepareRename` - probably useful
-        - And a lot more (just go through each for `clangd` and then expand to say `solargraph`).
-    - Make a universal plug for lsp. So focus more on making a general purpose solid communication interface. Instead of something specific.
-        - With a 4ish pass system. (more like each returned value from the lsp is used in 4 ways)
-            1. One for stuff like jump to x position. or rename symbol x to y. (stuff that explicitly requires user request to do something)
-                - Maybe even hover goes here
-            2. One for stuff that only affects highlighting and styles . like symbol highlighting etc.
-            3. One for Warnings/errors and inlay hints etc. (stuff that adds virtual text to the editor)
-            4. One for fromatting and stuff like that. (stuff that edits the buffer text)
 - [ ] Add codeium/copilot support for auto-completion (uses the VAI virtual text) as a test phase.
 - [ ] Add a whitespace highlighter (nerd font). for spaces and tabs at start/end of line. not as virtual but instead at render time.
 - [ ] Once renderer is proven to work well (i.e. redo this commit) merge `experimental` branch into `main`. commit `43f443e` on `experimental`.

grammar/h.scm

@@ -506,29 +506,29 @@
 ; Macros & directives
 ; ============================================================
 
-;; #F29CC3 #000000 0 0 0 0 2
+;; #F29CC3 #000000 0 0 0 0 3
 (preproc_def
   name: (_) @constant.macro)
 
-;; #F29CC3 #000000 0 0 0 0 2
+;; #F29CC3 #000000 0 0 0 0 3
 (preproc_call
   directive: (preproc_directive) @_u
   argument: (_) @constant.macro
   (#match? @_u "^#undef$"))
 
-;; #F29CC3 #000000 0 0 0 0 2
+;; #F29CC3 #000000 0 0 0 0 3
 (preproc_ifdef
   name: (identifier) @constant.macro)
 
-;; #F29CC3 #000000 0 0 0 0 2
+;; #F29CC3 #000000 0 0 0 0 3
 (preproc_elifdef
   name: (identifier) @constant.macro)
 
-;; #F29CC3 #000000 0 0 0 0 2
+;; #F29CC3 #000000 0 0 0 0 3
 (preproc_defined
   (identifier) @constant.macro)
 
-;; #F29CC3 #000000 0 0 0 0 2
+;; #F29CC3 #000000 0 0 0 0 3
 (preproc_defined) @function.macro
 
 ; ============================================================

include/editor/completions.h

@@ -0,0 +1,43 @@
+#ifndef EDITOR_COMPLETIONS_H
+#define EDITOR_COMPLETIONS_H
+
+#include "editor/decl.h"
+#include "pch.h"
+#include "ui/completionbox.h"
+#include "ui/hover.h"
+#include "utils/utils.h"
+
+struct CompletionItem {
+  std::string label;
+  uint8_t kind;
+  std::optional<std::string> detail;
+  std::optional<std::string> documentation;
+  bool is_markup = false;
+  bool deprecated = false;
+  std::string sort;
+  std::string filter;
+  bool snippet = false;
+  std::vector<TextEdit> edits;
+  json original;
+  std::vector<char> end_chars;
+};
+
+struct CompletionSession {
+  std::shared_mutex mtx;
+  bool active = false;
+  Coord hook;
+  std::optional<std::string> prefix;
+  uint8_t select = 0;
+  std::vector<CompletionItem> items;
+  std::vector<uint8_t> visible;
+  bool complete = true;
+  std::optional<char> trigger_char;
+  uint8_t trigger = 0;
+  CompletionBox box;
+  HoverBox hover;
+  uint32_t doc = UINT32_MAX;
+
+  CompletionSession() : box(this) {}
+};
+
+#endif

include/editor/decl.h

@@ -3,6 +3,12 @@
 
 #include "utils/utils.h"
 
+struct TextEdit {
+  Coord start;
+  Coord end;
+  std::string text;
+};
+
 struct Fold {
   uint32_t start;
   uint32_t end;

include/editor/editor.h

@@ -1,10 +1,12 @@
 #ifndef EDITOR_H
 #define EDITOR_H
 
+#include "editor/completions.h"
 #include "editor/spans.h"
 #include "io/knot.h"
 #include "io/sysio.h"
 #include "ts/decl.h"
+#include "ui/completionbox.h"
 #include "ui/diagnostics.h"
 #include "ui/hover.h"
 #include "utils/utils.h"
@@ -16,6 +18,8 @@
 #define EXTRA_META 4
 #define INDENT_WIDTH 2
 
+// autocomplete lua// Bracket closing / tab on enter
+
 struct Editor {
   std::string filename;
   std::string uri;
@@ -49,6 +53,7 @@ struct Editor {
   bool diagnostics_active;
   DiagnosticBox diagnostics;
   int lsp_version = 1;
+  CompletionSession completion;
 };
 
 Editor *new_editor(const char *filename_arg, Coord position, Coord size);
@@ -73,6 +78,8 @@ void ensure_scroll(Editor *editor);
 void handle_editor_event(Editor *editor, KeyEvent event);
 void edit_erase(Editor *editor, Coord pos, int64_t len);
 void edit_insert(Editor *editor, Coord pos, char *data, uint32_t len);
+void edit_replace(Editor *editor, Coord start, Coord end, const char *text,
+                  uint32_t len);
 Coord editor_hit_test(Editor *editor, uint32_t x, uint32_t y);
 char *get_selection(Editor *editor, uint32_t *out_len, Coord *out_start);
 void editor_worker(Editor *editor);
@@ -90,6 +97,13 @@ uint32_t leading_indent(const char *line, uint32_t len);
 uint32_t get_indent(Editor *editor, Coord cursor);
 bool closing_after_cursor(const char *line, uint32_t len, uint32_t col);
 void editor_lsp_handle(Editor *editor, json msg);
+void apply_lsp_edits(Editor *editor, std::vector<TextEdit> edits, bool move);
+void completion_resolve_doc(Editor *editor);
+void complete_accept(Editor *editor);
+void complete_next(Editor *editor);
+void complete_prev(Editor *editor);
+void complete_select(Editor *editor, uint8_t index);
+void handle_completion(Editor *editor, KeyEvent event);
 
 inline void apply_hook_insertion(Editor *editor, uint32_t line, uint32_t rows) {
   for (auto &hook : editor->hooks)
@@ -104,4 +118,56 @@ inline void apply_hook_deletion(Editor *editor, uint32_t removal_start,
       hook -= removal_end - removal_start + 1;
 }
 
+inline static void utf8_normalize_edit(Editor *editor, TextEdit *edit) {
+  std::shared_lock lock(editor->knot_mtx);
+  if (edit->start.row > editor->root->line_count) {
+    edit->start.row = editor->root->line_count;
+    edit->start.col = UINT32_MAX;
+  }
+  if (edit->end.row > editor->root->line_count) {
+    edit->end.row = editor->root->line_count;
+    edit->end.col = UINT32_MAX;
+  }
+  LineIterator *it = begin_l_iter(editor->root, edit->start.row);
+  if (!it)
+    return;
+  uint32_t len;
+  char *line = next_line(it, &len);
+  if (!line) {
+    free(it->buffer);
+    free(it);
+    return;
+  }
+  if (edit->start.col < len)
+    edit->start.col = utf16_offset_to_utf8(line, edit->start.col);
+  else
+    edit->start.col = len;
+  if (edit->end.row == edit->start.row) {
+    if (edit->end.col < len)
+      edit->end.col = utf16_offset_to_utf8(line, edit->end.col);
+    else
+      edit->end.col = len;
+    free(it->buffer);
+    free(it);
+    return;
+  }
+  free(it->buffer);
+  free(it);
+  it = begin_l_iter(editor->root, edit->end.row);
+  if (!it)
+    return;
+  line = next_line(it, &len);
+  if (!line) {
+    free(it->buffer);
+    free(it);
+    return;
+  }
+  if (edit->end.col < len)
+    edit->end.col = utf16_offset_to_utf8(line, edit->end.col);
+  else
+    edit->end.col = len;
+  free(it->buffer);
+  free(it);
+}
+
 #endif

include/io/sysio.h

@@ -62,19 +62,33 @@ struct ScreenCell {
 };
 
 struct KeyEvent {
+  /* KEY_CHAR, KEY_SPECIAL, KEY_MOUSE, KEY_PASTE, KEY_NONE */
   uint8_t key_type;
 
+  /* the character / string if key_type == KEY_CHAR or KEY_PASTE */
   char *c;
+  /* length of c */
   uint32_t len;
 
+  /* KEY_UP, KEY_DOWN, KEY_LEFT, KEY_RIGHT, KEY_DELETE if key_type ==
+   * KEY_SPECIAL */
   uint8_t special_key;
+  /* ALT, CNTRL, CNTRL_ALT, SHIFT if key_type == KEY_SPECIAL */
   uint8_t special_modifier;
 
+  /* column of mouse click */
   uint8_t mouse_x;
+  /* row of mouse click */
   uint8_t mouse_y;
+  /* LEFT_BTN, MIDDLE_BTN, RIGHT_BTN, SCROLL_BTN, NONE_BTN if key_type ==
+   * KEY_MOUSE */
   uint8_t mouse_button;
+  /* PRESS, RELEASE, DRAG, SCROLL if key_type == KEY_MOUSE */
   uint8_t mouse_state;
+  /* SCROLL_UP, SCROLL_DOWN, SCROLL_LEFT, SCROLL_RIGHT if key_type ==
+   * KEY_MOUSE and mouse_state == SCROLL */
   uint8_t mouse_direction;
+  /* ALT, CNTRL, CNTRL_ALT, SHIFT if key_type == KEY_MOUSE */
   uint8_t mouse_modifier;
 };
 
@@ -97,7 +111,8 @@ void update(uint32_t row, uint32_t col, std::string utf8, uint32_t fg,
             uint32_t bg, uint8_t flags, uint32_t ul_color);
 void update(uint32_t row, uint32_t col, const char *utf8, uint32_t fg,
             uint32_t bg, uint8_t flags, uint32_t ul_color);
-void set_cursor(int row, int col, int type, bool show_cursor_param);
+void set_cursor(uint8_t row, uint8_t col, uint32_t type,
+                bool show_cursor_param);
 void render();
 Coord get_size();
 

include/lsp/lsp.h

@@ -34,7 +34,12 @@ struct LSPInstance {
   bool incremental_sync = false;
   bool allow_hover = false;
   bool allow_completion = false;
-  std::string trigger_chars;
+  bool allow_resolve = false;
+  bool allow_formatting = false;
+  bool allow_formatting_on_type = false;
+  std::vector<char> format_chars;
+  std::vector<char> trigger_chars;
+  std::vector<char> end_chars;
   uint32_t last_id = 0;
   Queue<json> inbox;
   Queue<json> outbox;
@@ -51,15 +56,22 @@ static json client_capabilities = {
     {"textDocument",
      {{"publishDiagnostics", {{"relatedInformation", true}}},
       {"hover", {{"contentFormat", {"markdown", "plaintext"}}}},
+      {"formatting", {{"dynamicRegistration", false}}},
+      {"onTypeFormatting", {{"dynamicRegistration", false}}},
       {"completion",
        {{"completionItem",
-         {{"snippetSupport", true},
+         {{"commitCharactersSupport", true},
+          {"dynamicRegistration", false},
+          {"snippetSupport", true},
           {"documentationFormat", {"markdown", "plaintext"}},
-          {"resolveSupport", {{"properties", {"documentation", "detail"}}}},
+          {"resolveSupport", {{"properties", {"documentation"}}}},
           {"insertReplaceSupport", true},
           {"labelDetailsSupport", true},
-          {"insertTextModeSupport", {{"valueSet", {1}}}}}},
+          {"insertTextModeSupport", {{"valueSet", {1}}}},
-        {"completionItemKind", {{"valueSet", {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}}}},
+          {"deprecatedSupport", true}}},
+        {"completionItemKind",
+         {{"valueSet", {1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13,
+                        14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25}}}},
         {"contextSupport", true},
         {"insertTextMode", 1}}}}}};
 

include/pch.h

@@ -4,6 +4,9 @@
 #define PCRE2_CODE_UNIT_WIDTH 8
 #define PCRE_WORKSPACE_SIZE 512
 
+#include <magic.h>
+#include <nlohmann/json.hpp>
+#include <pcre2.h>
 extern "C" {
 #include "libgrapheme/grapheme.h"
 #include "unicode_width/unicode_width.h"
@@ -25,12 +28,9 @@ extern "C" {
 #include <fstream>
 #include <functional>
 #include <limits.h>
-#include <magic.h>
 #include <map>
 #include <mutex>
-#include <nlohmann/json.hpp>
 #include <optional>
-#include <pcre2.h>
 #include <queue>
 #include <shared_mutex>
 #include <signal.h>

include/ts/decl.h

@@ -7,9 +7,9 @@
 #define TS_DEF(name) extern "C" const TSLanguage *LANG(name)()
 
 struct Language {
-  std::string name;
+  std::string name = "unknown";
-  const TSLanguage *(*fn)();
+  const TSLanguage *(*fn)() = nullptr;
-  uint8_t lsp_id;
+  uint8_t lsp_id = 0;
   uint32_t color = 0xFFFFFF;
   const char *symbol = " ";
 };

include/ts/ts.h

@@ -11,8 +11,9 @@ extern std::unordered_map<std::string, pcre2_code *> regex_cache;
 
 TSQuery *load_query(const char *query_path, TSSetBase *set);
 void ts_collect_spans(Editor *editor);
-bool ts_predicate(TSQuery *query, const TSQueryMatch &match,
+bool ts_predicate(
-                  std::function<std::string(const TSNode *)> subject_fn);
+    TSQuery *query, const TSQueryMatch &match,
+    std::function<char *(const TSNode *, uint32_t *len)> subject_fn);
 void clear_regex_cache();
 
 #endif

include/ui/bar.h

@@ -8,7 +8,7 @@
 struct Bar {
   Coord screen;
   std::string command = "";
-  int cursor = 0;
+  uint32_t cursor = 0;
 
   Bar(Coord screen) : screen(screen) {}
   void render();

include/ui/completionbox.h

@@ -0,0 +1,21 @@
+#ifndef UI_COMPLETIONBOX_H
+#define UI_COMPLETIONBOX_H
+
+#include "io/sysio.h"
+#include "pch.h"
+#include "utils/utils.h"
+
+struct CompletionBox {
+  std::shared_mutex mtx;
+  struct CompletionSession *session;
+  bool hidden = true;
+  std::vector<ScreenCell> cells;
+  Coord size;
+  Coord position;
+
+  CompletionBox(CompletionSession *s) : session(s) {}
+  void render_update();
+  void render(Coord pos);
+};
+
+#endif

include/ui/diagnostics.h

@@ -9,8 +9,7 @@
 struct DiagnosticBox {
   std::vector<VWarn> warnings;
   std::vector<ScreenCell> cells;
-  uint32_t box_width;
+  Coord size;
-  uint32_t box_height;
 
   void clear();
   void render_first();

include/ui/hover.h

@@ -12,8 +12,7 @@ struct HoverBox {
   std::atomic<bool> is_markup;
   uint32_t scroll_;
   std::vector<ScreenCell> cells;
-  uint32_t box_width;
+  Coord size;
-  uint32_t box_height;
   std::vector<Highlight> highlights;
   std::vector<Span> hover_spans;
 

include/utils/utils.h

@@ -61,6 +61,8 @@ struct Match {
 
 #define MAX(a, b) ((a) > (b) ? (a) : (b))
 #define MIN(a, b) ((a) < (b) ? (a) : (b))
+#define UNUSED(x) (void)(x)
+#define USING(x) UNUSED(sizeof(x))
 
 std::string clean_text(const std::string &input);
 std::string percent_encode(const std::string &s);
@@ -74,6 +76,7 @@ uint32_t get_visual_col_from_bytes(const char *line, uint32_t len,
 uint32_t get_bytes_from_visual_col(const char *line, uint32_t len,
                                    uint32_t target_visual_col);
 int utf8_byte_offset_to_utf16(const char *s, size_t byte_pos);
+size_t utf16_offset_to_utf8(const char *s, int utf16_pos);
 
 void log(const char *fmt, ...);
 

samples/fish.fish

@@ -1,10 +1,10 @@
 #!/usr/bin/env fish
-# Fish highlighting torture test 🐟
+# Fish highlighting torture test
 
 # === Variables ===
-set normal_var "hello"
+set normal_var hello
 set -l local_var 123
-set -gx GLOBAL_VAR "world"
+set -gx GLOBAL_VAR world
 set PATH $PATH /usr/local/bin
 set --erase OLD_VAR
 
@@ -17,15 +17,15 @@ set double "double quoted $normal_var"
 set escaped "newline\n tab\t dollar\$"
 
 # === Conditionals ===
-if test $normal_var = "hello"
+if test $normal_var = hello
-    echo "equal"
+    echo equal
-else if test $normal_var != "world"
+else if test $normal_var != world
     echo "not equal"
 end
 
 # === Logical operators ===
-true and echo "yes"
+true and echo yes
-false or echo "fallback"
+false or echo fallback
 not false
 
 # === Arithmetic ===
@@ -50,14 +50,14 @@ function greet --argument name
     echo "Hello $name"
 end
 
-greet "world"
+greet world
 
 # === Command substitution ===
 set files (ls | grep ".fish")
 
 # === Redirections ===
-echo "output" > /tmp/fish_test.txt
+echo output >/tmp/fish_test.txt
-cat < /tmp/fish_test.txt >> /tmp/fish_log.txt
+cat </tmp/fish_test.txt >>/tmp/fish_log.txt
 
 # === Process substitution ===
 diff (ls /bin) (ls /usr/bin)
@@ -65,11 +65,11 @@ diff (ls /bin) (ls /usr/bin)
 # === Case statement ===
 switch $argv[1]
     case start
-        echo "Starting"
+        echo Starting
     case stop
-        echo "Stopping"
+        echo Stopping
     case '*'
-        echo "Unknown"
+        echo Unknown
 end
 
 # === Subshell ===
@@ -79,10 +79,10 @@ end
 
 # === Comments & operators ===
 # && || | & ! should all highlight
-true && echo "ok" || echo "fail"
+true && echo ok || echo fail
 
 # === Regex ===
-string match -r '^[a-z]+$' "hello"
+string match -r '^[a-z]+$' hello
 
 # === Test builtin ===
 test -f /etc/passwd
@@ -90,3 +90,4 @@ test ! -d /does/not/exist
 
 # === Exit ===
 exit 0
+

samples/php.php

@@ -1,5 +1,6 @@
 <!DOCTYPE html>
 <html lang="en">
+
 <head>
   <meta charset="UTF-8">
   <title>PHP Syntax Stress Test</title>
@@ -33,98 +34,103 @@
 
 <body>
 
-<?php
+  <?php
-// Basic variables
+  // Basic variables
-$number = 42;
+  $number = 42;
-$text = "Hello PHP";
+  $text = "Hello PHP";
-$truth = true;
+  $truth = true;
-$nothing = null;
+  $nothing = null;
 
-// Constants
+  // Constants
-define("APP_NAME", "SyntaxTester");
+  define("APP_NAME", "SyntaxTester");
 
-// Arrays
+  // Arrays
-$list = [1, 2, 3];
+  $list = [1, 2, 3];
-$assoc = [
+  $assoc = [
-  "one" => 1,
+    "one" => 1,
-  "two" => 2
+    "two" => 2
-];
+  ];
 
-// Function
+  // Function
-function add(int $a, int $b): int {
+  function add(int $a, int $b): int
-  return $a + $b;
+  {
-}
+    return $a + $b;
-
-// Class + methods
-class User {
-  private string $name;
-  public static int $count = 0;
-
-  public function __construct(string $name) {
-    $this->name = $name;
-    self::$count++;
   }
 
-  public function greet(): string {
+  // Class + methods
-    return "Hello {$this->name}";
+  class User
+  {
+    private string $name;
+    public static int $count = 0;
+
+    public function __construct(string $name)
+    {
+      $this->name = $name;
+      self::$count++;
+    }
+
+    public function greet(): string
+    {
+      return "Hello {$this->name}";
+    }
   }
-}
 
-// Object usage
+  // Object usage
-$user = new User("Alice");
+  $user = new User("Alice");
-echo $user->greet();
+  echo $user->greet();
 
-// Control flow
+  // Control flow
-if ($number > 10) {
+  if ($number > 10) {
-  echo "Big number";
+    echo "Big number";
-} elseif ($number === 10) {
+  } elseif ($number === 10) {
-  echo "Exactly ten";
+    echo "Exactly ten";
-} else {
+  } else {
-  echo "Small number";
+    echo "Small number";
-}
+  }
 
-// Loop
+  // Loop
-foreach ($list as $item) {
+  foreach ($list as $item) {
-  echo $item;
+    echo $item;
-}
+  }
 
-// Match expression
+  // Match expression
-$result = match ($number) {
+  $result = match ($number) {
-  1 => "one",
+    1 => "one",
-  2 => "two",
+    2 => "two",
-  default => "many"
+    default => "many"
-};
+  };
 
-// Try / catch
+  // Try / catch
-try {
+  try {
-  throw new Exception("Test exception");
+    throw new Exception("Test exception");
-} catch (Exception $e) {
+  } catch (Exception $e) {
-  echo $e->getMessage();
+    echo $e->getMessage();
-}
+  }
 
-// Anonymous function
+  // Anonymous function
-$double = fn($x) => $x * 2;
+  $double = fn($x) => $x * 2;
 
-// Nullsafe operator
+  // Nullsafe operator
-$len = $user?->name ? strlen($user->name) : 0;
+  $len = $user?->name ? strlen($user->name) : 0;
 
-// Ternary
+  // Ternary
-$status = $truth ? "yes" : "no";
+  $status = $truth ? "yes" : "no";
 
-// Include / require
+  // Include / require
-require_once "config.php";
+  require_once "config.php";
 
-// Output
+  // Output
-echo "<div class='box'>";
+  echo "<div class='box'>";
-echo htmlspecialchars($text);
+  echo htmlspecialchars($text);
-echo "</div>";
+  echo "</div>";
-?>
+  ?>
 
-<script>
+  <script>
-  // JS interacting with PHP output
+    // JS interacting with PHP output
-  const phpValue = <?= json_encode($number) ?>;
+    const phpValue = <?= json_encode($number) ?>;
-  console.log("Value from PHP:", phpValue);
+    console.log("Value from PHP:", phpValue);
-</script>
+  </script>
 
 </body>
+
 </html>

samples/python.py

@@ -1,3 +1,4 @@
+from __future__ import annotations
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 """Test file for Python Tree-sitter highlighting."""
@@ -17,7 +18,6 @@ __name__  # builtin constant
 import os
 import sys as system
 from re import compile as re_compile
-from __future__ import annotations
 from math import *
 
 # ==============================
@@ -94,7 +94,7 @@ while x > 0:
 
 try:
     1 / 0
-except ZeroDivisionError as e:
+except ZeroDivisionError as err:
     raise
 finally:
     pass
@@ -105,7 +105,7 @@ finally:
 a, b = 5, 10
 c = a + b * 2 // 3 % 4 ** 2
 d = (a << 2) & b | c ^ ~a
-e = not a or b and c
+ef = not a or b and c
 
 # ==============================
 # f-strings / interpolation
@@ -131,7 +131,7 @@ def static_func():
 def cls_func(cls):
     return cls
 
-@custom_decorator
+# @custom_decorator
 def decorated_func():
     return None
 

src/editor/boundaries.cc

@@ -42,8 +42,11 @@ void word_boundaries_exclusive(Editor *editor, Coord coord, uint32_t *prev_col,
     return;
   uint32_t line_len;
   char *line = next_line(it, &line_len);
-  if (!line)
+  if (!line) {
+    free(it->buffer);
+    free(it);
     return;
+  }
   if (line_len && line[line_len - 1] == '\n')
     line_len--;
   uint32_t col = coord.col;

src/editor/completions.cc

@@ -0,0 +1,404 @@
+#include "editor/decl.h"
+#include "editor/editor.h"
+#include "io/knot.h"
+#include "io/sysio.h"
+#include "lsp/lsp.h"
+#include "main.h"
+#include "utils/utils.h"
+#include <regex>
+
+inline static std::string completion_prefix(Editor *editor) {
+  Coord hook = editor->completion.hook;
+  Coord cur = editor->cursor;
+  if (hook.row != cur.row || cur.col < hook.col)
+    return "";
+  LineIterator *it = begin_l_iter(editor->root, hook.row);
+  char *line = next_line(it, nullptr);
+  if (!line) {
+    free(it->buffer);
+    free(it);
+    return "";
+  }
+  uint32_t start = utf16_offset_to_utf8(line, hook.col);
+  uint32_t end = editor->cursor.col;
+  std::string prefix(line + start, end - start);
+  free(it->buffer);
+  free(it);
+  return prefix;
+}
+
+void completion_filter(Editor *editor) {
+  auto &session = editor->completion;
+  std::string prefix = completion_prefix(editor);
+  session.visible.clear();
+  for (size_t i = 0; i < session.items.size(); ++i) {
+    const auto &item = session.items[i];
+    const std::string &key = item.filter;
+    if (key.size() >= prefix.size() &&
+        key.compare(0, prefix.size(), prefix) == 0)
+      session.visible.push_back(i);
+  }
+  if (session.visible.empty()) {
+    session.box.hidden = true;
+    return;
+  }
+  if (std::find(session.visible.begin(), session.visible.end(),
+                session.select) == session.visible.end())
+    session.select = session.visible[0];
+  session.box.hidden = false;
+  session.box.render_update();
+}
+
+void completion_request(Editor *editor) {
+  Coord hook = editor->cursor;
+  word_boundaries(editor, editor->cursor, &hook.col, nullptr, nullptr, nullptr);
+  editor->completion.active = true;
+  editor->completion.items.clear();
+  editor->completion.visible.clear();
+  editor->completion.select = 0;
+  editor->completion.hook = hook;
+  LSPPending *pending = new LSPPending();
+  pending->editor = editor;
+  pending->method = "textDocument/completion";
+  pending->callback = [](Editor *editor, std::string, json message) {
+    auto &session = editor->completion;
+    std::unique_lock lock(session.mtx);
+    std::vector<json> items_json;
+    std::vector<char> end_chars_def;
+    int insert_text_format = 1;
+    if (message.contains("result")) {
+      auto &result = message["result"];
+      if (result.is_array()) {
+        items_json = result.get<std::vector<json>>();
+        session.complete = true;
+      } else if (result.is_object() && result.contains("items")) {
+        auto &list = result;
+        items_json = list["items"].get<std::vector<json>>();
+        session.complete = !list.value("isIncomplete", false);
+        if (list.contains("itemDefaults") && list["itemDefaults"].is_object()) {
+          auto &defs = list["itemDefaults"];
+          if (defs.contains("insertTextFormat") &&
+              defs["insertTextFormat"].is_number())
+            insert_text_format = defs["insertTextFormat"].get<int>();
+          if (defs.contains("textEdit"))
+            if (defs["textEdit"].is_array())
+              for (auto &c : defs["textEdit"]) {
+                if (!c.is_string())
+                  continue;
+                std::string str = c.get<std::string>();
+                if (str.size() != 1)
+                  continue;
+                end_chars_def.push_back(str[0]);
+              }
+        }
+      }
+    }
+    session.items.reserve(items_json.size() + 1);
+    for (auto &item_json : items_json) {
+      CompletionItem item;
+      item.original = item_json;
+      item.label = item_json.value("label", "");
+      item.kind = item_json.value("kind", 0);
+      if (item_json.contains("detail") && item_json["detail"].is_string())
+        item.detail = item_json["detail"].get<std::string>();
+      if (item_json.contains("documentation")) {
+        if (item_json["documentation"].is_string()) {
+          item.documentation = item_json["documentation"].get<std::string>();
+        } else if (item_json["documentation"].contains("value") &&
+                   item_json["documentation"]["value"].is_string()) {
+          item.is_markup =
+              item_json["documentation"]["kind"].get<std::string>() ==
+              "markdown";
+          std::string documentation =
+              item_json["documentation"]["value"].get<std::string>();
+          if (item.is_markup) {
+            static const std::regex fence_no_lang("```(\\s*\\n)");
+            item.documentation = std::regex_replace(
+                documentation, fence_no_lang, "```" + editor->lang.name + "$1");
+          } else {
+            item.documentation = documentation;
+          }
+        }
+      }
+      if (item_json.contains("deprecated") &&
+          item_json["deprecated"].is_boolean())
+        item.deprecated = item_json["deprecated"].get<bool>();
+      auto tags = item_json.value("tags", std::vector<int>());
+      for (auto tag : tags)
+        if (tag == 1)
+          item.deprecated = true;
+      item.sort = item_json.value("sortText", item.label);
+      item.filter = item_json.value("filterText", item.label);
+      if (item_json.contains("preselect") &&
+          item_json["preselect"].is_boolean() &&
+          item_json["preselect"].get<bool>())
+        session.select = session.items.size() - 1;
+      TextEdit edit;
+      if (item_json.contains("textEdit")) {
+        auto &te = item_json["textEdit"];
+        if (te.contains("newText")) {
+          edit.text = te.value("newText", "");
+          if (te.contains("replace")) {
+            edit.start.row = te["replace"]["start"]["line"];
+            edit.start.col = te["replace"]["start"]["character"];
+            edit.end.row = te["replace"]["end"]["line"];
+            edit.end.col = te["replace"]["end"]["character"];
+          } else if (te.contains("insert")) {
+            edit.start.row = te["insert"]["start"]["line"];
+            edit.start.col = te["insert"]["start"]["character"];
+            edit.end.row = te["insert"]["end"]["line"];
+            edit.end.col = te["insert"]["end"]["character"];
+          } else if (te.contains("range")) {
+            edit.start.row = te["range"]["start"]["line"];
+            edit.start.col = te["range"]["start"]["character"];
+            edit.end.row = te["range"]["end"]["line"];
+            edit.end.col = te["range"]["end"]["character"];
+          } else {
+            edit.start = session.hook;
+            edit.end = editor->cursor;
+          }
+        }
+      } else if (item_json.contains("insertText") &&
+                 item_json["insertText"].is_string()) {
+        edit.text = item_json["insertText"].get<std::string>();
+        edit.start = session.hook;
+        edit.end = editor->cursor;
+      } else {
+        edit.text = item.label;
+        edit.start = session.hook;
+        edit.end = editor->cursor;
+      }
+      utf8_normalize_edit(editor, &edit);
+      item.edits.push_back(edit);
+      if (item_json.contains("additionalTextEdits")) {
+        for (auto &te : item_json["additionalTextEdits"]) {
+          TextEdit edit;
+          edit.text = te.value("newText", "");
+          edit.start.row = te["range"]["start"]["line"];
+          edit.start.col = te["range"]["start"]["character"];
+          edit.end.row = te["range"]["end"]["line"];
+          edit.end.col = te["range"]["end"]["character"];
+          utf8_normalize_edit(editor, &edit);
+          item.edits.push_back(edit);
+        }
+      }
+      item.snippet = insert_text_format == 2;
+      if (item_json.contains("insertTextFormat"))
+        item.snippet = item_json["insertTextFormat"].get<int>() == 2;
+      if (item_json.contains("commitCharacters"))
+        for (auto &c : item_json["commitCharacters"])
+          if (c.is_string() && c.get<std::string>().size() == 1)
+            item.end_chars.push_back(c.get<std::string>()[0]);
+      session.items.push_back(std::move(item));
+    }
+    completion_filter(editor);
+    session.box.hidden = false;
+    session.box.render_update();
+  };
+  std::shared_lock lock(editor->knot_mtx);
+  LineIterator *it = begin_l_iter(editor->root, hook.row);
+  char *line = next_line(it, nullptr);
+  if (!line) {
+    free(it->buffer);
+    free(it);
+    return;
+  }
+  uint32_t col = utf8_byte_offset_to_utf16(line, editor->cursor.col);
+  free(it->buffer);
+  free(it);
+  lock.unlock();
+  json message = {
+      {"jsonrpc", "2.0"},
+      {"method", "textDocument/completion"},
+      {"params",
+       {{"textDocument", {{"uri", editor->uri}}},
+        {"position", {{"line", editor->cursor.row}, {"character", col}}}}}};
+  if (editor->completion.trigger > 0) {
+    json context = {{"triggerKind", editor->completion.trigger}};
+    if (editor->completion.trigger == 2 && editor->completion.trigger_char)
+      context["triggerCharacter"] =
+          std::string(1, *editor->completion.trigger_char);
+    message["params"]["context"] = context;
+  }
+  lsp_send(editor->lsp, message, pending);
+}
+
+void handle_completion(Editor *editor, KeyEvent event) {
+  if (!editor->lsp || !editor->lsp->allow_completion)
+    return;
+  if (mode != INSERT) {
+    editor->completion.active = false;
+    return;
+  }
+  std::unique_lock lock(editor->completion.mtx);
+  if (event.key_type == KEY_PASTE) {
+    editor->completion.active = false;
+    return;
+  } else if (event.key_type == KEY_CHAR) {
+    char ch = *event.c;
+    if (!editor->completion.active) {
+      for (char c : editor->lsp->trigger_chars)
+        if (c == ch) {
+          editor->completion.trigger = 2;
+          editor->completion.trigger_char = c;
+          completion_request(editor);
+          return;
+        }
+    } else {
+      if (editor->completion.items.empty())
+        return;
+      const auto &item = editor->completion.items[editor->completion.select];
+      const std::vector<char> &end_chars =
+          item.end_chars.empty() ? editor->lsp->end_chars : item.end_chars;
+      for (char c : end_chars)
+        if (c == ch) {
+          complete_accept(editor);
+          return;
+        }
+    }
+    if ((ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') ||
+        (ch >= '0' && ch <= '9') || ch == '_') {
+      if (editor->completion.active) {
+        if (editor->completion.complete)
+          completion_filter(editor);
+        else
+          completion_request(editor);
+      } else {
+        editor->completion.trigger = 3;
+        completion_request(editor);
+      }
+    } else if (ch == CTRL('\\')) {
+      if (editor->completion.active) {
+        complete_accept(editor);
+      } else {
+        editor->completion.trigger = 1;
+        completion_request(editor);
+      }
+    } else if (ch == CTRL('p')) {
+      if (editor->completion.active)
+        complete_next(editor);
+    } else if (ch == CTRL('o')) {
+      if (editor->completion.active)
+        complete_prev(editor);
+    } else if (ch == 0x7F || ch == 0x08) {
+      if (editor->completion.active) {
+        if (editor->completion.complete) {
+          if (editor->cursor <= editor->completion.hook)
+            editor->completion.active = false;
+          else
+            completion_filter(editor);
+        } else {
+          completion_request(editor);
+        }
+      }
+    } else {
+      editor->completion.active = false;
+    }
+  } else if (event.key_type == KEY_MOUSE && event.mouse_modifier == 0) {
+    // Prolly remove mouse support here
+    // auto &box = editor->completion.box;
+    // if (event.mouse_y >= box.position.row &&
+    //     event.mouse_x >= box.position.col) {
+    //   uint32_t row = event.mouse_y - box.position.row;
+    //   uint32_t col = event.mouse_x - box.position.col;
+    //   if (row < box.size.row && col < box.size.col) {
+    //     uint8_t idx = 0;
+    //     /* TODO: fix index relative to scroll */
+    //     complete_select(editor, idx);
+    //   }
+    // }
+    // if it is being implemented then stop main event handler from processing
+    // when click inside the box
+    editor->completion.active = false;
+  } else {
+    editor->completion.active = false;
+  }
+}
+
+void completion_resolve_doc(Editor *editor) {
+  auto &item = editor->completion.items[editor->completion.select];
+  if (item.documentation)
+    return;
+  item.documentation = "";
+  LSPPending *pending = new LSPPending();
+  pending->editor = editor;
+  pending->method = "completionItem/resolve";
+  pending->callback = [](Editor *editor, std::string, json message) {
+    std::unique_lock lock(editor->completion.mtx);
+    auto &item = editor->completion.items[editor->completion.select];
+    if (message["result"].contains("documentation")) {
+      if (message["result"]["documentation"].is_string()) {
+        item.documentation =
+            message["result"]["documentation"].get<std::string>();
+      } else if (message["result"]["documentation"].contains("value") &&
+                 message["result"]["documentation"]["value"].is_string()) {
+        item.is_markup =
+            message["result"]["documentation"]["kind"].get<std::string>() ==
+            "markdown";
+        std::string documentation =
+            message["result"]["documentation"]["value"].get<std::string>();
+        if (item.is_markup) {
+          static const std::regex fence_no_lang("```(\\s*\\n)");
+          item.documentation = std::regex_replace(
+              documentation, fence_no_lang, "```" + editor->lang.name + "$1");
+        } else {
+          item.documentation = documentation;
+        }
+      }
+    }
+    editor->completion.box.render_update();
+  };
+  json message = {{"jsonrpc", "2.0"},
+                  {"method", "completionItem/resolve"},
+                  {"params", item.original}};
+  lsp_send(editor->lsp, message, pending);
+}
+
+void complete_accept(Editor *editor) {
+  if (!editor->completion.active || editor->completion.box.hidden)
+    return;
+  auto &item = editor->completion.items[editor->completion.select];
+  // TODO: support snippets here
+  apply_lsp_edits(editor, item.edits, true);
+  editor->completion.active = false;
+}
+
+inline static int visible_index(const CompletionSession &s) {
+  for (size_t i = 0; i < s.visible.size(); ++i)
+    if (s.visible[i] == s.select)
+      return (int)i;
+  return -1;
+}
+
+void complete_next(Editor *editor) {
+  auto &s = editor->completion;
+  if (!s.active || s.box.hidden || s.visible.empty())
+    return;
+  int vi = visible_index(s);
+  if (vi < 0)
+    vi = 0;
+  else
+    vi = (vi + 1) % s.visible.size();
+  s.select = s.visible[vi];
+  completion_resolve_doc(editor);
+  editor->completion.box.render_update();
+}
+
+void complete_prev(Editor *editor) {
+  auto &s = editor->completion;
+  if (!s.active || s.box.hidden || s.visible.empty())
+    return;
+  int vi = visible_index(s);
+  if (vi < 0)
+    vi = 0;
+  else
+    vi = (vi + s.visible.size() - 1) % s.visible.size();
+  s.select = s.visible[vi];
+  completion_resolve_doc(editor);
+  editor->completion.box.render_update();
+}
+
+void complete_select(Editor *editor, uint8_t index) {
+  editor->completion.select = index;
+  complete_accept(editor);
+}

src/editor/edit.cc

@@ -1,6 +1,7 @@
 #include "editor/editor.h"
 #include "editor/folds.h"
 #include "lsp/lsp.h"
+#include "utils/utils.h"
 
 void edit_erase(Editor *editor, Coord pos, int64_t len) {
   if (len == 0)
@@ -278,3 +279,22 @@ void edit_insert(Editor *editor, Coord pos, char *data, uint32_t len) {
     }
   }
 }
+
+void edit_replace(Editor *editor, Coord start, Coord end, const char *text,
+                  uint32_t len) {
+  std::shared_lock lock(editor->knot_mtx);
+  uint32_t start_byte =
+      line_to_byte(editor->root, start.row, nullptr) + start.col;
+  uint32_t end_byte = line_to_byte(editor->root, end.row, nullptr) + end.col;
+  lock.unlock();
+  char *buf = read(editor->root, start_byte, end_byte - start_byte);
+  if (!buf)
+    return;
+  uint32_t erase_len =
+      count_clusters(buf, end_byte - start_byte, 0, end_byte - start_byte);
+  free(buf);
+  if (erase_len != 0)
+    edit_erase(editor, start, erase_len);
+  if (len > 0)
+    edit_insert(editor, start, const_cast<char *>(text), len);
+}

src/editor/editor.cc

@@ -1,6 +1,8 @@
 #include "editor/editor.h"
+#include "editor/decl.h"
 #include "lsp/lsp.h"
 #include "utils/utils.h"
+#include <shared_mutex>
 
 Editor *new_editor(const char *filename_arg, Coord position, Coord size) {
   Editor *editor = new Editor();
@@ -66,15 +68,62 @@ void free_editor(Editor *editor) {
 void save_file(Editor *editor) {
   if (!editor || !editor->root)
     return;
+  std::shared_lock lock(editor->knot_mtx);
   char *str = read(editor->root, 0, editor->root->char_count);
   if (!str)
     return;
   std::ofstream out(editor->filename);
   out.write(str, editor->root->char_count);
+  out.close();
   free(str);
-  json msg = {{"jsonrpc", "2.0"},
+  lock.unlock();
-              {"method", "textDocument/didSave"},
+  if (editor->lsp) {
-              {"params", {{"textDocument", {{"uri", editor->uri}}}}}};
+    json save_msg = {{"jsonrpc", "2.0"},
-  if (editor->lsp)
+                     {"method", "textDocument/didSave"},
-    lsp_send(editor->lsp, msg, nullptr);
+                     {"params", {{"textDocument", {{"uri", editor->uri}}}}}};
+    lsp_send(editor->lsp, save_msg, nullptr);
+    if (editor->lsp->allow_formatting) {
+      json msg = {{"jsonrpc", "2.0"},
+                  {"method", "textDocument/formatting"},
+                  {"params",
+                   {{"textDocument", {{"uri", editor->uri}}},
+                    {"options",
+                     {{"tabSize", 2},
+                      {"insertSpaces", true},
+                      {"trimTrailingWhitespace", true},
+                      {"trimFinalNewlines", true}}}}}};
+      LSPPending *pending = new LSPPending();
+      pending->editor = editor;
+      pending->method = "textDocument/formatting";
+      pending->callback = [save_msg](Editor *editor, std::string,
+                                     json message) {
+        auto &edits = message["result"];
+        if (edits.is_array()) {
+          std::vector<TextEdit> t_edits;
+          t_edits.reserve(edits.size());
+          for (auto &edit : edits) {
+            TextEdit t_edit;
+            t_edit.text = edit.value("newText", "");
+            t_edit.start.row = edit["range"]["start"]["line"];
+            t_edit.start.col = edit["range"]["start"]["character"];
+            t_edit.end.row = edit["range"]["end"]["line"];
+            t_edit.end.col = edit["range"]["end"]["character"];
+            utf8_normalize_edit(editor, &t_edit);
+            t_edits.push_back(t_edit);
+          }
+          apply_lsp_edits(editor, t_edits, false);
+          ensure_scroll(editor);
+          std::unique_lock lock(editor->knot_mtx);
+          char *str = read(editor->root, 0, editor->root->char_count);
+          if (!str)
+            return;
+          std::ofstream out(editor->filename);
+          out.write(str, editor->root->char_count);
+          free(str);
+          lsp_send(editor->lsp, save_msg, nullptr);
+        }
+      };
+      lsp_send(editor->lsp, msg, pending);
+    }
+  }
 }

src/editor/events.cc

@@ -9,6 +9,7 @@ void handle_editor_event(Editor *editor, KeyEvent event) {
   static uint32_t click_count = 0;
   static Coord last_click_pos = {UINT32_MAX, UINT32_MAX};
   Coord start = editor->cursor;
+  uint8_t old_mode = mode;
   if (editor->hover_active)
     editor->hover_active = false;
   if (event.key_type == KEY_MOUSE) {
@@ -364,45 +365,6 @@ void handle_editor_event(Editor *editor, KeyEvent event) {
       case CTRL('s'):
         save_file(editor);
         break;
-      case CTRL(' '):
-        if (editor->lsp) {
-          json msg = {
-              {"jsonrpc", "2.0"},
-              {"method", "textDocument/completion"},
-              {
-                  "params",
-                  {
-                      {
-                          "textDocument",
-                          {
-                              {"uri", editor->uri},
-                          },
-                      },
-                      {
-                          "position",
-                          {
-                              {"line", editor->cursor.row},
-                              {"character", editor->cursor.col},
-                          },
-                      },
-                      {
-                          "context",
-                          {
-                              {"triggerKind", 1},
-                          },
-                      },
-                  },
-              },
-          };
-          LSPPending *pending = new LSPPending();
-          pending->editor = editor;
-          pending->method = "textDocument/completion";
-          pending->callback = [](Editor *editor, std::string, json completion) {
-            log("%s\n", completion.dump().c_str());
-          };
-          lsp_send(editor->lsp, msg, pending);
-        }
-        break;
       case 'p':
         uint32_t len;
         char *text = get_from_clipboard(&len);
@@ -650,6 +612,8 @@ void handle_editor_event(Editor *editor, KeyEvent event) {
     mode = NORMAL;
     break;
   }
+  if (old_mode == mode || mode != INSERT)
+    handle_completion(editor, event);
   ensure_scroll(editor);
   if ((event.key_type == KEY_CHAR || event.key_type == KEY_PASTE) && event.c)
     free(event.c);

src/editor/lsp.cc

@@ -1,5 +1,39 @@
+#include "editor/decl.h"
 #include "editor/editor.h"
 
+void apply_lsp_edits(Editor *editor, std::vector<TextEdit> edits, bool move) {
+  if (!edits.size())
+    return;
+  TextEdit first = edits[0];
+  Coord cursor = editor->cursor;
+  std::sort(
+      edits.begin(), edits.end(),
+      [](const TextEdit &a, const TextEdit &b) { return a.start > b.start; });
+  for (const auto &edit : edits)
+    edit_replace(editor, edit.start, edit.end, edit.text.c_str(),
+                 edit.text.size());
+  if (move) {
+    std::shared_lock lock(editor->knot_mtx);
+    editor->cursor = first.start;
+    editor->cursor =
+        move_right_pure(editor, editor->cursor,
+                        count_clusters(first.text.c_str(), first.text.size(), 0,
+                                       first.text.size()));
+  } else {
+    if (cursor.row >= editor->root->line_count) {
+      editor->cursor.row = editor->root->line_count - 1;
+      editor->cursor.col = 0;
+    } else {
+      std::shared_lock lock(editor->knot_mtx);
+      uint32_t len;
+      line_to_byte(editor->root, cursor.row, &len);
+      len--;
+      editor->cursor.row = cursor.row;
+      editor->cursor.col = cursor.col < len ? cursor.col : len;
+    }
+  }
+}
+
 void editor_lsp_handle(Editor *editor, json msg) {
   if (msg.contains("method") &&
       msg["method"] == "textDocument/publishDiagnostics") {

src/editor/renderer.cc

@@ -468,6 +468,12 @@ void render_editor(Editor *editor) {
     global_byte_offset += line_len + 1;
     line_index++;
   }
+  while (rendered_rows < editor->size.row) {
+    for (uint32_t col = 0; col < editor->size.col; col++)
+      update(editor->position.row + rendered_rows, editor->position.col + col,
+             " ", 0xFFFFFF, 0, 0);
+    rendered_rows++;
+  }
   if (cursor.row != UINT32_MAX && cursor.col != UINT32_MAX) {
     int type = 0;
     switch (mode) {
@@ -483,17 +489,13 @@ void render_editor(Editor *editor) {
       break;
     }
     set_cursor(cursor.row, cursor.col, type, true);
-    if (editor->hover_active)
+    if (editor->completion.active && !editor->completion.box.hidden)
+      editor->completion.box.render(cursor);
+    else if (editor->hover_active)
       editor->hover.render(cursor);
     else if (editor->diagnostics_active)
       editor->diagnostics.render(cursor);
   }
-  while (rendered_rows < editor->size.row) {
-    for (uint32_t col = 0; col < editor->size.col; col++)
-      update(editor->position.row + rendered_rows, editor->position.col + col,
-             " ", 0xFFFFFF, 0, 0);
-    rendered_rows++;
-  }
   free(it->buffer);
   free(it);
 }

src/io/renderer.cc

@@ -130,8 +130,8 @@ void render() {
     first_render = false;
   }
   for (uint32_t row = 0; row < rows; ++row) {
-    int first_change_col = -1;
+    int64_t first_change_col = -1;
-    int last_change_col = -1;
+    int64_t last_change_col = -1;
     for (uint32_t col = 0; col < cols; ++col) {
       uint32_t idx = row * cols + col;
       ScreenCell &old_cell = old_screen[idx];
@@ -144,7 +144,7 @@ void render() {
         if (first_change_col == -1) {
           first_change_col = col;
           if (first_change_col > 0) {
-            for (int back = 1; back <= 3 && first_change_col - back >= 0;
+            for (int64_t back = 1; back <= 4 && first_change_col - back >= 0;
                  ++back) {
               ScreenCell &prev_cell =
                   screen[row * cols + (first_change_col - back)];
@@ -161,17 +161,17 @@ void render() {
     if (first_change_col == -1)
       continue;
     char buf[64];
-    snprintf(buf, sizeof(buf), "\x1b[%d;%dH", row + 1, first_change_col + 1);
+    snprintf(buf, sizeof(buf), "\x1b[%d;%ldH", row + 1, first_change_col + 1);
     out.append(buf);
-    for (int col = first_change_col; col <= last_change_col; ++col) {
+    for (uint32_t col = first_change_col; col <= last_change_col; ++col) {
-      int idx = row * cols + col;
+      uint32_t idx = row * cols + col;
       ScreenCell &old_cell = old_screen[idx];
       ScreenCell &new_cell = screen[idx];
-      int width = new_cell.width > 0 ? new_cell.width : 1;
+      uint32_t width = new_cell.width > 0 ? new_cell.width : 1;
       bool overlap = false;
       if (width > 1) {
-        for (int i = 1; i < width; ++i) {
+        for (uint32_t i = 1; i < width; ++i) {
-          int next_col = col + i;
+          uint32_t next_col = col + i;
           if (next_col >= cols)
             break;
           const ScreenCell &next = screen[row * cols + next_col];
@@ -241,8 +241,16 @@ void render() {
         current_underline = underline;
       }
       if (width > 1 && overlap) {
-        for (int i = 1; i < width; ++i)
+        for (uint32_t i = 1; i < width; ++i) {
+          uint32_t next_col = col + i;
+          if (next_col >= cols)
+            break;
+          const ScreenCell &next = screen[row * cols + next_col];
+          if (!is_empty_cell(next))
+            break;
           out.push_back(' ');
+        }
+        out.push_back(' ');
       } else {
         if (!new_cell.utf8.empty()) {
           if (new_cell.utf8[0] == '\t')
@@ -282,10 +290,11 @@ void render() {
   }
 }
 
-void set_cursor(int row, int col, int type, bool show_cursor_param) {
+void set_cursor(uint8_t row, uint8_t col, uint32_t type,
+                bool show_cursor_param) {
   char buf[32];
-  int n = snprintf(buf, sizeof(buf), "\x1b[%d;%dH\x1b[%d q", row + 1, col + 1,
+  uint32_t n = snprintf(buf, sizeof(buf), "\x1b[%d;%dH\x1b[%d q", row + 1,
-                   type);
+                        col + 1, type);
   show_cursor = show_cursor_param;
   write(STDOUT_FILENO, buf, n);
 }

src/lsp/process.cc

@@ -72,12 +72,63 @@ std::shared_ptr<LSPInstance> get_or_init_lsp(uint8_t lsp_id) {
             lsp->incremental_sync = (change_type == 2);
           }
         }
-        if (caps.contains("hoverProvider"))
+        lsp->allow_formatting = caps.value("documentFormattingProvider", false);
-          lsp->allow_hover = caps["hoverProvider"].get<bool>();
+        if (caps.contains("documentOnTypeFormattingProvider")) {
-        else
+          auto &fmt = caps["documentOnTypeFormattingProvider"];
+          if (fmt.is_object()) {
+            if (fmt.contains("firstTriggerCharacter")) {
+              std::string s = fmt["firstTriggerCharacter"].get<std::string>();
+              if (s.size() == 1)
+                lsp->format_chars.push_back(s[0]);
+            }
+            if (fmt.contains("moreTriggerCharacter")) {
+              for (auto &c : fmt["moreTriggerCharacter"]) {
+                std::string s = c.get<std::string>();
+                if (s.size() == 1)
+                  lsp->format_chars.push_back(s[0]);
+              }
+            }
+            lsp->allow_formatting_on_type = true;
+          } else if (fmt.is_boolean()) {
+            lsp->allow_formatting_on_type = fmt.get<bool>();
+          }
+        }
+
+        if (caps.contains("hoverProvider")) {
+          auto &hover = caps["hoverProvider"];
+          lsp->allow_hover =
+              hover.is_boolean() ? hover.get<bool>() : hover.is_object();
+        } else {
           lsp->allow_hover = false;
-        if (caps.contains("completionProvider"))
+        }
+        if (caps.contains("completionProvider")) {
           lsp->allow_completion = true;
+          if (caps["completionProvider"].contains("resolveProvider"))
+            lsp->allow_resolve =
+                caps["completionProvider"]["resolveProvider"].get<bool>();
+          if (caps["completionProvider"].contains("triggerCharacters")) {
+            auto &chars = caps["completionProvider"]["triggerCharacters"];
+            if (chars.is_array()) {
+              for (auto &c : chars) {
+                std::string str = c.get<std::string>();
+                if (str.size() != 1)
+                  continue;
+                lsp->trigger_chars.push_back(str[0]);
+              }
+            }
+          }
+          if (caps["completionProvider"].contains("allCommitCharacters")) {
+            auto &chars = caps["completionProvider"]["allCommitCharacters"];
+            if (chars.is_array()) {
+              for (auto &c : chars) {
+                std::string str = c.get<std::string>();
+                if (str.size() != 1)
+                  continue;
+                lsp->end_chars.push_back(str[0]);
+              }
+            }
+          }
+        }
       }
       lsp->initialized = true;
       json initialized = {{"jsonrpc", "2.0"},

src/main.cc

@@ -24,6 +24,7 @@ void background_lsp() {
 }
 
 void input_listener() {
+
   while (running) {
     KeyEvent event = throttle(1ms, read_key);
     if (event.key_type == KEY_NONE)
@@ -130,3 +131,4 @@ int main(int argc, char *argv[]) {
   clear_regex_cache();
   return 0;
 }
+

src/ts/ts.cc

@@ -90,15 +90,17 @@ void ts_collect_spans(Editor *editor) {
     ts_query_cursor_exec(cursor, q, ts_tree_root_node(item.tsset->tree));
     std::unordered_map<std::string, PendingRanges> pending_injections;
     TSQueryMatch match;
+    auto subject_fn = [&](const TSNode *node, uint32_t *len) -> char * {
+      uint32_t start = ts_node_start_byte(*node);
+      uint32_t end = ts_node_end_byte(*node);
+      if (start == end || end > editor->root->char_count)
+        return nullptr;
+      std::shared_lock lock(editor->knot_mtx);
+      char *text = read(editor->root, start, end - start);
+      *len = end - start;
+      return text;
+    };
     while (ts_query_cursor_next_match(cursor, &match)) {
-      auto subject_fn = [&](const TSNode *node) -> std::string {
-        uint32_t start = ts_node_start_byte(*node);
-        uint32_t end = ts_node_end_byte(*node);
-        char *text = read(editor->root, start, end - start);
-        std::string final = std::string(text, end - start);
-        free(text);
-        return final;
-      };
       if (!ts_predicate(q, match, subject_fn))
         continue;
       for (uint32_t i = 0; i < match.capture_count; i++) {

src/ts/utils.cc

@@ -1,6 +1,7 @@
 #include "config.h"
 #include "io/sysio.h"
 #include "ts/ts.h"
+#include <cstdint>
 
 std::unordered_map<std::string, pcre2_code *> regex_cache;
 
@@ -114,8 +115,9 @@ TSQuery *load_query(const char *query_path, TSSetBase *set) {
   return q;
 }
 
-bool ts_predicate(TSQuery *query, const TSQueryMatch &match,
+bool ts_predicate(
-                  std::function<std::string(const TSNode *)> subject_fn) {
+    TSQuery *query, const TSQueryMatch &match,
+    std::function<char *(const TSNode *, uint32_t *len)> subject_fn) {
   uint32_t step_count;
   const TSQueryPredicateStep *steps =
       ts_query_predicates_for_pattern(query, match.pattern_index, &step_count);
@@ -149,11 +151,14 @@ bool ts_predicate(TSQuery *query, const TSQueryMatch &match,
   }
   const TSNode *node = find_capture_node(match, subject_id);
   pcre2_code *re = get_re(regex_txt);
-  std::string subject = subject_fn(node);
+  uint32_t len;
+  char *subject = subject_fn(node, &len);
+  if (!subject)
+    return false;
   pcre2_match_data *md = pcre2_match_data_create_from_pattern(re, nullptr);
-  int rc = pcre2_match(re, (PCRE2_SPTR)subject.c_str(), subject.size(), 0, 0,
+  int rc = pcre2_match(re, (PCRE2_SPTR)subject, len, 0, 0, md, nullptr);
-                       md, nullptr);
   pcre2_match_data_free(md);
   bool ok = (rc >= 0);
+  free(subject);
   return (command == "match?" ? ok : !ok);
 }

src/ui/bar.cc

@@ -7,6 +7,7 @@ void Bar::render() {
   uint32_t row = screen.row - 2;
   uint32_t col = 0;
   uint32_t width = screen.col;
+  UNUSED(width);
   uint32_t color = 0;
   uint32_t black = 0x0b0e14;
   uint32_t grey = 0x33363c;
@@ -80,7 +81,7 @@ void Bar::handle(KeyEvent event) {
         cursor--;
       break;
     case KEY_RIGHT:
-      if (cursor < command.length())
+      if (cursor < (uint32_t)command.length())
         cursor++;
       break;
     case KEY_UP:

src/ui/completionbox.cc

@@ -0,0 +1,172 @@
+#include "ui/completionbox.h"
+#include "editor/completions.h"
+#include "utils/utils.h"
+
+std::string item_kind_name(uint8_t kind) {
+  switch (kind) {
+  case 1:
+    return "Text";
+  case 2:
+    return "Method";
+  case 3:
+    return "Function";
+  case 4:
+    return "Constructor";
+  case 5:
+    return "Field";
+  case 6:
+    return "Variable";
+  case 7:
+    return "Class";
+  case 8:
+    return "Interface";
+  case 9:
+    return "Module";
+  case 10:
+    return "Property";
+  case 11:
+    return "Unit";
+  case 12:
+    return "Value";
+  case 13:
+    return "Enum";
+  case 14:
+    return "Keyword";
+  case 15:
+    return "Snippet";
+  case 16:
+    return "Color";
+  case 17:
+    return "File";
+  case 18:
+    return "Reference";
+  case 19:
+    return "Folder";
+  case 20:
+    return "EnumMember";
+  case 21:
+    return "Constant";
+  case 22:
+    return "Struct";
+  case 23:
+    return "Event";
+  case 24:
+    return "Operator";
+  case 25:
+    return "TypeParameter";
+  default:
+    return "Unknown";
+  }
+}
+
+const char *item_symbol(uint8_t kind) { return "●"; }
+
+uint32_t kind_color(uint8_t kind) { return 0x82AAFF; }
+
+void CompletionBox::render_update() {
+  if (hidden || session->visible.empty())
+    return;
+  std::unique_lock lock(mtx);
+  uint32_t max_label_len = 0;
+  uint32_t max_detail_len = 0;
+  uint32_t max_kind_len = 0;
+  for (auto i : session->visible) {
+    if (i >= session->items.size())
+      continue;
+    auto &item = session->items[i];
+    max_label_len = MAX(max_label_len, (uint32_t)item.label.size());
+    if (item.detail)
+      max_detail_len = MAX(max_detail_len, (uint32_t)item.detail->size());
+    max_kind_len =
+        MAX(max_kind_len, (uint32_t)item_kind_name(item.kind).size());
+  }
+  size.row = session->visible.size() + 2;
+  size.col = 2 + 2 + max_label_len + 1 + max_detail_len + 2 + max_kind_len + 1;
+  cells.assign(size.row * size.col, {" ", 0, 0, 0, 0, 0});
+  auto set = [&](uint32_t r, uint32_t c, const char *text, uint32_t fg,
+                 uint32_t bg, uint8_t flags) {
+    if (r < size.row && c < size.col)
+      cells[r * size.col + c] = {std::string(text), 0, fg, bg, flags, 0};
+  };
+  uint32_t border_fg = 0x82AAFF;
+  uint32_t sel_bg = 0x174225;
+  set(0, 0, "┌", border_fg, 0, 0);
+  for (uint32_t c = 1; c < size.col - 1; c++)
+    set(0, c, "─", border_fg, 0, 0);
+  set(0, size.col - 1, "┐", border_fg, 0, 0);
+  for (uint32_t row_idx = 0; row_idx < session->visible.size(); row_idx++) {
+    uint32_t r = row_idx + 1;
+    auto &item = session->items[session->visible[row_idx]];
+    uint32_t bg = (session->visible[row_idx] == session->select) ? sel_bg : 1;
+    uint32_t fg = 0xFFFFFF;
+    set(r, 0, "│", border_fg, 0, 0);
+    uint32_t c = 1;
+    const char *sym = item_symbol(item.kind);
+    set(r, c++, sym, kind_color(item.kind), bg, 0);
+    set(r, c++, " ", fg, bg, 0);
+    for (size_t i = 0; i < item.label.size(); i++)
+      set(r, c + i, (char[2]){item.label[i], 0}, fg, bg,
+          item.deprecated ? CF_STRIKETHROUGH : 0);
+    c += item.label.size();
+    set(r, c++, " ", fg, bg, 0);
+    uint32_t detail_fg = 0xAAAAAA;
+    if (item.detail) {
+      for (size_t i = 0; i < item.detail->size(); i++)
+        set(r, c + i, (char[2]){(*item.detail)[i], 0}, detail_fg, bg, 0);
+      c += item.detail->size();
+    }
+    uint32_t pad = size.col - 1 - c - max_kind_len;
+    for (uint32_t i = 0; i < pad; i++)
+      set(r, c + i, " ", fg, bg, 0);
+    c += pad;
+    std::string kind_name = item_kind_name(item.kind);
+    for (size_t i = 0; i < kind_name.size(); i++)
+      set(r, c + i, (char[2]){kind_name[i], 0}, kind_color(item.kind), bg, 0);
+    set(r, size.col - 1, "│", border_fg, 0, 0);
+  }
+  uint32_t bottom = size.row - 1;
+  set(bottom, 0, "└", border_fg, 0, 0);
+  for (uint32_t c = 1; c < size.col - 1; c++)
+    set(bottom, c, "─", border_fg, 0, 0);
+  set(bottom, size.col - 1, "┘", border_fg, 0, 0);
+}
+
+void CompletionBox::render(Coord pos) {
+  if (hidden || session->visible.empty())
+    return;
+  std::shared_lock lock(mtx);
+  int32_t start_row = (int32_t)pos.row - (int32_t)size.row;
+  if (start_row < 0)
+    start_row = pos.row + 1;
+  int32_t start_col = pos.col;
+  if (start_col + size.col > cols) {
+    start_col = cols - size.col;
+    if (start_col < 0)
+      start_col = 0;
+  }
+  position = {(uint32_t)start_row, (uint32_t)start_col};
+  for (uint32_t r = 0; r < size.row; r++)
+    for (uint32_t c = 0; c < size.col; c++)
+      update(start_row + r, start_col + c, cells[r * size.col + c].utf8,
+             cells[r * size.col + c].fg, cells[r * size.col + c].bg,
+             cells[r * size.col + c].flags);
+  if (session->items.size() > session->select &&
+      session->items[session->select].documentation &&
+      *session->items[session->select].documentation != "") {
+    if (session->doc != session->select) {
+      session->doc = session->select;
+      session->hover.clear();
+      session->hover.text = *session->items[session->select].documentation;
+      session->hover.is_markup = true;
+      session->hover.render_first();
+    } else {
+      if ((int32_t)position.col - (int32_t)session->hover.size.col > 0) {
+        session->hover.render({position.row + session->hover.size.row,
+                               position.col - session->hover.size.col});
+      } else {
+        session->hover.render(
+            {position.row + session->hover.size.row, position.col + size.col});
+      }
+    }
+  }
+}

src/ui/diagnostics.cc

@@ -3,9 +3,8 @@
 void DiagnosticBox::clear() {
   warnings.clear();
   cells.clear();
-  box_width = 0;
+  size = {0, 0};
-  box_height = 0;
+};
-}
 
 void DiagnosticBox::render_first() {
   if (warnings.empty())
@@ -18,11 +17,11 @@ void DiagnosticBox::render_first() {
       longest_line = MAX(longest_line, (uint32_t)see_also.length() + 4);
   }
   uint32_t content_width = MIN(longest_line, 150u);
-  box_width = content_width + 2;
+  size.col = content_width + 2;
-  cells.assign(box_width * 25, {" ", 0, 0, 0, 0, 0});
+  cells.assign(size.col * 25, {" ", 0, 0, 0, 0, 0});
   auto set = [&](uint32_t r, uint32_t c, const char *text, uint32_t fg,
                  uint32_t bg, uint8_t flags) {
-    cells[r * box_width + c] = {std::string(text), 0, fg, bg, flags, 0};
+    cells[r * size.col + c] = {std::string(text), 0, fg, bg, flags, 0};
   };
   uint32_t base_bg = 0;
   uint32_t border_fg = 0x82AAFF;
@@ -116,35 +115,35 @@ void DiagnosticBox::render_first() {
     };
     idx++;
   }
-  box_height = 2 + r;
+  size.row = 2 + r;
   set(0, 0, "┌", border_fg, base_bg, 0);
-  for (uint32_t i = 1; i < box_width - 1; i++)
+  for (uint32_t i = 1; i < size.col - 1; i++)
     set(0, i, "─", border_fg, base_bg, 0);
-  set(0, box_width - 1, "┐", border_fg, base_bg, 0);
+  set(0, size.col - 1, "┐", border_fg, base_bg, 0);
-  for (uint32_t r = 1; r < box_height - 1; r++) {
+  for (uint32_t r = 1; r < size.row - 1; r++) {
     set(r, 0, "│", border_fg, base_bg, 0);
-    set(r, box_width - 1, "│", border_fg, base_bg, 0);
+    set(r, size.col - 1, "│", border_fg, base_bg, 0);
   }
-  set(box_height - 1, 0, "└", border_fg, base_bg, 0);
+  set(size.row - 1, 0, "└", border_fg, base_bg, 0);
-  for (uint32_t i = 1; i < box_width - 1; i++)
+  for (uint32_t i = 1; i < size.col - 1; i++)
-    set(box_height - 1, i, "─", border_fg, base_bg, 0);
+    set(size.row - 1, i, "─", border_fg, base_bg, 0);
-  set(box_height - 1, box_width - 1, "┘", border_fg, base_bg, 0);
+  set(size.row - 1, size.col - 1, "┘", border_fg, base_bg, 0);
-  cells.resize(box_width * box_height);
+  cells.resize(size.col * size.row);
 }
 
 void DiagnosticBox::render(Coord pos) {
-  int32_t start_row = (int32_t)pos.row - (int32_t)box_height;
+  int32_t start_row = (int32_t)pos.row - (int32_t)size.row;
   if (start_row < 0)
     start_row = pos.row + 1;
   int32_t start_col = pos.col;
-  if (start_col + box_width > cols) {
+  if (start_col + size.col > cols) {
-    start_col = cols - box_width;
+    start_col = cols - size.col;
     if (start_col < 0)
       start_col = 0;
   }
-  for (uint32_t r = 0; r < box_height; r++)
+  for (uint32_t r = 0; r < size.row; r++)
-    for (uint32_t c = 0; c < box_width; c++)
+    for (uint32_t c = 0; c < size.col; c++)
-      update(start_row + r, start_col + c, cells[r * box_width + c].utf8,
+      update(start_row + r, start_col + c, cells[r * size.col + c].utf8,
-             cells[r * box_width + c].fg, cells[r * box_width + c].bg,
+             cells[r * size.col + c].fg, cells[r * size.col + c].bg,
-             cells[r * box_width + c].flags);
+             cells[r * size.col + c].flags);
 }

src/ui/hover.cc

@@ -5,8 +5,7 @@ void HoverBox::clear() {
   text = "";
   scroll_ = 0;
   is_markup = false;
-  box_width = 0;
+  size = {0, 0};
-  box_height = 0;
   cells.clear();
   highlights.clear();
   hover_spans.clear();
@@ -45,12 +44,13 @@ void HoverBox::render_first(bool scroll) {
       TSQueryCursor *cursor = ts_query_cursor_new();
       ts_query_cursor_exec(cursor, ts.query, ts_tree_root_node(ts.tree));
       TSQueryMatch match;
+      auto subject_fn = [&](const TSNode *node, uint32_t *len) -> char * {
+        uint32_t start = ts_node_start_byte(*node);
+        uint32_t end = ts_node_end_byte(*node);
+        *len = end - start;
+        return text.data() + start;
+      };
       while (ts_query_cursor_next_match(cursor, &match)) {
-        auto subject_fn = [&](const TSNode *node) -> std::string {
-          uint32_t start = ts_node_start_byte(*node);
-          uint32_t end = ts_node_end_byte(*node);
-          return text.substr(start, end - start);
-        };
         if (!ts_predicate(ts.query, match, subject_fn))
           continue;
         for (uint32_t i = 0; i < match.capture_count; i++) {
@@ -76,11 +76,6 @@ void HoverBox::render_first(bool scroll) {
                                  ts_tree_root_node(inj_ts.tree));
             TSQueryMatch inj_match;
             while (ts_query_cursor_next_match(inj_cursor, &inj_match)) {
-              auto subject_fn = [&](const TSNode *node) -> std::string {
-                uint32_t start = ts_node_start_byte(*node);
-                uint32_t end = ts_node_end_byte(*node);
-                return text.substr(start, end - start);
-              };
               if (!ts_predicate(inj_ts.query, inj_match, subject_fn))
                 continue;
               for (uint32_t i = 0; i < inj_match.capture_count; i++) {
@@ -139,7 +134,7 @@ void HoverBox::render_first(bool scroll) {
   //       in the loop instead as it was never meant to wrap in the first place
   longest_line = MAX(longest_line, current_width) + 1;
   uint32_t content_width = MIN(longest_line, 130u);
-  box_width = content_width + 2;
+  size.col = content_width + 2;
   size_t i = 0;
   size_t lines_skipped = 0;
   while (i < text.length() && lines_skipped < scroll_) {
@@ -153,10 +148,10 @@ void HoverBox::render_first(bool scroll) {
   uint32_t base_bg = 0;
   SpanCursor span_cursor(spans);
   span_cursor.sync(i);
-  cells.assign(box_width * 26, ScreenCell{" ", 0, 0, 0, 0, 0});
+  cells.assign(size.col * 26, ScreenCell{" ", 0, 0, 0, 0, 0});
   auto set = [&](uint32_t r, uint32_t c, const char *text, uint32_t fg,
                  uint32_t bg, uint8_t flags) {
-    cells[r * box_width + c] = {std::string(text), 0, fg, bg, flags, 0};
+    cells[r * size.col + c] = {std::string(text), 0, fg, bg, flags, 0};
   };
   uint32_t r = 0;
   while (i < text.length() && r < 24) {
@@ -186,35 +181,35 @@ void HoverBox::render_first(bool scroll) {
     r++;
   }
   if (!scroll)
-    box_height = r + 2;
+    size.row = r + 2;
   set(0, 0, "┌", border_fg, base_bg, 0);
-  for (uint32_t i = 1; i < box_width - 1; i++)
+  for (uint32_t i = 1; i < size.col - 1; i++)
     set(0, i, "─", border_fg, base_bg, 0);
-  set(0, box_width - 1, "┐", border_fg, base_bg, 0);
+  set(0, size.col - 1, "┐", border_fg, base_bg, 0);
-  for (uint32_t r = 1; r < box_height - 1; r++) {
+  for (uint32_t r = 1; r < size.row - 1; r++) {
     set(r, 0, "│", border_fg, base_bg, 0);
-    set(r, box_width - 1, "│", border_fg, base_bg, 0);
+    set(r, size.col - 1, "│", border_fg, base_bg, 0);
   }
-  set(box_height - 1, 0, "└", border_fg, base_bg, 0);
+  set(size.row - 1, 0, "└", border_fg, base_bg, 0);
-  for (uint32_t i = 1; i < box_width - 1; i++)
+  for (uint32_t i = 1; i < size.col - 1; i++)
-    set(box_height - 1, i, "─", border_fg, base_bg, 0);
+    set(size.row - 1, i, "─", border_fg, base_bg, 0);
-  set(box_height - 1, box_width - 1, "┘", border_fg, base_bg, 0);
+  set(size.row - 1, size.col - 1, "┘", border_fg, base_bg, 0);
-  cells.resize(box_width * box_height);
+  cells.resize(size.col * size.row);
 }
 
 void HoverBox::render(Coord pos) {
-  int32_t start_row = (int32_t)pos.row - (int32_t)box_height;
+  int32_t start_row = (int32_t)pos.row - (int32_t)size.row;
   if (start_row < 0)
     start_row = pos.row + 1;
   int32_t start_col = pos.col;
-  if (start_col + box_width > cols) {
+  if (start_col + size.col > cols) {
-    start_col = cols - box_width;
+    start_col = cols - size.col;
     if (start_col < 0)
       start_col = 0;
   }
-  for (uint32_t r = 0; r < box_height; r++)
+  for (uint32_t r = 0; r < size.row; r++)
-    for (uint32_t c = 0; c < box_width; c++)
+    for (uint32_t c = 0; c < size.col; c++)
-      update(start_row + r, start_col + c, cells[r * box_width + c].utf8,
+      update(start_row + r, start_col + c, cells[r * size.col + c].utf8,
-             cells[r * box_width + c].fg, cells[r * box_width + c].bg,
+             cells[r * size.col + c].fg, cells[r * size.col + c].bg,
-             cells[r * box_width + c].flags);
+             cells[r * size.col + c].flags);
 }

src/utils/system.cc

@@ -120,7 +120,7 @@ Language language_for_file(const char *filename) {
     if (it != kMimeToLang.end())
       return kLanguages.find(it->second)->second;
   }
-  return {"unknown", nullptr};
+  return Language{};
 }
 
 char *get_from_clipboard(uint32_t *out_len) {

src/utils/unicode.cc

@@ -107,3 +107,25 @@ int utf8_byte_offset_to_utf16(const char *s, size_t byte_pos) {
   }
   return utf16_units;
 }
+
+size_t utf16_offset_to_utf8(const char *s, int utf16_pos) {
+  int utf16_units = 0;
+  size_t i = 0;
+  while (utf16_units < utf16_pos) {
+    unsigned char c = s[i];
+    if ((c & 0x80) == 0x00) {
+      i += 1;
+      utf16_units += 1;
+    } else if ((c & 0xE0) == 0xC0) {
+      i += 2;
+      utf16_units += 1;
+    } else if ((c & 0xF0) == 0xE0) {
+      i += 3;
+      utf16_units += 1;
+    } else {
+      i += 4;
+      utf16_units += 2;
+    }
+  }
+  return i;
+}