Fix bugs with hl line data structures

TODO.md

@@ -34,9 +34,13 @@ Copyright 2025 Syed Daanish
 * make it faster for line inserts/deletes too (treeify the vector)
 * Try to make all functions better now that folds have been purged
 * Cleanup syntax and renderer files
-* Fix ruby regexp not living across lines when edits are made
 
-* for ruby regex use hueristic where is a space is seen after the / it is not a regexp
+* finish bash then do all the directive-like ones like jsonc (first to help with theme files) / toml / yaml / ini / nginx
+* then markdown / html
+* then gitignore / gitattributes
+* then fish then sql then css and [jt]sx? then python then lua (make with type annotations for lsp results)
+* then [ch](++)? then gdscript then erb then php
+* then haskell then gomod then go then rust
 
 * [ ] **Undo/Redo:** Add support for undo/redo history.
 

include/editor/editor.h

@@ -5,6 +5,7 @@
 #include "editor/indents.h"
 #include "io/knot.h"
 #include "io/sysio.h"
+#include "syntax/extras.h"
 #include "syntax/parser.h"
 #include "ui/completionbox.h"
 #include "ui/diagnostics.h"
@@ -50,6 +51,8 @@ struct Editor {
   CompletionSession completion;
   IndentationEngine indents;
   Parser *parser;
+  ExtraHighlighter extra_hl;
+  bool is_css_color;
 };
 
 Editor *new_editor(const char *filename_arg, Coord position, Coord size);

include/syntax/decl.h

@@ -3,69 +3,7 @@
 
 #include "io/knot.h"
 #include "io/sysio.h"
-
+#include "syntax/trie.h"
-struct Trie {
-  struct TrieNode {
-    bool is_word = false;
-    std::array<TrieNode *, 128> children{};
-    TrieNode() { children.fill(nullptr); }
-  };
-
-  Trie() : root(new TrieNode()) {}
-  ~Trie() { clear_trie(root); }
-
-  void build(const std::vector<std::string> &words) {
-    for (const auto &word : words) {
-      TrieNode *node = root;
-      for (char c : word) {
-        unsigned char uc = static_cast<unsigned char>(c);
-        if (!node->children[uc])
-          node->children[uc] = new TrieNode();
-        node = node->children[uc];
-      }
-      node->is_word = true;
-    }
-  }
-
-  uint32_t match(const char *text, uint32_t pos, uint32_t len,
-                 bool (*is_word_char)(char c)) const {
-    const TrieNode *node = root;
-    uint32_t max_len = 0;
-    for (uint32_t i = pos; i < len; ++i) {
-      unsigned char uc = static_cast<unsigned char>(text[i]);
-      if (uc >= 128)
-        return 0;
-      if (!node->children[uc]) {
-        if (node->is_word && !is_word_char(text[i]))
-          return i - pos;
-        break;
-      }
-      node = node->children[uc];
-      if (node->is_word)
-        max_len = i - pos + 1;
-    }
-    if (max_len > 0)
-      if (pos + max_len < len && is_word_char(text[pos + max_len]))
-        return 0;
-    return max_len;
-  }
-
-  void clear() {
-    clear_trie(root);
-    root = new TrieNode();
-  }
-
-private:
-  TrieNode *root;
-
-  void clear_trie(TrieNode *node) {
-    if (!node)
-      return;
-    for (auto *child : node->children)
-      clear_trie(child);
-    delete node;
-  }
-};
 
 struct Highlight {
   uint32_t fg;

include/syntax/extras.h

@@ -0,0 +1,490 @@
+#ifndef SYNTAX_EXTRAS_H
+#define SYNTAX_EXTRAS_H
+
+#include "io/knot.h"
+#include "syntax/decl.h"
+#include "utils/utils.h"
+
+inline static const std::vector<std::pair<std::string, uint32_t>> color_map = {
+    {"AliceBlue", 0xF0F8FF},
+    {"AntiqueWhite", 0xFAEBD7},
+    {"Aqua", 0x00FFFF},
+    {"Aquamarine", 0x7FFFD4},
+    {"Azure", 0xF0FFFF},
+    {"Beige", 0xF5F5DC},
+    {"Bisque", 0xFFE4C4},
+    {"Black", 0x000000},
+    {"BlanchedAlmond", 0xFFEBCD},
+    {"Blue", 0x0000FF},
+    {"BlueViolet", 0x8A2BE2},
+    {"Brown", 0xA52A2A},
+    {"BurlyWood", 0xDEB887},
+    {"CadetBlue", 0x5F9EA0},
+    {"Chartreuse", 0x7FFF00},
+    {"Chocolate", 0xD2691E},
+    {"Coral", 0xFF7F50},
+    {"CornflowerBlue", 0x6495ED},
+    {"Cornsilk", 0xFFF8DC},
+    {"Crimson", 0xDC143C},
+    {"Cyan", 0x00FFFF},
+    {"DarkBlue", 0x00008B},
+    {"DarkCyan", 0x008B8B},
+    {"DarkGoldenRod", 0xB8860B},
+    {"DarkGray", 0xA9A9A9},
+    {"DarkGrey", 0xA9A9A9},
+    {"DarkGreen", 0x006400},
+    {"DarkKhaki", 0xBDB76B},
+    {"DarkMagenta", 0x8B008B},
+    {"DarkOliveGreen", 0x556B2F},
+    {"DarkOrange", 0xFF8C00},
+    {"DarkOrchid", 0x9932CC},
+    {"DarkRed", 0x8B0000},
+    {"DarkSalmon", 0xE9967A},
+    {"DarkSeaGreen", 0x8FBC8F},
+    {"DarkSlateBlue", 0x483D8B},
+    {"DarkSlateGray", 0x2F4F4F},
+    {"DarkSlateGrey", 0x2F4F4F},
+    {"DarkTurquoise", 0x00CED1},
+    {"DarkViolet", 0x9400D3},
+    {"DeepPink", 0xFF1493},
+    {"DeepSkyBlue", 0x00BFFF},
+    {"DimGray", 0x696969},
+    {"DimGrey", 0x696969},
+    {"DodgerBlue", 0x1E90FF},
+    {"FireBrick", 0xB22222},
+    {"FloralWhite", 0xFFFAF0},
+    {"ForestGreen", 0x228B22},
+    {"Fuchsia", 0xFF00FF},
+    {"Gainsboro", 0xDCDCDC},
+    {"GhostWhite", 0xF8F8FF},
+    {"Gold", 0xFFD700},
+    {"GoldenRod", 0xDAA520},
+    {"Gray", 0x808080},
+    {"Grey", 0x808080},
+    {"Green", 0x008000},
+    {"GreenYellow", 0xADFF2F},
+    {"HoneyDew", 0xF0FFF0},
+    {"HotPink", 0xFF69B4},
+    {"IndianRed", 0xCD5C5C},
+    {"Indigo", 0x4B0082},
+    {"Ivory", 0xFFFFF0},
+    {"Khaki", 0xF0E68C},
+    {"Lavender", 0xE6E6FA},
+    {"LavenderBlush", 0xFFF0F5},
+    {"LawnGreen", 0x7CFC00},
+    {"LemonChiffon", 0xFFFACD},
+    {"LightBlue", 0xADD8E6},
+    {"LightCoral", 0xF08080},
+    {"LightCyan", 0xE0FFFF},
+    {"LightGoldenRodYellow", 0xFAFAD2},
+    {"LightGray", 0xD3D3D3},
+    {"LightGrey", 0xD3D3D3},
+    {"LightGreen", 0x90EE90},
+    {"LightPink", 0xFFB6C1},
+    {"LightSalmon", 0xFFA07A},
+    {"LightSeaGreen", 0x20B2AA},
+    {"LightSkyBlue", 0x87CEFA},
+    {"LightSlateGray", 0x778899},
+    {"LightSlateGrey", 0x778899},
+    {"LightSteelBlue", 0xB0C4DE},
+    {"LightYellow", 0xFFFFE0},
+    {"Lime", 0x00FF00},
+    {"LimeGreen", 0x32CD32},
+    {"Linen", 0xFAF0E6},
+    {"Magenta", 0xFF00FF},
+    {"Maroon", 0x800000},
+    {"MediumAquaMarine", 0x66CDAA},
+    {"MediumBlue", 0x0000CD},
+    {"MediumOrchid", 0xBA55D3},
+    {"MediumPurple", 0x9370DB},
+    {"MediumSeaGreen", 0x3CB371},
+    {"MediumSlateBlue", 0x7B68EE},
+    {"MediumSpringGreen", 0x00FA9A},
+    {"MediumTurquoise", 0x48D1CC},
+    {"MediumVioletRed", 0xC71585},
+    {"MidnightBlue", 0x191970},
+    {"MintCream", 0xF5FFFA},
+    {"MistyRose", 0xFFE4E1},
+    {"Moccasin", 0xFFE4B5},
+    {"NavajoWhite", 0xFFDEAD},
+    {"Navy", 0x000080},
+    {"OldLace", 0xFDF5E6},
+    {"Olive", 0x808000},
+    {"OliveDrab", 0x6B8E23},
+    {"Orange", 0xFFA500},
+    {"OrangeRed", 0xFF4500},
+    {"Orchid", 0xDA70D6},
+    {"PaleGoldenRod", 0xEEE8AA},
+    {"PaleGreen", 0x98FB98},
+    {"PaleTurquoise", 0xAFEEEE},
+    {"PaleVioletRed", 0xDB7093},
+    {"PapayaWhip", 0xFFEFD5},
+    {"PeachPuff", 0xFFDAB9},
+    {"Peru", 0xCD853F},
+    {"Pink", 0xFFC0CB},
+    {"Plum", 0xDDA0DD},
+    {"PowderBlue", 0xB0E0E6},
+    {"Purple", 0x800080},
+    {"RebeccaPurple", 0x663399},
+    {"Red", 0xFF0000},
+    {"RosyBrown", 0xBC8F8F},
+    {"RoyalBlue", 0x4169E1},
+    {"SaddleBrown", 0x8B4513},
+    {"Salmon", 0xFA8072},
+    {"SandyBrown", 0xF4A460},
+    {"SeaGreen", 0x2E8B57},
+    {"SeaShell", 0xFFF5EE},
+    {"Sienna", 0xA0522D},
+    {"Silver", 0xC0C0C0},
+    {"SkyBlue", 0x87CEEB},
+    {"SlateBlue", 0x6A5ACD},
+    {"SlateGray", 0x708090},
+    {"SlateGrey", 0x708090},
+    {"Snow", 0xFFFAFA},
+    {"SpringGreen", 0x00FF7F},
+    {"SteelBlue", 0x4682B4},
+    {"Tan", 0xD2B48C},
+    {"Teal", 0x008080},
+    {"Thistle", 0xD8BFD8},
+    {"Tomato", 0xFF6347},
+    {"Turquoise", 0x40E0D0},
+    {"Violet", 0xEE82EE},
+    {"Wheat", 0xF5DEB3},
+    {"White", 0xFFFFFF},
+    {"WhiteSmoke", 0xF5F5F5},
+    {"Yellow", 0xFFFF00},
+    {"YellowGreen", 0x9ACD32},
+};
+
+// Add word under cursor to this
+
+struct ExtraHighlighter {
+  std::vector<uint32_t> colors;
+  std::array<std::vector<uint32_t>, 50> lines;
+  Trie<uint32_t> css_colors = Trie<uint32_t>();
+  uint32_t start = 0;
+
+  ExtraHighlighter() { css_colors.build(color_map, false); }
+
+  void render(Knot *root, uint32_t n_start, std::string word, bool is_css) {
+    start = n_start;
+    for (auto &line : lines)
+      line.clear();
+    LineIterator *it = begin_l_iter(root, start);
+    if (!it)
+      return;
+    uint32_t idx = 0;
+    uint32_t len;
+    char *line;
+    while (idx < 50 && (line = next_line(it, &len))) {
+      lines[idx].assign(len, UINT32_MAX - 1);
+      uint32_t i = 0;
+      while (i < len) {
+        if (is_css) {
+          std::optional<uint32_t> color;
+          uint32_t color_len = css_colors.match(
+              line, i, len, [](char c) { return isalnum(c) || c == '_'; },
+              &color);
+          if (color) {
+            for (uint32_t j = 0; j < color_len; j++)
+              lines[idx][i + j] = *color;
+            i += color_len;
+            continue;
+          } else if (i + 5 < len && (line[i] == 'r' || line[i] == 'R') &&
+                     (line[i + 1] == 'g' || line[i + 1] == 'G') &&
+                     (line[i + 2] == 'b' || line[i + 2] == 'B')) {
+            uint32_t start = i;
+            i += 3;
+            if (line[i] == 'a' || line[i] == 'A')
+              i++;
+            if (line[i] == '(') {
+              i++;
+              bool is_percent = false;
+              std::string r = "";
+              while (i < len && line[i] >= '0' && line[i] <= '9')
+                r += line[i++];
+              if (r.empty())
+                continue;
+              while (i < len &&
+                     (line[i] == '.' || (line[i] >= '0' && line[i] <= '9')))
+                i++;
+              if (line[i] == '%') {
+                is_percent = true;
+                i++;
+              }
+              while (i < len && (line[i] == ',' || line[i] == ' '))
+                i++;
+              std::string g = "";
+              while (i < len && line[i] >= '0' && line[i] <= '9')
+                g += line[i++];
+              if (g.empty())
+                continue;
+              while (i < len &&
+                     (line[i] == '.' || (line[i] >= '0' && line[i] <= '9')))
+                i++;
+              while (i < len &&
+                     (line[i] == ',' || line[i] == ' ' || line[i] == '%'))
+                i++;
+              std::string b = "";
+              while (i < len && line[i] >= '0' && line[i] <= '9')
+                b += line[i++];
+              if (b.empty())
+                continue;
+              while (i < len &&
+                     (line[i] == ',' || line[i] == ' ' || line[i] == '.' ||
+                      line[i] == '/' || line[i] == '%' ||
+                      (line[i] >= '0' && line[i] <= '9')))
+                i++;
+              if (i < len && line[i] == ')')
+                i++;
+              else
+                continue;
+              uint32_t rr, gg, bb;
+              if (is_percent) {
+                rr = std::stoul(r) * 255 / 100;
+                gg = std::stoul(g) * 255 / 100;
+                bb = std::stoul(b) * 255 / 100;
+              } else {
+                rr = std::stoul(r);
+                gg = std::stoul(g);
+                bb = std::stoul(b);
+              }
+              rr = rr > 255 ? 255 : rr;
+              gg = gg > 255 ? 255 : gg;
+              bb = bb > 255 ? 255 : bb;
+              uint32_t color = (rr << 16) | (gg << 8) | bb;
+              for (uint32_t j = start; j < i; j++)
+                lines[idx][j] = color;
+            }
+            continue;
+          } else if (i + 5 < len && (line[i] == 'h' || line[i] == 'H') &&
+                     (line[i + 1] == 's' || line[i + 1] == 'S') &&
+                     (line[i + 2] == 'l' || line[i + 2] == 'L')) {
+            uint32_t start = i;
+            i += 3;
+            if (line[i] == 'a' || line[i] == 'A')
+              i++;
+            if (line[i] == '(') {
+              i++;
+              std::string h = "";
+              std::string h_unit = "";
+              enum unit : uint8_t { deg, grad, rad, turn };
+              unit u = deg;
+              bool negative = false;
+              if (i < len && (line[i] == '-' || line[i] == '+')) {
+                negative = line[i] == '-';
+                i++;
+              }
+              while (i < len && line[i] >= '0' && line[i] <= '9')
+                h += line[i++];
+              if (i < len && line[i] == '.') {
+                h += '.';
+                while (i < len && line[i] >= '0' && line[i] <= '9')
+                  h += line[i++];
+              }
+              if (h.empty())
+                continue;
+              while (i < len && ((line[i] >= 'a' && line[i] <= 'z') ||
+                                 (line[i] >= 'A' && line[i] <= 'Z')))
+                h_unit += line[i++];
+              for (size_t x = 0; x < h_unit.size(); x++)
+                h_unit[x] = tolower(h_unit[x]);
+              if (h_unit.empty())
+                u = deg;
+              else if (h_unit == "deg")
+                u = deg;
+              else if (h_unit == "grad")
+                u = grad;
+              else if (h_unit == "rad")
+                u = rad;
+              else if (h_unit == "turn")
+                u = turn;
+              else
+                continue;
+              double hue = std::stod(h);
+              if (negative)
+                hue = -hue;
+              switch (u) {
+              case deg:
+                break;
+              case grad:
+                hue = hue * 360.0 / 400.0;
+                break;
+              case rad:
+                hue = hue * 180.0 / M_PI;
+                break;
+              case turn:
+                hue = hue * 360.0;
+                break;
+              }
+              hue = fmod(hue, 360.0);
+              if (hue < 0)
+                hue += 360.0;
+              double h_final = hue / 360.0;
+              while (i < len && (line[i] == ',' || line[i] == ' '))
+                i++;
+              std::string s = "";
+              while (i < len && line[i] >= '0' && line[i] <= '9')
+                s += line[i++];
+              if (s.empty())
+                continue;
+              if (i < len && line[i] == '%')
+                i++;
+              else
+                continue;
+              while (i < len && (line[i] == ',' || line[i] == ' '))
+                i++;
+              std::string l = "";
+              while (i < len && line[i] >= '0' && line[i] <= '9')
+                l += line[i++];
+              if (l.empty())
+                continue;
+              if (i < len && line[i] == '%')
+                i++;
+              else
+                continue;
+              while (i < len &&
+                     (line[i] == ',' || line[i] == ' ' || line[i] == '.' ||
+                      line[i] == '/' || line[i] == '%' ||
+                      (line[i] >= '0' && line[i] <= '9')))
+                i++;
+              if (i < len && line[i] == ')')
+                i++;
+              double s_val = std::stod(s) / 100.0;
+              double l_val = std::stod(l) / 100.0;
+              uint32_t color = hslToRgb(h_final, s_val, l_val);
+              for (uint32_t j = start; j < i; j++)
+                lines[idx][j] = color;
+            }
+            continue;
+          }
+        }
+        if (i + 4 < len && line[i] == '#') {
+          i++;
+          uint32_t start = i;
+          while (i < len && isxdigit(line[i]))
+            i++;
+          uint32_t color = 0;
+          if (is_css && (i - start == 3 || i - start == 4)) {
+            uint32_t r =
+                std::stoul(std::string(line + start, 1), nullptr, 16) * 0x11;
+            uint32_t g =
+                std::stoul(std::string(line + start + 1, 1), nullptr, 16) *
+                0x11;
+            uint32_t b =
+                std::stoul(std::string(line + start + 2, 1), nullptr, 16) *
+                0x11;
+            color = (r << 16) | (g << 8) | b;
+          } else if ((is_css && (i - start == 8)) || i - start == 6) {
+            uint32_t r = std::stoul(std::string(line + start, 2), nullptr, 16);
+            uint32_t g =
+                std::stoul(std::string(line + start + 2, 2), nullptr, 16);
+            uint32_t b =
+                std::stoul(std::string(line + start + 4, 2), nullptr, 16);
+            color = (r << 16) | (g << 8) | b;
+          } else {
+            continue;
+          }
+          for (uint32_t j = start - 1; j < i; j++)
+            lines[idx][j] = color;
+          continue;
+        } else if (i + 5 < len && line[i] == '0' && line[i + 1] == 'x') {
+          i += 2;
+          uint32_t start = i;
+          while (i < len && isxdigit(line[i]))
+            i++;
+          uint32_t color = 0;
+          if (i - start == 6) {
+            uint32_t r = std::stoul(std::string(line + start, 2), nullptr, 16);
+            uint32_t g =
+                std::stoul(std::string(line + start + 2, 2), nullptr, 16);
+            uint32_t b =
+                std::stoul(std::string(line + start + 4, 2), nullptr, 16);
+            color = (r << 16) | (g << 8) | b;
+          } else {
+            continue;
+          }
+          if (color)
+            color--;
+          else
+            color++;
+          for (uint32_t j = start - 2; j < i; j++)
+            lines[idx][j] = color;
+          continue;
+        }
+        if (i < len && (isalnum(line[i]) || line[i] == '_')) {
+          uint32_t start = i;
+          uint32_t x = 0;
+          bool found = true;
+          while (i < len && (isalnum(line[i]) || line[i] == '_')) {
+            if (x < word.size() && line[i] == word[x]) {
+              i++;
+              x++;
+            } else {
+              found = false;
+              i++;
+            }
+          }
+          if (found && x == word.size())
+            for (uint32_t j = start; j < i; j++)
+              lines[idx][j] = UINT32_MAX;
+        } else {
+          i += utf8_codepoint_width(line[i]);
+        }
+      }
+      idx++;
+    }
+    free(it->buffer);
+    free(it);
+  }
+
+  std::optional<std::pair<uint32_t, uint32_t>> get(Coord pos) {
+    uint32_t val;
+    if (pos.row < start || pos.row >= start + 50 ||
+        pos.col >= lines[pos.row - start].size() ||
+        (val = lines[pos.row - start][pos.col]) == UINT32_MAX - 1)
+      return std::nullopt;
+    return (std::pair<uint32_t, uint32_t>){fg_for_bg(val), val};
+  }
+
+private:
+  uint32_t fg_for_bg(uint32_t color) {
+    uint8_t r = (color >> 16) & 0xFF;
+    uint8_t g = (color >> 8) & 0xFF;
+    uint8_t b = color & 0xFF;
+    double luminance = 0.299 * r + 0.587 * g + 0.114 * b;
+    return (luminance > 128) ? 0x000000 : 0xFFFFFF;
+  }
+
+  uint32_t hslToRgb(double h, double s, double l) {
+    double r, g, b;
+    if (s == 0.0) {
+      r = g = b = l;
+    } else {
+      auto hue2rgb = [](double p, double q, double t) -> double {
+        if (t < 0)
+          t += 1;
+        if (t > 1)
+          t -= 1;
+        if (t < 1.0 / 6)
+          return p + (q - p) * 6 * t;
+        if (t < 1.0 / 2)
+          return q;
+        if (t < 2.0 / 3)
+          return p + (q - p) * (2.0 / 3 - t) * 6;
+        return p;
+      };
+      double q = l < 0.5 ? l * (1 + s) : l + s - l * s;
+      double p = 2 * l - q;
+      r = hue2rgb(p, q, h + 1.0 / 3);
+      g = hue2rgb(p, q, h);
+      b = hue2rgb(p, q, h - 1.0 / 3);
+    }
+    uint32_t R = static_cast<uint32_t>(std::clamp(r, 0.0, 1.0) * 255);
+    uint32_t G = static_cast<uint32_t>(std::clamp(g, 0.0, 1.0) * 255);
+    uint32_t B = static_cast<uint32_t>(std::clamp(b, 0.0, 1.0) * 255);
+    return (R << 16) | (G << 8) | B;
+  }
+};
+
+#endif

include/syntax/line_tree.h

@@ -74,10 +74,14 @@ struct LineTree {
   }
   void insert(uint32_t x, uint32_t y) {
     std::unique_lock lock(mtx);
+    if (x > subtree_size(root))
+      x = subtree_size(root);
     root = insert_node(root, x, y);
   }
   void erase(uint32_t x, uint32_t y) {
     std::unique_lock lock(mtx);
+    if (x + y > subtree_size(root))
+      x = subtree_size(root) - y;
     root = erase_node(root, x, y);
   }
   uint32_t count() {
@@ -116,18 +120,26 @@ private:
   std::shared_mutex mtx;
   static constexpr uint32_t LEAF_TARGET = 256;
   LineTree::LineNode *erase_node(LineNode *n, uint32_t x, uint32_t y) {
-    if (!n)
+    if (!n || y == 0)
-      return nullptr;
-    if (!n->left && !n->right) {
-      n->data.erase(n->data.begin() + x, n->data.begin() + x + y);
-      fix(n);
       return n;
+    uint32_t left_sz = subtree_size(n->left);
+    uint32_t mid_sz = n->data.size();
+    if (x < left_sz) {
+      uint32_t len = std::min(y, left_sz - x);
+      n->left = erase_node(n->left, x, len);
+      y -= len;
+      x = left_sz;
+    }
+    if (y > 0 && x < left_sz + mid_sz) {
+      uint32_t mid_x = x - left_sz;
+      uint32_t len = std::min(y, mid_sz - mid_x);
+      n->data.erase(n->data.begin() + mid_x, n->data.begin() + mid_x + len);
+      y -= len;
+      x += len;
+    }
+    if (y > 0) {
+      n->right = erase_node(n->right, x - left_sz - n->data.size(), y);
     }
-    uint32_t left_size = subtree_size(n->left);
-    if (x < left_size)
-      n->left = erase_node(n->left, x, y);
-    else
-      n->right = erase_node(n->right, x - left_size - n->data.size(), y);
     if (n->left && n->right &&
         subtree_size(n->left) + subtree_size(n->right) < 256) {
       return merge(n->left, n->right);
@@ -142,7 +154,7 @@ private:
       return leaf;
     }
     if (!n->left && !n->right) {
-      n->data.insert(n->data.begin() + x, y, LineData{});
+      n->data.insert(n->data.begin() + x, y, LineData());
       fix(n);
       if (n->data.size() > 512)
         return split_leaf(n);

include/syntax/parser.h

@@ -5,8 +5,7 @@
 #include "syntax/line_tree.h"
 
 struct Parser {
-  Knot *root;
+  struct Editor *editor = nullptr;
-  std::shared_mutex *knot_mutex;
   std::string lang;
   std::shared_ptr<void> (*parse_func)(std::vector<Token> *tokens,
                                       std::shared_ptr<void> in_state,
@@ -19,10 +18,8 @@ struct Parser {
   LineTree line_tree;
   std::set<uint32_t> dirty_lines;
 
-  Parser(Knot *n_root, std::shared_mutex *n_knot_mutex, std::string n_lang,
+  Parser(Editor *editor, std::string n_lang, uint32_t n_scroll_max);
-         uint32_t n_scroll_max);
+  void edit(uint32_t start_line, uint32_t old_end_line, uint32_t inserted_rows);
-  void edit(Knot *n_root, uint32_t start_line, uint32_t old_end_line,
-            uint32_t new_end_line);
   void work();
   void scroll(uint32_t line);
 };

include/syntax/trie.h

@@ -0,0 +1,140 @@
+#ifndef SYNTAX_TRIE_H
+#define SYNTAX_TRIE_H
+
+#include "utils/utils.h"
+
+template <typename T> struct Trie {
+  struct TrieNode {
+    bool is_word = false;
+    std::array<TrieNode *, 128> children{};
+    std::conditional_t<std::is_void_v<T>, char, std::optional<T>> value;
+    TrieNode() { children.fill(nullptr); }
+  };
+
+  Trie() {}
+  ~Trie() { clear_trie(root); }
+
+  void build(const std::vector<std::string> &words, bool cs = true) {
+    static_assert(std::is_void_v<T>, "This build() is for Trie<void> only");
+    case_sensitive = cs;
+    for (auto &w : words)
+      insert(w);
+  }
+
+  template <typename U = T>
+  std::enable_if_t<!std::is_void_v<U>>
+  build(const std::vector<std::pair<std::string, U>> &words, bool cs = true) {
+    static_assert(!std::is_void_v<T>, "This build() is for typed Trie only");
+    case_sensitive = cs;
+    for (auto &[w, v] : words)
+      insert(w, v);
+  }
+
+  uint32_t match(const char *text, uint32_t pos, uint32_t len,
+                 bool (*is_word_char)(char c)) const {
+    const TrieNode *node = root;
+    uint32_t max_len = 0;
+    for (uint32_t i = pos; i < len; ++i) {
+      unsigned char uc = static_cast<unsigned char>(text[i]);
+      if (uc >= 128)
+        return 0;
+      if (!case_sensitive && uc >= 'A' && uc <= 'Z')
+        uc = uc - 'A' + 'a';
+      if (!node->children[uc]) {
+        if (node->is_word && !is_word_char(text[i]))
+          return i - pos;
+        break;
+      }
+      node = node->children[uc];
+      if (node->is_word)
+        max_len = i - pos + 1;
+    }
+    if (max_len > 0)
+      if (pos + max_len < len && is_word_char(text[pos + max_len]))
+        return 0;
+    return max_len;
+  }
+
+  template <typename U = T>
+  uint32_t
+  match(const char *text, uint32_t pos, uint32_t len,
+        bool (*is_word_char)(char c),
+        std::conditional_t<std::is_void_v<T>, void *, std::optional<T> *>
+            out_val = nullptr) const {
+    const TrieNode *node = root;
+    const TrieNode *last_word_node = nullptr;
+    uint32_t max_len = 0;
+    for (uint32_t i = pos; i < len; ++i) {
+      unsigned char uc = static_cast<unsigned char>(text[i]);
+      if (uc >= 128)
+        break;
+      if (!case_sensitive && uc >= 'A' && uc <= 'Z')
+        uc = uc - 'A' + 'a';
+      if (!node->children[uc])
+        break;
+      node = node->children[uc];
+      if (node->is_word) {
+        last_word_node = node;
+        max_len = i - pos + 1;
+      }
+    }
+    if (!last_word_node) {
+      if (out_val)
+        *out_val = std::nullopt;
+      return 0;
+    }
+    if (pos + max_len < len && is_word_char(text[pos + max_len])) {
+      if (out_val)
+        *out_val = std::nullopt;
+      return 0;
+    }
+    if (out_val)
+      *out_val = last_word_node->value;
+    return max_len;
+  }
+
+private:
+  TrieNode *root = new TrieNode();
+  bool case_sensitive = true;
+
+  void insert(const std::string &word) {
+    TrieNode *node = root;
+    for (char c : word) {
+      unsigned char uc = static_cast<unsigned char>(c);
+      if (uc >= 128)
+        return;
+      if (!case_sensitive && uc >= 'A' && uc <= 'Z')
+        uc = uc - 'A' + 'a';
+      if (!node->children[uc])
+        node->children[uc] = new TrieNode();
+      node = node->children[uc];
+    }
+    node->is_word = true;
+  }
+
+  template <typename U = T>
+  std::enable_if_t<!std::is_void_v<U>> insert(const std::string &word,
+                                              const U &val) {
+    TrieNode *node = root;
+    for (char c : word) {
+      unsigned char uc = static_cast<unsigned char>(c);
+      if (!case_sensitive && uc >= 'A' && uc <= 'Z')
+        uc = uc - 'A' + 'a';
+      if (!node->children[uc])
+        node->children[uc] = new TrieNode();
+      node = node->children[uc];
+    }
+    node->is_word = true;
+    node->value = val;
+  }
+
+  void clear_trie(TrieNode *node) {
+    if (!node)
+      return;
+    for (auto *child : node->children)
+      clear_trie(child);
+    delete node;
+  }
+};
+
+#endif

samples/bash.sh

@@ -1,8 +1,8 @@
 #!/usr/bin/env bash
 
-# ---------------------------------------------
+# ----------------------------------------------
 #   Bash Syntax Highlighter Test Specification
-# ---------------------------------------------
+# ----------------------------------------------
 
 VERSION="1.0.0"
 declare -a ITEMS=("alpha" "beta" "gamma" "delta")

samples/ruby.rb

@@ -21,6 +21,13 @@ cjk_samples = [
   "大量の文字列🚀🚀🚀"
 ]
 
+
+
+
+# a hex color: #FFFFFF hsl(147rad, 50%, 47%)
+
+0x603010 # another hex color
+
 # Ruby regex with unicode
 $unicode_regex_multiline = /[一-龯ぁ-ん#{0x3000}ァ
 \-ヶー
@@ -49,7 +56,7 @@ UNINITCORE = %{
 挨拶 = -> { "こんに \n ちは" }
 
 arr = Array.new()
-not_arr = NotArray.new()
+not_arr = NotABuiltin.new()
 
 raise NameError or SystemExit or CustomError or Errno or ErrorNotAtAll
 

src/editor/edit.cc

@@ -1,7 +1,6 @@
 #include "editor/editor.h"
 #include "lsp/lsp.h"
 #include "utils/utils.h"
-#include <cstdint>
 
 void edit_erase(Editor *editor, Coord pos, int64_t len) {
   if (len == 0)
@@ -52,7 +51,7 @@ void edit_erase(Editor *editor, Coord pos, int64_t len) {
     editor->root = erase(editor->root, start, byte_pos - start);
     lock_2.unlock();
     if (editor->parser)
-      editor->parser->edit(editor->root, start_row, end_row, start_row);
+      editor->parser->edit(start_row, end_row, 0);
     if (do_lsp) {
       if (editor->lsp->incremental_sync) {
         json message = {
@@ -124,7 +123,7 @@ void edit_erase(Editor *editor, Coord pos, int64_t len) {
     editor->root = erase(editor->root, byte_pos, end - byte_pos);
     lock_2.unlock();
     if (editor->parser)
-      editor->parser->edit(editor->root, start_row, end_row, start_row);
+      editor->parser->edit(start_row, end_row, 0);
     if (do_lsp) {
       if (editor->lsp->incremental_sync) {
         json message = {
@@ -175,7 +174,7 @@ void edit_insert(Editor *editor, Coord pos, char *data, uint32_t len) {
   apply_hook_insertion(editor, pos.row, rows);
   lock_2.unlock();
   if (editor->parser)
-    editor->parser->edit(editor->root, pos.row, pos.row, pos.row + rows);
+    editor->parser->edit(pos.row, pos.row, rows);
   if (editor->lsp) {
     if (editor->lsp->incremental_sync) {
       lock_1.lock();
@@ -218,19 +217,64 @@ void edit_insert(Editor *editor, Coord pos, char *data, uint32_t len) {
 
 void edit_replace(Editor *editor, Coord start, Coord end, const char *text,
                   uint32_t len) {
-  std::shared_lock lock(editor->knot_mtx);
+  std::unique_lock lock(editor->knot_mtx);
   uint32_t start_byte =
       line_to_byte(editor->root, start.row, nullptr) + start.col;
   uint32_t end_byte = line_to_byte(editor->root, end.row, nullptr) + end.col;
-  char *buf = read(editor->root, start_byte, end_byte - start_byte);
+  LineIterator *it = begin_l_iter(editor->root, start.row);
-  if (!buf)
+  char *line = next_line(it, nullptr);
-    return;
+  int utf16_start = 0;
-  lock.unlock();
+  if (line)
-  uint32_t erase_len =
+    utf16_start = utf8_byte_offset_to_utf16(line, start.col);
-      count_clusters(buf, end_byte - start_byte, 0, end_byte - start_byte);
+  free(it->buffer);
-  free(buf);
+  free(it);
-  if (erase_len != 0)
+  it = begin_l_iter(editor->root, end.row);
-    edit_erase(editor, start, erase_len);
+  line = next_line(it, nullptr);
+  int utf16_end = 0;
+  if (line)
+    utf16_end = utf8_byte_offset_to_utf16(line, end.col);
+  free(it->buffer);
+  free(it);
+  if (start_byte != end_byte)
+    editor->root = erase(editor->root, start_byte, end_byte - start_byte);
   if (len > 0)
-    edit_insert(editor, start, const_cast<char *>(text), len);
+    editor->root = insert(editor->root, start_byte, (char *)text, len);
+  uint32_t rows = 0;
+  for (uint32_t i = 0; i < len; i++)
+    if (text[i] == '\n')
+      rows++;
+  if (editor->parser) {
+    editor->parser->edit(start.row, end.row - 1, 0);
+    editor->parser->edit(start.row, start.row, rows);
+  }
+  if (editor->lsp) {
+    if (editor->lsp->incremental_sync) {
+      json message = {
+          {"jsonrpc", "2.0"},
+          {"method", "textDocument/didChange"},
+          {"params",
+           {{"textDocument",
+             {{"uri", editor->uri}, {"version", ++editor->lsp_version}}},
+            {"contentChanges",
+             json::array(
+                 {{{"range",
+                    {{"start",
+                      {{"line", start.row}, {"character", utf16_start}}},
+                     {"end", {{"line", end.row}, {"character", utf16_end}}}}},
+                   {"text", std::string(text, len)}}})}}}};
+      lsp_send(editor->lsp, message, nullptr);
+    } else {
+      char *buf = read(editor->root, 0, editor->root->char_count);
+      std::string full_text(buf);
+      free(buf);
+      json message = {
+          {"jsonrpc", "2.0"},
+          {"method", "textDocument/didChange"},
+          {"params",
+           {{"textDocument",
+             {{"uri", editor->uri}, {"version", ++editor->lsp_version}}},
+            {"contentChanges", json::array({{{"text", full_text}}})}}}};
+      lsp_send(editor->lsp, message, nullptr);
+    }
+  }
 }

src/editor/editor.cc

@@ -30,8 +30,11 @@ Editor *new_editor(const char *filename_arg, Coord position, Coord size) {
   free(str);
   editor->lang = language_for_file(filename.c_str());
   if (editor->lang.name != "unknown")
-    editor->parser = new Parser(editor->root, &editor->knot_mtx,
+    editor->parser = new Parser(editor, editor->lang.name, size.row + 5);
-                                editor->lang.name, size.row + 5);
+  if (editor->lang.name == "css" || editor->lang.name == "html" ||
+      editor->lang.name == "javascript" || editor->lang.name == "markdown" ||
+      editor->lang.name == "typescript")
+    editor->is_css_color = true;
   if (len <= (1024 * 28))
     request_add_to_lsp(editor->lang, editor);
   editor->indents.compute_indent(editor);
@@ -52,12 +55,13 @@ void save_file(Editor *editor) {
     return;
   std::shared_lock lock(editor->knot_mtx);
   int version = editor->lsp_version;
-  char *str = read(editor->root, 0, editor->root->char_count);
+  uint32_t char_count = editor->root->char_count;
+  char *str = read(editor->root, 0, char_count);
   if (!str)
     return;
   lock.unlock();
   std::ofstream out(editor->filename);
-  out.write(str, editor->root->char_count);
+  out.write(str, char_count);
   out.close();
   free(str);
   if (editor->lsp) {
@@ -99,12 +103,14 @@ void save_file(Editor *editor) {
           apply_lsp_edits(editor, t_edits, false);
           ensure_scroll(editor);
           std::shared_lock lock(editor->knot_mtx);
-          char *str = read(editor->root, 0, editor->root->char_count);
+          uint32_t char_count = editor->root->char_count;
+          char *str = read(editor->root, 0, char_count);
           if (!str)
             return;
           lock.unlock();
           std::ofstream out(editor->filename);
-          out.write(str, editor->root->char_count);
+          out.write(str, char_count);
+          out.close();
           free(str);
           lsp_send(editor->lsp, save_msg, nullptr);
         }

src/editor/renderer.cc

@@ -1,10 +1,13 @@
 #include "editor/editor.h"
+#include "io/sysio.h"
 #include "main.h"
 #include "syntax/decl.h"
 #include "syntax/parser.h"
+#include <cstdint>
 
 void render_editor(Editor *editor) {
   uint32_t sel_start = 0, sel_end = 0;
+  std::shared_lock knot_lock(editor->knot_mtx);
   uint32_t numlen =
       EXTRA_META + static_cast<int>(std::log10(editor->root->line_count + 1));
   uint32_t render_width = editor->size.col - numlen;
@@ -34,7 +37,6 @@ void render_editor(Editor *editor) {
         return (int)token.type;
     return 0;
   };
-  std::shared_lock knot_lock(editor->knot_mtx);
   if (editor->selection_active) {
     Coord start, end;
     if (editor->cursor >= editor->selection) {
@@ -88,6 +90,18 @@ void render_editor(Editor *editor) {
   LineIterator *it = begin_l_iter(editor->root, line_index);
   if (!it)
     return;
+  uint32_t prev_col, next_col;
+  std::string word;
+  word_boundaries_exclusive(editor, editor->cursor, &prev_col, &next_col);
+  if (next_col - prev_col > 0 && next_col - prev_col < 256 - 4) {
+    uint32_t offset = line_to_byte(editor->root, editor->cursor.row, nullptr);
+    char *word_ptr = read(editor->root, offset + prev_col, next_col - prev_col);
+    if (word_ptr) {
+      word = std::string(word_ptr, next_col - prev_col);
+      free(word_ptr);
+    }
+  }
+  editor->extra_hl.render(editor->root, line_index, word, editor->is_css_color);
   uint32_t rendered_rows = 0;
   uint32_t global_byte_offset = line_to_byte(editor->root, line_index, nullptr);
   while (rendered_rows < editor->size.row) {
@@ -157,9 +171,19 @@ void render_editor(Editor *editor) {
         const Highlight *hl = nullptr;
         if (editor->parser)
           hl = &highlights[get_type(current_byte_offset + local_render_offset)];
-        uint32_t fg = hl ? hl->fg : 0xFFFFFF;
+        std::optional<std::pair<uint32_t, uint32_t>> extra =
-        uint32_t bg = hl ? hl->bg : 0;
+            editor->extra_hl.get(
-        uint8_t fl = hl ? hl->flags : 0;
+                {line_index, current_byte_offset + local_render_offset});
+        uint32_t fg = extra && extra->second != UINT32_MAX
+                          ? extra->first
+                          : (hl ? hl->fg : 0xFFFFFF);
+        uint32_t bg = extra && extra->second != UINT32_MAX
+                          ? extra->second
+                          : (hl ? hl->bg : 0x000000);
+        uint8_t fl =
+            (hl ? hl->flags : 0) |
+            (extra ? (extra->second != UINT32_MAX ? CF_BOLD : CF_UNDERLINE)
+                   : 0);
         if (editor->selection_active && absolute_byte_pos >= sel_start &&
             absolute_byte_pos < sel_end)
           bg = 0x555555;

src/syntax/bash.cc

@@ -1,6 +1,6 @@
 #include "syntax/decl.h"
 #include "syntax/langs.h"
-#include "utils/utils.h"
+#include <cstdint>
 
 struct BashFullState {
   int brace_level = 0;
@@ -11,13 +11,11 @@ struct BashFullState {
   bool line_cont = false;
 
   struct Lit {
-    std::string delim = "";
+    std::string delim = ""; // Only 1 wide for strings
-    int brace_level = 1;
     bool allow_interp = false;
 
     bool operator==(const BashFullState::Lit &other) const {
-      return delim == other.delim && brace_level == other.brace_level &&
+      return delim == other.delim && allow_interp == other.allow_interp;
-             allow_interp == other.allow_interp;
     }
   } lit;
 
@@ -66,7 +64,35 @@ std::shared_ptr<void> bash_parse(std::vector<Token> *tokens,
   if (len == 0)
     return state;
   while (i < len) {
-    i += utf8_codepoint_width(text[i]);
+    if (state->full_state->in_state == BashFullState::STRING) {
+      uint32_t start = i;
+      while (i < len) {
+        if (text[i] == state->full_state->lit.delim[0]) {
+          tokens->push_back({start, i, TokenKind::String});
+          state->full_state->in_state = BashFullState::NONE;
+          break;
+        }
+        i++;
+      }
+      if (i == len)
+        tokens->push_back({start, i, TokenKind::String});
+      continue;
+    }
+    if (text[i] == '#') {
+      if (i == 0 && len > 4 && text[i + 1] == '!') {
+        tokens->push_back({0, len, TokenKind::Shebang});
+        return state;
+      }
+      tokens->push_back({i, len, TokenKind::Comment});
+      return state;
+    } else if (text[i] == '\'') {
+      state->full_state->in_state = BashFullState::STRING;
+      state->full_state->lit.delim = "'";
+      state->full_state->lit.allow_interp = false;
+      tokens->push_back({i, ++i, TokenKind::String});
+      continue;
+    }
+    i++;
   }
   return state;
 }
@@ -76,3 +102,6 @@ std::shared_ptr<void> bash_parse(std::vector<Token> *tokens,
 // ${var}  and $((math)) $(command) and `command` expansions ANSI-C quoted
 // stirngs - $''  backslash escapes but with \xHH and \uHHHH and \uHHHHHHHH \cX
 // too
+//
+// Lock edit_replace across both delete and insert instead of within to keep the
+// parser from glitching

src/syntax/parser.cc

@@ -1,4 +1,5 @@
 #include "syntax/parser.h"
+#include "editor/editor.h"
 #include "io/knot.h"
 #include "main.h"
 #include "syntax/decl.h"
@@ -6,10 +7,9 @@
 
 std::array<Highlight, TOKEN_KIND_COUNT> highlights = {};
 
-Parser::Parser(Knot *n_root, std::shared_mutex *n_knot_mutex,
+Parser::Parser(Editor *n_editor, std::string n_lang, uint32_t n_scroll_max) {
-               std::string n_lang, uint32_t n_scroll_max) {
+  editor = n_editor;
   scroll_max = n_scroll_max;
-  knot_mutex = n_knot_mutex;
   lang = n_lang;
   auto pair = parsers.find(n_lang);
   if (pair != parsers.end()) {
@@ -18,22 +18,26 @@ Parser::Parser(Knot *n_root, std::shared_mutex *n_knot_mutex,
   } else {
     assert("unknown lang should be checked by caller" && 0);
   }
-  edit(n_root, 0, 0, n_root->line_count);
+  edit(0, 0, editor->root->line_count);
 }
 
-void Parser::edit(Knot *n_root, uint32_t start_line, uint32_t old_end_line,
+void Parser::edit(uint32_t start_line, uint32_t old_end_line,
-                  uint32_t new_end_line) {
+                  uint32_t inserted_rows) {
   std::lock_guard lock(data_mutex);
-  root = n_root;
   if (((int64_t)old_end_line - (int64_t)start_line) > 0)
-    line_tree.erase(start_line + 1, old_end_line - start_line);
+    line_tree.erase(start_line, old_end_line - start_line);
-  if (((int64_t)new_end_line - (int64_t)old_end_line) > 0)
+  if (inserted_rows > 0)
-    line_tree.insert(start_line + 1, new_end_line - start_line);
+    line_tree.insert(start_line, inserted_rows);
+  if (start_line > 0)
+    dirty_lines.insert(start_line - 1);
   dirty_lines.insert(start_line);
+  dirty_lines.insert(start_line + 1);
 }
 
 void Parser::work() {
-  std::shared_lock k_lock(*knot_mutex);
+  if (!editor || !editor->root)
+    return;
+  std::shared_lock k_lock(editor->knot_mtx);
   k_lock.unlock();
   uint32_t capacity = 256;
   char *text = (char *)calloc((capacity + 1), sizeof(char));
@@ -45,14 +49,16 @@ void Parser::work() {
   std::unique_lock lock(mutex);
   lock.unlock();
   for (uint32_t c_line : tmp_dirty) {
-    if (c_line > scroll_max) {
+    if (c_line > scroll_max + 40) {
       remaining_dirty.insert(c_line);
       continue;
     }
     uint32_t line_count = line_tree.count();
     lock_data.lock();
     std::shared_ptr<void> prev_state =
-        (c_line > 0) ? line_tree.at(c_line - 1)->out_state : nullptr;
+        (c_line > 0) && c_line < line_tree.count()
+            ? line_tree.at(c_line - 1)->out_state
+            : nullptr;
     lock_data.unlock();
     while (c_line < line_count) {
       if (!running.load(std::memory_order_relaxed)) {
@@ -60,14 +66,18 @@ void Parser::work() {
         return;
       }
       k_lock.lock();
+      if (c_line > editor->root->line_count) {
+        k_lock.unlock();
+        continue;
+      }
       uint32_t r_offset, r_len;
-      r_offset = line_to_byte(root, c_line, &r_len);
+      r_offset = line_to_byte(editor->root, c_line, &r_len);
       if (r_len > capacity) {
         capacity = r_len;
         text = (char *)realloc(text, capacity + 1);
         memset(text, 0, capacity + 1);
       }
-      read_into(root, r_offset, r_len, text);
+      read_into(editor->root, r_offset, r_len, text);
       k_lock.unlock();
       if (c_line < scroll_max &&
           ((scroll_max > 100 && c_line > scroll_max - 100) || c_line < 100))
@@ -79,6 +89,12 @@ void Parser::work() {
       }
       lock_data.lock();
       LineData *line_data = line_tree.at(c_line);
+      if (!line_data) {
+        lock_data.unlock();
+        if (lock.owns_lock())
+          lock.unlock();
+        continue;
+      }
       std::shared_ptr<void> new_state =
           parse_func(&line_data->tokens, prev_state, text, r_len);
       line_data->in_state = prev_state;
@@ -98,8 +114,8 @@ void Parser::work() {
         remaining_dirty.insert(c_line);
         break;
       }
-      if (c_line < line_count &&
+      if (c_line < line_count && (line_data = line_tree.at(c_line)) &&
-          state_match_func(prev_state, line_tree.at(c_line)->in_state)) {
+          state_match_func(prev_state, line_data->in_state)) {
         lock_data.unlock();
         if (lock.owns_lock())
           lock.unlock();
@@ -129,7 +145,7 @@ void Parser::scroll(uint32_t line) {
     if (line_tree.at(c_line)->in_state || line_tree.at(c_line)->out_state)
       return;
     lock_data.unlock();
-    std::shared_lock k_lock(*knot_mutex);
+    std::shared_lock k_lock(editor->knot_mtx);
     k_lock.unlock();
     uint32_t capacity = 256;
     char *text = (char *)calloc((capacity + 1), sizeof(char));
@@ -144,14 +160,18 @@ void Parser::scroll(uint32_t line) {
         return;
       }
       k_lock.lock();
+      if (c_line > editor->root->line_count) {
+        k_lock.unlock();
+        continue;
+      }
       uint32_t r_offset, r_len;
-      r_offset = line_to_byte(root, c_line, &r_len);
+      r_offset = line_to_byte(editor->root, c_line, &r_len);
       if (r_len > capacity) {
         capacity = r_len;
         text = (char *)realloc(text, capacity + 1);
         memset(text, 0, capacity + 1);
       }
-      read_into(root, r_offset, r_len, text);
+      read_into(editor->root, r_offset, r_len, text);
       k_lock.unlock();
       if (c_line < scroll_max &&
           ((scroll_max > 100 && c_line > scroll_max - 100) || c_line < 100))
@@ -163,6 +183,12 @@ void Parser::scroll(uint32_t line) {
       }
       lock_data.lock();
       LineData *line_data = line_tree.at(c_line);
+      if (!line_data) {
+        lock_data.unlock();
+        if (lock.owns_lock())
+          lock.unlock();
+        continue;
+      }
       std::shared_ptr<void> new_state =
           parse_func(&line_data->tokens, prev_state, text, r_len);
       line_data->in_state = nullptr;

src/syntax/ruby.cc

@@ -290,15 +290,15 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
                                  std::shared_ptr<void> in_state,
                                  const char *text, uint32_t len) {
   static bool keywords_trie_init = false;
-  static Trie base_keywords_trie;
+  static Trie<void> base_keywords_trie;
-  static Trie expecting_keywords_trie;
+  static Trie<void> expecting_keywords_trie;
-  static Trie operator_keywords_trie;
+  static Trie<void> operator_keywords_trie;
-  static Trie expecting_operators_trie;
+  static Trie<void> expecting_operators_trie;
-  static Trie operator_trie;
+  static Trie<void> operator_trie;
-  static Trie types_trie;
+  static Trie<void> types_trie;
-  static Trie builtins_trie;
+  static Trie<void> builtins_trie;
-  static Trie methods_trie;
+  static Trie<void> methods_trie;
-  static Trie errors_trie;
+  static Trie<void> errors_trie;
   if (!keywords_trie_init) {
     base_keywords_trie.build(base_keywords);
     expecting_keywords_trie.build(expecting_keywords);

src/ui/completionbox.cc

@@ -1,5 +1,6 @@
 #include "ui/completionbox.h"
 #include "editor/completions.h"
+#include "io/sysio.h"
 #include "utils/utils.h"
 
 std::string item_kind_name(uint8_t kind) {
@@ -139,11 +140,12 @@ void CompletionBox::render(Coord pos) {
   if (start_row < 0)
     start_row = pos.row + 1;
   int32_t start_col = pos.col;
-  // if (start_col + size.col > cols) {
+  Coord screen_size = get_size();
-  //   start_col = cols - size.col;
+  if (start_col + size.col > screen_size.col) {
-  //   if (start_col < 0)
+    start_col = screen_size.col - size.col;
-  //     start_col = 0;
+    if (start_col < 0)
-  // }
+      start_col = 0;
+  }
   position = {(uint32_t)start_row, (uint32_t)start_col};
   for (uint32_t r = 0; r < size.row; r++)
     for (uint32_t c = 0; c < size.col; c++)

src/ui/diagnostics.cc

@@ -148,11 +148,12 @@ void DiagnosticBox::render(Coord pos) {
   if (start_row < 0)
     start_row = pos.row + 1;
   int32_t start_col = pos.col;
-  // if (start_col + size.col > cols) {
+  Coord screen_size = get_size();
-  //   start_col = cols - size.col;
+  if (start_col + size.col > screen_size.col) {
-  //   if (start_col < 0)
+    start_col = screen_size.col - size.col;
-  //     start_col = 0;
+    if (start_col < 0)
-  // }
+      start_col = 0;
+  }
   for (uint32_t r = 0; r < size.row; r++)
     for (uint32_t c = 0; c < size.col; c++)
       update(start_row + r, start_col + c, cells[r * size.col + c].utf8,

src/ui/hover.cc

@@ -104,11 +104,12 @@ void HoverBox::render(Coord pos) {
   if (start_row < 0)
     start_row = pos.row + 1;
   int32_t start_col = pos.col;
-  // if (start_col + size.col > cols) {
+  Coord screen_size = get_size();
-  //   start_col = cols - size.col;
+  if (start_col + size.col > screen_size.col) {
-  //   if (start_col < 0)
+    start_col = screen_size.col - size.col;
-  //     start_col = 0;
+    if (start_col < 0)
-  // }
+      start_col = 0;
+  }
   for (uint32_t r = 0; r < size.row; r++)
     for (uint32_t c = 0; c < size.col; c++)
       update(start_row + r, start_col + c, cells[r * size.col + c].utf8,