Improve highlighting structure

- switched to a sparse delta based map
- true lazy-loading to avoid any unneccessary allocations
- fixed windows management api

include/syntax/decl.h

@@ -6,6 +6,8 @@
 #include "pch.h"
 #include "syntax/trie.h"
 
+#define MAX_LINES_LOOKBEHIND 80
+
 struct Highlight {
   uint32_t fg{0xFFFFFF};
   uint32_t bg{0x000000};
@@ -35,10 +37,23 @@ struct Token {
   TokenKind type;
 };
 
+struct StateBase {
+  virtual ~StateBase() = default;
+  virtual std::unique_ptr<StateBase> clone() const = 0;
+};
+
+struct CustomState : StateBase {
+  std::string value;
+  explicit CustomState(std::string v) : value(std::move(v)) {}
+  std::unique_ptr<StateBase> clone() const override {
+    return std::make_unique<CustomState>(value);
+  }
+};
+
 struct LineData {
-  std::shared_ptr<void> in_state{nullptr};
+  std::unique_ptr<StateBase> out_state;
   std::vector<Token> tokens;
-  std::shared_ptr<void> out_state{nullptr};
+  std::unique_ptr<StateBase> in_state;
 };
 
 #endif

include/syntax/langs.h

@@ -4,29 +4,19 @@
 #include "syntax/decl.h"
 
 #define DEF_LANG(name)                                                         \
-  std::shared_ptr<void> name##_parse(                                          \
-      std::vector<Token> *tokens, std::shared_ptr<void> in_state,              \
-      const char *text, uint32_t len, uint32_t line_num);                      \
-  bool name##_state_match(std::shared_ptr<void> state_1,                       \
-                          std::shared_ptr<void> state_2);
+  std::unique_ptr<StateBase> name##_parse(                                     \
+      std::vector<Token> *tokens, StateBase *in_state, const char *text,       \
+      uint32_t len, uint32_t line_num);                                        \
+  bool name##_state_match(StateBase *state_1, StateBase *state_2);
 
 #define LANG_A(name)                                                           \
   {                                                                            \
     #name, { name##_parse, name##_state_match }                                \
   }
 
-template <typename T>
-inline std::shared_ptr<T> ensure_state(std::shared_ptr<T> state) {
-  using U = typename T::full_state_type;
-  if (!state)
-    state = std::make_shared<T>();
-  if (!state.unique())
-    state = std::make_shared<T>(*state);
-  if (!state->full_state)
-    state->full_state = std::make_shared<U>();
-  else if (!state->full_state.unique())
-    state->full_state = std::make_shared<U>(*state->full_state);
-  return state;
+template <typename B, typename A>
+std::unique_ptr<B> static_unique_ptr_cast(std::unique_ptr<A> &&p) {
+  return std::unique_ptr<B>(static_cast<B *>(p.release()));
 }
 
 DEF_LANG(ruby);
@@ -34,11 +24,10 @@ DEF_LANG(bash);
 
 inline static const std::unordered_map<
     std::string,
-    std::tuple<std::shared_ptr<void> (*)(
-                   std::vector<Token> *tokens, std::shared_ptr<void> in_state,
+    std::tuple<std::unique_ptr<StateBase> (*)(
+                   std::vector<Token> *tokens, StateBase *in_state,
                    const char *text, uint32_t len, uint32_t line_num),
-               bool (*)(std::shared_ptr<void> state_1,
-                        std::shared_ptr<void> state_2)>>
+               bool (*)(StateBase *state_1, StateBase *state_2)>>
     parsers = {
         LANG_A(ruby),
         LANG_A(bash),

include/syntax/line_map.h

@@ -0,0 +1,104 @@
+#ifndef LINE_MAP_H
+#define LINE_MAP_H
+
+#include "syntax/decl.h"
+
+struct LineKey {
+  uint32_t line;
+  uint32_t version;
+  bool operator==(const LineKey &other) const {
+    return line == other.line && version == other.version;
+  }
+};
+
+struct LineKeyHash {
+  size_t operator()(const LineKey &key) const {
+    uint64_t combined = (uint64_t(key.line) << 32) | key.version;
+    return std::hash<uint64_t>()(combined);
+  }
+};
+
+struct EditDelta {
+  uint32_t start_line;
+  int64_t delta;
+};
+
+struct LineMap {
+  LineMap() : current_version(0) { edit_log.push_back({0, 0}); }
+
+  LineData *at(uint32_t line) {
+    auto key_opt = resolve_line(line);
+    if (!key_opt)
+      return nullptr;
+    LineKey key = *key_opt;
+    if (key.version == current_version)
+      return lines[key].get();
+    auto data_ptr = std::move(lines[key]);
+    lines.erase(key);
+    key = {line, current_version};
+    lines[key] = std::move(data_ptr);
+    return lines[key].get();
+  }
+
+  LineData *create_at(uint32_t line) {
+    auto key_opt = resolve_line(line);
+    LineKey key;
+    std::unique_ptr<LineData> data_ptr;
+    if (key_opt) {
+      key = *key_opt;
+      if (key.version == current_version)
+        return lines[key].get();
+      data_ptr = std::move(lines[key]);
+      lines.erase(key);
+    } else {
+      data_ptr = std::make_unique<LineData>();
+    }
+    key = {line, current_version};
+    lines[key] = std::move(data_ptr);
+    return lines[key].get();
+  }
+
+  void apply_edit(uint32_t start, int64_t delta) {
+    if (delta < 0) {
+      int64_t count = -delta;
+      for (int64_t i = 0; i < count; i++) {
+        auto key = resolve_line(start + i);
+        if (!key)
+          continue;
+        lines.erase(*key);
+      }
+    }
+    current_version++;
+    edit_log.push_back({start, delta});
+  }
+
+private:
+  std::unordered_map<LineKey, std::unique_ptr<LineData>, LineKeyHash> lines;
+
+  std::vector<EditDelta> edit_log;
+
+  uint32_t current_version;
+
+  std::optional<LineKey> resolve_line(uint32_t line) {
+    uint32_t current_line = line;
+    for (int64_t v = current_version; v >= 0; v--) {
+      LineKey key = {current_line, (uint32_t)v};
+      if (lines.find(key) != lines.end())
+        return key;
+      const auto &edit = edit_log[v];
+      if (edit.delta > 0) {
+        if (current_line >= edit.start_line) {
+          if (current_line < edit.start_line + edit.delta)
+            return std::nullopt;
+          current_line -= edit.delta;
+        }
+      } else {
+        if (current_line >= edit.start_line)
+          current_line -= edit.delta;
+      }
+    }
+    return std::nullopt;
+  }
+};
+
+#endif

include/syntax/line_tree.h

@@ -1,232 +0,0 @@
-#ifndef LINE_TREE_H
-#define LINE_TREE_H
-
-#include "syntax/decl.h"
-
-struct LineTree {
-  void clear() {
-    clear_node(root);
-    root = nullptr;
-    stack_size = 0;
-  }
-  void build(uint32_t x) { root = build_node(x); }
-  LineData *at(uint32_t x) {
-    LineNode *n = root;
-    while (n) {
-      uint32_t left_size = n->left ? n->left->size : 0;
-      if (x < left_size) {
-        n = n->left;
-      } else if (x < left_size + n->data.size()) {
-        return &n->data[x - left_size];
-      } else {
-        x -= left_size + n->data.size();
-        n = n->right;
-      }
-    }
-    return nullptr;
-  }
-  LineData *start_iter(uint32_t x) {
-    stack_size = 0;
-    LineNode *n = root;
-    while (n) {
-      uint32_t left_size = n->left ? n->left->size : 0;
-      if (x < left_size) {
-        push(n, 0);
-        n = n->left;
-      } else if (x < left_size + n->data.size()) {
-        push(n, x - left_size + 1);
-        return &n->data[x - left_size];
-      } else {
-        x -= left_size + n->data.size();
-        push(n, UINT32_MAX);
-        n = n->right;
-      }
-    }
-    return nullptr;
-  }
-  void end_iter() { stack_size = 0; }
-  LineData *next() {
-    while (stack_size) {
-      auto &f = stack[stack_size - 1];
-      LineNode *n = f.node;
-      if (f.index < n->data.size())
-        return &n->data[f.index++];
-      stack_size--;
-      if (n->right) {
-        n = n->right;
-        while (n) {
-          push(n, 0);
-          if (!n->left)
-            break;
-          n = n->left;
-        }
-        return &stack[stack_size - 1].node->data[0];
-      }
-    }
-    return nullptr;
-  }
-  void insert(uint32_t x, uint32_t y) {
-    if (x > subtree_size(root))
-      x = subtree_size(root);
-    root = insert_node(root, x, y);
-  }
-  void erase(uint32_t x, uint32_t y) {
-    if (x + y > subtree_size(root))
-      x = subtree_size(root) - y;
-    root = erase_node(root, x, y);
-  }
-  uint32_t count() { return subtree_size(root); }
-  ~LineTree() { clear(); }
-
-private:
-  struct LineNode {
-    LineNode *left = nullptr;
-    LineNode *right = nullptr;
-    uint8_t depth = 1;
-    uint32_t size = 0;
-    std::vector<LineData> data;
-  };
-  struct Frame {
-    LineNode *node;
-    uint32_t index;
-  };
-  void push(LineNode *n, uint32_t x) {
-    stack[stack_size].node = n;
-    stack[stack_size].index = x;
-    stack_size++;
-  }
-  static void clear_node(LineNode *n) {
-    if (!n)
-      return;
-    clear_node(n->left);
-    clear_node(n->right);
-    delete n;
-  }
-  LineNode *root = nullptr;
-  Frame stack[32];
-  std::atomic<uint8_t> stack_size = 0;
-  static constexpr uint32_t LEAF_TARGET = 256;
-  LineTree::LineNode *erase_node(LineNode *n, uint32_t x, uint32_t y) {
-    if (!n || y == 0)
-      return n;
-    uint32_t left_sz = subtree_size(n->left);
-    uint32_t mid_sz = n->data.size();
-    if (x < left_sz) {
-      uint32_t len = std::min(y, left_sz - x);
-      n->left = erase_node(n->left, x, len);
-      y -= len;
-      x = left_sz;
-    }
-    if (y > 0 && x < left_sz + mid_sz) {
-      uint32_t mid_x = x - left_sz;
-      uint32_t len = std::min(y, mid_sz - mid_x);
-      n->data.erase(n->data.begin() + mid_x, n->data.begin() + mid_x + len);
-      y -= len;
-      x += len;
-    }
-    if (y > 0) {
-      n->right = erase_node(n->right, x - left_sz - n->data.size(), y);
-    }
-    if (n->left && n->right &&
-        subtree_size(n->left) + subtree_size(n->right) < 256) {
-      return merge(n->left, n->right);
-    }
-    return rebalance(n);
-  }
-  LineTree::LineNode *insert_node(LineNode *n, uint32_t x, uint32_t y) {
-    if (!n) {
-      auto *leaf = new LineNode();
-      leaf->data.resize(y);
-      leaf->size = y;
-      return leaf;
-    }
-    if (!n->left && !n->right) {
-      n->data.insert(n->data.begin() + x, y, LineData());
-      fix(n);
-      if (n->data.size() > 512)
-        return split_leaf(n);
-      return n;
-    }
-    uint32_t left_size = subtree_size(n->left);
-    if (x <= left_size)
-      n->left = insert_node(n->left, x, y);
-    else
-      n->right = insert_node(n->right, x - left_size - n->data.size(), y);
-    return rebalance(n);
-  }
-  LineNode *build_node(uint32_t count) {
-    if (count <= LEAF_TARGET) {
-      auto *n = new LineNode();
-      n->data.resize(count);
-      n->size = count;
-      return n;
-    }
-    uint32_t left_count = count / 2;
-    uint32_t right_count = count - left_count;
-    auto *n = new LineNode();
-    n->left = build_node(left_count);
-    n->right = build_node(right_count);
-    fix(n);
-    return n;
-  }
-  static LineNode *split_leaf(LineNode *n) {
-    auto *right = new LineNode();
-    size_t mid = n->data.size() / 2;
-    right->data.assign(n->data.begin() + mid, n->data.end());
-    n->data.resize(mid);
-    fix(n);
-    fix(right);
-    auto *parent = new LineNode();
-    parent->left = n;
-    parent->right = right;
-    fix(parent);
-    return parent;
-  }
-  static LineNode *merge(LineNode *a, LineNode *b) {
-    a->data.insert(a->data.end(), b->data.begin(), b->data.end());
-    delete b;
-    fix(a);
-    return a;
-  }
-  static void fix(LineNode *n) {
-    n->depth = 1 + MAX(height(n->left), height(n->right));
-    n->size = subtree_size(n->left) + n->data.size() + subtree_size(n->right);
-  }
-  static LineNode *rotate_right(LineNode *y) {
-    LineNode *x = y->left;
-    LineNode *T2 = x->right;
-    x->right = y;
-    y->left = T2;
-    fix(y);
-    fix(x);
-    return x;
-  }
-  static LineNode *rotate_left(LineNode *x) {
-    LineNode *y = x->right;
-    LineNode *T2 = y->left;
-    y->left = x;
-    x->right = T2;
-    fix(x);
-    fix(y);
-    return y;
-  }
-  static LineNode *rebalance(LineNode *n) {
-    fix(n);
-    int balance = int(height(n->left)) - int(height(n->right));
-    if (balance > 1) {
-      if (height(n->left->left) < height(n->left->right))
-        n->left = rotate_left(n->left);
-      return rotate_right(n);
-    }
-    if (balance < -1) {
-      if (height(n->right->right) < height(n->right->left))
-        n->right = rotate_right(n->right);
-      return rotate_left(n);
-    }
-    return n;
-  }
-  static uint8_t height(LineNode *n) { return n ? n->depth : 0; }
-  static uint32_t subtree_size(LineNode *n) { return n ? n->size : 0; }
-};
-
-#endif

include/syntax/parser.h

@@ -3,24 +3,22 @@
 
 #include "ruby/decl.h"
 #include "syntax/decl.h"
-#include "syntax/line_tree.h"
+#include "syntax/line_map.h"
 
 struct Parser {
   struct Editor *editor = nullptr;
   std::string lang;
-  std::shared_ptr<void> (*parse_func)(std::vector<Token> *tokens,
-                                      std::shared_ptr<void> in_state,
-                                      const char *text, uint32_t len,
-                                      uint32_t line_num);
-  bool (*state_match_func)(std::shared_ptr<void> state_1,
-                           std::shared_ptr<void> state_2);
+  std::unique_ptr<StateBase> (*parse_func)(std::vector<Token> *tokens,
+                                           StateBase *in_state,
+                                           const char *text, uint32_t len,
+                                           uint32_t line_num);
+  bool (*state_match_func)(StateBase *state_1, StateBase *state_2);
   mrb_value parser_block = mrb_nil_value();
   mrb_value match_block = mrb_nil_value();
   bool is_custom{false};
   std::atomic<uint32_t> scroll_max{0};
   std::atomic<bool> scroll_dirty{false};
-  LineTree line_tree;
-  UniqueQueue<uint32_t> dirty_lines;
+  LineMap line_map = LineMap();
 
   Parser(Editor *editor, std::string n_lang, uint32_t n_scroll_max);
   void edit(uint32_t start_line, uint32_t removed_rows, uint32_t inserted_rows);