Make syntax highlighting smoother

.gitignore

@@ -14,4 +14,6 @@ bin
 
 grammar/.*.scm
 
+.thinlto-cache/
+
 __old__

Makefile

@@ -13,15 +13,21 @@ CCACHE := ccache
 CXX_DEBUG := $(CCACHE) g++
 CXX_RELEASE := $(CCACHE) clang++
 
-CFLAGS_DEBUG := -std=c++20 -Wall -Wextra -O0 -fno-inline -gsplit-dwarf -g -fsanitize=address -fno-omit-frame-pointer -I./include -I./libs
+CFLAGS_DEBUG := -std=c++20 -Wall -Wextra \
-CFLAGS_RELEASE := -std=c++20 -O3 -march=native -flto=thin \
+								-O0 -fno-inline -gsplit-dwarf\
-	-fno-exceptions -fno-rtti -fstrict-aliasing \
+								-g -fsanitize=address -fno-omit-frame-pointer\
-	-ffast-math -funroll-loops \
+								-Wno-unused-command-line-argument \
-	-fvisibility=hidden \
+								-I./include -I./libs
-	-fomit-frame-pointer -DNDEBUG -s \
+CFLAGS_RELEASE := -std=c++20 -O3 -march=native \
-	-mllvm -vectorize-loops \
+									-fno-exceptions -fno-rtti -fstrict-aliasing \
-	-fno-unwind-tables -fno-asynchronous-unwind-tables\
+									-ffast-math \
-	-I./include -I./libs
+									-fvisibility=hidden -fuse-ld=lld \
+									-flto=thin -Wl,--thinlto-cache-dir=.thinlto-cache \
+									-fomit-frame-pointer -DNDEBUG -s \
+									-mllvm -vectorize-loops \
+									-fno-unwind-tables -fno-asynchronous-unwind-tables\
+									-Wno-unused-command-line-argument \
+									-I./include -I./libs
 
 PCH_CFLAGS_DEBUG := $(CFLAGS_DEBUG) -x c++-header
 PCH_CFLAGS_RELEASE := $(CFLAGS_RELEASE) -x c++-header

README.md

@@ -6,10 +6,6 @@ A TUI IDE.
 
 # TODO
 
-- [ ] Locking knot while getting hex is not good idea . make knot true knot so i can copy it super fast with refcount on a pool of nodes.
-    - also edits are somehow still leaking . as in they are not applying properly for very quick edits.
-    - Look into ts_collect_spans and edit_insert (on large files)
-- [ ] Add strikethrough support
 - [ ] Add status bar & RUNNER mode
 - [ ] Fix indentation logic
 - [ ] Fix bug where closing immediately while lsp is loading hangs and then segfaults.

include/editor/editor.h

@@ -33,9 +33,8 @@ struct Editor {
   Queue<TSInputEdit> edit_queue;
   std::vector<Fold> folds;
   Spans spans;
-  // TODO: Split into 2 groups to have their own mutex's . one for word hl and
+  Spans word_spans;
-  //       one for hex colors
+  Spans hex_color_spans;
-  Spans def_spans;
   uint32_t hooks[94];
   bool jumper_set;
   std::shared_mutex v_mtx;

include/editor/spans.h

@@ -7,7 +7,7 @@
 struct Spans {
   std::vector<Span> spans;
   Queue<std::pair<uint32_t, int64_t>> edits;
-  bool mid_parse = false;
+  std::atomic<bool> mid_parse = false;
   std::shared_mutex mtx;
 };
 

include/io/knot.h

@@ -159,8 +159,10 @@ char *leaf_from_offset(Knot *root, uint32_t start_offset, uint32_t *out_len);
 // compliant) I.e some forms of backtracking etc. are not supported
 // root is the root of the rope to be searched
 // Returns a vector of pairs of start and length offsets (in bytes)
-std::vector<std::pair<size_t, size_t>> search_rope(Knot *root,
+std::vector<std::pair<size_t, size_t>> search_rope_dfa(Knot *root,
-                                                   const char *pattern);
+                                                       const char *pattern);
+
+std::vector<Match> search_rope(Knot *root, const char *pattern);
 
 // Helper function to free the rope
 // root is the root of the rope

src/editor/edit.cc

@@ -51,7 +51,6 @@ void edit_erase(Editor *editor, Coord pos, int64_t len) {
     apply_hook_deletion(editor, start_row + 1, end_row);
     std::unique_lock lock_2(editor->knot_mtx);
     editor->root = erase(editor->root, start, byte_pos - start);
-    lock_2.unlock();
     if (editor->ts.tree) {
       TSInputEdit edit = {
           .start_byte = start,
@@ -63,6 +62,15 @@ void edit_erase(Editor *editor, Coord pos, int64_t len) {
       };
       editor->edit_queue.push(edit);
     }
+    std::unique_lock lock_3(editor->spans.mtx);
+    apply_edit(editor->spans.spans, start, start - byte_pos);
+    if (editor->spans.mid_parse)
+      editor->spans.edits.push({start, start - byte_pos});
+    lock_3.unlock();
+    lock_2.unlock();
+    std::unique_lock lock_4(editor->hex_color_spans.mtx);
+    apply_edit(editor->hex_color_spans.spans, byte_pos, start - byte_pos);
+    lock_4.unlock();
     if (do_lsp) {
       if (editor->lsp->incremental_sync) {
         json message = {
@@ -88,12 +96,6 @@ void edit_erase(Editor *editor, Coord pos, int64_t len) {
         lsp_send(editor->lsp, message, nullptr);
       }
     }
-    std::unique_lock lock_3(editor->spans.mtx);
-    apply_edit(editor->spans.spans, start, start - byte_pos);
-    if (editor->spans.mid_parse)
-      editor->spans.edits.push({start, start - byte_pos});
-    std::unique_lock lock_4(editor->def_spans.mtx);
-    apply_edit(editor->def_spans.spans, byte_pos, start - byte_pos);
   } else {
     std::shared_lock lock_1(editor->knot_mtx);
     uint32_t cursor_original =
@@ -140,7 +142,6 @@ void edit_erase(Editor *editor, Coord pos, int64_t len) {
     apply_hook_deletion(editor, start_row + 1, end_row);
     std::unique_lock lock_2(editor->knot_mtx);
     editor->root = erase(editor->root, byte_pos, end - byte_pos);
-    lock_2.unlock();
     if (editor->ts.tree) {
       TSInputEdit edit = {
           .start_byte = byte_pos,
@@ -152,6 +153,15 @@ void edit_erase(Editor *editor, Coord pos, int64_t len) {
       };
       editor->edit_queue.push(edit);
     }
+    std::unique_lock lock_3(editor->spans.mtx);
+    apply_edit(editor->spans.spans, byte_pos, byte_pos - end);
+    if (editor->spans.mid_parse)
+      editor->spans.edits.push({byte_pos, byte_pos - end});
+    lock_3.unlock();
+    lock_2.unlock();
+    std::unique_lock lock_4(editor->hex_color_spans.mtx);
+    apply_edit(editor->hex_color_spans.spans, byte_pos, byte_pos - end);
+    lock_4.unlock();
     if (do_lsp) {
       if (editor->lsp->incremental_sync) {
         json message = {
@@ -177,12 +187,6 @@ void edit_erase(Editor *editor, Coord pos, int64_t len) {
         lsp_send(editor->lsp, message, nullptr);
       }
     }
-    std::unique_lock lock_3(editor->spans.mtx);
-    apply_edit(editor->spans.spans, byte_pos, byte_pos - end);
-    if (editor->spans.mid_parse)
-      editor->spans.edits.push({byte_pos, byte_pos - end});
-    std::unique_lock lock_4(editor->def_spans.mtx);
-    apply_edit(editor->def_spans.spans, byte_pos, byte_pos - end);
   }
 }
 
@@ -202,7 +206,6 @@ void edit_insert(Editor *editor, Coord pos, char *data, uint32_t len) {
   lock_1.unlock();
   std::unique_lock lock_2(editor->knot_mtx);
   editor->root = insert(editor->root, byte_pos, data, len);
-  lock_2.unlock();
   uint32_t cols = 0;
   uint32_t rows = 0;
   for (uint32_t i = 0; i < len; i++) {
@@ -227,6 +230,15 @@ void edit_insert(Editor *editor, Coord pos, char *data, uint32_t len) {
     };
     editor->edit_queue.push(edit);
   }
+  std::unique_lock lock_3(editor->spans.mtx);
+  apply_edit(editor->spans.spans, byte_pos, len);
+  if (editor->spans.mid_parse)
+    editor->spans.edits.push({byte_pos, len});
+  lock_3.unlock();
+  lock_2.unlock();
+  std::unique_lock lock_4(editor->hex_color_spans.mtx);
+  apply_edit(editor->hex_color_spans.spans, byte_pos, len);
+  lock_4.unlock();
   if (editor->lsp) {
     if (editor->lsp->incremental_sync) {
       lock_1.lock();
@@ -265,10 +277,4 @@ void edit_insert(Editor *editor, Coord pos, char *data, uint32_t len) {
       lsp_send(editor->lsp, message, nullptr);
     }
   }
-  std::unique_lock lock_3(editor->spans.mtx);
-  apply_edit(editor->spans.spans, byte_pos, len);
-  if (editor->spans.mid_parse)
-    editor->spans.edits.push({byte_pos, len});
-  std::unique_lock lock_4(editor->def_spans.mtx);
-  apply_edit(editor->def_spans.spans, byte_pos, len);
 }

src/editor/editor.cc

@@ -33,8 +33,9 @@ Editor *new_editor(const char *filename_arg, Coord position, Coord size) {
     ts_parser_set_language(editor->ts.parser, editor->ts.language);
     editor->ts.query_file =
         get_exe_dir() + "/../grammar/" + language.name + ".scm";
-    request_add_to_lsp(language, editor);
   }
+  if (len <= (1024 * 28))
+    request_add_to_lsp(language, editor);
   return editor;
 }
 

src/editor/renderer.cc

@@ -1,6 +1,7 @@
 #include "editor/editor.h"
 #include "editor/folds.h"
 #include "main.h"
+#include "ts/decl.h"
 
 void render_editor(Editor *editor) {
   uint32_t sel_start = 0, sel_end = 0;
@@ -73,14 +74,16 @@ void render_editor(Editor *editor) {
   Coord cursor = {UINT32_MAX, UINT32_MAX};
   uint32_t line_index = editor->scroll.row;
   SpanCursor span_cursor(editor->spans);
-  SpanCursor def_span_cursor(editor->def_spans);
+  SpanCursor word_span_cursor(editor->word_spans);
+  SpanCursor hex_span_cursor(editor->hex_color_spans);
   LineIterator *it = begin_l_iter(editor->root, line_index);
   if (!it)
     return;
   uint32_t rendered_rows = 0;
   uint32_t global_byte_offset = line_to_byte(editor->root, line_index, nullptr);
   span_cursor.sync(global_byte_offset);
-  def_span_cursor.sync(global_byte_offset);
+  word_span_cursor.sync(global_byte_offset);
+  hex_span_cursor.sync(global_byte_offset);
   while (rendered_rows < editor->size.row) {
     const Fold *fold = fold_for_line(editor->folds, line_index);
     if (fold) {
@@ -180,16 +183,23 @@ void render_editor(Editor *editor) {
         uint32_t absolute_byte_pos =
             global_byte_offset + current_byte_offset + local_render_offset;
         Highlight *hl = span_cursor.get_highlight(absolute_byte_pos);
-        Highlight *def_hl = def_span_cursor.get_highlight(absolute_byte_pos);
+        Highlight *word_hl = word_span_cursor.get_highlight(absolute_byte_pos);
+        Highlight *hex_hl = hex_span_cursor.get_highlight(absolute_byte_pos);
         uint32_t fg = hl ? hl->fg : 0xFFFFFF;
         uint32_t bg = hl ? hl->bg : 0;
         uint8_t fl = hl ? hl->flags : 0;
-        if (def_hl) {
+        if (hex_hl) {
-          if (def_hl->fg != 0)
+          if (hex_hl->fg != 0)
-            fg = def_hl->fg;
+            fg = hex_hl->fg;
-          if (def_hl->bg != 0)
+          if (hex_hl->bg != 0)
-            bg = def_hl->bg;
+            bg = hex_hl->bg;
-          fl |= def_hl->flags;
+          fl |= hex_hl->flags;
+        } else if (word_hl) {
+          if (word_hl->fg != 0)
+            fg |= word_hl->fg;
+          if (word_hl->bg != 0)
+            bg |= word_hl->bg;
+          fl |= word_hl->flags;
         }
         if (editor->selection_active && absolute_byte_pos >= sel_start &&
             absolute_byte_pos < sel_end)

src/editor/worker.cc

@@ -30,16 +30,19 @@ void hover_diagnostic(Editor *editor) {
 void editor_worker(Editor *editor) {
   if (!editor || !editor->root)
     return;
-  if (editor->root->char_count > (1024 * 200))
+  if (editor->root->char_count > (1024 * 128))
     return;
   if (editor->ts.query_file != "" && !editor->ts.query)
     editor->ts.query = load_query(editor->ts.query_file.c_str(), &editor->ts);
   if (editor->ts.parser && editor->ts.query)
     ts_collect_spans(editor);
+  if (editor->root->char_count > (1024 * 32))
+    return;
   uint32_t prev_col, next_col;
   word_boundaries_exclusive(editor, editor->cursor, &prev_col, &next_col);
-  std::unique_lock lock(editor->def_spans.mtx);
+  std::unique_lock lock(editor->word_spans.mtx);
-  editor->def_spans.spans.clear();
+  editor->word_spans.spans.clear();
+  lock.unlock();
   if (next_col - prev_col > 0 && next_col - prev_col < 256 - 4) {
     std::shared_lock lockk(editor->knot_mtx);
     uint32_t offset = line_to_byte(editor->root, editor->cursor.row, nullptr);
@@ -48,43 +51,56 @@ void editor_worker(Editor *editor) {
     if (word) {
       char buf[256];
       snprintf(buf, sizeof(buf), "\\b%s\\b", word);
+      std::shared_lock lockk(editor->knot_mtx);
       std::vector<std::pair<size_t, size_t>> results =
-          search_rope(editor->root, buf);
+          search_rope_dfa(editor->root, buf);
+      lockk.unlock();
+      std::unique_lock lock2(editor->word_spans.mtx);
+      editor->word_spans.spans.reserve(results.size());
       for (const auto &match : results) {
         Span s;
         s.start = match.first;
         s.end = match.first + match.second;
         s.hl = &HL_UNDERLINE;
-        editor->def_spans.spans.push_back(s);
+        editor->word_spans.spans.push_back(s);
       }
       free(word);
+      lock2.unlock();
     }
   }
-  uint8_t top = 0;
+  static uint16_t limit = 150;
-  static Highlight *hl_s = (Highlight *)calloc(200, sizeof(Highlight));
+  static Highlight *hl_s = (Highlight *)calloc(limit, sizeof(Highlight));
   if (!hl_s)
     exit(ENOMEM);
   std::shared_lock lockk(editor->knot_mtx);
-  std::vector<std::pair<size_t, size_t>> results =
+  std::vector<Match> results =
-      search_rope(editor->root, "(0x|#)[0-9a-fA-F]{6}");
+      search_rope(editor->root, "(?:0x|#)[0-9a-fA-F]{6}");
-  for (int i = 0; i < results.size() && top < 200; i++) {
+  if (results.size() > limit) {
+    limit = results.size() + 50;
+    free(hl_s);
+    hl_s = (Highlight *)calloc(limit, sizeof(Highlight));
+    if (!hl_s)
+      exit(ENOMEM);
+  }
+  lockk.unlock();
+  std::unique_lock lock2(editor->hex_color_spans.mtx);
+  editor->hex_color_spans.spans.clear();
+  editor->hex_color_spans.spans.reserve(results.size());
+  for (size_t i = 0; i < results.size(); ++i) {
     Span s;
-    s.start = results[i].first;
+    s.start = results[i].start;
-    s.end = results[i].first + results[i].second;
+    s.end = results[i].end;
-    char *buf = read(editor->root, s.start, s.end - s.start);
+    int x = results[i].text[0] == '#' ? 1 : 2;
-    int x = buf[0] == '#' ? 1 : 2;
+    uint32_t bg = HEX(results[i].text.substr(x));
-    uint32_t bg = HEX(buf + x);
+    uint8_t r = bg >> 16;
-    free(buf);
+    uint8_t g = (bg >> 8) & 0xFF;
-    uint8_t r = bg >> 16, g = (bg >> 8) & 0xFF, b = bg & 0xFF;
+    uint8_t b = bg & 0xFF;
     double luminance = 0.299 * r + 0.587 * g + 0.114 * b;
     uint32_t fg = (luminance > 128) ? 0x010101 : 0xFEFEFE;
-    hl_s[top] = {fg, bg, CF_BOLD, UINT8_MAX};
+    hl_s[i] = {fg, bg, CF_BOLD, UINT8_MAX};
-    s.hl = &hl_s[top];
+    s.hl = &hl_s[i];
-    editor->def_spans.spans.push_back(s);
+    editor->hex_color_spans.spans.push_back(s);
-    top++;
   }
-  std::sort(editor->def_spans.spans.begin(), editor->def_spans.spans.end());
+  lock2.unlock();
-  lock.unlock();
-  lockk.unlock();
   hover_diagnostic(editor);
 }

src/io/knot.cc

@@ -788,8 +788,8 @@ char *leaf_from_offset(Knot *root, uint32_t start_offset, uint32_t *out_len) {
   return nullptr;
 }
 
-std::vector<std::pair<size_t, size_t>> search_rope(Knot *root,
+std::vector<std::pair<size_t, size_t>> search_rope_dfa(Knot *root,
-                                                   const char *pattern) {
+                                                       const char *pattern) {
   std::vector<std::pair<size_t, size_t>> results;
   int errorcode;
   PCRE2_SIZE erroffset;
@@ -807,15 +807,17 @@ std::vector<std::pair<size_t, size_t>> search_rope(Knot *root,
     pcre2_match_data_free(mdata);
     return results;
   }
+  size_t limit = 256;
+  results.reserve(limit);
   size_t chunk_abs_offset = 0;
   size_t saved_match_start = 0;
   bool match_in_progress = false;
   int flags = PCRE2_PARTIAL_SOFT;
   while (1) {
-    const char *chunk_start = next_leaf(it, nullptr);
+    uint32_t chunk_len;
+    const char *chunk_start = next_leaf(it, &chunk_len);
     if (!chunk_start)
       break;
-    size_t chunk_len = strlen(chunk_start);
     const char *current_ptr = chunk_start;
     size_t remaining_len = chunk_len;
     while (remaining_len > 0) {
@@ -837,6 +839,10 @@ std::vector<std::pair<size_t, size_t>> search_rope(Knot *root,
               chunk_abs_offset + (current_ptr - chunk_start) + ov[1];
         }
         size_t total_len = match_end_abs - match_start_abs;
+        if (results.size() >= limit) {
+          limit *= 2;
+          results.reserve(limit);
+        }
         results.push_back(std::make_pair(match_start_abs, total_len));
         size_t consumed = ov[1];
         if (consumed == 0)
@@ -868,7 +874,6 @@ std::vector<std::pair<size_t, size_t>> search_rope(Knot *root,
         } else {
           break;
         }
-        // if (rc != PCRE2_ERROR_NOMATCH) {} // handle error
       }
     }
     chunk_abs_offset += chunk_len;
@@ -881,125 +886,69 @@ std::vector<std::pair<size_t, size_t>> search_rope(Knot *root,
   return results;
 }
 
-// TODO: Optimize and make it actually utilize capture groups etc.
+static const size_t MAX_OVERLAP = 1024;
-//
+
-// static const size_t MAX_OVERLAP = 1024;
+std::vector<Match> search_rope(Knot *root, const char *pattern) {
-//
+  std::vector<Match> results;
-// std::vector<std::pair<size_t, size_t>> search_rope_new(Knot *root,
+  int errorcode;
-//                                                        const char *pattern) {
+  PCRE2_SIZE erroffset;
-//   std::vector<std::pair<size_t, size_t>> results;
+  pcre2_code *re = pcre2_compile((PCRE2_SPTR)pattern, PCRE2_ZERO_TERMINATED, 0,
-//   int errorcode;
+                                 &errorcode, &erroffset, nullptr);
-//   PCRE2_SIZE erroffset;
+  if (!re)
-//
+    return results;
-//   // 1. Compile (Standard compilation)
+  pcre2_match_data *mdata = pcre2_match_data_create_from_pattern(re, nullptr);
-//   pcre2_code *re = pcre2_compile((PCRE2_SPTR)pattern, PCRE2_ZERO_TERMINATED,
+  LeafIterator *it = begin_k_iter(root, 0);
-//   0,
+  if (!it) {
-//                                  &errorcode, &erroffset, nullptr);
+    pcre2_match_data_free(mdata);
-//   if (!re) {
+    pcre2_code_free(re);
-//     fprintf(stderr, "PCRE2 compile error: %d\n", errorcode);
+    return results;
-//     return results;
+  }
-//   }
+  size_t limit = 256;
-//
+  results.reserve(limit);
-//   pcre2_match_data *mdata = pcre2_match_data_create_from_pattern(re,
+  std::string buffer;
-//   nullptr);
+  buffer.reserve(MAX_OVERLAP * 2);
-//
+  size_t buffer_abs_offset = 0;
-//   LeafIterator *it = begin_k_iter(root, 0);
+  size_t processed_up_to = 0;
-//   if (!it) {
+  while (true) {
-//     pcre2_code_free(re);
+    uint32_t chunk_len;
-//     pcre2_match_data_free(mdata);
+    const char *chunk = next_leaf(it, &chunk_len);
-//     return results;
+    if (!chunk)
-//   }
+      break;
-//
+    buffer.append(chunk, chunk_len);
-//   // Buffer to hold (Last X chars) + (Current Chunk)
+    PCRE2_SPTR subject = (PCRE2_SPTR)buffer.data();
-//   std::string buffer;
+    size_t subject_len = buffer.size();
-//
+    size_t start_offset = 0;
-//   // Tracks where the *start* of the current buffer is located relative to
+    while (true) {
-//   the
+      int rc = pcre2_match(re, subject, subject_len, start_offset, 0, mdata,
-//   // whole rope
+                           nullptr);
-//   size_t buffer_abs_offset = 0;
+      if (rc < 0)
-//
+        break;
-//   // Tracks the absolute offset up to which we have already "cleared"
+      PCRE2_SIZE *ov = pcre2_get_ovector_pointer(mdata);
-//   matches.
+      size_t local_start = ov[0];
-//   // This prevents reporting a match twice if it sits inside the overlap
+      size_t local_end = ov[1];
-//   region. size_t processed_up_to_abs = 0;
+      size_t abs_start = buffer_abs_offset + local_start;
-//
+      if (abs_start >= processed_up_to) {
-//   while (1) {
+        if (results.size() >= limit) {
-//     // 2. Get next chunk
+          limit *= 2;
-//     const char *chunk_start = next_leaf(it, nullptr);
+          results.reserve(limit);
-//     if (!chunk_start)
+        }
-//       break;
+        results.push_back({abs_start, abs_start + local_end - local_start,
-//
+                           std::string(buffer.data() + local_start,
-//     // 3. Update Buffer: Append new data
+                                       local_end - local_start)});
-//     size_t chunk_len = strlen(chunk_start);
+        processed_up_to = abs_start + 1;
-//     buffer.append(chunk_start, chunk_len);
+      }
-//
+      start_offset = (local_end > local_start) ? local_end : local_start + 1;
-//     PCRE2_SPTR subject = (PCRE2_SPTR)buffer.c_str();
+      if (start_offset >= subject_len)
-//     size_t subject_len = buffer.length();
+        break;
-//     size_t start_offset = 0;
+    }
-//
+    if (buffer.size() > MAX_OVERLAP) {
-//     // 4. Run pcre2_match loop on the current window
+      size_t trim = buffer.size() - MAX_OVERLAP;
-//     while (true) {
+      buffer.erase(0, trim);
-//       int rc = pcre2_match(re, subject, subject_len, start_offset,
+      buffer_abs_offset += trim;
-//                            0, // Default options
+    }
-//                            mdata, nullptr);
+  }
-//
+  pcre2_match_data_free(mdata);
-//       if (rc < 0) {
+  pcre2_code_free(re);
-//         // No match (or error) in the rest of this buffer
+  free(it);
-//         break;
+  return results;
-//       }
+}
-//
-//       PCRE2_SIZE *ovector = pcre2_get_ovector_pointer(mdata);
-//       size_t match_local_start = ovector[0];
-//       size_t match_local_end = ovector[1];
-//
-//       // Calculate Absolute Coordinates
-//       size_t match_abs_start = buffer_abs_offset + match_local_start;
-//       size_t match_len = match_local_end - match_local_start;
-//
-//       // 5. Deduplication Check
-//       // If we find a match that starts *before* where we finished processing
-//       // the previous chunk, it means this match is entirely inside the
-//       // overlap region and was reported in the previous iteration.
-//       if (match_abs_start >= processed_up_to_abs) {
-//         results.push_back(std::make_pair(match_abs_start, match_len));
-//         // Update processed marker so we don't report this again
-//         // (Using start + 1 ensures we allow overlapping matches if regex
-//         // allows, but strictly prevents the exact same start index being
-//         // reported twice)
-//         processed_up_to_abs = match_abs_start + 1;
-//       }
-//
-//       // Prepare for next match in this buffer
-//       start_offset = match_local_end;
-//
-//       // Handle empty matches (e.g. "a*" matching empty string) to prevent
-//       // infinite loop
-//       if (match_local_end == match_local_start) {
-//         if (start_offset < subject_len) {
-//           start_offset++;
-//         } else {
-//           break; // End of buffer
-//         }
-//       }
-//     }
-//
-//     // 6. Maintenance: Shrink buffer to keep only the last MAX_OVERLAP
-//     // characters
-//     if (buffer.length() > MAX_OVERLAP) {
-//       size_t to_remove = buffer.length() - MAX_OVERLAP;
-//
-//       // Remove from the beginning of the string
-//       buffer.erase(0, to_remove);
-//
-//       // The buffer's start has now moved forward in absolute terms
-//       buffer_abs_offset += to_remove;
-//     }
-//   }
-//
-//   // Cleanup
-//   pcre2_match_data_free(mdata);
-//   pcre2_code_free(re);
-//   free(it); // Assuming iter needs free based on original code usage
-//
-//   return results;
-// }

src/ts/ts.cc

@@ -13,10 +13,10 @@ const char *read_ts(void *payload, uint32_t byte_index, TSPoint,
 }
 
 void ts_collect_spans(Editor *editor) {
-  static int parse_counter = 0;
+  static int parse_counter = 64;
   if (!editor->ts.parser || !editor->root || !editor->ts.query)
     return;
-  const bool injections_enabled = editor->root->char_count < (1024 * 32);
+  const bool injections_enabled = editor->root->char_count < (1024 * 20);
   for (auto &inj : editor->ts.injections)
     inj.second.ranges.clear();
   TSInput tsinput{
@@ -30,32 +30,19 @@ void ts_collect_spans(Editor *editor) {
   if (!editor->edit_queue.empty()) {
     while (editor->edit_queue.pop(edit))
       edits.push_back(edit);
-    if (editor->ts.tree) {
+    if (editor->ts.tree)
       for (auto &e : edits)
         ts_tree_edit(editor->ts.tree, &e);
-    }
+    for (auto &inj : editor->ts.injections)
-    for (auto &inj : editor->ts.injections) {
+      if (inj.second.tree)
-      if (inj.second.tree) {
+        for (auto &e : edits)
-        for (auto &e : edits) {
+          ts_tree_edit(inj.second.tree, &e);
-          TSInputEdit inj_edit = e;
+  } else if (editor->ts.tree && parse_counter++ < 64) {
-          for (auto &r : inj.second.ranges) {
-            if (e.start_byte >= r.start_byte && e.start_byte <= r.end_byte) {
-              inj_edit.start_byte -= r.start_byte;
-              inj_edit.old_end_byte -= r.start_byte;
-              inj_edit.new_end_byte -= r.start_byte;
-            }
-          }
-          ts_tree_edit(inj.second.tree, &inj_edit);
-        }
-      }
-    }
-  } else if (editor->ts.tree && parse_counter < 64) {
-    parse_counter++;
     return;
   }
   parse_counter = 0;
-  editor->spans.mid_parse = true;
   std::shared_lock lock(editor->knot_mtx);
+  editor->spans.mid_parse = true;
   TSTree *tree = ts_parser_parse(editor->ts.parser, editor->ts.tree, tsinput);
   if (!tree)
     return;
@@ -169,11 +156,12 @@ void ts_collect_spans(Editor *editor) {
       }
     }
   }
+  lock.lock();
   std::pair<uint32_t, int64_t> span_edit;
   while (editor->spans.edits.pop(span_edit))
     apply_edit(new_spans, span_edit.first, span_edit.second);
   std::sort(new_spans.begin(), new_spans.end());
-  std::unique_lock span_mtx(editor->spans.mtx);
   editor->spans.mid_parse = false;
+  std::unique_lock span_mtx(editor->spans.mtx);
   editor->spans.spans.swap(new_spans);
 }

src/utils/system.cc

@@ -38,19 +38,30 @@ char *load_file(const char *path, uint32_t *out_len) {
   if (!file.is_open())
     return nullptr;
   std::streamsize len = file.tellg();
-  if (len < 0 || (std::uint32_t)len > 0xFFFFFFFF)
+  if (len < 0 || static_cast<uint32_t>(len) > 0xFFFFFFFF)
     return nullptr;
   file.seekg(0, std::ios::beg);
-  char *buf = (char *)malloc(static_cast<std::uint32_t>(len));
+  bool add_newline = false;
+  if (len > 0) {
+    file.seekg(-1, std::ios::end);
+    char last_char;
+    file.read(&last_char, 1);
+    if (last_char != '\n')
+      add_newline = true;
+  }
+  file.seekg(0, std::ios::beg);
+  uint32_t alloc_size = static_cast<uint32_t>(len) + (add_newline ? 1 : 0);
+  char *buf = (char *)malloc(alloc_size);
   if (!buf)
     return nullptr;
-  if (file.read(buf, len)) {
+  if (!file.read(buf, len)) {
-    *out_len = static_cast<uint32_t>(len);
-    return buf;
-  } else {
     free(buf);
     return nullptr;
   }
+  if (add_newline)
+    buf[len++] = '\n';
+  *out_len = static_cast<uint32_t>(len);
+  return buf;
 }
 
 static std::string file_extension(const char *filename) {