Major changes to api and structure.

- Remove newline caching
- limit petal sizes
- make public facing api use points only
- fix iterators (chunk & line)

include/utils/utils.h

@@ -0,0 +1,8 @@
+#pragma once
+
+#include "pch.h"
+
+struct Point {
+  uint32_t row;
+  uint32_t col;
+};

include/vase/buffer/append.h

@@ -12,31 +12,15 @@ struct AppendBuffer : Buffer {
   uint32_t current_offset{0};
   uint32_t t_offset{0};
 
-  using LChunk = uint32_t[CHUNK_SIZE];
-  std::vector<LChunk *> newlines;
-  LChunk *l_current;
-  uint32_t l_offset{0};
-
   AppendBuffer();
 
   ~AppendBuffer();
 
   uint32_t key(char c);
-  uint32_t append(const char *text, uint32_t length, uint32_t *line_count);
+  uint32_t append(const char *text, uint32_t length);
 
   const char *read(uint32_t pos, uint32_t *out_len);
-  uint32_t count_lines(uint32_t pos, uint32_t length);
+  inline uint32_t length();
-
-  uint32_t next_newline(uint32_t pos);
-  uint32_t nth_newline(uint32_t pos, uint32_t n);
-
-private:
-  inline uint32_t newline_value_at(uint32_t flat_index);
 
   inline void new_text_chunk();
-  inline void new_line_chunk();
-
-  inline uint32_t find_newline(uint32_t target, uint32_t low, uint32_t high);
-
-  inline uint32_t _append(const char *text, uint32_t length);
 };

include/vase/buffer/buffer.h

@@ -4,8 +4,6 @@
 
 struct Buffer {
   virtual const char *read(uint32_t pos, uint32_t *out_len) = 0;
-  virtual uint32_t count_lines(uint32_t pos, uint32_t length) = 0;
+  virtual inline uint32_t length() = 0;
-  virtual uint32_t next_newline(uint32_t pos) = 0;
-  virtual uint32_t nth_newline(uint32_t pos, uint32_t n) = 0;
   virtual ~Buffer() = default;
 };

include/vase/buffer/original.h

@@ -5,16 +5,13 @@
 
 struct OriginalBuffer : Buffer {
   char *buf;
-  uint32_t length;
+  uint32_t len;
-  std::vector<uint32_t> newlines;
 
   // TODO: replace with file io or even lazy loaded file io for large files/logs etc. later.
-  OriginalBuffer(char *buf, uint32_t length);
+  OriginalBuffer(char *buf, uint32_t len);
 
   ~OriginalBuffer();
 
   const char *read(uint32_t pos, uint32_t *out_len);
-  uint32_t count_lines(uint32_t pos, uint32_t length);
+  uint32_t length();
-  uint32_t next_newline(uint32_t pos);
-  uint32_t nth_newline(uint32_t pos, uint32_t n);
 };

include/vase/iterators/chunk.h

@@ -8,11 +8,14 @@ struct ChunkIterator {
   std::vector<Shard *> stack;
   Petal *petal = nullptr;
   uint32_t petal_offset = 0;
+  uint32_t global_offset;
 
-  ChunkIterator(Shard *r, uint32_t offset = 0);
+  ChunkIterator(Shard *r);
 
   ~ChunkIterator();
 
-  void seek(uint32_t offset);
+  void seek_offset(uint32_t offset);
+  void seek_line(uint32_t offset);
   bool next(const char **data, uint32_t *out_len);
+  uint32_t byte_offset();
 };

include/vase/iterators/line.h

@@ -5,16 +5,17 @@
 #include "pch.h"
 
 struct LineIterator {
-  uint32_t offset;
   ChunkIterator it;
 
   const char *cursor = nullptr;
-  uint32_t remaining = 0;
   bool eof = false;
+  uint32_t remaining = 0;
+  uint32_t line_offset = 0;
 
   LineIterator(Shard *r, uint32_t start_line);
 
-  ~LineIterator();
+  ~LineIterator() = default;
 
   bool next(std::string &line);
+  uint32_t byte_offset();
 };

include/vase/shard.h

@@ -1,7 +1,9 @@
 #pragma once
 
 #include "buffer/buffer.h"
+#include "buffer/original.h"
 #include "pch.h"
+#include "utils/utils.h"
 
 struct Shard {
   enum struct ShardKind : uint8_t {
@@ -19,17 +21,8 @@ struct Shard {
   Shard(ShardKind kind, uint32_t length, uint32_t lines, uint8_t height)
       : kind(kind), height(height), length(length), lines(lines), refs(1) {};
 
-  virtual ~Shard() = default;
+  static void retain(Shard *n);
-
+  static void release(Shard *n);
-  static void retain(Shard *n) {
-    n->refs++;
-  }
-
-  static void release(Shard *n) {
-    if (!n || --n->refs > 0)
-      return;
-    delete n;
-  }
 };
 
 struct Branch : Shard {
@@ -47,11 +40,6 @@ struct Branch : Shard {
     retain(left);
     retain(right);
   };
-
-  ~Branch() {
-    release(left);
-    release(right);
-  }
 };
 
 struct Petal : Shard {
@@ -64,12 +52,11 @@ struct Petal : Shard {
         source(source), pos(pos) {};
 };
 
+Shard *create_shards(Buffer *o);
+
 std::pair<Shard *, Shard *> split_shard(Shard *n, uint32_t offset);
 Shard *concat_shard(Shard *left, Shard *right);
 Shard *merge(Shard *a, Shard *b);
 Shard *merge_leaves(Shard *a, Shard *b);
 Shard *append_leaf(Shard *root, Shard *leaf);
-
+Shard *build_balanced(Shard **pieces, uint32_t lo, uint32_t hi);
-uint32_t offset_of(Shard *s, uint32_t line_number, uint32_t col);
-
-void print_shard(const Shard *shard, int depth = 0);

include/vase/vase.h

@@ -2,46 +2,44 @@
 
 #include "buffer/append.h"
 #include "buffer/original.h"
-#include "iterators/chunk.h"
 #include "iterators/line.h"
 #include "pch.h"
 #include "search.h"
 #include "shard.h"
+#include "utils/utils.h"
 
 struct Vase {
-  OriginalBuffer original;
-  AppendBuffer append;
-
-  /*std::vector<Shard *> undo; // TODO: later
-  uint8_t top; // of the undo stack.
-  uint8_t max; // for redo when no edits have been done after some undo.*/
-
-  Shard *root;
-
   Vase(char *data, uint32_t length);
 
   ~Vase();
 
   uint32_t length();
-
   std::string to_string();
-  static void flatten(Shard *s, std::string &out);
 
-  void type(uint32_t offset, char key);
+  Point resolve_column(uint32_t line, uint32_t column);
-  void insert(uint32_t offset, const char *data, uint32_t len);
-  void erase(uint32_t cursor, int64_t amount);
 
-  // Grapheme clusters or the specail cluster "\r\n'
+  LineIterator iterate(uint32_t line);
-  void erase_clusters(uint32_t cursor, int64_t amount);
-
-  // moves in clusters from line,col and sets new position to line,col returns false if not enough space to jump.
-  bool jump_clusters(uint32_t *line, uint32_t *col, int64_t amount);
-
-  // need to also make helpers for line width.
 
+  void input(Point *point, char key);
+  void insert(Point *point, const char *data, uint32_t len);
+  void erase(Point *point, int64_t amount);
+  bool jump(Point *point, int64_t amount);
+  bool clamp(Point *point);
   void regex_search_replace(
     std::string_view pattern,
-    uint32_t start_offset, uint32_t end_offset,
+    Point start, Point end,
     std::string_view replace, std::string_view options
   );
+
+private:
+  OriginalBuffer original;
+  AppendBuffer append;
+  Shard *root;
+
+  /*std::vector<Shard *> undo; // TODO: later
+  uint16_t top; // of the undo stack.
+  uint16_t max; // for redo when no edits have been done after some undo.*/
+
+  uint32_t offset_of(Point point);
+  static void flatten(Shard *s, std::string &out);
 };

src/main.cc

@@ -1,17 +1,24 @@
 #include "io/file.h"
 #include "pch.h"
+#include "vase/iterators/chunk.h"
+#include "vase/iterators/line.h"
 #include "vase/shard.h"
 #include "vase/vase.h"
 
 int main(int argc, char *argv[]) {
-  /*const char *text_o =
+
-    "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz\n"
+#ifdef DEBUG
-    "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz\n"
+
-    "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz\n"
+  const char *text_o =
-    "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz";
+    "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz1\n"
+    "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz2\n"
+    "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz3\n"
+    "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz4";
 
   uint32_t len = strlen(text_o);
-  char *text = strdup(text_o);*/
+  char *text = strdup(text_o);
+
+#else
 
   const char *path;
 
@@ -24,65 +31,9 @@ int main(int argc, char *argv[]) {
   char *text;
   read_file(path, &text, &len);
 
-  auto start = std::chrono::steady_clock::now();
+#endif
+
   Vase vase = Vase(text, len);
-  auto end = std::chrono::steady_clock::now();
-  std::cout << "Time to load: "
-            << std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count()
-            << " ms\n";
-
-  std::cout << "Press Enter to continue...";
-  std::cin.get();
-
-  vase.insert(1, "bcgr\ntt", 5);
-  vase.insert(2, "cgr\ntt", 5);
-  vase.insert(58, "gr\ntt", 5);
-  vase.insert(100, "gr\ntt", 5);
-  vase.insert(20, "gr\ntt", 5);
-
-  vase.type(5, '1');
-  vase.type(6, '2');
-  vase.type(7, '3');
-  vase.type(8, '4');
-
-  vase.erase(offset_of(vase.root, 7, 3), -3);
-
-  // supports gisxUn and m as inverse (as multiline is default.)
-  // print_regex(matches); // debug
-  start = std::chrono::steady_clock::now();
-  std::vector<RegexMatch> matches = regex_search(vase.root, "here", 0, vase.length(), "");
-  end = std::chrono::steady_clock::now();
-  std::cout << "Time search: "
-            << std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count()
-            << " ms\n";
-
-  // print_shard(vase.root); // debug
-
-  /*LineIterator it(vase.root, 0);
-  std::string line;
-  while (it.next(line))
-    std::cout << line << "\n";*/
-
-  /*start = std::chrono::steady_clock::now();
-  vase.regex_search_replace("here", 0, vase.length(), "hello", "");
-  end = std::chrono::steady_clock::now();
-  std::cout << "Time replace: "
-            << std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count()
-            << " ms\n";*/
-
-  std::cout << "\n\n"
-            << (int)vase.root->height << " append:" << (int)vase.append.current_offset;
-
-  std::cout << "Press Enter to continue...";
-  std::cin.get();
-
-  /*ChunkIterator it2(vase.root);
-  const char *data;
-  uint32_t length;
-  while (it2.next(&data, &length))
-    std::cout << std::string(data, length);*/
-
-  // std::cout          << vase.to_string();
 
   return 0;
 }

src/vase/buffer/append.cc

@@ -2,140 +2,49 @@
 
 AppendBuffer::AppendBuffer() {
   new_text_chunk();
-  new_line_chunk();
 }
 
 AppendBuffer::~AppendBuffer() {
   for (auto p : buf)
     free(p);
-  for (auto p : newlines)
-    free(p);
 }
 
 uint32_t AppendBuffer::key(char c) {
   if (t_offset == CHUNK_SIZE)
     new_text_chunk();
-
   (*t_current)[t_offset++] = c;
-
+  return current_offset++;
-  if (c == '\n') {
-    if (l_offset == CHUNK_SIZE)
-      new_line_chunk();
-    (*l_current)[l_offset++] = current_offset;
-  }
-  ++current_offset;
-
-  return current_offset - 1;
 }
 
-uint32_t AppendBuffer::append(const char *text, uint32_t length, uint32_t *line_count) {
+uint32_t AppendBuffer::append(const char *text, uint32_t length) {
   uint32_t start = current_offset;
-
+  while (length > 0) {
-  while (length) {
+    if (t_offset == CHUNK_SIZE)
-    uint32_t space = CHUNK_SIZE - t_offset;
-
-    if (space == 0) {
       new_text_chunk();
-      space = CHUNK_SIZE;
+    uint32_t copy = std::min(length, CHUNK_SIZE - t_offset);
-    }
+    memcpy((*t_current) + t_offset, text, copy);
-
+    t_offset += copy;
-    uint32_t copy = length < space ? length : space;
+    current_offset += copy;
-    *line_count += _append(text, copy);
-
     text += copy;
     length -= copy;
   }
-
   return start;
 }
 
 const char *AppendBuffer::read(uint32_t pos, uint32_t *out_len) {
   if (pos >= current_offset)
     return nullptr;
-
   uint32_t local_offset = pos % CHUNK_SIZE;
-
   if (out_len) {
     uint32_t remaining = current_offset - pos;
     uint32_t until_chunk_end = CHUNK_SIZE - local_offset;
     *out_len = std::min(remaining, until_chunk_end);
   }
-
   return &(*buf[pos / CHUNK_SIZE])[local_offset];
 }
 
-uint32_t AppendBuffer::find_newline(uint32_t target, uint32_t low, uint32_t high) {
+inline uint32_t AppendBuffer::length() {
-  if (low >= high)
+  return current_offset;
-    return low;
-
-  uint32_t first_chunk = low / CHUNK_SIZE;
-  uint32_t last_chunk = (high - 1) / CHUNK_SIZE;
-  uint32_t left = first_chunk;
-  uint32_t right = last_chunk;
-
-  while (left < right) {
-    uint32_t mid = left + (right - left) / 2;
-    uint32_t first_value = (*newlines[mid])[0];
-    if (first_value < target)
-      left = mid + 1;
-    else
-      right = mid;
-  }
-
-  uint32_t chunk = left;
-  if (chunk > first_chunk) {
-    LChunk &prev = *newlines[chunk - 1];
-    uint32_t prev_size =
-      (chunk - 1 == newlines.size() - 1) ? l_offset : CHUNK_SIZE;
-    if (prev[prev_size - 1] >= target)
-      chunk--;
-  }
-
-  LChunk &c = *newlines[chunk];
-  uint32_t chunk_size =
-    (chunk == newlines.size() - 1) ? l_offset : CHUNK_SIZE;
-  uint32_t l = 0;
-  uint32_t r = chunk_size;
-
-  while (l < r) {
-    uint32_t m = l + (r - l) / 2;
-    if (c[m] < target)
-      l = m + 1;
-    else
-      r = m;
-  }
-
-  return chunk * CHUNK_SIZE + l;
-}
-
-inline uint32_t AppendBuffer::newline_value_at(uint32_t flat_index) {
-  uint32_t chunk = flat_index / CHUNK_SIZE;
-  uint32_t off = flat_index % CHUNK_SIZE;
-  return (*newlines[chunk])[off];
-}
-
-uint32_t AppendBuffer::next_newline(uint32_t pos) {
-  uint32_t total = CHUNK_SIZE * (uint32_t)(newlines.size() - 1) + l_offset;
-  uint32_t idx = find_newline(pos, 0, total);
-  return idx >= total ? UINT32_MAX : newline_value_at(idx);
-}
-
-uint32_t AppendBuffer::nth_newline(uint32_t pos, uint32_t n) {
-  uint32_t total = CHUNK_SIZE * (uint32_t)(newlines.size() - 1) + l_offset;
-  uint32_t idx0 = find_newline(pos, 0, total);
-  uint32_t idx = idx0 + (n - 1);
-  return idx >= total ? UINT32_MAX : newline_value_at(idx);
-}
-
-uint32_t AppendBuffer::count_lines(uint32_t pos, uint32_t length) {
-  if (newlines.empty())
-    return 0;
-
-  uint32_t total_newlines = CHUNK_SIZE * (newlines.size() - 1) + l_offset;
-  uint32_t pos_index = find_newline(pos, 0, total_newlines);
-  uint32_t end_index = find_newline(pos + length, pos_index, total_newlines);
-
-  return end_index - pos_index;
 }
 
 inline void AppendBuffer::new_text_chunk() {
@@ -143,32 +52,3 @@ inline void AppendBuffer::new_text_chunk() {
   buf.push_back(t_current);
   t_offset = 0;
 }
-
-inline void AppendBuffer::new_line_chunk() {
-  l_current = (LChunk *)malloc(sizeof(LChunk));
-  newlines.push_back(l_current);
-  l_offset = 0;
-}
-
-inline uint32_t AppendBuffer::_append(const char *text, uint32_t length) {
-  memcpy((*t_current) + t_offset, text, length);
-
-  const char *p = text;
-  const char *end = text + length;
-
-  uint32_t start_index = l_offset;
-  while (p < end) {
-    p = (const char *)memchr(p, '\n', end - p);
-    if (!p)
-      break;
-    if (l_offset == CHUNK_SIZE)
-      new_line_chunk();
-    (*l_current)[l_offset++] = current_offset + uint32_t(p - text);
-    p++;
-  }
-
-  t_offset += length;
-  current_offset += length;
-
-  return l_offset - start_index;
-}

src/vase/buffer/original.cc

@@ -1,52 +1,19 @@
 #include "vase/buffer/original.h"
 
-OriginalBuffer::OriginalBuffer(char *buf, uint32_t length) : buf(buf), length(length) {
+OriginalBuffer::OriginalBuffer(char *buf, uint32_t len) : buf(buf), len(len) {}
-  const char *p = buf;
-  const char *end = buf + length;
-  while (p < end) {
-    p = (const char *)memchr(p, '\n', end - p);
-    if (!p)
-      break;
-    newlines.push_back(uint32_t(p - buf));
-    p++;
-  }
-}
 
 OriginalBuffer::~OriginalBuffer() {
   free(buf);
 }
 
 const char *OriginalBuffer::read(uint32_t pos, uint32_t *out_len) {
-  if (pos >= length)
+  if (pos >= len)
     return nullptr;
-  *out_len = length - pos;
+  if (out_len)
+    *out_len = len - pos;
   return buf + pos;
 }
 
-uint32_t OriginalBuffer::count_lines(uint32_t pos, uint32_t length) {
+uint32_t OriginalBuffer::length() {
-  auto start = std::lower_bound(
+  return len;
-    newlines.begin(),
-    newlines.end(),
-    pos
-  );
-
-  auto end = std::lower_bound(
-    start,
-    newlines.end(),
-    pos + length
-  );
-
-  return (uint32_t)(end - start);
-}
-
-uint32_t OriginalBuffer::next_newline(uint32_t pos) {
-  auto it = std::lower_bound(newlines.begin(), newlines.end(), pos);
-  return it == newlines.end() ? UINT32_MAX : *it;
-}
-
-uint32_t OriginalBuffer::nth_newline(uint32_t pos, uint32_t n) {
-  auto it = std::lower_bound(newlines.begin(), newlines.end(), pos);
-  if (uint32_t(newlines.end() - it) < n)
-    return UINT32_MAX;
-  return *(it + (n - 1));
 }

src/vase/iterators/chunk.cc

@@ -1,26 +1,24 @@
 #include "vase/iterators/chunk.h"
 
-ChunkIterator::ChunkIterator(Shard *r, uint32_t offset) : root(r) {
+ChunkIterator::ChunkIterator(Shard *r) : root(r) {
   if (!root)
     return;
   Shard::retain(root);
-  seek(offset);
 }
 
 ChunkIterator::~ChunkIterator() {
   Shard::release(root);
 }
 
-void ChunkIterator::seek(uint32_t offset) {
+void ChunkIterator::seek_offset(uint32_t offset) {
   stack.clear();
   petal = nullptr;
   petal_offset = 0;
+  global_offset = 0;
   if (!root || offset >= root->length)
     return;
-
   uint32_t target = offset;
   Shard *curr = root;
-
   while (curr) {
     if (curr->kind == Shard::ShardKind::Petal) {
       petal = (Petal *)curr;
@@ -34,28 +32,87 @@ void ChunkIterator::seek(uint32_t offset) {
         curr = b->left;
       } else {
         target -= left_len;
+        global_offset += b->left->length;
         curr = b->right;
       }
     }
   }
 }
 
+void ChunkIterator::seek_line(uint32_t line) {
+  stack.clear();
+  petal = nullptr;
+  petal_offset = 0;
+  global_offset = 0;
+  if (!root || line > root->lines)
+    return;
+  Shard *curr = root;
+  while (curr) {
+    if (curr->kind == Shard::ShardKind::Petal) {
+      petal = (Petal *)curr;
+      const char *text = nullptr;
+      uint32_t remaining = 0;
+      uint32_t offset = 0;
+      while (line) {
+        if (!text) {
+          uint32_t got = 0;
+          text = petal->source->read(petal->pos + offset, &got);
+          remaining = std::min(got, petal->length - offset);
+        }
+        const char *c = (const char *)memchr(text, '\n', remaining);
+        if (!c) {
+          offset += remaining;
+          text = nullptr;
+          continue;
+        }
+        remaining -= (c - text) + 1;
+        offset += (c - text) + 1;
+        text = c + 1;
+        line--;
+      }
+      petal_offset = offset;
+      return;
+    } else {
+      auto *b = (Branch *)curr;
+      uint32_t left_lines = b->left->lines;
+      if (line <= left_lines) {
+        stack.push_back(b->right);
+        curr = b->left;
+      } else {
+        line -= left_lines;
+        global_offset += b->left->length;
+        curr = b->right;
+      }
+    }
+  }
+}
+
+uint32_t ChunkIterator::byte_offset() {
+  return global_offset + petal_offset;
+}
+
 bool ChunkIterator::next(const char **data, uint32_t *out_len) {
   while (true) {
-    if (petal && petal_offset < petal->length) {
+    if (petal) {
-      uint32_t remaining = petal->length - petal_offset;
+      if (petal_offset < petal->length) {
-      uint32_t read_pos = petal->pos + petal_offset;
+        uint32_t remaining = petal->length - petal_offset;
-      uint32_t got = 0;
+        uint32_t read_pos = petal->pos + petal_offset;
-      const char *chunk = petal->source->read(read_pos, &got);
+        uint32_t got = 0;
-      if (got == 0) {
+        const char *chunk = petal->source->read(read_pos, &got);
-        petal = nullptr;
+        if (got == 0) {
-        continue;
+          petal_offset = petal->length;
+          petal = nullptr;
+          continue;
+        }
+        uint32_t take = std::min(got, remaining);
+        *data = chunk;
+        *out_len = take;
+        petal_offset += take;
+        return true;
       }
-      uint32_t take = std::min(got, remaining);
+      global_offset += petal->length;
-      *data = chunk;
+      petal = nullptr;
-      *out_len = take;
+      petal_offset = 0;
-      petal_offset += take;
-      return true;
     }
     if (stack.empty())
       return false;

src/vase/iterators/line.cc

@@ -1,12 +1,12 @@
 #include "vase/iterators/line.h"
 
-LineIterator::LineIterator(Shard *r, uint32_t start_line)
+LineIterator::LineIterator(Shard *r, uint32_t start_line) : it(r) {
-    : offset(offset_of(r, start_line, 0)), it(r, offset) {}
+  it.seek_line(start_line);
-
+}
-LineIterator::~LineIterator() {};
 
 bool LineIterator::next(std::string &line) {
   line.clear();
+  line_offset = it.byte_offset() - remaining;
   while (true) {
     if (remaining == 0) {
       uint32_t got;
@@ -14,12 +14,15 @@ bool LineIterator::next(std::string &line) {
         eof = true;
         break;
       }
+      line_offset = it.byte_offset() - remaining;
       remaining = got;
     }
     const char *nl = (const char *)memchr(cursor, '\n', remaining);
     if (nl) {
       const char *end = nl;
-      size_t len = end - cursor;
+      uint32_t len = nl - cursor;
+      if (len && cursor[len - 1] == '\r')
+        --len;
       line.append(cursor, len);
       cursor = end + 1;
       remaining -= (uint32_t)(len + 1);
@@ -31,3 +34,7 @@ bool LineIterator::next(std::string &line) {
   }
   return !line.empty();
 }
+
+uint32_t LineIterator::byte_offset() {
+  return line_offset;
+}

src/vase/search.cc

@@ -9,6 +9,8 @@ std::vector<RegexMatch> regex_search(
   bool global = false;
   uint32_t flags = PCRE2_MULTILINE;
 
+  const char *dot = "(?:(?!\\n)\\X)";
+
   for (char c : options) {
     switch (c) {
     case 'g':
@@ -18,7 +20,7 @@ std::vector<RegexMatch> regex_search(
       flags |= PCRE2_CASELESS;
       break;
     case 's':
-      flags |= PCRE2_DOTALL;
+      dot = "(?:\\X)";
       break;
     case 'x':
       flags |= PCRE2_EXTENDED;
@@ -40,11 +42,36 @@ std::vector<RegexMatch> regex_search(
   int errornumber;
   PCRE2_SIZE erroroffset;
 
-  PCRE2_SPTR pattern = (PCRE2_SPTR)pattern_str.data();
+  std::string out;
+  out.reserve(pattern_str.size() * 2);
+
+  bool escaped = false;
+
+  for (char c : pattern_str) {
+    // TODO: also dont switch out in [.] style groups.
+    if (escaped) {
+      out += c;
+      escaped = false;
+      continue;
+    }
+
+    if (c == '\\') {
+      out += c;
+      escaped = true;
+      continue;
+    }
+
+    if (c == '.') {
+      out += dot;
+      continue;
+    }
+
+    out += c;
+  }
 
   pcre2_code *re = pcre2_compile(
-    pattern,
+    (PCRE2_SPTR)out.data(),
-    pattern_str.size(),
+    out.size(),
     flags,
     &errornumber,
     &erroroffset,
@@ -56,7 +83,8 @@ std::vector<RegexMatch> regex_search(
 
   pcre2_match_data *match_data = pcre2_match_data_create_from_pattern(re, NULL);
 
-  ChunkIterator it(root, start_offset);
+  ChunkIterator it(root);
+  it.seek_offset(start_offset);
 
   const char *data;
   uint32_t length;

src/vase/shard.cc

@@ -1,5 +1,21 @@
 #include "vase/shard.h"
 
+void Shard::retain(Shard *n) {
+  n->refs++;
+};
+
+void Shard::release(Shard *n) {
+  if (!n || --n->refs > 0)
+    return;
+  if (n->kind == Shard::ShardKind::Branch) {
+    release(((Branch *)n)->left);
+    release(((Branch *)n)->right);
+    delete (Branch *)n;
+  } else {
+    delete (Petal *)n;
+  }
+}
+
 int height(Shard *n) {
   return n ? n->height : 0;
 }
@@ -123,15 +139,37 @@ std::pair<Shard *, Shard *> split_shard(Shard *n, uint32_t offset) {
     }
   } else {
     Petal *p = (Petal *)n;
+    uint32_t count[2]{0};
+    uint32_t read_offset = 0;
+    while (read_offset < p->length) {
+      uint32_t got = 0;
+      const char *c = p->source->read(p->pos + read_offset, &got);
+      const char *end = c + std::min(got, p->length - read_offset);
+      const char *cursor = c;
+      while (cursor < end) {
+        const char *nl = (const char *)memchr(cursor, '\n', end - cursor);
+        if (!nl) {
+          cursor = end;
+          break;
+        }
+        uint32_t nl_pos = read_offset + (uint32_t)(nl - c);
+        if (nl_pos < offset)
+          count[0]++;
+        else
+          count[1]++;
+        cursor = nl + 1;
+      }
+      read_offset += (uint32_t)(cursor - c);
+    }
     auto left = new Petal(
       offset,
-      p->source->count_lines(p->pos, offset),
+      count[0],
       p->source,
       p->pos
     );
     auto right = new Petal(
       p->length - offset,
-      p->source->count_lines(p->pos + offset, p->length - offset),
+      count[1],
       p->source,
       p->pos + offset
     );
@@ -157,7 +195,7 @@ Shard *merge_leaves(Shard *a, Shard *b) {
 Shard *append_leaf(Shard *root, Shard *leaf) {
   if (!root)
     return leaf;
-  if (root->kind == Shard::ShardKind::Petal)
+  if (root->kind == Shard::ShardKind::Petal && root->length < 32 * 1024)
     return merge_leaves(root, leaf);
   Branch *b = (Branch *)root;
   auto new_right = append_leaf(b->right, leaf);
@@ -170,67 +208,47 @@ Shard *concat_shard(Shard *left, Shard *right) {
   return merge(left, right);
 }
 
-uint32_t offset_of(Shard *s, uint32_t line_number, uint32_t col) {
+Shard *build_balanced(Shard **pieces, uint32_t lo, uint32_t hi) {
-  if (line_number == 0)
+  if (hi - lo == 1)
-    return col;
+    return pieces[lo];
+  size_t mid = lo + (hi - lo) / 2;
+  Shard *left = build_balanced(pieces, lo, mid);
+  Shard *right = build_balanced(pieces, mid, hi);
+  Shard *node = new Branch(left, right);
+  Shard::release(left);
+  Shard::release(right);
+  return node;
+}
 
-  uint32_t nth = line_number;
+Shard *create_shards(Buffer *o) {
-  uint32_t base_offset = 0;
+  constexpr uint32_t PETAL_SIZE = 32 * 1024;
+  std::vector<Shard *> pieces;
+  uint32_t pos = 0;
+  const uint32_t total = o->length();
+  pieces.reserve((total + PETAL_SIZE - 1) / PETAL_SIZE);
 
-  while (s->kind == Shard::ShardKind::Branch) {
+  while (pos < total) {
-    auto *b = (Branch *)s;
+    uint32_t len = 0;
-    if (nth <= b->left->lines) {
+    const char *data = o->read(pos, &len);
-      s = b->left;
+    uint32_t take = std::min(len, PETAL_SIZE);
-    } else {
+    uint32_t lines = 0;
-      nth -= b->left->lines;
+    const char *p = data;
-      base_offset += b->left->length;
+    const char *end = data + take;
-      s = b->right;
+    while (p < end) {
+      const void *nl = memchr(p, '\n', end - p);
+      if (!nl)
+        break;
+      lines++;
+      p = (const char *)nl + 1;
     }
+    pieces.push_back(new Petal(take, lines, o, pos));
+    pos += take;
   }
 
-  auto *petal = (Petal *)s;
+  if (pieces.empty())
-  uint32_t nl_pos = petal->source->nth_newline(petal->pos, nth);
+    return nullptr;
-  uint32_t offset_in_petal = (nl_pos - petal->pos) + 1;
+  if (pieces.size() == 1)
-  return base_offset + offset_in_petal + col;
+    return pieces[0];
-}
 
-void print_shard(const Shard *shard, int depth) {
+  return build_balanced(pieces.data(), 0, pieces.size());
-  if (!shard) {
-    std::cout << std::string(depth * 2, ' ') << "<null>\n";
-    return;
-  }
-
-  std::string indent(depth * 2, ' ');
-  std::cout << indent;
-
-  if (shard->kind == Shard::ShardKind::Branch) {
-    auto *branch = (const Branch *)shard;
-
-    std::cout
-      << "Branch"
-      << " @" << shard
-      << " len=" << shard->length
-      << " lines=" << shard->lines
-      << " height=" << (int)shard->height
-      << " refs=" << shard->refs.load()
-      << "\n";
-
-    std::cout << indent << "├─ left:\n";
-    print_shard(branch->left, depth + 2);
-
-    std::cout << indent << "└─ right:\n";
-    print_shard(branch->right, depth + 2);
-  } else {
-    auto *petal = (const Petal *)(shard);
-
-    std::cout
-      << "Petal"
-      << " @" << shard
-      << " len=" << shard->length
-      << " lines=" << shard->lines
-      << " refs=" << shard->refs.load()
-      << " source=@" << petal->source
-      << " pos=" << petal->pos
-      << "\n";
-  }
 }

src/vase/vase.cc

@@ -1,9 +1,11 @@
 #include "vase/vase.h"
+#include "utils/utils.h"
+#include "vase/iterators/line.h"
 #include "vase/search.h"
 
-Vase::Vase(char *data, uint32_t length) : original(data, length), append() {
+Vase::Vase(char *data, uint32_t length)
-  root = new Petal(length, original.newlines.size(), &original, 0);
+    : original(data, length), append(),
-}
+      root(create_shards(&original)) {}
 
 Vase::~Vase() {
   Shard::release(root);
@@ -19,10 +21,15 @@ std::string Vase::to_string() {
   return out;
 }
 
-void Vase::type(uint32_t offset, char key) {
+void Vase::input(Point *point, char key) {
+  if (key == '\n')
+    *point = {point->row + 1, 0};
+  else
+    point->col++;
   uint32_t pos = append.key(key);
+  // limit petal size to 32KiB here later.
   Shard *inserted = new Petal(1, key == '\n', &append, pos);
-  auto [left, right] = split_shard(root, offset);
+  auto [left, right] = split_shard(root, offset_of(*point));
   Shard *left2 = append_leaf(left, inserted);
   Shard *new_root = concat_shard(left2, right);
   Shard::release(left);
@@ -33,9 +40,33 @@ void Vase::type(uint32_t offset, char key) {
   root = new_root;
 }
 
-void Vase::insert(uint32_t offset, const char *data, uint32_t len) {
+void Vase::insert(Point *point, const char *data, uint32_t len) {
+  if (len == 0)
+    return;
+  uint32_t offset = offset_of(*point);
+  uint32_t pos = append.append(data, len);
   uint32_t lines = 0;
-  uint32_t pos = append.append(data, len, &lines);
+  const char *start = data;
+  const char *last_line = start;
+  const char *end = start + len;
+  while ((data = (const char *)memchr(data, '\n', end - data))) {
+    ++lines;
+    last_line = ++data;
+  }
+  uint32_t col = 0;
+  uint32_t remaining = end - last_line;
+  while (remaining) {
+    uint32_t n = grapheme_next_character_break_utf8(last_line, remaining);
+    last_line += n;
+    remaining -= n;
+    ++col;
+  }
+  if (lines) {
+    point->row += lines;
+    point->col = col;
+  } else {
+    point->col += col;
+  }
   Shard *inserted = new Petal(len, lines, &append, pos);
   auto [left, right] = split_shard(root, offset);
   Shard *left2 = append_leaf(left, inserted);
@@ -48,16 +79,18 @@ void Vase::insert(uint32_t offset, const char *data, uint32_t len) {
   root = new_root;
 }
 
-void Vase::erase(uint32_t cursor, int64_t amount) {
+void Vase::erase(Point *point, int64_t amount) {
+  // wrong: havta make amount be in clusters not bytes
+  uint32_t offset = offset_of(*point);
   if (amount == 0)
     return;
   uint32_t start;
   uint32_t count;
   if (amount < 0) {
-    count = std::min<uint32_t>(-amount, cursor);
+    count = std::min<uint32_t>(-amount, offset);
-    start = cursor - count;
+    start = offset - count;
   } else {
-    start = cursor;
+    start = offset;
     count = amount;
   }
   auto [a, b] = split_shard(root, start);
@@ -91,6 +124,34 @@ void Vase::flatten(Shard *s, std::string &out) {
   }
 }
 
+uint32_t Vase::offset_of(Point point) {
+  LineIterator it(root, point.row);
+  std::string line;
+  uint32_t offset = 0;
+  if (it.next(line)) {
+    const char *ptr = line.data();
+    uint32_t remaining = line.length();
+    while (point.col && remaining) {
+      uint32_t next_len = grapheme_next_character_break_utf8(ptr, remaining);
+      remaining -= next_len;
+      ptr += next_len;
+      offset += next_len;
+      point.col--;
+    }
+  }
+  return it.byte_offset() + offset;
+}
+
+bool Vase::jump(Point *, int64_t) {
+  // todo.
+  return false;
+}
+
+bool Vase::clamp(Point *) {
+  // todo.
+  return false;
+}
+
 struct ReplacePart {
   enum struct PartType {
     FullMatch,
@@ -107,7 +168,7 @@ std::vector<ReplacePart> parse_replace(AppendBuffer &buf, std::string_view s) {
   auto flush_constant = [&]() {
     if (!constant.empty()) {
       uint32_t lines = 0;
-      uint32_t pos = buf.append(constant.data(), (uint32_t)constant.size(), &lines);
+      uint32_t pos = buf.append(constant.data(), (uint32_t)constant.size());
       parts.push_back(
         ReplacePart{
           .type = ReplacePart::PartType::Constant,
@@ -155,24 +216,12 @@ std::vector<ReplacePart> parse_replace(AppendBuffer &buf, std::string_view s) {
   return parts;
 }
 
-Shard *build_balanced(Shard **pieces, size_t lo, size_t hi) {
-  if (hi - lo == 1)
-    return pieces[lo];
-  size_t mid = lo + (hi - lo) / 2;
-  Shard *left = build_balanced(pieces, lo, mid);
-  Shard *right = build_balanced(pieces, mid, hi);
-  Shard *node = new Branch(left, right);
-  Shard::release(left);
-  Shard::release(right);
-  return node;
-}
-
 void Vase::regex_search_replace(
   std::string_view pattern,
-  uint32_t start_offset, uint32_t end_offset,
+  Point start, Point end,
   std::string_view replace, std::string_view options
 ) {
-  std::vector<RegexMatch> matches = regex_search(root, pattern, start_offset, end_offset, options);
+  std::vector<RegexMatch> matches = regex_search(root, pattern, offset_of(start), offset_of(end), options);
   if (matches.empty())
     return;