Make append buffer contiguous and update iterators

include/vase/buffer/append.h

@@ -5,19 +5,19 @@
 #include "pch.h"
 
 struct AppendBuffer : Buffer {
-  using TChunk = char[APPEND_CHUNK_SIZE];
+  char *buf = nullptr;
-  std::vector<TChunk *> buf;
+  uint64_t allocated_capacity = 0;
-  TChunk *t_current;
+  uint64_t current_size = 0;
-  uint64_t current_offset{0};
+  int fd = -1;
-  uint64_t t_offset{0};
-
-  AppendBuffer();
 
+  AppendBuffer(std::filesystem::path base_dir);
   ~AppendBuffer();
 
-  uint64_t append(char c);
+  uint64_t append(const char c);
-  uint64_t append(const char *text, uint64_t length);
+  uint64_t append(const char *text, uint64_t len);
+  const char *read(uint64_t pos) override;
+  uint64_t length() override;
 
-  const char *read(uint64_t pos, uint64_t *out_len);
+private:
-  inline uint64_t length();
+  void grow(uint64_t len);
 };

include/vase/buffer/buffer.h

@@ -3,7 +3,7 @@
 #include "pch.h"
 
 struct Buffer {
-  virtual const char *read(uint64_t pos, uint64_t *out_len) = 0;
+  virtual const char *read(uint64_t pos) = 0;
-  virtual inline uint64_t length() = 0;
+  virtual uint64_t length() = 0;
   virtual ~Buffer() = default;
 };

include/vase/buffer/original.h

@@ -12,6 +12,6 @@ struct OriginalBuffer : Buffer {
   ~OriginalBuffer();
 
   void initialize();
-  const char *read(uint64_t pos, uint64_t *out_len);
+  const char *read(uint64_t pos) override;
-  uint64_t length();
+  uint64_t length() override;
 };

include/vase/constants.h

@@ -2,10 +2,4 @@
 
 #include "pch.h"
 
-constexpr uint64_t APPEND_CHUNK_SIZE = 64 * 1024;
 constexpr uint64_t PETAL_SIZE_MAX = 32 * 1024;
-
-static_assert(
-  APPEND_CHUNK_SIZE > PETAL_SIZE_MAX,
-  "PETAL_SIZE_MAX must be smaller than APPEND_CHUNK_SIZE"
-);

include/vase/iterators/iter.h

@@ -0,0 +1,9 @@
+#pragma once
+
+#include "../shard.h"
+#include "pch.h"
+
+enum struct Direction : uint8_t {
+  Forward,
+  Backward
+};

include/vase/iterators/line.h

@@ -1,22 +1,20 @@
 #pragma once
 
 #include "../shard.h"
-#include "chunk.h"
 #include "pch.h"
+#include "petal.h"
 
 struct LineIterator {
-  ChunkIterator it;
+  PetalIterator it;
-  Direction dir;
-
   const char *chunk;
   uint64_t len;
-  uint64_t offset = 0;
+  Direction dir;
-  uint64_t chunk_offset = 0;
+  uint64_t current_offset = 0;
-  bool at_end = false;
+  uint64_t last_line_offset = 0;
 
-  LineIterator(Shard *r, uint64_t start_line, Direction dir);
+  LineIterator(Shard *root, uint64_t line_num, Direction dir);
   ~LineIterator() = default;
 
-  bool next(std::string &line);
+  bool next(std::string *line);
   uint64_t byte_offset();
 };

include/vase/iterators/petal.h

@@ -1,30 +1,27 @@
 #pragma once
 
 #include "../shard.h"
+#include "iter.h"
 #include "pch.h"
 
-enum struct Direction : uint8_t {
+struct PetalIterator {
-  Forward,
-  Backward
-};
-
-struct ChunkIterator {
   Direction dir;
   Shard *root = nullptr;
   std::vector<Shard *> stack;
   Petal *petal = nullptr;
-  uint64_t petal_offset = 0;
   uint64_t global_offset = 0;
+  uint64_t last_offset = 0;
+  uint64_t petal_offset = 0;
   uint64_t global_line = 0;
-  bool at_end = false;
 
-  ChunkIterator(Shard *r, Direction dir);
+  PetalIterator(Shard *r, Direction dir);
-
+  ~PetalIterator() = default;
-  ~ChunkIterator();
 
   void seek_offset(uint64_t offset);
   void seek_line(uint64_t offset);
   bool next(const char **data, uint64_t *out_len);
   uint64_t byte_offset();
-  uint64_t line_offset();
+
+private:
+  Petal *_next(uint64_t *offset);
 };

include/vase/shard.h

@@ -7,7 +7,7 @@
 #include "utils/utils.h"
 
 struct Shard {
-  enum struct ShardKind : uint8_t {
+  enum struct Kind : uint8_t {
     Branch,
     Petal
   } kind;
@@ -19,7 +19,7 @@ struct Shard {
 
   std::atomic_uint64_t refs;
 
-  Shard(ShardKind kind, uint64_t length, uint64_t lines, uint8_t height)
+  Shard(Kind kind, uint64_t length, uint64_t lines, uint8_t height)
       : kind(kind), height(height), length(length), lines(lines), refs(1) {};
 
   static void retain(Shard *n);
@@ -27,14 +27,13 @@ struct Shard {
 
   static Shard *from_file(std::filesystem::path path, OriginalBuffer *b);
   static std::vector<Shard *> from_swap(std::filesystem::path path, OriginalBuffer *b);
-  static Shard *new_empty(OriginalBuffer *b);
 
   static std::pair<Shard *, Shard *> split(Shard *n, uint64_t offset);
   static Shard *concat(Shard *left, Shard *right);
   static Shard *merge(Shard *a, Shard *b);
   static Shard *merge_leaves(Shard *a, Shard *b);
-  static Shard *append_leaf(Shard *root, Shard *leaf);
+  static Shard *append(Shard *root, Shard *leaf);
-  static Shard *build_balanced(Shard **pieces, uint64_t lo, uint64_t hi);
+  static Shard *build(Shard **pieces, uint64_t lo, uint64_t hi);
 };
 
 struct Branch : Shard {
@@ -43,7 +42,7 @@ struct Branch : Shard {
 
   Branch(Shard *l, Shard *r)
       : Shard(
-          ShardKind::Branch,
+          Kind::Branch,
           l->length + r->length,
           l->lines + r->lines,
           1 + std::max(l->height, r->height)
@@ -56,10 +55,60 @@ struct Branch : Shard {
 
 struct Petal : Shard {
   Buffer *source;
-
   uint64_t pos;
 
   Petal(uint64_t length, uint64_t lines, Buffer *source, uint64_t pos)
-      : Shard(ShardKind::Petal, length, lines, 1),
+      : Shard(Kind::Petal, length, lines, 1),
         source(source), pos(pos) {};
 };
+
+extern inline void dump_shard(Shard *node, int depth = 0) {
+  if (!node) {
+    std::cout << std::string(depth * 2, ' ') << "<null>\n";
+    return;
+  }
+
+  std::string indent(depth * 2, ' ');
+
+  std::cout << indent
+            << "Shard@" << node
+            << " kind=";
+
+  switch (node->kind) {
+  case Shard::Kind::Branch:
+    std::cout << "Branch";
+    break;
+  case Shard::Kind::Petal:
+    std::cout << "Petal";
+    break;
+  }
+
+  std::cout
+    << " height=" << unsigned(node->height)
+    << " length=" << node->length
+    << " lines=" << node->lines
+    << " refs=" << node->refs.load()
+    << "\n";
+
+  if (node->kind == Shard::Kind::Branch) {
+    auto *branch = static_cast<Branch *>(node);
+
+    std::cout << indent << "  left:\n";
+    dump_shard(branch->left, depth + 2);
+
+    std::cout << indent << "  right:\n";
+    dump_shard(branch->right, depth + 2);
+  } else {
+    auto *petal = static_cast<Petal *>(node);
+
+    std::cout
+      << indent << "  source=" << petal->source
+      << " pos=" << petal->pos
+      << " length=" << petal->length
+      << " lines=" << petal->lines
+      << "\n";
+  }
+
+  if (!depth)
+    std::cout << "\n\n";
+}

include/vase/vase.h

@@ -48,7 +48,6 @@ struct Vase {
   std::filesystem::path swapdir;
 
   Vase(std::filesystem::path path, std::filesystem::path swapdir);
-
   ~Vase();
 
   uint64_t length();
@@ -80,7 +79,6 @@ struct Vase {
 
   bool undo();
   bool redo();
-
   void snapshot();
   void prune_history(uint64_t n);
 

src/main.cc

@@ -1,5 +1,4 @@
 #include "pch.h"
-#include "vase/iterators/chunk.h"
 #include "vase/iterators/line.h"
 #include "vase/vase.h"
 
@@ -9,11 +8,12 @@ int main(int argc, char *argv[]) {
 
   Vase vase = Vase(argv[1], "/tmp");
 
-  Point p = {500, 1000};
+  Point p = {UINT64_MAX, UINT64_MAX};
-  vase.insert(&p, 'x');
+  vase.insert(&p, 'c');
 
-  printf("\n"
+  printf("%s\n\n", vase.to_string().c_str());
-         "%lu bytes\n"
+
+  printf("%lu bytes\n"
          "%lu lines\n"
          "\n",
          vase.length(),

src/vase/buffer/append.cc

@@ -1,57 +1,70 @@
 #include "vase/buffer/append.h"
 
-AppendBuffer::AppendBuffer() {
+AppendBuffer::AppendBuffer(std::filesystem::path base_dir) {
-  t_current = (TChunk *)malloc(sizeof(TChunk));
+  base_dir /= "tapp.XXXXXX";
-  buf.push_back(t_current);
+  char *s = strdup(base_dir.c_str());
+  fd = mkstemp(s);
+  if (fd == -1)
+    throw std::runtime_error("mkstemp failed");
+  unlink(s);
+  free(s);
+  allocated_capacity = 1ull << 30;
+  if (ftruncate(fd, allocated_capacity) == -1)
+    throw std::runtime_error("ftruncate failed");
+  buf = (char *)mmap(nullptr, allocated_capacity, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+  if (buf == MAP_FAILED)
+    throw std::runtime_error("mmap failed");
 }
 
 AppendBuffer::~AppendBuffer() {
-  for (auto p : buf)
+  if (buf && buf != MAP_FAILED)
-    free(p);
+    munmap(buf, allocated_capacity);
+  if (fd != -1)
+    close(fd);
 }
 
-uint64_t AppendBuffer::append(char c) {
+void AppendBuffer::grow(uint64_t len) {
-  if (t_offset == APPEND_CHUNK_SIZE) {
+  if (current_size + len > allocated_capacity) {
-    t_current = (TChunk *)malloc(sizeof(TChunk));
+    uint64_t new_capacity = allocated_capacity * 2;
-    if (!t_current)
+    if (new_capacity < current_size + len)
-      throw std::runtime_error("malloc failed");
+      new_capacity = current_size + len + (1ull << 30);
-    buf.push_back(t_current);
+    if (ftruncate(fd, new_capacity) == -1)
-    t_offset = 0;
+      throw std::runtime_error("ftruncate failed");
+#if defined(__linux__)
+    void *new_buf = mremap(buf, allocated_capacity, new_capacity, MREMAP_MAYMOVE);
+    if (new_buf == MAP_FAILED)
+      throw std::runtime_error("mremap failed");
+#else
+    munmap(buf, allocated_capacity);
+    void *new_buf = mmap(nullptr, new_capacity, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+    if (new_buf == MAP_FAILED)
+      throw std::runtime_error("mmap failed");
+#endif
+    allocated_capacity = new_capacity;
+    buf = (char *)new_buf;
   }
-  (*t_current)[t_offset++] = c;
-  return current_offset++;
 }
 
-uint64_t AppendBuffer::append(const char *text, uint64_t length) {
+uint64_t AppendBuffer::append(const char c) {
-  uint64_t start = current_offset;
+  grow(1);
-  while (length > 0) {
+  buf[current_size++] = c;
-    if (t_offset == APPEND_CHUNK_SIZE) {
+  return current_size - 1;
-      t_current = (TChunk *)malloc(sizeof(TChunk));
-      buf.push_back(t_current);
-      t_offset = 0;
-    }
-    uint64_t copy = std::min(length, APPEND_CHUNK_SIZE - t_offset);
-    memcpy((*t_current) + t_offset, text, copy);
-    t_offset += copy;
-    current_offset += copy;
-    text += copy;
-    length -= copy;
-  }
-  return start;
 }
 
-const char *AppendBuffer::read(uint64_t pos, uint64_t *out_len) {
+uint64_t AppendBuffer::append(const char *text, uint64_t len) {
-  if (pos >= current_offset)
+  grow(len);
+  memcpy(buf + current_size, text, len);
+  uint64_t old_pos = current_size;
+  current_size += len;
+  return old_pos;
+}
+
+const char *AppendBuffer::read(uint64_t pos) {
+  if (pos >= current_size)
     return nullptr;
-  uint64_t local_offset = pos % APPEND_CHUNK_SIZE;
+  return buf + pos;
-  if (out_len) {
-    uint64_t remaining = current_offset - pos;
-    uint64_t until_chunk_end = APPEND_CHUNK_SIZE - local_offset;
-    *out_len = std::min(remaining, until_chunk_end);
-  }
-  return &(*buf[pos / APPEND_CHUNK_SIZE])[local_offset];
 }
 
-inline uint64_t AppendBuffer::length() {
+uint64_t AppendBuffer::length() {
-  return current_offset;
+  return current_size;
 }

src/vase/buffer/original.cc

@@ -5,13 +5,12 @@ OriginalBuffer::OriginalBuffer(std::filesystem::path base_dir) {
   if (!std::filesystem::exists(base_dir) || !std::filesystem::is_directory(base_dir))
     throw std::runtime_error("Swap directory does not exist or is not a directory.");
   base_dir /= "tbuf.XXXXXX";
-  auto s = base_dir.string();
+  char *s = strdup(base_dir.c_str());
-  std::vector<char> mutable_path(s.begin(), s.end());
+  fd = mkstemp(s);
-  mutable_path.push_back('\0');
-  fd = mkstemp(mutable_path.data());
   if (fd == -1)
     throw std::runtime_error("mkstemp failed");
-  unlink(mutable_path.data());
+  unlink(s);
+  free(s);
 }
 
 OriginalBuffer::~OriginalBuffer() {
@@ -36,11 +35,9 @@ void OriginalBuffer::initialize() {
   fd = -1;
 }
 
-const char *OriginalBuffer::read(uint64_t pos, uint64_t *out_len) {
+const char *OriginalBuffer::read(uint64_t pos) {
   if (pos >= len)
     return nullptr;
-  if (out_len)
-    *out_len = len - pos;
   return buf + pos;
 }
 

src/vase/iterators/chunk.cc

@@ -1,213 +0,0 @@
-#include "vase/iterators/chunk.h"
-
-ChunkIterator::ChunkIterator(Shard *r, Direction dir)
-    : dir(dir), root(r) {
-  if (!root)
-    return;
-  Shard::retain(root);
-}
-
-ChunkIterator::~ChunkIterator() {
-  Shard::release(root);
-}
-
-void ChunkIterator::seek_offset(uint64_t offset) {
-  stack.clear();
-  petal = nullptr;
-  petal_offset = 0;
-  global_offset = 0;
-  global_line = 0;
-  at_end = false;
-  if (!root)
-    return;
-  if (offset >= root->length)
-    offset = root->length - 1;
-  uint64_t target = offset;
-  Shard *curr = root;
-  while (curr) {
-    if (curr->kind == Shard::ShardKind::Petal) {
-      petal = (Petal *)curr;
-      petal_offset = target;
-      global_offset += target;
-      return;
-    } else {
-      auto *b = (Branch *)curr;
-      uint64_t left_len = b->left->length;
-      if (target < left_len) {
-        if (dir == Direction::Forward)
-          stack.push_back(b->right);
-        curr = b->left;
-      } else {
-        target -= left_len;
-        if (dir == Direction::Backward)
-          stack.push_back(b->left);
-        global_offset += left_len;
-        global_line += b->left->lines;
-        curr = b->right;
-      }
-    }
-  }
-  std::unreachable();
-}
-
-void ChunkIterator::seek_line(uint64_t line) {
-  stack.clear();
-  petal = nullptr;
-  petal_offset = 0;
-  global_offset = 0;
-  at_end = false;
-  bool last_line = false;
-  if (!root)
-    return;
-  if (dir == Direction::Backward) {
-    if (line > root->lines + 1)
-      line = root->lines + 1;
-    if (line == root->lines + 1)
-      last_line = true;
-  } else {
-    if (line > root->lines)
-      line = root->lines;
-    if (line == root->lines)
-      last_line = true;
-  }
-  Shard *curr = root;
-  while (curr) {
-    if (curr->kind == Shard::ShardKind::Petal) {
-      petal = (Petal *)curr;
-      if (last_line && dir == Direction::Backward) {
-        petal_offset = petal->length;
-        global_offset += petal->length;
-      } else {
-        const char *text = nullptr;
-        uint64_t remaining = 0;
-        uint64_t offset = 0;
-        while (line) {
-          if (!text) {
-            uint64_t got = 0;
-            text = petal->source->read(petal->pos + offset, &got);
-            remaining = std::min(got, petal->length - offset);
-          }
-          const char *c = (const char *)memchr(text, '\n', remaining);
-          if (!c) {
-            offset += remaining;
-            text = nullptr;
-            continue;
-          }
-          remaining -= (c - text) + 1;
-          offset += (c - text) + 1;
-          text = c + 1;
-          line--;
-        }
-        if (dir == Direction::Backward)
-          --offset;
-        if (last_line && dir == Direction::Forward)
-          at_end = true;
-        petal_offset = offset;
-        global_offset += offset;
-      }
-      return;
-    } else {
-      auto *b = (Branch *)curr;
-      uint64_t left_lines = b->left->lines;
-      if (line <= left_lines) {
-        if (dir == Direction::Forward)
-          stack.push_back(b->right);
-        curr = b->left;
-      } else {
-        line -= left_lines;
-        if (dir == Direction::Backward)
-          stack.push_back(b->left);
-        global_offset += b->left->length;
-        curr = b->right;
-      }
-    }
-  }
-  std::unreachable();
-}
-
-uint64_t ChunkIterator::byte_offset() {
-  return global_offset;
-}
-
-uint64_t ChunkIterator::line_offset() {
-  return global_line;
-}
-
-bool ChunkIterator::next(const char **data, uint64_t *out_len) {
-  if (dir == Direction::Forward) {
-    while (true) {
-      if (petal) {
-        if (petal_offset == petal->length) {
-          if (at_end) {
-            *data = nullptr;
-            *out_len = 0;
-            at_end = false;
-            return true;
-          }
-          petal = nullptr;
-          continue;
-        }
-        uint64_t remaining = petal->length - petal_offset;
-        uint64_t got = 0;
-        const char *chunk = petal->source->read(petal->pos + petal_offset, &got);
-        if (got == 0) {
-          petal = nullptr;
-          petal_offset = 0;
-          return false;
-        }
-        uint64_t take = std::min(got, remaining);
-        *data = chunk;
-        *out_len = take;
-        petal_offset += take;
-        global_offset += take;
-        return true;
-      }
-      if (stack.empty())
-        return false;
-      auto s = stack.back();
-      stack.pop_back();
-      while (s->kind == Shard::ShardKind::Branch) {
-        Branch *b = (Branch *)s;
-        stack.push_back(b->right);
-        s = b->left;
-      }
-      petal = (Petal *)s;
-      petal_offset = 0;
-    }
-  } else {
-    while (true) {
-      if (petal) {
-        if (petal_offset == 0) {
-          petal = nullptr;
-          continue;
-        }
-        uint64_t got = 0;
-        *data = petal->source->read(petal->pos, &got);
-        if (petal_offset > got) {
-          uint64_t got2 = 0;
-          *data = petal->source->read(petal->pos + got, &got2);
-          *out_len = std::min(got2, petal_offset - got);
-          petal_offset = got;
-          global_offset -= *out_len;
-          return true;
-        } else {
-          *out_len = std::min(got, petal_offset);
-          petal_offset = 0;
-          global_offset -= *out_len;
-          return true;
-        }
-      }
-      if (stack.empty())
-        return false;
-      auto s = stack.back();
-      stack.pop_back();
-      while (s->kind == Shard::ShardKind::Branch) {
-        Branch *b = (Branch *)s;
-        stack.push_back(b->left);
-        s = b->right;
-      }
-      petal = (Petal *)s;
-      petal_offset = petal->length;
-    }
-  }
-}

src/vase/iterators/line.cc

@@ -1,97 +1,73 @@
 #include "vase/iterators/line.h"
 
-LineIterator::LineIterator(Shard *r, uint64_t start_line, Direction dir)
+LineIterator::LineIterator(Shard *root, uint64_t line_num, Direction dir)
-    : it(r, dir), dir(dir) {
+    : it(root, dir), dir(dir) {
-  int b = start_line == UINT64_MAX ? 0 : dir == Direction::Backward;
+  it.seek_line(line_num);
-  it.seek_line(start_line + b);
+  it.next(&chunk, &len);
-  if (!it.next(&chunk, &len)) {
+  current_offset = it.byte_offset();
-    chunk = nullptr;
+  last_line_offset = current_offset;
-    return;
-  }
-  if (chunk == nullptr)
-    at_end = true;
-  if (dir == Direction::Forward)
-    chunk_offset = it.byte_offset() - len;
-  else
-    chunk_offset = it.byte_offset();
 }
 
-uint64_t LineIterator::byte_offset() {
+bool LineIterator::next(std::string *line) {
-  return offset;
+  if (!line || !chunk)
-}
-
-bool LineIterator::next(std::string &line) {
-  line.clear();
-  if (at_end) {
-    offset = chunk_offset;
-    at_end = false;
-    return true;
-  }
-  if (!chunk)
     return false;
+  last_line_offset = current_offset;
+  line->clear();
   if (dir == Direction::Forward) {
-    offset = chunk_offset;
+  retry_forward:
-    while (true) {
     const char *nl = (const char *)memchr(chunk, '\n', len);
     if (!nl) {
       if (len && chunk[len - 1] == '\r')
-          --len;
+        len--;
-        line.append(chunk, len);
+      line->append(chunk, len);
-        chunk_offset += len;
+      current_offset += len;
-        if (!it.next(&chunk, &len)) {
+      if (!it.next(&chunk, &len))
-          chunk = nullptr;
         return true;
-        }
+      goto retry_forward;
-        if (chunk == nullptr)
-          return true;
-        chunk_offset = it.byte_offset() - len;
-        continue;
     }
     const char *end = nl;
-      if (end > chunk && *(end - 1) == '\r')
+    if (end - chunk > 0 && *(end - 1) == '\r')
-        --end;
+      end--;
-      line.append(chunk, end);
+    line->append(chunk, end);
-      uint64_t consumed = (nl - chunk) + 1;
+    size_t consumed = (nl - chunk) + 1;
     chunk += consumed;
     len -= consumed;
-      chunk_offset += consumed;
+    current_offset += consumed;
     return true;
-    }
   } else {
-    while (true) {
+  retry_backward:
-#if defined(__GLIBC__) || defined(__APPLE__)
+#if defined(__GLIBC__) || defined(__APPLE__) || defined(__FreeBSD__)
     const char *nl = (const char *)memrchr(chunk, '\n', len);
 #else
     const char *nl = nullptr;
-      uint64_t i = len;
+    const char *p = chunk + len;
-      while (i--) {
+    while (!nl && p != chunk)
-        if (chunk[i] == '\n') {
+      if (*(--p) == '\n')
-          nl = chunk + i;
+        nl = p;
-          break;
-        }
-      }
 #endif
     if (!nl) {
       if (len && chunk[len - 1] == '\r')
-          --len;
+        len--;
-        line.insert(0, chunk, len);
+      line->insert(0, chunk, len);
-        chunk_offset -= len;
+      current_offset -= len;
-        if (!it.next(&chunk, &len)) {
+      if (!it.next(&chunk, &len))
-          chunk = nullptr;
         return true;
+      goto retry_backward;
     }
-        chunk_offset = it.byte_offset();
-        continue;
-      }
-      const char *end = chunk + len;
     const char *start = nl + 1;
-      const char *line_end = end;
+    const char *line_end = chunk + len;
-      if (line_end > start && *(line_end - 1) == '\r')
+    const size_t consumed = line_end - nl;
-        --line_end;
+    const char *end = line_end;
-      line.insert(0, start, line_end - start);
+    if (end > start && *(end - 1) == '\r')
+      end--;
+    if (end > start)
+      line->insert(0, start, end - start);
     len = nl - chunk;
-      offset = chunk_offset + (start - chunk);
+    current_offset -= consumed;
     return true;
   }
 }
+
+uint64_t LineIterator::byte_offset() {
+  return last_line_offset;
 }

src/vase/iterators/petal.cc

@@ -0,0 +1,158 @@
+#include "vase/iterators/petal.h"
+
+PetalIterator::PetalIterator(Shard *r, Direction dir)
+    : dir(dir), root(r) {
+  if (!root)
+    return;
+}
+
+void PetalIterator::seek_offset(uint64_t offset) {
+  stack.clear();
+  petal = nullptr;
+  global_offset = 0;
+  last_offset = 0;
+  global_line = 0;
+  if (!root)
+    return;
+  if (offset >= root->length)
+    offset = root->length - 1;
+  uint64_t target = offset;
+  Shard *curr = root;
+  while (curr) {
+    if (curr->kind == Shard::Kind::Petal) {
+      petal = (Petal *)curr;
+      petal_offset = target;
+      global_offset += target;
+      last_offset = global_offset;
+      return;
+    } else {
+      auto *b = (Branch *)curr;
+      uint64_t left_len = b->left->length;
+      if (target < left_len) {
+        if (dir == Direction::Forward)
+          stack.push_back(b->right);
+        curr = b->left;
+      } else {
+        target -= left_len;
+        if (dir == Direction::Backward)
+          stack.push_back(b->left);
+        global_offset += left_len;
+        global_line += b->left->lines;
+        curr = b->right;
+      }
+    }
+  }
+  std::unreachable();
+}
+
+void PetalIterator::seek_line(uint64_t line) {
+  stack.clear();
+  petal = nullptr;
+  global_offset = 0;
+  last_offset = 0;
+  bool last_line = false;
+  if (!root)
+    return;
+  if (line > root->lines)
+    line = root->lines;
+  if (line == root->lines)
+    last_line = true;
+  if (!last_line && dir == Direction::Backward)
+    line++;
+  Shard *curr = root;
+  while (curr) {
+    if (curr->kind == Shard::Kind::Petal) {
+      petal = (Petal *)curr;
+      if (last_line && dir == Direction::Backward) {
+        petal_offset = petal->length;
+        global_offset += petal->length;
+      } else {
+        const char *text = petal->source->read(petal->pos);
+        uint64_t offset = 0;
+        while (line) {
+          const char *c = (const char *)memchr(text, '\n', petal->length - offset);
+          if (!c)
+            throw std::runtime_error("leaf line count is wrong.");
+          offset += (c - text) + 1;
+          text = c + 1;
+          line--;
+        }
+        if (dir == Direction::Backward)
+          --offset;
+        petal_offset = offset;
+        global_offset += offset;
+      }
+      last_offset = global_offset;
+      return;
+    } else {
+      auto *b = (Branch *)curr;
+      uint64_t left_lines = b->left->lines;
+      if (line <= left_lines) {
+        if (dir == Direction::Forward)
+          stack.push_back(b->right);
+        curr = b->left;
+      } else {
+        line -= left_lines;
+        if (dir == Direction::Backward)
+          stack.push_back(b->left);
+        global_offset += b->left->length;
+        curr = b->right;
+      }
+    }
+  }
+  std::unreachable();
+}
+
+Petal *PetalIterator::_next(uint64_t *offset) {
+  while (true) {
+    if (petal) {
+      auto ret = petal;
+      last_offset = global_offset;
+      if (dir == Direction::Forward)
+        global_offset += petal_offset;
+      else
+        global_offset -= petal_offset;
+      petal = nullptr;
+      *offset = petal_offset;
+      return ret;
+    }
+    if (stack.empty())
+      return nullptr;
+    auto s = stack.back();
+    stack.pop_back();
+    while (s->kind == Shard::Kind::Branch) {
+      Branch *b = (Branch *)s;
+      if (dir == Direction::Forward) {
+        stack.push_back(b->right);
+        s = b->left;
+      } else {
+        stack.push_back(b->left);
+        s = b->right;
+      }
+    }
+    petal = (Petal *)s;
+    petal_offset = dir == Direction::Forward ? 0 : petal->length;
+  }
+}
+
+bool PetalIterator::next(const char **data, uint64_t *out_len) {
+  uint64_t offset = 0;
+  Petal *p = _next(&offset);
+  if (!p) {
+    *data = nullptr;
+    *out_len = 0;
+    return false;
+  }
+  if (dir == Direction::Forward) {
+    *out_len = p->length - offset;
+    *data = p->source->read(p->pos + offset);
+  } else {
+    *out_len = offset;
+    *data = p->source->read(p->pos);
+  }
+  return true;
+}
+
+uint64_t PetalIterator::byte_offset() {
+  return last_offset;
+}

src/vase/regex/api.cc

@@ -74,13 +74,13 @@ void Vase::regex_search_replace(
   for (const RegexMatch &match : matches) {
     uint64_t gap = match.start - cursor;
     if (gap > 0) {
-      auto [keep, rest] = split_shard(remaining, gap);
+      auto [keep, rest] = Shard::split(remaining, gap);
       Shard::release(remaining);
       pieces.push_back(keep);
       remaining = rest;
     }
 
-    auto [dropped, rest2] = split_shard(remaining, match.end - match.start);
+    auto [dropped, rest2] = Shard::split(remaining, match.end - match.start);
     Shard::release(remaining);
     remaining = rest2;
 
@@ -102,8 +102,8 @@ void Vase::regex_search_replace(
           if (group.start != UINT32_MAX) {
             uint64_t ls = group.start - match.start;
             uint64_t le = group.end - match.start;
-            auto [a, b] = split_shard(dropped, ls);
+            auto [a, b] = Shard::split(dropped, ls);
-            auto [g, c] = split_shard(b, le - ls);
+            auto [g, c] = Shard::split(b, le - ls);
             Shard::release(a);
             Shard::release(b);
             Shard::release(c);
@@ -132,7 +132,7 @@ void Vase::regex_search_replace(
       Shard::release(p);
   }
 
-  Shard *new_root = compact.empty() ? nullptr : build_balanced(compact.data(), 0, compact.size());
+  Shard *new_root = compact.empty() ? nullptr : Shard::build(compact.data(), 0, compact.size());
   Shard::release(root);
   root = new_root;
 }

src/vase/regex/core.cc

@@ -1,10 +1,11 @@
+#include "vase/iterators/petal.h"
 #include "vase/vase.h"
 
 std::vector<RegexMatch> Vase::_regex_search(
   std::string_view pattern, Range range, std::string_view options
 ) {
   bool global = false;
-  uint64_t flags = PCRE2_MULTILINE;
+  uint64_t flags = PCRE2_MULTILINE | PCRE2_UTF;
 
   const char *dot = "(?:(?!\\n)\\X)";
 
@@ -104,7 +105,7 @@ std::vector<RegexMatch> Vase::_regex_search(
   uint64_t start_offset = offset_of(range.start);
   uint64_t end_offset = offset_of(range.end);
 
-  ChunkIterator it(root, Direction::Forward);
+  PetalIterator it(root, Direction::Forward);
   it.seek_offset(start_offset);
 
   const char *data;

src/vase/shard.cc

@@ -1,13 +1,14 @@
 #include "vase/shard.h"
 
 void Shard::retain(Shard *n) {
+  if (n)
     n->refs++;
 };
 
 void Shard::release(Shard *n) {
   if (!n || --n->refs > 0)
     return;
-  if (n->kind == Shard::ShardKind::Branch) {
+  if (n->kind == Shard::Kind::Branch) {
     release(((Branch *)n)->left);
     release(((Branch *)n)->right);
     delete (Branch *)n;
@@ -50,7 +51,7 @@ Shard *rotate_left(Branch *z) {
 }
 
 Shard *balance(Shard *node) {
-  if (!node || node->kind == Shard::ShardKind::Petal)
+  if (!node || node->kind == Shard::Kind::Petal)
     return node;
 
   Branch *b = (Branch *)node;
@@ -123,8 +124,7 @@ std::pair<Shard *, Shard *> Shard::split(Shard *n, uint64_t offset) {
     Shard::retain(n);
     return {n, nullptr};
   }
-
+  if (n->kind == Shard::Kind::Branch) {
-  if (n->kind == Shard::ShardKind::Branch) {
     Branch *b = (Branch *)n;
     if (offset < b->left->length) {
       auto [a, b2] = split(b->left, offset);
@@ -140,45 +140,27 @@ std::pair<Shard *, Shard *> Shard::split(Shard *n, uint64_t offset) {
   } else {
     Petal *p = (Petal *)n;
     uint64_t count[2]{0};
-    uint64_t read_offset = 0;
+    const char *c = p->source->read(p->pos);
-    while (read_offset < p->length) {
+    const char *start = c;
-      uint64_t got = 0;
+    const char *end = c + p->length;
-      const char *c = p->source->read(p->pos + read_offset, &got);
+    while (c < end) {
-      const char *end = c + std::min(got, p->length - read_offset);
+      const char *nl = (const char *)memchr(c, '\n', end - c);
-      const char *cursor = c;
+      if (!nl)
-      while (cursor < end) {
-        const char *nl = (const char *)memchr(cursor, '\n', end - cursor);
-        if (!nl) {
-          cursor = end;
         break;
-        }
+      if ((uint64_t)(nl - start) < offset)
-        uint64_t nl_pos = read_offset + (uint64_t)(nl - c);
-        if (nl_pos < offset)
         count[0]++;
       else
         count[1]++;
-        cursor = nl + 1;
+      c = nl + 1;
     }
-      read_offset += (uint64_t)(cursor - c);
+    auto left = new Petal(offset, count[0], p->source, p->pos);
-    }
+    auto right = new Petal(p->length - offset, count[1], p->source, p->pos + offset);
-    auto left = new Petal(
-      offset,
-      count[0],
-      p->source,
-      p->pos
-    );
-    auto right = new Petal(
-      p->length - offset,
-      count[1],
-      p->source,
-      p->pos + offset
-    );
     return {left, right};
   }
 }
 
 Shard *Shard::merge_leaves(Shard *a, Shard *b) {
-  if (a->kind != Shard::ShardKind::Petal || b->kind != Shard::ShardKind::Petal)
+  if (a->kind != Shard::Kind::Petal || b->kind != Shard::Kind::Petal)
     return merge(a, b);
   Petal *pa = (Petal *)a;
   Petal *pb = (Petal *)b;
@@ -192,13 +174,15 @@ Shard *Shard::merge_leaves(Shard *a, Shard *b) {
   );
 }
 
-Shard *Shard::append_leaf(Shard *root, Shard *leaf) {
+Shard *Shard::append(Shard *root, Shard *leaf) {
-  if (!root)
+  if (!root) {
+    Shard::retain(leaf);
     return leaf;
-  if (root->kind == Shard::ShardKind::Petal && root->length < PETAL_SIZE_MAX)
+  }
+  if (root->kind == Shard::Kind::Petal && root->length < PETAL_SIZE_MAX)
     return merge_leaves(root, leaf);
   Branch *b = (Branch *)root;
-  auto new_right = append_leaf(b->right, leaf);
+  auto new_right = append(b->right, leaf);
   auto out = balance(new Branch(b->left, new_right));
   Shard::release(new_right);
   return out;
@@ -208,12 +192,12 @@ Shard *Shard::concat(Shard *left, Shard *right) {
   return merge(left, right);
 }
 
-Shard *Shard::build_balanced(Shard **pieces, uint64_t lo, uint64_t hi) {
+Shard *Shard::build(Shard **pieces, uint64_t lo, uint64_t hi) {
   if (hi - lo == 1)
     return pieces[lo];
   size_t mid = lo + (hi - lo) / 2;
-  Shard *left = build_balanced(pieces, lo, mid);
+  Shard *left = build(pieces, lo, mid);
-  Shard *right = build_balanced(pieces, mid, hi);
+  Shard *right = build(pieces, mid, hi);
   Shard *node = new Branch(left, right);
   Shard::release(left);
   Shard::release(right);
@@ -270,5 +254,5 @@ Shard *Shard::from_file(std::filesystem::path path, OriginalBuffer *o) {
 
   if (pieces.size() == 1)
     return pieces[0];
-  return build_balanced(pieces.data(), 0, pieces.size());
+  return build(pieces.data(), 0, pieces.size());
 }

src/vase/vase.cc

@@ -1,17 +1,18 @@
 #include "vase/vase.h"
 #include "utils/utils.h"
 #include "vase/iterators/line.h"
+#include "vase/iterators/petal.h"
 
 Vase::Vase(std::filesystem::path path, std::filesystem::path swapdir)
     : path(path), swapdir(swapdir) {
   if (!std::filesystem::exists(swapdir) || !std::filesystem::is_directory(swapdir))
     throw std::runtime_error("Swap directory does not exist or is not a directory.");
-  append = new AppendBuffer();
+  append = new AppendBuffer(swapdir);
   original = new OriginalBuffer(swapdir);
   if (std::filesystem::is_regular_file(path))
     root = Shard::from_file(path, original);
   else
-    root = Shard::new_empty(original);
+    root = nullptr;
   history.push_back(root);
   Shard::retain(root);
   history_top = 0;
@@ -31,7 +32,7 @@ uint64_t Vase::length() {
 
 std::string Vase::to_string() {
   std::string out;
-  ChunkIterator it(root, Direction::Forward);
+  PetalIterator it(root, Direction::Forward);
   it.seek_offset(0);
   const char *data;
   uint64_t len;
@@ -42,7 +43,7 @@ std::string Vase::to_string() {
 
 std::string Vase::to_string(Range range) {
   std::string out;
-  ChunkIterator it(root, Direction::Forward);
+  PetalIterator it(root, Direction::Forward);
   uint64_t start = offset_of(range.start);
   it.seek_offset(start);
   const char *data;
@@ -104,7 +105,7 @@ bool Vase::save() {
   std::ofstream file(path, std::ios::binary);
   if (!file)
     return false;
-  ChunkIterator it(root, Direction::Forward);
+  PetalIterator it(root, Direction::Forward);
   it.seek_offset(0);
   const char *data;
   uint64_t len;
@@ -117,24 +118,25 @@ bool Vase::save() {
 }
 
 bool Vase::save_swap() {
+  return false;
 }
 
 void Vase::insert(Point *point, char key) {
+  uint64_t pos = append->append(key);
+  Shard *inserted = new Petal(1, key == '\n', append, pos);
+  auto [left, right] = Shard::split(root, offset_of(*point));
+  Shard *left2 = Shard::append(left, inserted);
+  Shard::release(left);
+  Shard::release(inserted);
+  Shard *new_root = Shard::concat(left2, right);
+  Shard::release(left2);
+  Shard::release(right);
+  Shard::release(root);
+  root = new_root;
   if (key == '\n')
     *point = {point->row + 1, 0};
   else
     point->col++;
-  uint64_t pos = append->append(key);
-  Shard *inserted = new Petal(1, key == '\n', append, pos);
-  auto [left, right] = Shard::split(root, offset_of(*point));
-  Shard *left2 = Shard::append_leaf(left, inserted);
-  Shard *new_root = Shard::concat(left2, right);
-  Shard::release(left);
-  Shard::release(right);
-  Shard::release(left2);
-  Shard::release(inserted);
-  Shard::release(root);
-  root = new_root;
 }
 
 void Vase::insert(Point *point, const char *data, uint64_t len) {
@@ -175,12 +177,12 @@ void Vase::_insert(Point *point, const char *data, uint64_t len) {
   }
   Shard *inserted = new Petal(len, lines, append, pos);
   auto [left, right] = Shard::split(root, offset);
-  Shard *left2 = Shard::append_leaf(left, inserted);
+  Shard *left2 = Shard::append(left, inserted);
-  Shard *new_root = Shard::concat(left2, right);
   Shard::release(left);
-  Shard::release(right);
-  Shard::release(left2);
   Shard::release(inserted);
+  Shard *new_root = Shard::concat(left2, right);
+  Shard::release(left2);
+  Shard::release(right);
   Shard::release(root);
   root = new_root;
 }
@@ -236,10 +238,9 @@ void Vase::replace(Range range, const char *data, uint64_t len) {
 
 uint64_t Vase::offset_of(Point point) {
   LineIterator it(root, point.row, Direction::Forward);
-
   std::string line;
   uint64_t offset = 0;
-  if (it.next(line)) {
+  if (it.next(&line)) {
     const char *ptr = line.data();
     uint64_t remaining = line.length();
     while (point.col && remaining) {
@@ -254,10 +255,10 @@ uint64_t Vase::offset_of(Point point) {
 }
 
 Point Vase::point_of(uint64_t offset) {
-  ChunkIterator it(root, Direction::Backward);
+  PetalIterator it(root, Direction::Backward);
   it.seek_offset(offset);
   Point p;
-  p.row = it.line_offset();
+  p.row = it.global_line;
   std::string line;
   const char *chunk;
   uint64_t len = 0;
@@ -268,13 +269,10 @@ Point Vase::point_of(uint64_t offset) {
     const char *nl = (const char *)memrchr(chunk, '\n', len);
 #else
     const char *nl = nullptr;
-    uint64_t i = len;
+    const char *p = chunk + len;
-    while (i--) {
+    while (!nl && p != chunk)
-      if (chunk[i] == '\n') {
+      if (*(--p) == '\n')
-        nl = chunk + i;
+        nl = p;
-        break;
-      }
-    }
 #endif
     if (!nl) {
       if (len && chunk[len - 1] == '\r')
@@ -310,7 +308,7 @@ void Vase::move_clusters(Point *point, uint64_t amount, Direction dir) {
     LineIterator it(root, point->row, Direction::Backward);
     while (amount) {
       std::string line;
-      if (!it.next(line))
+      if (!it.next(&line))
         return;
       std::vector<uint64_t> clusters;
       const char *ptr = line.data();
@@ -340,7 +338,7 @@ void Vase::move_clusters(Point *point, uint64_t amount, Direction dir) {
     LineIterator it(root, point->row, Direction::Forward);
     while (amount) {
       std::string line;
-      if (!it.next(line))
+      if (!it.next(&line))
         return;
       if (point->col || amount) {
         const char *ptr = line.data();
@@ -374,19 +372,17 @@ void Vase::clamp(Point *point) {
     point->row = root->lines;
   LineIterator it(root, point->row, Direction::Forward);
   std::string line;
-  if (!it.next(line)) {
-    point->col = 0;
-    return;
-  }
   uint64_t clusters = 0;
+  if (it.next(&line)) {
     const char *ptr = line.data();
-  uint64_t remaining = line.size();
+    uint64_t remaining = line.length();
     while (remaining) {
-    uint64_t len = grapheme_next_character_break_utf8(ptr, remaining);
+      uint64_t next_len = grapheme_next_character_break_utf8(ptr, remaining);
-    ptr += len;
+      remaining -= next_len;
-    remaining -= len;
+      ptr += next_len;
       clusters++;
     }
+  }
   if (point->col > clusters)
     point->col = clusters;
 }