Perfect

- Make original buffer use a disk backed mapping to save memory.
- rearrange api function headers for search etc.
- fix bugs with iterators.
- support uint64_t for all byte and line offsets to allow for much larger files to be opened.
- and more minor bug fixes.

include/pch.h

@@ -40,9 +40,12 @@ extern "C" {
 #include <shared_mutex>
 #include <signal.h>
 #include <stack>
+#include <stdexcept>
 #include <string.h>
 #include <string>
 #include <sys/ioctl.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
 #include <sys/types.h>
 #include <sys/wait.h>
 #include <termios.h>

include/vase/buffer/append.h

@@ -8,16 +8,16 @@ struct AppendBuffer : Buffer {
   using TChunk = char[APPEND_CHUNK_SIZE];
   std::vector<TChunk *> buf;
   TChunk *t_current;
-  uint32_t current_offset{0};
-  uint32_t t_offset{0};
+  uint64_t current_offset{0};
+  uint64_t t_offset{0};
 
   AppendBuffer();
 
   ~AppendBuffer();
 
-  uint32_t key(char c);
-  uint32_t append(const char *text, uint32_t length);
+  uint64_t append(char c);
+  uint64_t append(const char *text, uint64_t length);
 
-  const char *read(uint32_t pos, uint32_t *out_len);
-  inline uint32_t length();
+  const char *read(uint64_t pos, uint64_t *out_len);
+  inline uint64_t length();
 };

include/vase/buffer/buffer.h

@@ -3,7 +3,7 @@
 #include "pch.h"
 
 struct Buffer {
-  virtual const char *read(uint32_t pos, uint32_t *out_len) = 0;
-  virtual inline uint32_t length() = 0;
+  virtual const char *read(uint64_t pos, uint64_t *out_len) = 0;
+  virtual inline uint64_t length() = 0;
   virtual ~Buffer() = default;
 };

include/vase/buffer/original.h

@@ -4,14 +4,14 @@
 #include "pch.h"
 
 struct OriginalBuffer : Buffer {
-  char *buf;
-  uint32_t len;
-
-  // Add 2 modes here, normal and lazy, lazy copies the file to a safe location and uses mmap.
-  OriginalBuffer(std::string path);
+  const char *buf;
+  uint64_t len;
+  int fd = -1;
 
+  OriginalBuffer();
   ~OriginalBuffer();
 
-  const char *read(uint32_t pos, uint32_t *out_len);
-  uint32_t length();
+  void initialize();
+  const char *read(uint64_t pos, uint64_t *out_len);
+  uint64_t length();
 };

include/vase/constants.h

@@ -2,8 +2,8 @@
 
 #include "pch.h"
 
-constexpr uint32_t APPEND_CHUNK_SIZE = 64 * 1024;
-constexpr uint32_t PETAL_SIZE_MAX = 32 * 1024;
+constexpr uint64_t APPEND_CHUNK_SIZE = 64 * 1024;
+constexpr uint64_t PETAL_SIZE_MAX = 32 * 1024;
 
 static_assert(
   APPEND_CHUNK_SIZE > PETAL_SIZE_MAX,

include/vase/iterators/chunk.h

@@ -13,15 +13,18 @@ struct ChunkIterator {
   Shard *root = nullptr;
   std::vector<Shard *> stack;
   Petal *petal = nullptr;
-  uint32_t petal_offset = 0;
-  uint32_t global_offset = 0;
+  uint64_t petal_offset = 0;
+  uint64_t global_offset = 0;
+  uint64_t global_line = 0;
+  bool at_end = false;
 
   ChunkIterator(Shard *r, Direction dir);
 
   ~ChunkIterator();
 
-  void seek_offset(uint32_t offset);
-  void seek_line(uint32_t offset);
-  bool next(const char **data, uint32_t *out_len);
-  uint32_t byte_offset();
+  void seek_offset(uint64_t offset);
+  void seek_line(uint64_t offset);
+  bool next(const char **data, uint64_t *out_len);
+  uint64_t byte_offset();
+  uint64_t line_offset();
 };

include/vase/iterators/line.h

@@ -9,11 +9,14 @@ struct LineIterator {
   Direction dir;
 
   const char *chunk;
-  uint32_t len;
+  uint64_t len;
+  uint64_t offset = 0;
+  uint64_t chunk_offset = 0;
+  bool at_end = false;
 
-  LineIterator(Shard *r, uint32_t start_line, Direction dir);
+  LineIterator(Shard *r, uint64_t start_line, Direction dir);
   ~LineIterator() = default;
 
   bool next(std::string &line);
-  uint32_t byte_offset();
+  uint64_t byte_offset();
 };

include/vase/search.h

@@ -1,21 +0,0 @@
-#pragma once
-
-#include "pch.h"
-#include "vase/shard.h"
-
-struct RegexGroup {
-  uint32_t start{UINT32_MAX};
-  uint32_t end{UINT32_MAX};
-};
-
-struct RegexMatch {
-  uint32_t start;
-  uint32_t end;
-  RegexGroup groups[9]{};
-};
-
-std::vector<RegexMatch> regex_search(
-  Shard *root, std::string_view pattern_str,
-  uint32_t start_offset, uint32_t end_offset,
-  std::string_view options
-);

include/vase/shard.h

@@ -14,12 +14,12 @@ struct Shard {
 
   uint8_t height;
 
-  uint32_t length;
-  uint32_t lines;
+  uint64_t length;
+  uint64_t lines;
 
-  std::atomic_uint32_t refs;
+  std::atomic_uint64_t refs;
 
-  Shard(ShardKind kind, uint32_t length, uint32_t lines, uint8_t height)
+  Shard(ShardKind kind, uint64_t length, uint64_t lines, uint8_t height)
       : kind(kind), height(height), length(length), lines(lines), refs(1) {};
 
   static void retain(Shard *n);
@@ -46,18 +46,19 @@ struct Branch : Shard {
 struct Petal : Shard {
   Buffer *source;
 
-  uint32_t pos;
+  uint64_t pos;
 
-  Petal(uint32_t length, uint32_t lines, Buffer *source, uint32_t pos)
+  Petal(uint64_t length, uint64_t lines, Buffer *source, uint64_t pos)
       : Shard(ShardKind::Petal, length, lines, 1),
         source(source), pos(pos) {};
 };
 
-Shard *create_shards(Buffer *o);
+Shard *create_file_shards(std::string &path, OriginalBuffer *b);
+Shard *create_swap_shards(std::string &path, OriginalBuffer *b);
 
-std::pair<Shard *, Shard *> split_shard(Shard *n, uint32_t offset);
+std::pair<Shard *, Shard *> split_shard(Shard *n, uint64_t offset);
 Shard *concat_shard(Shard *left, Shard *right);
 Shard *merge(Shard *a, Shard *b);
 Shard *merge_leaves(Shard *a, Shard *b);
 Shard *append_leaf(Shard *root, Shard *leaf);
-Shard *build_balanced(Shard **pieces, uint32_t lo, uint32_t hi);
+Shard *build_balanced(Shard **pieces, uint64_t lo, uint64_t hi);

include/vase/vase.h

@@ -10,8 +10,8 @@
 #include "utils/utils.h"
 
 struct Point {
-  uint32_t row;
-  uint32_t col;
+  uint64_t row;
+  uint64_t col;
 };
 
 struct Range {
@@ -19,30 +19,52 @@ struct Range {
   Point end;
 };
 
+struct RegexGroup {
+  uint64_t start{UINT32_MAX};
+  uint64_t end{UINT32_MAX};
+};
+
+struct RegexMatch {
+  uint64_t start;
+  uint64_t end;
+  RegexGroup groups[9]{};
+};
+
+struct ReplacePart {
+  enum struct PartType {
+    FullMatch,
+    CaptureGroup,
+    Constant
+  } type;
+
+  std::variant<uint8_t, Shard *> value;
+};
+
 struct Vase {
-  OriginalBuffer original;
-  AppendBuffer append;
+  OriginalBuffer *original;
+  AppendBuffer *append;
   Shard *root;
 
   Vase(std::string path);
 
   ~Vase();
 
-  uint32_t length();
+  uint64_t length();
   std::string to_string();
+  std::string to_string(Range range);
 
-  Point resolve_column(uint32_t line, uint32_t column);
+  Point resolve_column(uint64_t line, uint64_t column);
 
-  LineIterator iterate(uint32_t line);
+  LineIterator iterate(uint64_t line);
 
   void insert(Point *point, char key);
-  void insert(Point *point, const char *data, uint32_t len);
-  void erase(Point *point, int64_t amount);
+  void insert(Point *point, const char *data, uint64_t len);
+  void erase(Point *point, uint64_t amount, Direction dir);
   void erase(Range range);
-  void replace(Range range, const char *data, uint32_t len);
+  void replace(Range range, const char *data, uint64_t len);
 
-  void move_clusters(Point *point, int64_t amount);
-  void move_lines(Point *point, int64_t amount);
+  void move_clusters(Point *point, uint64_t amount, Direction dir);
+  void move_lines(Point *point, uint64_t amount, Direction dir);
   void clamp(Point *point);
 
   void regex_search_replace(
@@ -50,17 +72,30 @@ struct Vase {
     std::string_view replace, std::string_view options
   );
 
+  std::vector<Range> regex_search(
+    std::string_view pattern, Range range, std::string_view options
+  );
+
   bool undo();
   bool redo();
 
   void snapshot();
-  void prune_history(uint32_t n);
+  void prune_history(uint64_t n);
+
+  bool save(std::string path);
+  bool save_swap(std::string path);
+
+  uint64_t offset_of(Point point);
+  Point point_of(uint64_t offset);
 
 private:
   std::vector<Shard *> history;
-  uint32_t history_top;
+  uint64_t history_top;
 
-  uint32_t offset_of(Point point);
-  static void flatten(Shard *s, std::string &out);
-  void _insert(Point *point, const char *data, uint32_t len);
+  void _insert(Point *point, const char *data, uint64_t len);
+
+  std::vector<ReplacePart> parse_replace(std::string_view s);
+  std::vector<RegexMatch> _regex_search(
+    std::string_view pattern, Range range, std::string_view options
+  );
 };