SyedM/crib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 5 Wiki Activity

Fix bugs with hl line data structures

Browse Source
This commit is contained in:
Syed Daanish
2026-01-20 09:37:54 +00:00
parent fd894e4e9f
commit 81da75dc15
19 changed files with 884 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
TODO.md
View File

@@ -34,9 +34,13 @@ Copyright 2025 Syed Daanish
* make it faster for line inserts/deletes too (treeify the vector)
* Try to make all functions better now that folds have been purged
* Cleanup syntax and renderer files
* Fix ruby regexp not living across lines when edits are made
* for ruby regex use hueristic where is a space is seen after the / it is not a regexp
* finish bash then do all the directive-like ones like jsonc (first to help with theme files) / toml / yaml / ini / nginx
* then markdown / html
* then gitignore / gitattributes
* then fish then sql then css and [jt]sx? then python then lua (make with type annotations for lsp results)
* then [ch](++)? then gdscript then erb then php
* then haskell then gomod then go then rust
* [ ] **Undo/Redo:** Add support for undo/redo history.

3
include/editor/editor.h
View File

@@ -5,6 +5,7 @@
#include "editor/indents.h"
#include "io/knot.h"
#include "io/sysio.h"
#include "syntax/extras.h"
#include "syntax/parser.h"
#include "ui/completionbox.h"
#include "ui/diagnostics.h"
@@ -50,6 +51,8 @@ struct Editor {
CompletionSession completion;
IndentationEngine indents;
Parser *parser;
ExtraHighlighter extra_hl;
bool is_css_color;
};
Editor *new_editor(const char *filename_arg, Coord position, Coord size);

64
include/syntax/decl.h
View File

@@ -3,69 +3,7 @@
#include "io/knot.h"
#include "io/sysio.h"
struct Trie {
struct TrieNode {
bool is_word = false;
std::array<TrieNode *, 128> children{};
TrieNode() { children.fill(nullptr); }
};
Trie() : root(new TrieNode()) {}
~Trie() { clear_trie(root); }
void build(const std::vector<std::string> &words) {
for (const auto &word : words) {
TrieNode *node = root;
for (char c : word) {
unsigned char uc = static_cast<unsigned char>(c);
if (!node->children[uc])
node->children[uc] = new TrieNode();
node = node->children[uc];
}
node->is_word = true;
}
}
uint32_t match(const char *text, uint32_t pos, uint32_t len,
bool (*is_word_char)(char c)) const {
const TrieNode *node = root;
uint32_t max_len = 0;
for (uint32_t i = pos; i < len; ++i) {
unsigned char uc = static_cast<unsigned char>(text[i]);
if (uc >= 128)
return 0;
if (!node->children[uc]) {
if (node->is_word && !is_word_char(text[i]))
return i - pos;
break;
}
node = node->children[uc];
if (node->is_word)
max_len = i - pos + 1;
}
if (max_len > 0)
if (pos + max_len < len && is_word_char(text[pos + max_len]))
return 0;
return max_len;
}
void clear() {
clear_trie(root);
root = new TrieNode();
}
private:
TrieNode *root;
void clear_trie(TrieNode *node) {
if (!node)
return;
for (auto *child : node->children)
clear_trie(child);
delete node;
}
};
#include "syntax/trie.h"
struct Highlight {
uint32_t fg;

490
include/syntax/extras.h Normal file
View File

@@ -0,0 +1,490 @@
#ifndef SYNTAX_EXTRAS_H
#define SYNTAX_EXTRAS_H
#include "io/knot.h"
#include "syntax/decl.h"
#include "utils/utils.h"
inline static const std::vector<std::pair<std::string, uint32_t>> color_map = {
{"AliceBlue", 0xF0F8FF},
{"AntiqueWhite", 0xFAEBD7},
{"Aqua", 0x00FFFF},
{"Aquamarine", 0x7FFFD4},
{"Azure", 0xF0FFFF},
{"Beige", 0xF5F5DC},
{"Bisque", 0xFFE4C4},
{"Black", 0x000000},
{"BlanchedAlmond", 0xFFEBCD},
{"Blue", 0x0000FF},
{"BlueViolet", 0x8A2BE2},
{"Brown", 0xA52A2A},
{"BurlyWood", 0xDEB887},
{"CadetBlue", 0x5F9EA0},
{"Chartreuse", 0x7FFF00},
{"Chocolate", 0xD2691E},
{"Coral", 0xFF7F50},
{"CornflowerBlue", 0x6495ED},
{"Cornsilk", 0xFFF8DC},
{"Crimson", 0xDC143C},
{"Cyan", 0x00FFFF},
{"DarkBlue", 0x00008B},
{"DarkCyan", 0x008B8B},
{"DarkGoldenRod", 0xB8860B},
{"DarkGray", 0xA9A9A9},
{"DarkGrey", 0xA9A9A9},
{"DarkGreen", 0x006400},
{"DarkKhaki", 0xBDB76B},
{"DarkMagenta", 0x8B008B},
{"DarkOliveGreen", 0x556B2F},
{"DarkOrange", 0xFF8C00},
{"DarkOrchid", 0x9932CC},
{"DarkRed", 0x8B0000},
{"DarkSalmon", 0xE9967A},
{"DarkSeaGreen", 0x8FBC8F},
{"DarkSlateBlue", 0x483D8B},
{"DarkSlateGray", 0x2F4F4F},
{"DarkSlateGrey", 0x2F4F4F},
{"DarkTurquoise", 0x00CED1},
{"DarkViolet", 0x9400D3},
{"DeepPink", 0xFF1493},
{"DeepSkyBlue", 0x00BFFF},
{"DimGray", 0x696969},
{"DimGrey", 0x696969},
{"DodgerBlue", 0x1E90FF},
{"FireBrick", 0xB22222},
{"FloralWhite", 0xFFFAF0},
{"ForestGreen", 0x228B22},
{"Fuchsia", 0xFF00FF},
{"Gainsboro", 0xDCDCDC},
{"GhostWhite", 0xF8F8FF},
{"Gold", 0xFFD700},
{"GoldenRod", 0xDAA520},
{"Gray", 0x808080},
{"Grey", 0x808080},
{"Green", 0x008000},
{"GreenYellow", 0xADFF2F},
{"HoneyDew", 0xF0FFF0},
{"HotPink", 0xFF69B4},
{"IndianRed", 0xCD5C5C},
{"Indigo", 0x4B0082},
{"Ivory", 0xFFFFF0},
{"Khaki", 0xF0E68C},
{"Lavender", 0xE6E6FA},
{"LavenderBlush", 0xFFF0F5},
{"LawnGreen", 0x7CFC00},
{"LemonChiffon", 0xFFFACD},
{"LightBlue", 0xADD8E6},
{"LightCoral", 0xF08080},
{"LightCyan", 0xE0FFFF},
{"LightGoldenRodYellow", 0xFAFAD2},
{"LightGray", 0xD3D3D3},
{"LightGrey", 0xD3D3D3},
{"LightGreen", 0x90EE90},
{"LightPink", 0xFFB6C1},
{"LightSalmon", 0xFFA07A},
{"LightSeaGreen", 0x20B2AA},
{"LightSkyBlue", 0x87CEFA},
{"LightSlateGray", 0x778899},
{"LightSlateGrey", 0x778899},
{"LightSteelBlue", 0xB0C4DE},
{"LightYellow", 0xFFFFE0},
{"Lime", 0x00FF00},
{"LimeGreen", 0x32CD32},
{"Linen", 0xFAF0E6},
{"Magenta", 0xFF00FF},
{"Maroon", 0x800000},
{"MediumAquaMarine", 0x66CDAA},
{"MediumBlue", 0x0000CD},
{"MediumOrchid", 0xBA55D3},
{"MediumPurple", 0x9370DB},
{"MediumSeaGreen", 0x3CB371},
{"MediumSlateBlue", 0x7B68EE},
{"MediumSpringGreen", 0x00FA9A},
{"MediumTurquoise", 0x48D1CC},
{"MediumVioletRed", 0xC71585},
{"MidnightBlue", 0x191970},
{"MintCream", 0xF5FFFA},
{"MistyRose", 0xFFE4E1},
{"Moccasin", 0xFFE4B5},
{"NavajoWhite", 0xFFDEAD},
{"Navy", 0x000080},
{"OldLace", 0xFDF5E6},
{"Olive", 0x808000},
{"OliveDrab", 0x6B8E23},
{"Orange", 0xFFA500},
{"OrangeRed", 0xFF4500},
{"Orchid", 0xDA70D6},
{"PaleGoldenRod", 0xEEE8AA},
{"PaleGreen", 0x98FB98},
{"PaleTurquoise", 0xAFEEEE},
{"PaleVioletRed", 0xDB7093},
{"PapayaWhip", 0xFFEFD5},
{"PeachPuff", 0xFFDAB9},
{"Peru", 0xCD853F},
{"Pink", 0xFFC0CB},
{"Plum", 0xDDA0DD},
{"PowderBlue", 0xB0E0E6},
{"Purple", 0x800080},
{"RebeccaPurple", 0x663399},
{"Red", 0xFF0000},
{"RosyBrown", 0xBC8F8F},
{"RoyalBlue", 0x4169E1},
{"SaddleBrown", 0x8B4513},
{"Salmon", 0xFA8072},
{"SandyBrown", 0xF4A460},
{"SeaGreen", 0x2E8B57},
{"SeaShell", 0xFFF5EE},
{"Sienna", 0xA0522D},
{"Silver", 0xC0C0C0},
{"SkyBlue", 0x87CEEB},
{"SlateBlue", 0x6A5ACD},
{"SlateGray", 0x708090},
{"SlateGrey", 0x708090},
{"Snow", 0xFFFAFA},
{"SpringGreen", 0x00FF7F},
{"SteelBlue", 0x4682B4},
{"Tan", 0xD2B48C},
{"Teal", 0x008080},
{"Thistle", 0xD8BFD8},
{"Tomato", 0xFF6347},
{"Turquoise", 0x40E0D0},
{"Violet", 0xEE82EE},
{"Wheat", 0xF5DEB3},
{"White", 0xFFFFFF},
{"WhiteSmoke", 0xF5F5F5},
{"Yellow", 0xFFFF00},
{"YellowGreen", 0x9ACD32},
};
// Add word under cursor to this
struct ExtraHighlighter {
std::vector<uint32_t> colors;
std::array<std::vector<uint32_t>, 50> lines;
Trie<uint32_t> css_colors = Trie<uint32_t>();
uint32_t start = 0;
ExtraHighlighter() { css_colors.build(color_map, false); }
void render(Knot *root, uint32_t n_start, std::string word, bool is_css) {
start = n_start;
for (auto &line : lines)
line.clear();
LineIterator *it = begin_l_iter(root, start);
if (!it)
return;
uint32_t idx = 0;
uint32_t len;
char *line;
while (idx < 50 && (line = next_line(it, &len))) {
lines[idx].assign(len, UINT32_MAX - 1);
uint32_t i = 0;
while (i < len) {
if (is_css) {
std::optional<uint32_t> color;
uint32_t color_len = css_colors.match(
line, i, len, [](char c) { return isalnum(c) || c == '_'; },
&color);
if (color) {
for (uint32_t j = 0; j < color_len; j++)
lines[idx][i + j] = *color;
i += color_len;
continue;
} else if (i + 5 < len && (line[i] == 'r' || line[i] == 'R') &&
(line[i + 1] == 'g' || line[i + 1] == 'G') &&
(line[i + 2] == 'b' || line[i + 2] == 'B')) {
uint32_t start = i;
i += 3;
if (line[i] == 'a' || line[i] == 'A')
i++;
if (line[i] == '(') {
i++;
bool is_percent = false;
std::string r = "";
while (i < len && line[i] >= '0' && line[i] <= '9')
r += line[i++];
if (r.empty())
continue;
while (i < len &&
(line[i] == '.' || (line[i] >= '0' && line[i] <= '9')))
i++;
if (line[i] == '%') {
is_percent = true;
i++;
}
while (i < len && (line[i] == ',' || line[i] == ' '))
i++;
std::string g = "";
while (i < len && line[i] >= '0' && line[i] <= '9')
g += line[i++];
if (g.empty())
continue;
while (i < len &&
(line[i] == '.' || (line[i] >= '0' && line[i] <= '9')))
i++;
while (i < len &&
(line[i] == ',' || line[i] == ' ' || line[i] == '%'))
i++;
std::string b = "";
while (i < len && line[i] >= '0' && line[i] <= '9')
b += line[i++];
if (b.empty())
continue;
while (i < len &&
(line[i] == ',' || line[i] == ' ' || line[i] == '.' ||
line[i] == '/' || line[i] == '%' ||
(line[i] >= '0' && line[i] <= '9')))
i++;
if (i < len && line[i] == ')')
i++;
else
continue;
uint32_t rr, gg, bb;
if (is_percent) {
rr = std::stoul(r) * 255 / 100;
gg = std::stoul(g) * 255 / 100;
bb = std::stoul(b) * 255 / 100;
} else {
rr = std::stoul(r);
gg = std::stoul(g);
bb = std::stoul(b);
}
rr = rr > 255 ? 255 : rr;
gg = gg > 255 ? 255 : gg;
bb = bb > 255 ? 255 : bb;
uint32_t color = (rr << 16) | (gg << 8) | bb;
for (uint32_t j = start; j < i; j++)
lines[idx][j] = color;
}
continue;
} else if (i + 5 < len && (line[i] == 'h' || line[i] == 'H') &&
(line[i + 1] == 's' || line[i + 1] == 'S') &&
(line[i + 2] == 'l' || line[i + 2] == 'L')) {
uint32_t start = i;
i += 3;
if (line[i] == 'a' || line[i] == 'A')
i++;
if (line[i] == '(') {
i++;
std::string h = "";
std::string h_unit = "";
enum unit : uint8_t { deg, grad, rad, turn };
unit u = deg;
bool negative = false;
if (i < len && (line[i] == '-' || line[i] == '+')) {
negative = line[i] == '-';
i++;
}
while (i < len && line[i] >= '0' && line[i] <= '9')
h += line[i++];
if (i < len && line[i] == '.') {
h += '.';
while (i < len && line[i] >= '0' && line[i] <= '9')
h += line[i++];
}
if (h.empty())
continue;
while (i < len && ((line[i] >= 'a' && line[i] <= 'z') ||
(line[i] >= 'A' && line[i] <= 'Z')))
h_unit += line[i++];
for (size_t x = 0; x < h_unit.size(); x++)
h_unit[x] = tolower(h_unit[x]);
if (h_unit.empty())
u = deg;
else if (h_unit == "deg")
u = deg;
else if (h_unit == "grad")
u = grad;
else if (h_unit == "rad")
u = rad;
else if (h_unit == "turn")
u = turn;
else
continue;
double hue = std::stod(h);
if (negative)
hue = -hue;
switch (u) {
case deg:
break;
case grad:
hue = hue * 360.0 / 400.0;
break;
case rad:
hue = hue * 180.0 / M_PI;
break;
case turn:
hue = hue * 360.0;
break;
}
hue = fmod(hue, 360.0);
if (hue < 0)
hue += 360.0;
double h_final = hue / 360.0;
while (i < len && (line[i] == ',' || line[i] == ' '))
i++;
std::string s = "";
while (i < len && line[i] >= '0' && line[i] <= '9')
s += line[i++];
if (s.empty())
continue;
if (i < len && line[i] == '%')
i++;
else
continue;
while (i < len && (line[i] == ',' || line[i] == ' '))
i++;
std::string l = "";
while (i < len && line[i] >= '0' && line[i] <= '9')
l += line[i++];
if (l.empty())
continue;
if (i < len && line[i] == '%')
i++;
else
continue;
while (i < len &&
(line[i] == ',' || line[i] == ' ' || line[i] == '.' ||
line[i] == '/' || line[i] == '%' ||
(line[i] >= '0' && line[i] <= '9')))
i++;
if (i < len && line[i] == ')')
i++;
double s_val = std::stod(s) / 100.0;
double l_val = std::stod(l) / 100.0;
uint32_t color = hslToRgb(h_final, s_val, l_val);
for (uint32_t j = start; j < i; j++)
lines[idx][j] = color;
}
continue;
}
}
if (i + 4 < len && line[i] == '#') {
i++;
uint32_t start = i;
while (i < len && isxdigit(line[i]))
i++;
uint32_t color = 0;
if (is_css && (i - start == 3 || i - start == 4)) {
uint32_t r =
std::stoul(std::string(line + start, 1), nullptr, 16) * 0x11;
uint32_t g =
std::stoul(std::string(line + start + 1, 1), nullptr, 16) *
0x11;
uint32_t b =
std::stoul(std::string(line + start + 2, 1), nullptr, 16) *
0x11;
color = (r << 16) | (g << 8) | b;
} else if ((is_css && (i - start == 8)) || i - start == 6) {
uint32_t r = std::stoul(std::string(line + start, 2), nullptr, 16);
uint32_t g =
std::stoul(std::string(line + start + 2, 2), nullptr, 16);
uint32_t b =
std::stoul(std::string(line + start + 4, 2), nullptr, 16);
color = (r << 16) | (g << 8) | b;
} else {
continue;
}
for (uint32_t j = start - 1; j < i; j++)
lines[idx][j] = color;
continue;
} else if (i + 5 < len && line[i] == '0' && line[i + 1] == 'x') {
i += 2;
uint32_t start = i;
while (i < len && isxdigit(line[i]))
i++;
uint32_t color = 0;
if (i - start == 6) {
uint32_t r = std::stoul(std::string(line + start, 2), nullptr, 16);
uint32_t g =
std::stoul(std::string(line + start + 2, 2), nullptr, 16);
uint32_t b =
std::stoul(std::string(line + start + 4, 2), nullptr, 16);
color = (r << 16) | (g << 8) | b;
} else {
continue;
}
if (color)
color--;
else
color++;
for (uint32_t j = start - 2; j < i; j++)
lines[idx][j] = color;
continue;
}
if (i < len && (isalnum(line[i]) || line[i] == '_')) {
uint32_t start = i;
uint32_t x = 0;
bool found = true;
while (i < len && (isalnum(line[i]) || line[i] == '_')) {
if (x < word.size() && line[i] == word[x]) {
i++;
x++;
} else {
found = false;
i++;
}
}
if (found && x == word.size())
for (uint32_t j = start; j < i; j++)
lines[idx][j] = UINT32_MAX;
} else {
i += utf8_codepoint_width(line[i]);
}
}
idx++;
}
free(it->buffer);
free(it);
}
std::optional<std::pair<uint32_t, uint32_t>> get(Coord pos) {
uint32_t val;
if (pos.row < start || pos.row >= start + 50 ||
pos.col >= lines[pos.row - start].size() ||
(val = lines[pos.row - start][pos.col]) == UINT32_MAX - 1)
return std::nullopt;
return (std::pair<uint32_t, uint32_t>){fg_for_bg(val), val};
}
private:
uint32_t fg_for_bg(uint32_t color) {
uint8_t r = (color >> 16) & 0xFF;
uint8_t g = (color >> 8) & 0xFF;
uint8_t b = color & 0xFF;
double luminance = 0.299 * r + 0.587 * g + 0.114 * b;
return (luminance > 128) ? 0x000000 : 0xFFFFFF;
}
uint32_t hslToRgb(double h, double s, double l) {
double r, g, b;
if (s == 0.0) {
r = g = b = l;
} else {
auto hue2rgb = [](double p, double q, double t) -> double {
if (t < 0)
t += 1;
if (t > 1)
t -= 1;
if (t < 1.0 / 6)
return p + (q - p) * 6 * t;
if (t < 1.0 / 2)
return q;
if (t < 2.0 / 3)
return p + (q - p) * (2.0 / 3 - t) * 6;
return p;
};
double q = l < 0.5 ? l * (1 + s) : l + s - l * s;
double p = 2 * l - q;
r = hue2rgb(p, q, h + 1.0 / 3);
g = hue2rgb(p, q, h);
b = hue2rgb(p, q, h - 1.0 / 3);
}
uint32_t R = static_cast<uint32_t>(std::clamp(r, 0.0, 1.0) * 255);
uint32_t G = static_cast<uint32_t>(std::clamp(g, 0.0, 1.0) * 255);
uint32_t B = static_cast<uint32_t>(std::clamp(b, 0.0, 1.0) * 255);
return (R << 16) | (G << 8) | B;
}
};
#endif

34
include/syntax/line_tree.h
View File

@@ -74,10 +74,14 @@ struct LineTree {
}
void insert(uint32_t x, uint32_t y) {
std::unique_lock lock(mtx);
if (x > subtree_size(root))
x = subtree_size(root);
root = insert_node(root, x, y);
}
void erase(uint32_t x, uint32_t y) {
std::unique_lock lock(mtx);
if (x + y > subtree_size(root))
x = subtree_size(root) - y;
root = erase_node(root, x, y);
}
uint32_t count() {
@@ -116,18 +120,26 @@ private:
std::shared_mutex mtx;
static constexpr uint32_t LEAF_TARGET = 256;
LineTree::LineNode *erase_node(LineNode *n, uint32_t x, uint32_t y) {
if (!n)
return nullptr;
if (!n->left && !n->right) {
n->data.erase(n->data.begin() + x, n->data.begin() + x + y);
fix(n);
if (!n || y == 0)
return n;
uint32_t left_sz = subtree_size(n->left);
uint32_t mid_sz = n->data.size();
if (x < left_sz) {
uint32_t len = std::min(y, left_sz - x);
n->left = erase_node(n->left, x, len);
y -= len;
x = left_sz;
}
if (y > 0 && x < left_sz + mid_sz) {
uint32_t mid_x = x - left_sz;
uint32_t len = std::min(y, mid_sz - mid_x);
n->data.erase(n->data.begin() + mid_x, n->data.begin() + mid_x + len);
y -= len;
x += len;
}
if (y > 0) {
n->right = erase_node(n->right, x - left_sz - n->data.size(), y);
}
uint32_t left_size = subtree_size(n->left);
if (x < left_size)
n->left = erase_node(n->left, x, y);
else
n->right = erase_node(n->right, x - left_size - n->data.size(), y);
if (n->left && n->right &&
subtree_size(n->left) + subtree_size(n->right) < 256) {
return merge(n->left, n->right);
@@ -142,7 +154,7 @@ private:
return leaf;
}
if (!n->left && !n->right) {
n->data.insert(n->data.begin() + x, y, LineData{});
n->data.insert(n->data.begin() + x, y, LineData());
fix(n);
if (n->data.size() > 512)
return split_leaf(n);

9
include/syntax/parser.h
View File

@@ -5,8 +5,7 @@
#include "syntax/line_tree.h"
struct Parser {
Knot *root;
std::shared_mutex *knot_mutex;
struct Editor *editor = nullptr;
std::string lang;
std::shared_ptr<void> (*parse_func)(std::vector<Token> *tokens,
std::shared_ptr<void> in_state,
@@ -19,10 +18,8 @@ struct Parser {
LineTree line_tree;
std::set<uint32_t> dirty_lines;
Parser(Knot *n_root, std::shared_mutex *n_knot_mutex, std::string n_lang,
uint32_t n_scroll_max);
void edit(Knot *n_root, uint32_t start_line, uint32_t old_end_line,
uint32_t new_end_line);
Parser(Editor *editor, std::string n_lang, uint32_t n_scroll_max);
void edit(uint32_t start_line, uint32_t old_end_line, uint32_t inserted_rows);
void work();
void scroll(uint32_t line);
};

140
include/syntax/trie.h Normal file
View File

@@ -0,0 +1,140 @@
#ifndef SYNTAX_TRIE_H
#define SYNTAX_TRIE_H
#include "utils/utils.h"
template <typename T> struct Trie {
struct TrieNode {
bool is_word = false;
std::array<TrieNode *, 128> children{};
std::conditional_t<std::is_void_v<T>, char, std::optional<T>> value;
TrieNode() { children.fill(nullptr); }
};
Trie() {}
~Trie() { clear_trie(root); }
void build(const std::vector<std::string> &words, bool cs = true) {
static_assert(std::is_void_v<T>, "This build() is for Trie<void> only");
case_sensitive = cs;
for (auto &w : words)
insert(w);
}
template <typename U = T>
std::enable_if_t<!std::is_void_v<U>>
build(const std::vector<std::pair<std::string, U>> &words, bool cs = true) {
static_assert(!std::is_void_v<T>, "This build() is for typed Trie only");
case_sensitive = cs;
for (auto &[w, v] : words)
insert(w, v);
}
uint32_t match(const char *text, uint32_t pos, uint32_t len,
bool (*is_word_char)(char c)) const {
const TrieNode *node = root;
uint32_t max_len = 0;
for (uint32_t i = pos; i < len; ++i) {
unsigned char uc = static_cast<unsigned char>(text[i]);
if (uc >= 128)
return 0;
if (!case_sensitive && uc >= 'A' && uc <= 'Z')
uc = uc - 'A' + 'a';
if (!node->children[uc]) {
if (node->is_word && !is_word_char(text[i]))
return i - pos;
break;
}
node = node->children[uc];
if (node->is_word)
max_len = i - pos + 1;
}
if (max_len > 0)
if (pos + max_len < len && is_word_char(text[pos + max_len]))
return 0;
return max_len;
}
template <typename U = T>
uint32_t
match(const char *text, uint32_t pos, uint32_t len,
bool (*is_word_char)(char c),
std::conditional_t<std::is_void_v<T>, void *, std::optional<T> *>
out_val = nullptr) const {
const TrieNode *node = root;
const TrieNode *last_word_node = nullptr;
uint32_t max_len = 0;
for (uint32_t i = pos; i < len; ++i) {
unsigned char uc = static_cast<unsigned char>(text[i]);
if (uc >= 128)
break;
if (!case_sensitive && uc >= 'A' && uc <= 'Z')
uc = uc - 'A' + 'a';
if (!node->children[uc])
break;
node = node->children[uc];
if (node->is_word) {
last_word_node = node;
max_len = i - pos + 1;
}
}
if (!last_word_node) {
if (out_val)
*out_val = std::nullopt;
return 0;
}
if (pos + max_len < len && is_word_char(text[pos + max_len])) {
if (out_val)
*out_val = std::nullopt;
return 0;
}
if (out_val)
*out_val = last_word_node->value;
return max_len;
}
private:
TrieNode *root = new TrieNode();
bool case_sensitive = true;
void insert(const std::string &word) {
TrieNode *node = root;
for (char c : word) {
unsigned char uc = static_cast<unsigned char>(c);
if (uc >= 128)
return;
if (!case_sensitive && uc >= 'A' && uc <= 'Z')
uc = uc - 'A' + 'a';
if (!node->children[uc])
node->children[uc] = new TrieNode();
node = node->children[uc];
}
node->is_word = true;
}
template <typename U = T>
std::enable_if_t<!std::is_void_v<U>> insert(const std::string &word,
const U &val) {
TrieNode *node = root;
for (char c : word) {
unsigned char uc = static_cast<unsigned char>(c);
if (!case_sensitive && uc >= 'A' && uc <= 'Z')
uc = uc - 'A' + 'a';
if (!node->children[uc])
node->children[uc] = new TrieNode();
node = node->children[uc];
}
node->is_word = true;
node->value = val;
}
void clear_trie(TrieNode *node) {
if (!node)
return;
for (auto *child : node->children)
clear_trie(child);
delete node;
}
};
#endif

4
samples/bash.sh
View File

@@ -1,8 +1,8 @@
#!/usr/bin/env bash
# ---------------------------------------------
# ----------------------------------------------
# Bash Syntax Highlighter Test Specification
# ---------------------------------------------
# ----------------------------------------------
VERSION="1.0.0"
declare -a ITEMS=("alpha" "beta" "gamma" "delta")

9
samples/ruby.rb
View File

@@ -21,6 +21,13 @@ cjk_samples = [
"大量の文字列🚀🚀🚀"
]
# a hex color: #FFFFFF hsl(147rad, 50%, 47%)
0x603010 # another hex color
# Ruby regex with unicode
$unicode_regex_multiline = /[一-龯ぁ-ん#{0x3000}ァ
\-ヶー
@@ -49,7 +56,7 @@ UNINITCORE = %{
挨拶 = -> { "こんに \n ちは" }
arr = Array.new()
not_arr = NotArray.new()
not_arr = NotABuiltin.new()
raise NameError or SystemExit or CustomError or Errno or ErrorNotAtAll

74
src/editor/edit.cc
View File

@@ -1,7 +1,6 @@
#include "editor/editor.h"
#include "lsp/lsp.h"
#include "utils/utils.h"
#include <cstdint>
void edit_erase(Editor *editor, Coord pos, int64_t len) {
if (len == 0)
@@ -52,7 +51,7 @@ void edit_erase(Editor *editor, Coord pos, int64_t len) {
editor->root = erase(editor->root, start, byte_pos - start);
lock_2.unlock();
if (editor->parser)
editor->parser->edit(editor->root, start_row, end_row, start_row);
editor->parser->edit(start_row, end_row, 0);
if (do_lsp) {
if (editor->lsp->incremental_sync) {
json message = {
@@ -124,7 +123,7 @@ void edit_erase(Editor *editor, Coord pos, int64_t len) {
editor->root = erase(editor->root, byte_pos, end - byte_pos);
lock_2.unlock();
if (editor->parser)
editor->parser->edit(editor->root, start_row, end_row, start_row);
editor->parser->edit(start_row, end_row, 0);
if (do_lsp) {
if (editor->lsp->incremental_sync) {
json message = {
@@ -175,7 +174,7 @@ void edit_insert(Editor *editor, Coord pos, char *data, uint32_t len) {
apply_hook_insertion(editor, pos.row, rows);
lock_2.unlock();
if (editor->parser)
editor->parser->edit(editor->root, pos.row, pos.row, pos.row + rows);
editor->parser->edit(pos.row, pos.row, rows);
if (editor->lsp) {
if (editor->lsp->incremental_sync) {
lock_1.lock();
@@ -218,19 +217,64 @@ void edit_insert(Editor *editor, Coord pos, char *data, uint32_t len) {
void edit_replace(Editor *editor, Coord start, Coord end, const char *text,
uint32_t len) {
std::shared_lock lock(editor->knot_mtx);
std::unique_lock lock(editor->knot_mtx);
uint32_t start_byte =
line_to_byte(editor->root, start.row, nullptr) + start.col;
uint32_t end_byte = line_to_byte(editor->root, end.row, nullptr) + end.col;
char *buf = read(editor->root, start_byte, end_byte - start_byte);
if (!buf)
return;
lock.unlock();
uint32_t erase_len =
count_clusters(buf, end_byte - start_byte, 0, end_byte - start_byte);
free(buf);
if (erase_len != 0)
edit_erase(editor, start, erase_len);
LineIterator *it = begin_l_iter(editor->root, start.row);
char *line = next_line(it, nullptr);
int utf16_start = 0;
if (line)
utf16_start = utf8_byte_offset_to_utf16(line, start.col);
free(it->buffer);
free(it);
it = begin_l_iter(editor->root, end.row);
line = next_line(it, nullptr);
int utf16_end = 0;
if (line)
utf16_end = utf8_byte_offset_to_utf16(line, end.col);
free(it->buffer);
free(it);
if (start_byte != end_byte)
editor->root = erase(editor->root, start_byte, end_byte - start_byte);
if (len > 0)
edit_insert(editor, start, const_cast<char *>(text), len);
editor->root = insert(editor->root, start_byte, (char *)text, len);
uint32_t rows = 0;
for (uint32_t i = 0; i < len; i++)
if (text[i] == '\n')
rows++;
if (editor->parser) {
editor->parser->edit(start.row, end.row - 1, 0);
editor->parser->edit(start.row, start.row, rows);
}
if (editor->lsp) {
if (editor->lsp->incremental_sync) {
json message = {
{"jsonrpc", "2.0"},
{"method", "textDocument/didChange"},
{"params",
{{"textDocument",
{{"uri", editor->uri}, {"version", ++editor->lsp_version}}},
{"contentChanges",
json::array(
{{{"range",
{{"start",
{{"line", start.row}, {"character", utf16_start}}},
{"end", {{"line", end.row}, {"character", utf16_end}}}}},
{"text", std::string(text, len)}}})}}}};
lsp_send(editor->lsp, message, nullptr);
} else {
char *buf = read(editor->root, 0, editor->root->char_count);
std::string full_text(buf);
free(buf);
json message = {
{"jsonrpc", "2.0"},
{"method", "textDocument/didChange"},
{"params",
{{"textDocument",
{{"uri", editor->uri}, {"version", ++editor->lsp_version}}},
{"contentChanges", json::array({{{"text", full_text}}})}}}};
lsp_send(editor->lsp, message, nullptr);
}
}
}

18
src/editor/editor.cc
View File

@@ -30,8 +30,11 @@ Editor *new_editor(const char *filename_arg, Coord position, Coord size) {
free(str);
editor->lang = language_for_file(filename.c_str());
if (editor->lang.name != "unknown")
editor->parser = new Parser(editor->root, &editor->knot_mtx,
editor->lang.name, size.row + 5);
editor->parser = new Parser(editor, editor->lang.name, size.row + 5);
if (editor->lang.name == "css" || editor->lang.name == "html" ||
editor->lang.name == "javascript" || editor->lang.name == "markdown" ||
editor->lang.name == "typescript")
editor->is_css_color = true;
if (len <= (1024 * 28))
request_add_to_lsp(editor->lang, editor);
editor->indents.compute_indent(editor);
@@ -52,12 +55,13 @@ void save_file(Editor *editor) {
return;
std::shared_lock lock(editor->knot_mtx);
int version = editor->lsp_version;
char *str = read(editor->root, 0, editor->root->char_count);
uint32_t char_count = editor->root->char_count;
char *str = read(editor->root, 0, char_count);
if (!str)
return;
lock.unlock();
std::ofstream out(editor->filename);
out.write(str, editor->root->char_count);
out.write(str, char_count);
out.close();
free(str);
if (editor->lsp) {
@@ -99,12 +103,14 @@ void save_file(Editor *editor) {
apply_lsp_edits(editor, t_edits, false);
ensure_scroll(editor);
std::shared_lock lock(editor->knot_mtx);
char *str = read(editor->root, 0, editor->root->char_count);
uint32_t char_count = editor->root->char_count;
char *str = read(editor->root, 0, char_count);
if (!str)
return;
lock.unlock();
std::ofstream out(editor->filename);
out.write(str, editor->root->char_count);
out.write(str, char_count);
out.close();
free(str);
lsp_send(editor->lsp, save_msg, nullptr);
}

32
src/editor/renderer.cc
View File

@@ -1,10 +1,13 @@
#include "editor/editor.h"
#include "io/sysio.h"
#include "main.h"
#include "syntax/decl.h"
#include "syntax/parser.h"
#include <cstdint>
void render_editor(Editor *editor) {
uint32_t sel_start = 0, sel_end = 0;
std::shared_lock knot_lock(editor->knot_mtx);
uint32_t numlen =
EXTRA_META + static_cast<int>(std::log10(editor->root->line_count + 1));
uint32_t render_width = editor->size.col - numlen;
@@ -34,7 +37,6 @@ void render_editor(Editor *editor) {
return (int)token.type;
return 0;
};
std::shared_lock knot_lock(editor->knot_mtx);
if (editor->selection_active) {
Coord start, end;
if (editor->cursor >= editor->selection) {
@@ -88,6 +90,18 @@ void render_editor(Editor *editor) {
LineIterator *it = begin_l_iter(editor->root, line_index);
if (!it)
return;
uint32_t prev_col, next_col;
std::string word;
word_boundaries_exclusive(editor, editor->cursor, &prev_col, &next_col);
if (next_col - prev_col > 0 && next_col - prev_col < 256 - 4) {
uint32_t offset = line_to_byte(editor->root, editor->cursor.row, nullptr);
char *word_ptr = read(editor->root, offset + prev_col, next_col - prev_col);
if (word_ptr) {
word = std::string(word_ptr, next_col - prev_col);
free(word_ptr);
}
}
editor->extra_hl.render(editor->root, line_index, word, editor->is_css_color);
uint32_t rendered_rows = 0;
uint32_t global_byte_offset = line_to_byte(editor->root, line_index, nullptr);
while (rendered_rows < editor->size.row) {
@@ -157,9 +171,19 @@ void render_editor(Editor *editor) {
const Highlight *hl = nullptr;
if (editor->parser)
hl = &highlights[get_type(current_byte_offset + local_render_offset)];
uint32_t fg = hl ? hl->fg : 0xFFFFFF;
uint32_t bg = hl ? hl->bg : 0;
uint8_t fl = hl ? hl->flags : 0;
std::optional<std::pair<uint32_t, uint32_t>> extra =
editor->extra_hl.get(
{line_index, current_byte_offset + local_render_offset});
uint32_t fg = extra && extra->second != UINT32_MAX
? extra->first
: (hl ? hl->fg : 0xFFFFFF);
uint32_t bg = extra && extra->second != UINT32_MAX
? extra->second
: (hl ? hl->bg : 0x000000);
uint8_t fl =
(hl ? hl->flags : 0) |
(extra ? (extra->second != UINT32_MAX ? CF_BOLD : CF_UNDERLINE)
: 0);
if (editor->selection_active && absolute_byte_pos >= sel_start &&
absolute_byte_pos < sel_end)
bg = 0x555555;

41
src/syntax/bash.cc
View File

@@ -1,6 +1,6 @@
#include "syntax/decl.h"
#include "syntax/langs.h"
#include "utils/utils.h"
#include <cstdint>
struct BashFullState {
int brace_level = 0;
@@ -11,13 +11,11 @@ struct BashFullState {
bool line_cont = false;
struct Lit {
std::string delim = "";
int brace_level = 1;
std::string delim = ""; // Only 1 wide for strings
bool allow_interp = false;
bool operator==(const BashFullState::Lit &other) const {
return delim == other.delim && brace_level == other.brace_level &&
allow_interp == other.allow_interp;
return delim == other.delim && allow_interp == other.allow_interp;
}
} lit;
@@ -66,7 +64,35 @@ std::shared_ptr<void> bash_parse(std::vector<Token> *tokens,
if (len == 0)
return state;
while (i < len) {
i += utf8_codepoint_width(text[i]);
if (state->full_state->in_state == BashFullState::STRING) {
uint32_t start = i;
while (i < len) {
if (text[i] == state->full_state->lit.delim[0]) {
tokens->push_back({start, i, TokenKind::String});
state->full_state->in_state = BashFullState::NONE;
break;
}
i++;
}
if (i == len)
tokens->push_back({start, i, TokenKind::String});
continue;
}
if (text[i] == '#') {
if (i == 0 && len > 4 && text[i + 1] == '!') {
tokens->push_back({0, len, TokenKind::Shebang});
return state;
}
tokens->push_back({i, len, TokenKind::Comment});
return state;
} else if (text[i] == '\'') {
state->full_state->in_state = BashFullState::STRING;
state->full_state->lit.delim = "'";
state->full_state->lit.allow_interp = false;
tokens->push_back({i, ++i, TokenKind::String});
continue;
}
i++;
}
return state;
}
@@ -76,3 +102,6 @@ std::shared_ptr<void> bash_parse(std::vector<Token> *tokens,
// ${var} and $((math)) $(command) and `command` expansions ANSI-C quoted
// stirngs - $'' backslash escapes but with \xHH and \uHHHH and \uHHHHHHHH \cX
// too
//
// Lock edit_replace across both delete and insert instead of within to keep the
// parser from glitching

0
src/syntax/extras.cc Normal file
View File

66
src/syntax/parser.cc
View File

@@ -1,4 +1,5 @@
#include "syntax/parser.h"
#include "editor/editor.h"
#include "io/knot.h"
#include "main.h"
#include "syntax/decl.h"
@@ -6,10 +7,9 @@
std::array<Highlight, TOKEN_KIND_COUNT> highlights = {};
Parser::Parser(Knot *n_root, std::shared_mutex *n_knot_mutex,
std::string n_lang, uint32_t n_scroll_max) {
Parser::Parser(Editor *n_editor, std::string n_lang, uint32_t n_scroll_max) {
editor = n_editor;
scroll_max = n_scroll_max;
knot_mutex = n_knot_mutex;
lang = n_lang;
auto pair = parsers.find(n_lang);
if (pair != parsers.end()) {
@@ -18,22 +18,26 @@ Parser::Parser(Knot *n_root, std::shared_mutex *n_knot_mutex,
} else {
assert("unknown lang should be checked by caller" && 0);
}
edit(n_root, 0, 0, n_root->line_count);
edit(0, 0, editor->root->line_count);
}
void Parser::edit(Knot *n_root, uint32_t start_line, uint32_t old_end_line,
uint32_t new_end_line) {
void Parser::edit(uint32_t start_line, uint32_t old_end_line,
uint32_t inserted_rows) {
std::lock_guard lock(data_mutex);
root = n_root;
if (((int64_t)old_end_line - (int64_t)start_line) > 0)
line_tree.erase(start_line + 1, old_end_line - start_line);
if (((int64_t)new_end_line - (int64_t)old_end_line) > 0)
line_tree.insert(start_line + 1, new_end_line - start_line);
line_tree.erase(start_line, old_end_line - start_line);
if (inserted_rows > 0)
line_tree.insert(start_line, inserted_rows);
if (start_line > 0)
dirty_lines.insert(start_line - 1);
dirty_lines.insert(start_line);
dirty_lines.insert(start_line + 1);
}
void Parser::work() {
std::shared_lock k_lock(*knot_mutex);
if (!editor || !editor->root)
return;
std::shared_lock k_lock(editor->knot_mtx);
k_lock.unlock();
uint32_t capacity = 256;
char *text = (char *)calloc((capacity + 1), sizeof(char));
@@ -45,14 +49,16 @@ void Parser::work() {
std::unique_lock lock(mutex);
lock.unlock();
for (uint32_t c_line : tmp_dirty) {
if (c_line > scroll_max) {
if (c_line > scroll_max + 40) {
remaining_dirty.insert(c_line);
continue;
}
uint32_t line_count = line_tree.count();
lock_data.lock();
std::shared_ptr<void> prev_state =
(c_line > 0) ? line_tree.at(c_line - 1)->out_state : nullptr;
(c_line > 0) && c_line < line_tree.count()
? line_tree.at(c_line - 1)->out_state
: nullptr;
lock_data.unlock();
while (c_line < line_count) {
if (!running.load(std::memory_order_relaxed)) {
@@ -60,14 +66,18 @@ void Parser::work() {
return;
}
k_lock.lock();
if (c_line > editor->root->line_count) {
k_lock.unlock();
continue;
}
uint32_t r_offset, r_len;
r_offset = line_to_byte(root, c_line, &r_len);
r_offset = line_to_byte(editor->root, c_line, &r_len);
if (r_len > capacity) {
capacity = r_len;
text = (char *)realloc(text, capacity + 1);
memset(text, 0, capacity + 1);
}
read_into(root, r_offset, r_len, text);
read_into(editor->root, r_offset, r_len, text);
k_lock.unlock();
if (c_line < scroll_max &&
((scroll_max > 100 && c_line > scroll_max - 100) || c_line < 100))
@@ -79,6 +89,12 @@ void Parser::work() {
}
lock_data.lock();
LineData *line_data = line_tree.at(c_line);
if (!line_data) {
lock_data.unlock();
if (lock.owns_lock())
lock.unlock();
continue;
}
std::shared_ptr<void> new_state =
parse_func(&line_data->tokens, prev_state, text, r_len);
line_data->in_state = prev_state;
@@ -98,8 +114,8 @@ void Parser::work() {
remaining_dirty.insert(c_line);
break;
}
if (c_line < line_count &&
state_match_func(prev_state, line_tree.at(c_line)->in_state)) {
if (c_line < line_count && (line_data = line_tree.at(c_line)) &&
state_match_func(prev_state, line_data->in_state)) {
lock_data.unlock();
if (lock.owns_lock())
lock.unlock();
@@ -129,7 +145,7 @@ void Parser::scroll(uint32_t line) {
if (line_tree.at(c_line)->in_state || line_tree.at(c_line)->out_state)
return;
lock_data.unlock();
std::shared_lock k_lock(*knot_mutex);
std::shared_lock k_lock(editor->knot_mtx);
k_lock.unlock();
uint32_t capacity = 256;
char *text = (char *)calloc((capacity + 1), sizeof(char));
@@ -144,14 +160,18 @@ void Parser::scroll(uint32_t line) {
return;
}
k_lock.lock();
if (c_line > editor->root->line_count) {
k_lock.unlock();
continue;
}
uint32_t r_offset, r_len;
r_offset = line_to_byte(root, c_line, &r_len);
r_offset = line_to_byte(editor->root, c_line, &r_len);
if (r_len > capacity) {
capacity = r_len;
text = (char *)realloc(text, capacity + 1);
memset(text, 0, capacity + 1);
}
read_into(root, r_offset, r_len, text);
read_into(editor->root, r_offset, r_len, text);
k_lock.unlock();
if (c_line < scroll_max &&
((scroll_max > 100 && c_line > scroll_max - 100) || c_line < 100))
@@ -163,6 +183,12 @@ void Parser::scroll(uint32_t line) {
}
lock_data.lock();
LineData *line_data = line_tree.at(c_line);
if (!line_data) {
lock_data.unlock();
if (lock.owns_lock())
lock.unlock();
continue;
}
std::shared_ptr<void> new_state =
parse_func(&line_data->tokens, prev_state, text, r_len);
line_data->in_state = nullptr;

18
src/syntax/ruby.cc
View File

@@ -290,15 +290,15 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
std::shared_ptr<void> in_state,
const char *text, uint32_t len) {
static bool keywords_trie_init = false;
static Trie base_keywords_trie;
static Trie expecting_keywords_trie;
static Trie operator_keywords_trie;
static Trie expecting_operators_trie;
static Trie operator_trie;
static Trie types_trie;
static Trie builtins_trie;
static Trie methods_trie;
static Trie errors_trie;
static Trie<void> base_keywords_trie;
static Trie<void> expecting_keywords_trie;
static Trie<void> operator_keywords_trie;
static Trie<void> expecting_operators_trie;
static Trie<void> operator_trie;
static Trie<void> types_trie;
static Trie<void> builtins_trie;
static Trie<void> methods_trie;
static Trie<void> errors_trie;
if (!keywords_trie_init) {
base_keywords_trie.build(base_keywords);
expecting_keywords_trie.build(expecting_keywords);

12
src/ui/completionbox.cc
View File

@@ -1,5 +1,6 @@
#include "ui/completionbox.h"
#include "editor/completions.h"
#include "io/sysio.h"
#include "utils/utils.h"
std::string item_kind_name(uint8_t kind) {
@@ -139,11 +140,12 @@ void CompletionBox::render(Coord pos) {
if (start_row < 0)
start_row = pos.row + 1;
int32_t start_col = pos.col;
// if (start_col + size.col > cols) {
// start_col = cols - size.col;
// if (start_col < 0)
// start_col = 0;
// }
Coord screen_size = get_size();
if (start_col + size.col > screen_size.col) {
start_col = screen_size.col - size.col;
if (start_col < 0)
start_col = 0;
}
position = {(uint32_t)start_row, (uint32_t)start_col};
for (uint32_t r = 0; r < size.row; r++)
for (uint32_t c = 0; c < size.col; c++)

11
src/ui/diagnostics.cc
View File

@@ -148,11 +148,12 @@ void DiagnosticBox::render(Coord pos) {
if (start_row < 0)
start_row = pos.row + 1;
int32_t start_col = pos.col;
// if (start_col + size.col > cols) {
// start_col = cols - size.col;
// if (start_col < 0)
// start_col = 0;
// }
Coord screen_size = get_size();
if (start_col + size.col > screen_size.col) {
start_col = screen_size.col - size.col;
if (start_col < 0)
start_col = 0;
}
for (uint32_t r = 0; r < size.row; r++)
for (uint32_t c = 0; c < size.col; c++)
update(start_row + r, start_col + c, cells[r * size.col + c].utf8,

11
src/ui/hover.cc
View File

@@ -104,11 +104,12 @@ void HoverBox::render(Coord pos) {
if (start_row < 0)
start_row = pos.row + 1;
int32_t start_col = pos.col;
// if (start_col + size.col > cols) {
// start_col = cols - size.col;
// if (start_col < 0)
// start_col = 0;
// }
Coord screen_size = get_size();
if (start_col + size.col > screen_size.col) {
start_col = screen_size.col - size.col;
if (start_col < 0)
start_col = 0;
}
for (uint32_t r = 0; r < size.row; r++)
for (uint32_t c = 0; c < size.col; c++)
update(start_row + r, start_col + c, cells[r * size.col + c].utf8,