Implement syntax highlighting

- Create a generic incremental syntax highlighting system.
- Implement a ruby syntax highlighter for it.
This commit is contained in:
2026-08-20 02:23:11 +01:00
parent 48fb8e3501
commit 3a41af01f9
18 changed files with 2631 additions and 28 deletions
+2
View File
@@ -5,6 +5,7 @@
#include "internal/commands/commands.h"
#include "internal/commands/suffixes.h"
#include "internal/marks/marks.h"
#include "internal/theme/theme.h"
#include "pch.h"
namespace bed {
@@ -13,6 +14,7 @@ struct BEd {
internal::commands::Command no_op;
internal::commands::Command eof_op;
std::array<std::optional<internal::commands::Suffix>, 26> suffixes;
internal::theme::Theme theme;
bool help_mode = false;
std::string last_help = "";
+8 -3
View File
@@ -2,12 +2,17 @@
#include "definitions.h"
#include "internal/marks/marks.h"
#include "internal/syntax/parser.h"
#include "internal/syntax/ruby/parser.h"
#include "internal/theme/theme.h"
#include "pch.h"
namespace bed::internal::buffer {
struct Buffer {
internal::vase::Vase vase;
internal::marks::MarksEngine marks;
vase::Vase vase;
marks::MarksEngine marks;
syntax::Language lang;
syntax::Parser parser;
uint64_t line = 0;
bool modified;
std::filesystem::path save_path = "";
@@ -28,7 +33,7 @@ struct Buffer {
void join(uint64_t start_line, uint64_t end_line);
void remove(uint64_t start_line, uint64_t end_line);
void append(std::string text, uint64_t line);
void print(uint64_t start_line, uint64_t end_line);
void print(BEd &ctx, uint64_t start_line, uint64_t end_line);
std::string list_string(std::string_view s);
};
} // namespace bed::internal::buffer
+182
View File
@@ -0,0 +1,182 @@
#pragma once
#include "internal/trie/trie.h"
#include "pch.h"
namespace bed::internal::syntax {
struct Token {
uint32_t start;
uint32_t end;
enum : uint8_t {
Data,
Shebang,
Comment,
Error,
String,
Escape,
Interpolation,
Regexp,
Number,
True,
False,
Char,
Keyword,
KeywordOperator,
Operator,
Function,
Namespace,
Class,
Module,
Type,
Constant,
VariableInstance,
VariableGlobal,
Annotation,
Directive,
Label,
Brace1,
Brace2,
Brace3,
Brace4,
Brace5,
Heading1,
Heading2,
Heading3,
Heading4,
Heading5,
Heading6,
Blockquote,
List,
ListItem,
Code,
LanguageName,
LinkLabel,
ImageLabel,
Link,
Table,
TableHeader,
Italic,
Bold,
Underline,
Strikethrough,
HorizontalRule,
Tag,
Attribute,
CheckDone,
CheckNotDone,
Count
} type;
};
struct Language {
std::function<void *()> none_state;
std::function<void(void **, std::string_view, bool, std::vector<Token> *)> parse;
std::function<void *(void *)> copy;
std::function<bool(void *, void *)> equal;
std::function<void(void *)> destroy;
};
struct ParseState {
static constexpr uint64_t BRANCH_BIT = 1ull << 63;
static constexpr uint64_t LINES_MASK = ~BRANCH_BIT;
uint64_t header;
bool is_branch() const {
return header & BRANCH_BIT;
}
uint64_t lines() const {
return header & LINES_MASK;
}
};
struct ParseStateBranch : ParseState {
ParseState *left;
ParseState *right;
};
struct ParseStateLeaf : ParseState {
void *state;
};
struct TreeCursor {
ParseStateLeaf *leaf = nullptr;
ParseStateBranch *stack[64];
uint8_t depth = 0;
bool went_left[64];
TreeCursor(ParseState *root, uint64_t target_line, uint64_t *relative) {
ParseState *node = root;
while (node->is_branch()) {
auto *branch = (ParseStateBranch *)node;
auto *left = branch->left;
stack[depth] = branch;
if (target_line < left->lines()) {
went_left[depth] = true;
++depth;
node = left;
} else {
target_line -= left->lines();
went_left[depth] = false;
++depth;
node = branch->right;
}
}
*relative = target_line;
leaf = (ParseStateLeaf *)node;
}
void next() {
while (depth > 0) {
auto *branch = stack[depth - 1];
bool from_left = went_left[depth - 1];
--depth;
if (!from_left)
continue;
ParseState *node = branch->right;
while (node->is_branch()) {
auto *b = (ParseStateBranch *)node;
stack[depth] = b;
went_left[depth] = true;
++depth;
node = b->left;
}
leaf = (ParseStateLeaf *)node;
return;
}
leaf = nullptr;
}
void prev() {
while (depth > 0) {
auto *branch = stack[depth - 1];
bool from_left = went_left[depth - 1];
--depth;
if (from_left)
continue;
ParseState *node = branch->left;
while (node->is_branch()) {
auto *b = (ParseStateBranch *)node;
stack[depth] = b;
went_left[depth] = false;
++depth;
node = b->right;
}
leaf = (ParseStateLeaf *)node;
return;
}
leaf = nullptr;
}
};
/*struct Symbol {
uint64_t definition;
std::vector<uint64_t> references;
};
struct Space {
uint64_t len;
};
struct Scope {
uint64_t len;
uint32_t type;
trie::Trie<Symbol> symbols;
std::vector<std::variant<Space, Scope>> children;
};*/
} // namespace bed::internal::syntax
+40
View File
@@ -0,0 +1,40 @@
#pragma once
#include "decl.h"
#include "internal/vase/vase.h"
#include "pch.h"
namespace bed::internal::syntax {
struct Parser {
ParseState *root;
Language &lang;
Parser(vase::Vase &, uint64_t, Language &);
~Parser();
void reset(vase::Vase &, uint64_t, Language &);
void erase(vase::Vase &, uint64_t, uint64_t);
void insert(vase::Vase &, uint64_t, uint64_t);
void modify(vase::Vase &, uint64_t, uint64_t);
std::pair<ParseState *, ParseState *> split_tree(ParseState *node, uint64_t line);
ParseState *join_tree(ParseState *a, ParseState *b);
// Scope root;
// Scope &get_scope(uint64_t);
struct Iterator {
Parser *p;
std::optional<vase::Iterator> it;
void *state;
uint64_t at;
std::vector<Token> tokens;
Iterator(uint64_t, Parser *, vase::Vase &);
~Iterator();
Iterator(const Iterator &) = delete;
Iterator &operator=(const Iterator &) = delete;
Iterator(Iterator &&other);
Iterator &operator=(Iterator &&other);
void next();
};
std::optional<Iterator> get_hl(vase::Vase &, uint64_t);
};
} // namespace bed::internal::syntax
+50
View File
@@ -0,0 +1,50 @@
#pragma once
#include "../decl.h"
#include "pch.h"
#include "tries.h"
namespace bed::internal::syntax::ruby {
struct alignas(2) RubyState {
struct RubyInternalState {
uint16_t brace_level;
uint16_t lit_brace_level;
enum : uint8_t {
NONE,
STRING,
REGEXP,
HEREDOC,
COMMENT,
END
} state;
static constexpr const uint8_t EXPECTING_EXPRESSION = 0b00010000;
static constexpr const uint8_t ALLOW_INTERPOLATION = 0b00000001;
uint8_t flags;
char delim_start;
char delim_end;
};
struct Heredocs {
// header masks
static constexpr const uint8_t ALLOW_INDENTATION = 0b10000000;
static constexpr const uint8_t ALLOW_INTERPOLATION = 0b01000000;
static constexpr const uint8_t LEN_MASK = 0b00111111;
// the rest will be len number of bytes with the actual name.
};
uint8_t top;
uint8_t docs;
// the stack (RubyInternalState * top)
// the docs queue (docs)
inline RubyInternalState *stack() {
return (RubyInternalState *)(this + 1);
}
inline uint8_t *heredocs() {
return (uint8_t *)(stack() + top);
}
};
Language lang_ruby();
} // namespace bed::internal::syntax::ruby
+103
View File
@@ -0,0 +1,103 @@
#pragma once
#include "decl.h"
#include "pch.h"
namespace bed::internal::syntax::ruby {
struct RubyParser {
void **v_state;
RubyState *state;
std::string_view line;
uint32_t i = 0;
bool heredoc_start_line = false;
RubyParser(void **v_state, std::string_view line)
: v_state(v_state), state((RubyState *)*v_state), line(line) {}
uint32_t len() const {
return line.size();
}
RubyState::RubyInternalState &current() {
return state->stack()[state->top - 1];
}
char peek(uint32_t offset = 0) {
uint32_t pos = i + offset;
return pos < line.size() ? line[pos] : '\0';
}
std::string_view peek_str(uint32_t len) {
return line.substr(i, len);
}
void advance() {
++i;
}
void advance(uint32_t n) {
i += n;
}
void push_state() {
state->top++;
*v_state = (RubyState *)realloc(
*v_state,
sizeof(RubyState)
+ sizeof(RubyState::RubyInternalState) * state->top
+ state->docs
);
state = (RubyState *)*v_state;
memmove(
state->heredocs(),
state->stack() + state->top - 1,
state->docs
);
current() = {
.brace_level = 1,
.lit_brace_level = 0,
.state = RubyState::RubyInternalState::NONE,
.flags = 0,
.delim_start = '\0',
.delim_end = '\0'
};
}
void pop_state() {
state->top--;
memmove(
state->heredocs(),
state->stack() + state->top + 1,
state->docs
);
}
void enqueue_doc(uint8_t header, std::string_view name) {
uint32_t bytes = 1 + name.size();
uint32_t old_docs = state->docs;
state->docs += bytes;
*v_state = (RubyState *)realloc(
*v_state,
sizeof(RubyState)
+ sizeof(RubyState::RubyInternalState) * state->top
+ state->docs
);
state = (RubyState *)*v_state;
uint8_t *heredocs = state->heredocs();
heredocs[old_docs] = header;
memcpy(heredocs + old_docs + 1, name.data(), name.size());
}
bool dequeue_doc(uint8_t heredoc_len) {
uint32_t bytes = heredoc_len + 1;
uint8_t *heredocs = state->heredocs();
if (state->docs -= bytes)
memmove(heredocs, heredocs + bytes, state->docs);
else
return false;
return true;
}
};
void ruby_parse(void **v_state, std::string_view line, bool first_line, std::vector<Token> *);
} // namespace bed::internal::syntax::ruby
+336
View File
@@ -0,0 +1,336 @@
#pragma once
#include "internal/trie/trie.h"
#include "pch.h"
namespace bed::internal::syntax::ruby {
const static std::vector<std::string> types = {
"BasicObject",
"Object",
"NilClass",
"TrueClass",
"FalseClass",
"Integer",
"Fixnum",
"Bignum",
"Float",
"Rational",
"Complex",
"Numeric",
"String",
"Symbol",
"Array",
"Hash",
"Range",
"Regexp",
"Struct",
"Enumarator",
"Enumerable",
"Time",
"Date",
"IO",
"File",
"Dir",
"Thread",
"Proc",
"Method",
"Module",
"Class",
"Mutex",
"ConditionVariable",
"MatchData",
"Encoding",
"Fiber",
};
const static std::vector<std::string> builtins = {
"ARGF",
"ARGV",
"ENV",
"STDIN",
"STDOUT",
"STDERR",
"DATA",
"TOPLEVEL_BINDING",
"RUBY_PLATFORM",
"RUBY_VERSION",
"RUBY_RELEASE_DATE",
"RUBY_PATCHLEVEL",
"RUBY_ENGINE",
"__LINE__",
"__FILE__",
"__ENCODING__",
"__dir__",
"__callee__",
"__method__",
"__id__",
"__send__",
};
const static std::vector<std::string> methods = {
"abort",
"at_exit",
"binding",
"block_given?",
"caller",
"catch",
"chomp",
"chomp!",
"chop",
"chop!",
"eval",
"exec",
"exit",
"exit!",
"fail",
"fork",
"format",
"gets",
"global_variables",
"gsub",
"gsub!",
"iterator?",
"lambda",
"load",
"loop",
"open",
"print",
"printf",
"proc",
"putc",
"puts",
"raise",
"rand",
"readline",
"readlines",
"require",
"require_relative",
"select",
"sleep",
"spawn",
"split",
"sprintf",
"srand",
"sub",
"sub!",
"syscall",
"system",
"test",
"throw",
"trace_var",
"trap",
"untrace_var",
"attr",
"attr_reader",
"attr_writer",
"attr_accessor",
"class_variable_get",
"class_variable_set",
"define_method",
"instance_variable_get",
"instance_variable_set",
"private",
"protected",
"public",
"public_class_method",
"module_function",
"remove_method",
"undef_method",
"method",
"methods",
"singleton_methods",
"private_methods",
"protected_methods",
"public_methods",
"send",
"extend",
"include",
"prepend",
"clone",
"dup",
"freeze",
"taint",
"untaint",
"trust",
"untrust",
"untaint?",
"trust?",
"each",
"each_with_index",
"each_with_object",
"map",
"collect",
"select",
"reject",
"reduce",
"inject",
"find",
"detect",
"all?",
"any?",
"none?",
"one?",
"count",
"cycle",
"drop",
"drop_while",
"take",
"take_while",
"chunk",
"chunk_while",
"group_by",
"partition",
"slice_before",
"slice_after",
"nil?",
"is_a?",
"kind_of?",
"instance_of?",
"respond_to?",
"equal?",
"object_id",
"class",
"singleton_class",
"clone",
"freeze",
"tap",
"then",
};
const static std::vector<std::string> errors = {
"Error",
"Exception",
"SignalException",
"Interrupt",
"StopIteration",
"Errno",
"SystemExit",
"fatal",
};
const static std::vector<std::string> base_keywords = {
"class",
"module",
"begin",
"end",
"else",
"rescue",
"ensure",
"do",
"when",
};
const static std::vector<std::string> expecting_keywords = {
"if",
"elsif",
"case",
"for",
"while",
"until",
"unless",
};
const static std::vector<std::string> operator_keywords = {
"alias",
"BEGIN",
"break",
"catch",
"defined?",
"in",
"next",
"redo",
"rescue",
"retry",
"super",
"self",
"nil",
"undef",
};
const static std::vector<std::string> expecting_operators = {
"and",
"return",
"not",
"yield",
"or",
};
const static std::vector<std::string> operators = {
"+",
"-",
"*",
"/",
"%",
"**",
"==",
"!=",
"===",
"<=>",
">",
">=",
"<",
"<=",
"&&",
"||",
"!",
"&",
"|",
"^",
"~",
"<<",
">>",
"=",
"+=",
"-=",
"*=",
"/=",
"%=",
"**=",
"&=",
"|=",
"^=",
"<<=",
">>=",
"..",
"...",
"===",
"=",
"=>",
"&",
"`",
"->",
"=~",
};
struct RubyTries {
trie::Trie<void> base_keywords_trie;
trie::Trie<void> expecting_keywords_trie;
trie::Trie<void> operator_keywords_trie;
trie::Trie<void> expecting_operators_trie;
trie::Trie<void> operator_trie;
trie::Trie<void> types_trie;
trie::Trie<void> builtins_trie;
trie::Trie<void> methods_trie;
trie::Trie<void> errors_trie;
RubyTries() {
for (auto &keyword : base_keywords)
base_keywords_trie.insert(keyword);
for (auto &keyword : expecting_keywords)
expecting_keywords_trie.insert(keyword);
for (auto &keyword : operator_keywords)
operator_keywords_trie.insert(keyword);
for (auto &keyword : expecting_operators)
expecting_operators_trie.insert(keyword);
for (auto &keyword : operators)
operator_trie.insert(keyword);
for (auto &keyword : types)
types_trie.insert(keyword);
for (auto &keyword : builtins)
builtins_trie.insert(keyword);
for (auto &keyword : methods)
methods_trie.insert(keyword);
for (auto &keyword : errors)
errors_trie.insert(keyword);
}
};
} // namespace bed::internal::syntax::ruby
+31
View File
@@ -0,0 +1,31 @@
#pragma once
#include "definitions.h"
#include "internal/syntax/parser.h"
#include "pch.h"
namespace bed::internal::theme {
struct Highlight {
enum : uint8_t {
None = 0,
Bold = 1 << 0,
Italic = 1 << 1,
Strikethrough = 1 << 2,
Underline = 1 << 3,
};
uint32_t fg;
uint32_t bg;
uint8_t flags;
};
struct Theme {
std::array<Highlight, internal::syntax::Token::Count> hl;
Theme();
Highlight get(internal::syntax::Token token) const;
static Theme default_theme();
static Theme from_name(std::string_view name);
};
} // namespace bed::internal::theme
+9 -9
View File
@@ -157,14 +157,14 @@ struct Trie {
}
}
uint64_t longest_match(std::string_view input) {
Node *current = &root;
uint64_t longest_match(std::string_view input) const {
const Node *current = &root;
uint64_t pos = 0;
uint64_t longest = 0;
if (current->value)
longest = 0;
while (pos < input.size()) {
Node *child = find_child(*current, input[pos]);
const Node *child = find_child(*current, input[pos]);
if (!child)
break;
const auto remaining = input.substr(pos);
@@ -179,7 +179,7 @@ struct Trie {
return longest;
}
bool matches(std::string_view key)
bool matches(std::string_view key) const
requires(std::is_void_v<T>)
{
Node *current = &root;
@@ -198,13 +198,13 @@ struct Trie {
return current->value.has_value();
}
std::optional<V> get(std::string_view key)
std::optional<V> get(std::string_view key) const
requires(!std::is_void_v<T>)
{
Node *current = &root;
const Node *current = &root;
uint64_t pos = 0;
while (pos < key.size()) {
Node *child = find_child(*current, key[pos]);
const Node *child = find_child(*current, key[pos]);
if (!child)
return std::nullopt;
const auto remaining = key.substr(pos);
@@ -228,7 +228,7 @@ struct Trie {
uint64_t common_prefix(
std::string_view a,
std::string_view b
) {
) const {
const auto n = std::min(a.size(), b.size());
uint64_t i = 0;
while (i < n && equal_char(a[i], b[i]))
@@ -236,7 +236,7 @@ struct Trie {
return i;
}
Node *find_child(Node &node, char first) {
Node *find_child(const Node &node, char first) const {
for (auto *child : node.children)
if (equal_char(child->edge.front(), first))
return child;
+2 -3
View File
@@ -13,14 +13,13 @@ struct Shard {
} kind;
uint8_t height;
std::atomic_uint32_t refs;
uint64_t length;
uint64_t lines;
std::atomic_uint64_t refs;
Shard(Kind kind, uint64_t length, uint64_t lines, uint8_t height)
: kind(kind), height(height), length(length), lines(lines), refs(1) {};
: kind(kind), height(height), refs(1), length(length), lines(lines) {};
static void retain(Shard *n);
static void release(Shard *n);