syedm/bed
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update syntax parser system.

Browse Source
This commit is contained in:
syedm
2026-08-20 19:33:45 +01:00
parent 1489b08d57
commit 6ea03d651a
8 changed files with 357 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/internal/syntax/decl.h
+41 -71
View File
@@ -7,7 +7,7 @@ namespace bed::internal::syntax {
struct Token {
uint32_t start;
uint32_t end;
enum : uint8_t {
enum Kind : uint8_t {
Data,
Shebang,
Comment,
@@ -68,9 +68,20 @@ struct Token {
} type;
};
struct ParseEvent {
std::string_view name;
enum : uint8_t {
Opening,
Closing,
SymbolDef,
Symbol
} ev_type;
uint8_t type; // type.
};
struct Language {
std::function<void *()> none_state;
std::function<void(void **, std::string_view, bool, std::vector<Token> *)> parse;
std::function<void(void **, std::string_view, bool, std::vector<Token> *, std::vector<ParseEvent> *)> parse;
std::function<void *(void *)> copy;
std::function<bool(void *, void *)> equal;
std::function<void(void *)> destroy;
@@ -102,81 +113,40 @@ struct TreeCursor {
ParseStateBranch *stack[64];
uint8_t depth = 0;
bool went_left[64];
TreeCursor(ParseState *root, uint64_t target_line, uint64_t *relative) {
ParseState *node = root;
while (node->is_branch()) {
auto *branch = (ParseStateBranch *)node;
auto *left = branch->left;
stack[depth] = branch;
if (target_line < left->lines()) {
went_left[depth] = true;
++depth;
node = left;
} else {
target_line -= left->lines();
went_left[depth] = false;
++depth;
node = branch->right;
}
}
*relative = target_line;
leaf = (ParseStateLeaf *)node;
TreeCursor(ParseState *root, uint64_t target_line, uint64_t *relative);
void next();
void prev();
};
struct ScopeNode {
static constexpr uint64_t SCOPE_BIT = 1ull << 63;
static constexpr uint64_t LINES_MASK = ~SCOPE_BIT;
uint64_t header;
bool is_scope() const {
return header & SCOPE_BIT;
}
void next() {
while (depth > 0) {
auto *branch = stack[depth - 1];
bool from_left = went_left[depth - 1];
--depth;
if (!from_left)
continue;
ParseState *node = branch->right;
while (node->is_branch()) {
auto *b = (ParseStateBranch *)node;
stack[depth] = b;
went_left[depth] = true;
++depth;
node = b->left;
}
leaf = (ParseStateLeaf *)node;
return;
}
leaf = nullptr;
}
void prev() {
while (depth > 0) {
auto *branch = stack[depth - 1];
bool from_left = went_left[depth - 1];
--depth;
if (from_left)
continue;
ParseState *node = branch->left;
while (node->is_branch()) {
auto *b = (ParseStateBranch *)node;
stack[depth] = b;
went_left[depth] = false;
++depth;
node = b->right;
}
leaf = (ParseStateLeaf *)node;
return;
}
leaf = nullptr;
uint64_t lines() const {
return header & LINES_MASK;
}
};
struct Symbol {
uint64_t definition;
std::vector<uint64_t> references;
uint32_t definition;
uint16_t reference_count;
uint8_t type;
uint8_t len;
// first chars of count len, padded to 4 bytes.
// then a set of references. 32bit
// references later as modifying at end is faster than shifting the name.
};
struct Space {
uint64_t len;
};
struct Scope {
uint64_t len;
uint32_t type;
trie::Trie<Symbol> symbols;
std::vector<std::variant<Space, Scope>> children;
struct alignas(8) Scope : ScopeNode {
uint16_t children_count;
uint16_t symbol_count;
uint8_t type;
uint8_t len;
uint32_t : 32; // if more stuff is needed use the padding first.
// first chars of count len, padded to 8 bytes.
// followed by that many number of pointers.
};
} // namespace bed::internal::syntax
include/internal/syntax/parser.h
+3 -2
View File
@@ -6,7 +6,7 @@
namespace bed::internal::syntax {
struct Parser {
static constexpr const uint64_t MAX_CHUNK = 256;
static constexpr uint64_t MAX_CHUNK = 256;
ParseState *root;
Language lang;
@@ -25,7 +25,7 @@ struct Parser {
ParseState *join_tree(ParseState *a, ParseState *b);
Scope scope_root;
Scope &get_scope(uint64_t);
Scope *get_scope(uint64_t);
struct Iterator {
Parser *p;
@@ -33,6 +33,7 @@ struct Parser {
void *state;
uint64_t at;
std::vector<Token> tokens;
std::vector<ParseEvent> events;
Iterator(uint64_t, Parser *, vase::Vase &);
~Iterator();
Iterator(const Iterator &) = delete;
include/internal/syntax/ruby/decl.h
+19 -3
View File
@@ -5,6 +5,15 @@
#include "tries.h"
namespace bed::internal::syntax::ruby {
enum struct ScopeTypes : uint8_t {
None,
Comment,
Method,
Class,
Module,
Block
};
struct alignas(2) RubyState {
struct RubyInternalState {
uint16_t brace_level;
@@ -17,9 +26,16 @@ struct alignas(2) RubyState {
COMMENT,
END
} state;
static constexpr const uint8_t EXPECTING_EXPRESSION = 0b00010000;
static constexpr const uint8_t ALLOW_INTERPOLATION = 0b00000001;
uint8_t flags;
static constexpr uint8_t NAME_MASK = 0b00000011;
enum : uint8_t {
NONE_NAME = 0b00,
CLASS_NAME = 0b01,
DEF_NAME = 0b10,
MODULE_NAME = 0b11
};
static constexpr uint8_t ALLOW_INTERPOLATION = 1 << 6;
static constexpr uint8_t EXPECTING_EXPRESSION = 1 << 7;
uint8_t flags = 0;
char delim_start;
char delim_end;
};
include/internal/syntax/ruby/parser.h
+4 -1
View File
@@ -99,5 +99,8 @@ struct RubyParser {
}
};
void ruby_parse(void **v_state, std::string_view line, bool first_line, std::vector<Token> *);
void ruby_parse(
void **v_state, std::string_view line, bool first_line,
std::vector<Token> *tokens, std::vector<ParseEvent> *events
);
} // namespace bed::internal::syntax::ruby
include/internal/syntax/ruby/tries.h
+22 -7
View File
@@ -209,25 +209,31 @@ const static std::vector<std::string> errors = {
};
const static std::vector<std::string> base_keywords = {
"class",
"module",
"begin",
"end",
"else",
"rescue",
"ensure",
"do",
"when",
};
const static std::vector<std::string> expecting_keywords = {
"if",
"elsif",
};
const static std::vector<std::string> expecting_end_keywords = {
"case",
"for",
};
const static std::vector<std::string> conditional_keywords = {
"if",
"unless",
"while",
"until",
"unless",
};
const static std::vector<std::string> end_keywords = {
"begin",
"do",
};
const static std::vector<std::string> operator_keywords = {
@@ -312,6 +318,9 @@ struct RubyTries {
trie::Trie<void> builtins_trie;
trie::Trie<void> methods_trie;
trie::Trie<void> errors_trie;
trie::Trie<void> expecting_end_keywords_trie;
trie::Trie<void> conditional_keywords_trie;
trie::Trie<void> end_keywords_trie;
RubyTries() {
for (auto &keyword : base_keywords)
base_keywords_trie.insert(keyword);
@@ -331,6 +340,12 @@ struct RubyTries {
methods_trie.insert(keyword);
for (auto &keyword : errors)
errors_trie.insert(keyword);
for (auto &keyword : expecting_end_keywords)
expecting_end_keywords_trie.insert(keyword);
for (auto &keyword : conditional_keywords)
conditional_keywords_trie.insert(keyword);
for (auto &keyword : end_keywords)
end_keywords_trie.insert(keyword);
}
};
} // namespace bed::internal::syntax::ruby