syedm/bed
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update syntax parser system.

Browse Source
This commit is contained in:
syedm
2026-08-20 19:33:45 +01:00
parent 1489b08d57
commit 6ea03d651a
8 changed files with 357 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/internal/syntax/decl.h
+41 -71
View File
@@ -7,7 +7,7 @@ namespace bed::internal::syntax {
struct Token { struct Token {
uint32_t start; uint32_t start;
uint32_t end; uint32_t end;
enum : uint8_t { enum Kind : uint8_t {
Data, Data,
Shebang, Shebang,
Comment, Comment,
@@ -68,9 +68,20 @@ struct Token {
} type; } type;
}; };
struct ParseEvent {
std::string_view name;
enum : uint8_t {
Opening,
Closing,
SymbolDef,
Symbol
} ev_type;
uint8_t type; // type.
};
struct Language { struct Language {
std::function<void *()> none_state; std::function<void *()> none_state;
std::function<void(void **, std::string_view, bool, std::vector<Token> *)> parse; std::function<void(void **, std::string_view, bool, std::vector<Token> *, std::vector<ParseEvent> *)> parse;
std::function<void *(void *)> copy; std::function<void *(void *)> copy;
std::function<bool(void *, void *)> equal; std::function<bool(void *, void *)> equal;
std::function<void(void *)> destroy; std::function<void(void *)> destroy;
@@ -102,81 +113,40 @@ struct TreeCursor {
ParseStateBranch *stack[64]; ParseStateBranch *stack[64];
uint8_t depth = 0; uint8_t depth = 0;
bool went_left[64]; bool went_left[64];
TreeCursor(ParseState *root, uint64_t target_line, uint64_t *relative) { TreeCursor(ParseState *root, uint64_t target_line, uint64_t *relative);
ParseState *node = root; void next();
while (node->is_branch()) { void prev();
auto *branch = (ParseStateBranch *)node; };
auto *left = branch->left;
stack[depth] = branch; struct ScopeNode {
if (target_line < left->lines()) { static constexpr uint64_t SCOPE_BIT = 1ull << 63;
went_left[depth] = true; static constexpr uint64_t LINES_MASK = ~SCOPE_BIT;
++depth; uint64_t header;
node = left; bool is_scope() const {
} else { return header & SCOPE_BIT;
target_line -= left->lines();
went_left[depth] = false;
++depth;
node = branch->right;
} }
} uint64_t lines() const {
*relative = target_line; return header & LINES_MASK;
leaf = (ParseStateLeaf *)node;
}
void next() {
while (depth > 0) {
auto *branch = stack[depth - 1];
bool from_left = went_left[depth - 1];
--depth;
if (!from_left)
continue;
ParseState *node = branch->right;
while (node->is_branch()) {
auto *b = (ParseStateBranch *)node;
stack[depth] = b;
went_left[depth] = true;
++depth;
node = b->left;
}
leaf = (ParseStateLeaf *)node;
return;
}
leaf = nullptr;
}
void prev() {
while (depth > 0) {
auto *branch = stack[depth - 1];
bool from_left = went_left[depth - 1];
--depth;
if (from_left)
continue;
ParseState *node = branch->left;
while (node->is_branch()) {
auto *b = (ParseStateBranch *)node;
stack[depth] = b;
went_left[depth] = false;
++depth;
node = b->right;
}
leaf = (ParseStateLeaf *)node;
return;
}
leaf = nullptr;
} }
}; };
struct Symbol { struct Symbol {
uint64_t definition; uint32_t definition;
std::vector<uint64_t> references; uint16_t reference_count;
uint8_t type;
uint8_t len;
// first chars of count len, padded to 4 bytes.
// then a set of references. 32bit
// references later as modifying at end is faster than shifting the name.
}; };
struct Space { struct alignas(8) Scope : ScopeNode {
uint64_t len; uint16_t children_count;
}; uint16_t symbol_count;
uint8_t type;
struct Scope { uint8_t len;
uint64_t len; uint32_t : 32; // if more stuff is needed use the padding first.
uint32_t type; // first chars of count len, padded to 8 bytes.
trie::Trie<Symbol> symbols; // followed by that many number of pointers.
std::vector<std::variant<Space, Scope>> children;
}; };
} // namespace bed::internal::syntax } // namespace bed::internal::syntax
include/internal/syntax/parser.h
+3 -2
View File
@@ -6,7 +6,7 @@
namespace bed::internal::syntax { namespace bed::internal::syntax {
struct Parser { struct Parser {
static constexpr const uint64_t MAX_CHUNK = 256; static constexpr uint64_t MAX_CHUNK = 256;
ParseState *root; ParseState *root;
Language lang; Language lang;
@@ -25,7 +25,7 @@ struct Parser {
ParseState *join_tree(ParseState *a, ParseState *b); ParseState *join_tree(ParseState *a, ParseState *b);
Scope scope_root; Scope scope_root;
Scope &get_scope(uint64_t); Scope *get_scope(uint64_t);
struct Iterator { struct Iterator {
Parser *p; Parser *p;
@@ -33,6 +33,7 @@ struct Parser {
void *state; void *state;
uint64_t at; uint64_t at;
std::vector<Token> tokens; std::vector<Token> tokens;
std::vector<ParseEvent> events;
Iterator(uint64_t, Parser *, vase::Vase &); Iterator(uint64_t, Parser *, vase::Vase &);
~Iterator(); ~Iterator();
Iterator(const Iterator &) = delete; Iterator(const Iterator &) = delete;
include/internal/syntax/ruby/decl.h
+19 -3
View File
@@ -5,6 +5,15 @@
#include "tries.h" #include "tries.h"
namespace bed::internal::syntax::ruby { namespace bed::internal::syntax::ruby {
enum struct ScopeTypes : uint8_t {
None,
Comment,
Method,
Class,
Module,
Block
};
struct alignas(2) RubyState { struct alignas(2) RubyState {
struct RubyInternalState { struct RubyInternalState {
uint16_t brace_level; uint16_t brace_level;
@@ -17,9 +26,16 @@ struct alignas(2) RubyState {
COMMENT, COMMENT,
END END
} state; } state;
static constexpr const uint8_t EXPECTING_EXPRESSION = 0b00010000; static constexpr uint8_t NAME_MASK = 0b00000011;
static constexpr const uint8_t ALLOW_INTERPOLATION = 0b00000001; enum : uint8_t {
uint8_t flags; NONE_NAME = 0b00,
CLASS_NAME = 0b01,
DEF_NAME = 0b10,
MODULE_NAME = 0b11
};
static constexpr uint8_t ALLOW_INTERPOLATION = 1 << 6;
static constexpr uint8_t EXPECTING_EXPRESSION = 1 << 7;
uint8_t flags = 0;
char delim_start; char delim_start;
char delim_end; char delim_end;
}; };
include/internal/syntax/ruby/parser.h
+4 -1
View File
@@ -99,5 +99,8 @@ struct RubyParser {
} }
}; };
void ruby_parse(void **v_state, std::string_view line, bool first_line, std::vector<Token> *); void ruby_parse(
void **v_state, std::string_view line, bool first_line,
std::vector<Token> *tokens, std::vector<ParseEvent> *events
);
} // namespace bed::internal::syntax::ruby } // namespace bed::internal::syntax::ruby
include/internal/syntax/ruby/tries.h
+22 -7
View File
@@ -209,25 +209,31 @@ const static std::vector<std::string> errors = {
}; };
const static std::vector<std::string> base_keywords = { const static std::vector<std::string> base_keywords = {
"class",
"module",
"begin",
"end",
"else", "else",
"rescue", "rescue",
"ensure", "ensure",
"do",
"when", "when",
}; };
const static std::vector<std::string> expecting_keywords = { const static std::vector<std::string> expecting_keywords = {
"if",
"elsif", "elsif",
};
const static std::vector<std::string> expecting_end_keywords = {
"case", "case",
"for", "for",
};
const static std::vector<std::string> conditional_keywords = {
"if",
"unless",
"while", "while",
"until", "until",
"unless", };
const static std::vector<std::string> end_keywords = {
"begin",
"do",
}; };
const static std::vector<std::string> operator_keywords = { const static std::vector<std::string> operator_keywords = {
@@ -312,6 +318,9 @@ struct RubyTries {
trie::Trie<void> builtins_trie; trie::Trie<void> builtins_trie;
trie::Trie<void> methods_trie; trie::Trie<void> methods_trie;
trie::Trie<void> errors_trie; trie::Trie<void> errors_trie;
trie::Trie<void> expecting_end_keywords_trie;
trie::Trie<void> conditional_keywords_trie;
trie::Trie<void> end_keywords_trie;
RubyTries() { RubyTries() {
for (auto &keyword : base_keywords) for (auto &keyword : base_keywords)
base_keywords_trie.insert(keyword); base_keywords_trie.insert(keyword);
@@ -331,6 +340,12 @@ struct RubyTries {
methods_trie.insert(keyword); methods_trie.insert(keyword);
for (auto &keyword : errors) for (auto &keyword : errors)
errors_trie.insert(keyword); errors_trie.insert(keyword);
for (auto &keyword : expecting_end_keywords)
expecting_end_keywords_trie.insert(keyword);
for (auto &keyword : conditional_keywords)
conditional_keywords_trie.insert(keyword);
for (auto &keyword : end_keywords)
end_keywords_trie.insert(keyword);
} }
}; };
} // namespace bed::internal::syntax::ruby } // namespace bed::internal::syntax::ruby
src/internal/syntax/parser.cc
+6 -4
View File
@@ -61,6 +61,7 @@ void Parser::reset(vase::Vase &vase, uint64_t lines, Language lang_) {
it.next(); it.next();
std::vector<ParseStateLeaf *> leaves; std::vector<ParseStateLeaf *> leaves;
std::vector<Token> tokens; std::vector<Token> tokens;
std::vector<ParseEvent> events;
leaves.reserve((lines + MAX_CHUNK - 1) / MAX_CHUNK); leaves.reserve((lines + MAX_CHUNK - 1) / MAX_CHUNK);
void *state = lang.none_state(); void *state = lang.none_state();
uint32_t consumed = 0; uint32_t consumed = 0;
@@ -71,7 +72,7 @@ void Parser::reset(vase::Vase &vase, uint64_t lines, Language lang_) {
uint32_t chunk_lines = 0; uint32_t chunk_lines = 0;
while (chunk_lines < MAX_CHUNK && consumed < lines) { while (chunk_lines < MAX_CHUNK && consumed < lines) {
tokens.clear(); tokens.clear();
lang.parse(&state, it.line, consumed == 0, &tokens); lang.parse(&state, it.line, consumed == 0, &tokens, &events);
++chunk_lines; ++chunk_lines;
++consumed; ++consumed;
if (!it.next() && consumed < lines) if (!it.next() && consumed < lines)
@@ -163,6 +164,7 @@ void Parser::modify(vase::Vase &vase, uint64_t target, uint64_t count) {
if (count == 0 || !root) if (count == 0 || !root)
return; return;
std::vector<Token> tokens; std::vector<Token> tokens;
std::vector<ParseEvent> events;
uint64_t offset; uint64_t offset;
TreeCursor c = TreeCursor(root, target, &offset); TreeCursor c = TreeCursor(root, target, &offset);
uint64_t at = target - offset; uint64_t at = target - offset;
@@ -203,7 +205,7 @@ void Parser::modify(vase::Vase &vase, uint64_t target, uint64_t count) {
c.leaf->state = lang.copy(state); c.leaf->state = lang.copy(state);
} }
tokens.clear(); tokens.clear();
lang.parse(&state, it.line, at == 0, &tokens); lang.parse(&state, it.line, at == 0, &tokens, &events);
at++; at++;
} }
lang.destroy(state); lang.destroy(state);
@@ -241,7 +243,7 @@ Parser::Iterator::Iterator(uint64_t target, Parser *p, vase::Vase &vase) : p(p)
while (at < target) { while (at < target) {
it->next(); it->next();
tokens.clear(); tokens.clear();
p->lang.parse(&state, it->line, at == 0, &tokens); p->lang.parse(&state, it->line, at == 0, &tokens, &events);
at++; at++;
} }
} }
@@ -275,6 +277,6 @@ Parser::Iterator &Parser::Iterator::operator=(Iterator &&other) {
void Parser::Iterator::next() { void Parser::Iterator::next() {
it->next(); it->next();
tokens.clear(); tokens.clear();
p->lang.parse(&state, it->line, at++ == 0, &tokens); p->lang.parse(&state, it->line, at++ == 0, &tokens, &events);
} }
} // namespace bed::internal::syntax } // namespace bed::internal::syntax
src/internal/syntax/ruby/parse.cc
+196 -67
View File
@@ -220,10 +220,11 @@ void handle_regex(RubyParser &p, std::vector<Token> *tokens) {
tokens->push_back({start, p.len(), Token::Regexp}); tokens->push_back({start, p.len(), Token::Regexp});
} }
bool handle_line_markers(RubyParser &p, std::vector<Token> *tokens) { bool handle_line_markers(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseEvent> *events) {
if (p.len() == 6 && p.peek_str(6) == "=begin") { if (p.len() == 6 && p.peek_str(6) == "=begin") {
p.current().state = RubyState::RubyInternalState::COMMENT; p.current().state = RubyState::RubyInternalState::COMMENT;
p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION; p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
events->push_back({p.peek_str(6), ParseEvent::Opening, (uint8_t)ScopeTypes::Comment});
tokens->push_back({0, p.len(), Token::Comment}); tokens->push_back({0, p.len(), Token::Comment});
return true; return true;
} }
@@ -248,8 +249,98 @@ bool handle_comment(RubyParser &p, std::vector<Token> *tokens, bool first_line)
return false; return false;
} }
void handle_syntax(RubyParser &p, std::vector<Token> *tokens) { void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseEvent> *events) {
static const RubyTries tries = RubyTries(); static const RubyTries tries = RubyTries();
if (p.current().flags & RubyState::RubyInternalState::NAME_MASK) {
switch (p.current().flags & RubyState::RubyInternalState::NAME_MASK) {
case RubyState::RubyInternalState::CLASS_NAME: {
while (p.peek() == ' ' || p.peek() == '\t')
p.advance();
if (p.peek() == '\0')
return;
if (p.peek() == '<' && p.peek(1) == '<') {
p.advance(2);
events->push_back({"singleton", ParseEvent::Opening, (uint8_t)ScopeTypes::Class});
p.current().flags = (p.current().flags & ~RubyState::RubyInternalState::NAME_MASK);
p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
return;
}
uint32_t k = p.i;
uint32_t j = 0;
if (identifier_start_char(p.peek(j))) {
j++;
while (identifier_char(p.peek(j)))
j++;
tokens->push_back({p.i, p.i + j, Token::Constant});
p.advance(j);
while (p.peek() == ' ' || p.peek() == '\t')
p.advance();
if (p.peek() == ':' && p.peek(1) == ':') {
p.advance(2);
return;
}
}
events->push_back({p.line.substr(k, j), ParseEvent::Opening, (uint8_t)ScopeTypes::Class});
p.current().flags = (p.current().flags & ~RubyState::RubyInternalState::NAME_MASK);
return;
}
case RubyState::RubyInternalState::DEF_NAME: {
while (p.peek() == ' ' || p.peek() == '\t')
p.advance();
if (p.peek() == '\0')
return;
uint32_t k = p.i;
uint32_t j = 0;
if (identifier_start_char(p.peek(j))) {
j++;
while (identifier_char(p.peek(j)))
j++;
if (p.peek(j) == '!' || p.peek(j) == '?')
j++;
if ('A' <= p.peek() && p.peek() <= 'Z')
tokens->push_back({p.i, p.i + j, Token::Constant});
else if (j == 4 && p.peek_str(4) == "self")
tokens->push_back({p.i, p.i + j, Token::Keyword});
else
tokens->push_back({p.i, p.i + j, Token::Function});
p.advance(j);
while (p.peek() == ' ' || p.peek() == '\t')
p.advance();
if (p.peek() == '.') {
p.advance();
return;
}
}
events->push_back({p.line.substr(k, j), ParseEvent::Opening, (uint8_t)ScopeTypes::Method});
p.current().flags = (p.current().flags & ~RubyState::RubyInternalState::NAME_MASK);
return;
}
case RubyState::RubyInternalState::MODULE_NAME: {
while (p.peek() == ' ' || p.peek() == '\t')
p.advance();
if (p.peek() == '\0')
return;
uint32_t k = p.i;
uint32_t j = 0;
if (identifier_start_char(p.peek(j))) {
j++;
while (identifier_char(p.peek(j)))
j++;
tokens->push_back({p.i, p.i + j, Token::Constant});
p.advance(j);
while (p.peek() == ' ' || p.peek() == '\t')
p.advance();
if (p.peek() == ':' && p.peek(1) == ':') {
p.advance(2);
return;
}
}
events->push_back({p.line.substr(k, j), ParseEvent::Opening, (uint8_t)ScopeTypes::Module});
p.current().flags = (p.current().flags & ~RubyState::RubyInternalState::NAME_MASK);
return;
}
}
}
if (p.i + 3 <= p.len() && p.peek_str(2) == "<<") { if (p.i + 3 <= p.len() && p.peek_str(2) == "<<") {
uint32_t j = 2; uint32_t j = 2;
bool indented = false; bool indented = false;
@@ -515,9 +606,9 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens) {
} }
case '{': { case '{': {
p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION; p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
/*uint8_t brace_color = uint8_t brace_color =
(uint8_t)Token::K_BRACE1 + (state->full_state.brace_level % 5); (uint8_t)Token::Brace1 + (p.current().brace_level % 5);
tokens->push_back({p.i, p.i + 1, (Token)brace_color});*/ tokens->push_back({p.i, p.i + 1, (Token::Kind)brace_color});
p.current().brace_level++; p.current().brace_level++;
p.advance(); p.advance();
return; return;
@@ -528,18 +619,18 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens) {
p.pop_state(); p.pop_state();
tokens->push_back({p.i, p.i + 1, Token::Interpolation}); tokens->push_back({p.i, p.i + 1, Token::Interpolation});
} else { } else {
/*uint8_t brace_color = uint8_t brace_color =
(uint8_t)Token::K_BRACE1 + (state->full_state.brace_level % 5); (uint8_t)Token::Brace1 + (p.current().brace_level % 5);
tokens->push_back({p.i, p.i + 1, (Token)brace_color});*/ tokens->push_back({p.i, p.i + 1, (Token::Kind)brace_color});
} }
p.advance(); p.advance();
return; return;
} }
case '(': { case '(': {
p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION; p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
/*uint8_t brace_color = uint8_t brace_color =
(uint8_t)Token::K_BRACE1 + (state->full_state.brace_level % 5); (uint8_t)Token::Brace1 + (p.current().brace_level % 5);
tokens->push_back({p.i, p.i + 1, (Token)brace_color});*/ tokens->push_back({p.i, p.i + 1, (Token::Kind)brace_color});
p.current().brace_level++; p.current().brace_level++;
p.advance(); p.advance();
return; return;
@@ -547,17 +638,17 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens) {
case ')': { case ')': {
p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION; p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
p.current().brace_level--; p.current().brace_level--;
/*uint8_t brace_color = uint8_t brace_color =
(uint8_t)Token::K_BRACE1 + (state->full_state.brace_level % 5); (uint8_t)Token::Brace1 + (p.current().brace_level % 5);
tokens->push_back({p.i, p.i + 1, (Token)brace_color});*/ tokens->push_back({p.i, p.i + 1, (Token::Kind)brace_color});
p.advance(); p.advance();
return; return;
} }
case '[': { case '[': {
p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION; p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
/*uint8_t brace_color = uint8_t brace_color =
(uint8_t)Token::K_BRACE1 + (state->full_state.brace_level % 5); (uint8_t)Token::Brace1 + (p.current().brace_level % 5);
tokens->push_back({p.i, p.i + 1, (Token)brace_color});*/ tokens->push_back({p.i, p.i + 1, (Token::Kind)brace_color});
p.current().brace_level++; p.current().brace_level++;
p.advance(); p.advance();
return; return;
@@ -565,9 +656,9 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens) {
case ']': { case ']': {
p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION; p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
p.current().brace_level--; p.current().brace_level--;
/*uint8_t brace_color = uint8_t brace_color =
(uint8_t)Token::K_BRACE1 + (state->full_state.brace_level % 5); (uint8_t)Token::Brace1 + (p.current().brace_level % 5);
tokens->push_back({p.i, p.i + 1, (Token)brace_color});*/ tokens->push_back({p.i, p.i + 1, (Token::Kind)brace_color});
p.advance(); p.advance();
return; return;
} }
@@ -602,7 +693,7 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens) {
return; return;
} }
case '%': { case '%': {
if (p.current().flags & RubyState::RubyInternalState::EXPECTING_EXPRESSION || p.i + 1 >= p.len()) { if (p.i + 1 >= p.len()) {
tokens->push_back({p.i, p.i + 1, Token::Operator}); tokens->push_back({p.i, p.i + 1, Token::Operator});
p.advance(); p.advance();
p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION; p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
@@ -680,11 +771,16 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens) {
p.advance(prefix_len + 1); p.advance(prefix_len + 1);
return; return;
} }
case ';':
p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
p.advance();
return;
default: default:
if ('0' <= p.peek() && p.peek() <= '9') { if ('0' <= p.peek() && p.peek() <= '9') {
p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION; p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
uint32_t start = p.i; uint32_t start = p.i;
if (p.peek() == '0') { if (p.peek() == '0'
&& (p.peek(1) == 'X' || p.peek(1) == 'x' || p.peek(1) == 'b' || p.peek(1) == 'B' || p.peek(1) == 'o' || p.peek(1) == 'O')) {
p.advance(); p.advance();
if (p.peek() == 'x' || p.peek() == 'X') { if (p.peek() == 'x' || p.peek() == 'X') {
p.advance(); p.advance();
@@ -716,15 +812,6 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens) {
else else
break; break;
} }
} else {
while (true) {
while (p.peek() >= '0' && p.peek() <= '7')
p.advance();
if (p.peek() == '_')
p.advance();
else
break;
}
} }
} else { } else {
while (true) { while (true) {
@@ -763,51 +850,83 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens) {
tokens->push_back({start, p.i, Token::Number}); tokens->push_back({start, p.i, Token::Number});
return; return;
} else if (identifier_start_char(p.peek())) { } else if (identifier_start_char(p.peek())) {
p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
uint32_t j = 1; uint32_t j = 1;
while (identifier_char(p.peek(j))) while (identifier_char(p.peek(j)))
j++; j++;
if (p.peek(j) == '!' || p.peek(j) == '?') if (p.peek(j) == '!' || p.peek(j) == '?')
j++; j++;
if (j == tries.base_keywords_trie.longest_match(p.peek_str(p.len() - p.i))) { if (j == tries.base_keywords_trie.longest_match(p.peek_str(j))) {
p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
tokens->push_back({p.i, p.i + j, Token::Keyword}); tokens->push_back({p.i, p.i + j, Token::Keyword});
p.advance(j); p.advance(j);
return; return;
} else if (j == tries.expecting_keywords_trie.longest_match(p.peek_str(p.len() - p.i))) { } else if (j == tries.expecting_keywords_trie.longest_match(p.peek_str(j))) {
p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION; p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
tokens->push_back({p.i, p.i + j, Token::Keyword}); tokens->push_back({p.i, p.i + j, Token::Keyword});
p.advance(j); p.advance(j);
return; return;
} else if (j == tries.operator_keywords_trie.longest_match(p.peek_str(p.len() - p.i))) { } else if (j == tries.operator_keywords_trie.longest_match(p.peek_str(j))) {
p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
tokens->push_back({p.i, p.i + j, Token::KeywordOperator}); tokens->push_back({p.i, p.i + j, Token::KeywordOperator});
p.advance(j); p.advance(j);
return; return;
} else if (j == tries.expecting_operators_trie.longest_match(p.peek_str(p.len() - p.i))) { } else if (j == tries.expecting_operators_trie.longest_match(p.peek_str(j))) {
p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION; p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
tokens->push_back({p.i, p.i + j, Token::KeywordOperator}); tokens->push_back({p.i, p.i + j, Token::KeywordOperator});
p.advance(j); p.advance(j);
return; return;
} else if (j == tries.types_trie.longest_match(p.peek_str(p.len() - p.i))) { } else if (j == tries.types_trie.longest_match(p.peek_str(j))) {
p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
tokens->push_back({p.i, p.i + j, Token::Type}); tokens->push_back({p.i, p.i + j, Token::Type});
p.advance(j); p.advance(j);
return; return;
} else if (j == tries.methods_trie.longest_match(p.peek_str(p.len() - p.i))) { } else if (j == tries.methods_trie.longest_match(p.peek_str(j))) {
p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
tokens->push_back({p.i, p.i + j, Token::Function}); tokens->push_back({p.i, p.i + j, Token::Function});
p.advance(j); p.advance(j);
return; return;
} else if (j == tries.builtins_trie.longest_match(p.peek_str(p.len() - p.i))) { } else if (j == tries.expecting_end_keywords_trie.longest_match(p.peek_str(j))) {
events->push_back({p.peek_str(j), ParseEvent::Opening, 0});
p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
tokens->push_back({p.i, p.i + j, Token::Keyword});
p.advance(j);
return;
} else if (j == tries.conditional_keywords_trie.longest_match(p.peek_str(j))) {
if (p.current().flags & RubyState::RubyInternalState::EXPECTING_EXPRESSION)
events->push_back({p.peek_str(j), ParseEvent::Opening, 0});
p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
tokens->push_back({p.i, p.i + j, Token::Keyword});
p.advance(j);
return;
} else if (j == tries.end_keywords_trie.longest_match(p.peek_str(j))) {
p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
ScopeTypes kind = p.peek() == 'd' ? ScopeTypes::Block : ScopeTypes::None;
events->push_back({p.peek_str(j), ParseEvent::Opening, (uint8_t)kind});
tokens->push_back({p.i, p.i + j, Token::Keyword});
p.advance(j);
return;
} else if (j == tries.builtins_trie.longest_match(p.peek_str(j))) {
p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
tokens->push_back({p.i, p.i + j, Token::Constant}); tokens->push_back({p.i, p.i + j, Token::Constant});
p.advance(j); p.advance(j);
return; return;
} else if (j == tries.errors_trie.longest_match(p.peek_str(p.len() - p.i))) { } else if (j == tries.errors_trie.longest_match(p.peek_str(j))) {
p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
tokens->push_back({p.i, p.i + j, Token::Error}); tokens->push_back({p.i, p.i + j, Token::Error});
p.advance(j); p.advance(j);
return; return;
} else if ('A' <= p.peek() && p.peek() <= 'Z' && !(p.peek(j) == '!' || p.peek(j) == '?')) { } else if ('A' <= p.peek() && p.peek() <= 'Z' && !(p.peek(j) == '!' || p.peek(j) == '?')) {
p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
if (j >= 5 && p.peek_str(j).substr(j - 5) == "Error") {
tokens->push_back({p.i, p.i + j, Token::Error});
p.advance(j);
return;
}
tokens->push_back({p.i, p.i + j, Token::Constant}); tokens->push_back({p.i, p.i + j, Token::Constant});
p.advance(j); p.advance(j);
return; return;
} else { } else {
p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
if (j == 4 && p.peek_str(4) == "true") { if (j == 4 && p.peek_str(4) == "true") {
tokens->push_back({p.i, p.i + j, Token::True}); tokens->push_back({p.i, p.i + j, Token::True});
p.advance(4); p.advance(4);
@@ -818,32 +937,34 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens) {
p.advance(5); p.advance(5);
return; return;
} }
if (j == 3 && p.peek_str(3) == "end") {
events->push_back({p.peek_str(3), ParseEvent::Closing, 0});
tokens->push_back({p.i, p.i + j, Token::Keyword});
p.advance(3);
return;
}
if (j == 5 && p.peek_str(5) == "class") {
tokens->push_back({p.i, p.i + j, Token::Keyword});
p.advance(5);
p.current().flags =
(p.current().flags & ~RubyState::RubyInternalState::NAME_MASK)
| RubyState::RubyInternalState::CLASS_NAME;
return;
}
if (j == 6 && p.peek_str(6) == "module") {
tokens->push_back({p.i, p.i + j, Token::Keyword});
p.advance(6);
p.current().flags =
(p.current().flags & ~RubyState::RubyInternalState::NAME_MASK)
| RubyState::RubyInternalState::MODULE_NAME;
return;
}
if (j == 3 && p.peek_str(3) == "def") { if (j == 3 && p.peek_str(3) == "def") {
tokens->push_back({p.i, p.i + j, Token::Keyword}); tokens->push_back({p.i, p.i + j, Token::Keyword});
p.advance(3); p.advance(3);
while (p.peek() == ' ' || p.peek() == '\t') p.current().flags =
p.advance(); (p.current().flags & ~RubyState::RubyInternalState::NAME_MASK)
while (p.i < p.len()) { | RubyState::RubyInternalState::DEF_NAME;
if (identifier_start_char(p.peek())) {
uint32_t j = 1;
while (identifier_char(p.peek(j)))
j++;
if (p.peek(j) == '!' || p.peek(j) == '?')
j++;
if ('A' <= p.peek() && p.peek() <= 'Z')
tokens->push_back({p.i, p.i + j, Token::Constant});
else if (j == 4 && p.peek_str(4) == "self")
tokens->push_back({p.i, p.i + j, Token::Keyword});
else
tokens->push_back({p.i, p.i + j, Token::Function});
p.advance(j);
if (p.peek() == '.') {
p.advance();
continue;
}
}
break;
}
return; return;
} }
uint32_t start = p.i; uint32_t start = p.i;
@@ -913,15 +1034,23 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens) {
} }
} }
void ruby_parse(void **v_state, std::string_view line, bool fl, std::vector<Token> *tokens) { void ruby_parse(
void **v_state,
std::string_view line,
bool fl,
std::vector<Token> *tokens,
std::vector<ParseEvent> *events
) {
RubyParser p(v_state, line); RubyParser p(v_state, line);
while (p.i < p.len()) { while (p.i < p.len()) {
if (p.current().state == RubyState::RubyInternalState::END) if (p.current().state == RubyState::RubyInternalState::END)
return; return;
if (p.current().state == RubyState::RubyInternalState::COMMENT) { if (p.current().state == RubyState::RubyInternalState::COMMENT) {
tokens->push_back({p.i, p.len(), Token::Comment}); tokens->push_back({p.i, p.len(), Token::Comment});
if (p.i == 0 && p.peek_str(4) == "=end") if (p.i == 0 && p.peek_str(4) == "=end") {
p.current().state = RubyState::RubyInternalState::NONE; p.current().state = RubyState::RubyInternalState::NONE;
events->push_back({{nullptr, 0}, ParseEvent::Closing, 0});
}
return; return;
} }
if (!p.heredoc_start_line if (!p.heredoc_start_line
@@ -939,11 +1068,11 @@ void ruby_parse(void **v_state, std::string_view line, bool fl, std::vector<Toke
handle_regex(p, tokens); handle_regex(p, tokens);
continue; continue;
} }
if (!p.i && handle_line_markers(p, tokens)) if (!p.i && handle_line_markers(p, tokens, events))
return; return;
if (handle_comment(p, tokens, fl)) if (handle_comment(p, tokens, fl))
return; return;
handle_syntax(p, tokens); handle_syntax(p, tokens, events);
continue; continue;
} }
} }
src/internal/syntax/tree_cursor.cc
+66
View File
@@ -0,0 +1,66 @@
#include "internal/syntax/decl.h"
namespace bed::internal::syntax {
TreeCursor::TreeCursor(ParseState *root, uint64_t target_line, uint64_t *relative) {
ParseState *node = root;
while (node->is_branch()) {
auto *branch = (ParseStateBranch *)node;
auto *left = branch->left;
stack[depth] = branch;
if (target_line < left->lines()) {
went_left[depth] = true;
++depth;
node = left;
} else {
target_line -= left->lines();
went_left[depth] = false;
++depth;
node = branch->right;
}
}
*relative = target_line;
leaf = (ParseStateLeaf *)node;
}
void TreeCursor::next() {
while (depth > 0) {
auto *branch = stack[depth - 1];
bool from_left = went_left[depth - 1];
--depth;
if (!from_left)
continue;
ParseState *node = branch->right;
while (node->is_branch()) {
auto *b = (ParseStateBranch *)node;
stack[depth] = b;
went_left[depth] = true;
++depth;
node = b->left;
}
leaf = (ParseStateLeaf *)node;
return;
}
leaf = nullptr;
}
void TreeCursor::prev() {
while (depth > 0) {
auto *branch = stack[depth - 1];
bool from_left = went_left[depth - 1];
--depth;
if (from_left)
continue;
ParseState *node = branch->left;
while (node->is_branch()) {
auto *b = (ParseStateBranch *)node;
stack[depth] = b;
went_left[depth] = false;
++depth;
node = b->right;
}
leaf = (ParseStateLeaf *)node;
return;
}
leaf = nullptr;
}
} // namespace bed::internal::syntax