syedm/bed
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve ruby parsing heuristics.

Browse Source
This commit is contained in:
syedm
2026-08-21 12:28:16 +01:00
parent 6ea03d651a
commit 739d2e450e
6 changed files with 82 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/internal/syntax/decl.h
+20 -33
View File
@@ -70,13 +70,16 @@ struct Token {
struct ParseEvent {
std::string_view name;
enum : uint8_t {
enum T : uint8_t {
Opening,
Closing,
SymbolDef,
Symbol
} ev_type;
uint8_t type; // type.
ParseEvent(T t) : ev_type(t) {}
ParseEvent(std::string_view s, T e_t, uint8_t type) : name(s), ev_type(e_t), type(type) {}
};
struct Language {
@@ -87,6 +90,16 @@ struct Language {
std::function<void(void *)> destroy;
};
struct Symbol {
uint32_t definition;
uint16_t reference_count;
uint8_t type;
uint8_t len;
// first chars of count len, padded to 4 bytes.
// then a set of references. 32bit
// references later as modifying at end is faster than shifting the name.
};
struct ParseState {
static constexpr uint64_t BRANCH_BIT = 1ull << 63;
static constexpr uint64_t LINES_MASK = ~BRANCH_BIT;
@@ -106,6 +119,12 @@ struct ParseStateBranch : ParseState {
struct ParseStateLeaf : ParseState {
void *state;
uint64_t n;
static constexpr uint32_t IS_OPENING = 1ull << 31;
static constexpr uint32_t LINE_MASK = ~IS_OPENING;
// followed by n number of relative offsets
// stored as uint32_t with 1 bit for if it is start or end
// and rest as number.
};
struct TreeCursor {
@@ -117,36 +136,4 @@ struct TreeCursor {
void next();
void prev();
};
struct ScopeNode {
static constexpr uint64_t SCOPE_BIT = 1ull << 63;
static constexpr uint64_t LINES_MASK = ~SCOPE_BIT;
uint64_t header;
bool is_scope() const {
return header & SCOPE_BIT;
}
uint64_t lines() const {
return header & LINES_MASK;
}
};
struct Symbol {
uint32_t definition;
uint16_t reference_count;
uint8_t type;
uint8_t len;
// first chars of count len, padded to 4 bytes.
// then a set of references. 32bit
// references later as modifying at end is faster than shifting the name.
};
struct alignas(8) Scope : ScopeNode {
uint16_t children_count;
uint16_t symbol_count;
uint8_t type;
uint8_t len;
uint32_t : 32; // if more stuff is needed use the padding first.
// first chars of count len, padded to 8 bytes.
// followed by that many number of pointers.
};
} // namespace bed::internal::syntax
include/internal/syntax/parser.h
-3
View File
@@ -24,9 +24,6 @@ struct Parser {
std::pair<ParseState *, ParseState *> split_tree(ParseState *node, uint64_t line);
ParseState *join_tree(ParseState *a, ParseState *b);
Scope scope_root;
Scope *get_scope(uint64_t);
struct Iterator {
Parser *p;
std::optional<vase::Iterator> it;
include/internal/syntax/ruby/parser.h
+3 -2
View File
@@ -10,6 +10,7 @@ struct RubyParser {
std::string_view line;
uint32_t i = 0;
bool heredoc_start_line = false;
bool ending = true;
RubyParser(void **v_state, std::string_view line)
: v_state(v_state), state((RubyState *)*v_state), line(line) {}
@@ -22,7 +23,7 @@ struct RubyParser {
return state->stack()[state->top - 1];
}
char peek(uint32_t offset = 0) {
char peek(int32_t offset = 0) {
uint32_t pos = i + offset;
return pos < line.size() ? line[pos] : '\0';
}
@@ -57,7 +58,7 @@ struct RubyParser {
.brace_level = 1,
.lit_brace_level = 0,
.state = RubyState::RubyInternalState::NONE,
.flags = 0,
.flags = RubyState::RubyInternalState::EXPECTING_EXPRESSION,
.delim_start = '\0',
.delim_end = '\0'
};
include/internal/syntax/ruby/tries.h
+2 -2
View File
@@ -212,10 +212,10 @@ const static std::vector<std::string> base_keywords = {
"else",
"rescue",
"ensure",
"when",
};
const static std::vector<std::string> expecting_keywords = {
"when",
"elsif",
};
@@ -247,13 +247,13 @@ const static std::vector<std::string> operator_keywords = {
"redo",
"rescue",
"retry",
"super",
"self",
"nil",
"undef",
};
const static std::vector<std::string> expecting_operators = {
"super",
"and",
"return",
"not",
src/internal/syntax/ruby/parse.cc
+54 -41
View File
@@ -115,6 +115,7 @@ bool handle_heredoc(RubyParser &p, std::vector<Token> *tokens) {
if (!p.dequeue_doc(heredoc_len))
p.current().state = RubyState::RubyInternalState::NONE;
tokens->push_back({p.i, p.len(), Token::Annotation});
p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
return true;
}
}
@@ -130,13 +131,13 @@ bool handle_heredoc(RubyParser &p, std::vector<Token> *tokens) {
tokens->push_back({p.i, p.i + 2, Token::Interpolation});
p.advance(2);
p.push_state();
break;
return false;
}
p.advance();
}
if (p.i >= p.len())
tokens->push_back({start, p.len(), Token::String});
return false;
return true;
}
}
@@ -151,7 +152,7 @@ void handle_string(RubyParser &p, std::vector<Token> *tokens) {
tokens->push_back({p.i, p.i + 2, Token::Interpolation});
p.advance(2);
p.push_state();
break;
return;
}
if (p.peek() == p.current().delim_start
&& p.current().delim_start != p.current().delim_end)
@@ -191,7 +192,7 @@ void handle_regex(RubyParser &p, std::vector<Token> *tokens) {
tokens->push_back({p.i, p.i + 2, Token::Interpolation});
p.advance(2);
p.push_state();
break;
return;
}
if (p.peek() == p.current().delim_start
&& p.current().delim_start != p.current().delim_end)
@@ -224,7 +225,7 @@ bool handle_line_markers(RubyParser &p, std::vector<Token> *tokens, std::vector<
if (p.len() == 6 && p.peek_str(6) == "=begin") {
p.current().state = RubyState::RubyInternalState::COMMENT;
p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
events->push_back({p.peek_str(6), ParseEvent::Opening, (uint8_t)ScopeTypes::Comment});
events->push_back(ParseEvent::Opening);
tokens->push_back({0, p.len(), Token::Comment});
return true;
}
@@ -251,21 +252,22 @@ bool handle_comment(RubyParser &p, std::vector<Token> *tokens, bool first_line)
void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseEvent> *events) {
static const RubyTries tries = RubyTries();
if (p.current().flags & RubyState::RubyInternalState::NAME_MASK) {
switch (p.current().flags & RubyState::RubyInternalState::NAME_MASK) {
case RubyState::RubyInternalState::CLASS_NAME: {
while (p.peek() == ' ' || p.peek() == '\t')
p.advance();
if (p.current().flags & RubyState::RubyInternalState::NAME_MASK) {
if (p.peek() == '\0')
return;
if (p.peek() == '\\' && p.peek(1) == '\0')
return;
switch (p.current().flags & RubyState::RubyInternalState::NAME_MASK) {
case RubyState::RubyInternalState::CLASS_NAME: {
if (p.peek() == '<' && p.peek(1) == '<') {
p.advance(2);
events->push_back({"singleton", ParseEvent::Opening, (uint8_t)ScopeTypes::Class});
events->push_back(ParseEvent::Opening);
p.current().flags = (p.current().flags & ~RubyState::RubyInternalState::NAME_MASK);
p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
return;
}
uint32_t k = p.i;
uint32_t j = 0;
if (identifier_start_char(p.peek(j))) {
j++;
@@ -280,16 +282,11 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
return;
}
}
events->push_back({p.line.substr(k, j), ParseEvent::Opening, (uint8_t)ScopeTypes::Class});
events->push_back(ParseEvent::Opening);
p.current().flags = (p.current().flags & ~RubyState::RubyInternalState::NAME_MASK);
return;
}
case RubyState::RubyInternalState::DEF_NAME: {
while (p.peek() == ' ' || p.peek() == '\t')
p.advance();
if (p.peek() == '\0')
return;
uint32_t k = p.i;
uint32_t j = 0;
if (identifier_start_char(p.peek(j))) {
j++;
@@ -311,16 +308,11 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
return;
}
}
events->push_back({p.line.substr(k, j), ParseEvent::Opening, (uint8_t)ScopeTypes::Method});
events->push_back(ParseEvent::Opening);
p.current().flags = (p.current().flags & ~RubyState::RubyInternalState::NAME_MASK);
return;
}
case RubyState::RubyInternalState::MODULE_NAME: {
while (p.peek() == ' ' || p.peek() == '\t')
p.advance();
if (p.peek() == '\0')
return;
uint32_t k = p.i;
uint32_t j = 0;
if (identifier_start_char(p.peek(j))) {
j++;
@@ -335,7 +327,7 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
return;
}
}
events->push_back({p.line.substr(k, j), ParseEvent::Opening, (uint8_t)ScopeTypes::Module});
events->push_back(ParseEvent::Opening);
p.current().flags = (p.current().flags & ~RubyState::RubyInternalState::NAME_MASK);
return;
}
@@ -349,7 +341,7 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
if (p.peek(j) == '~' || p.peek(j) == '-')
j++;
tokens->push_back({p.i, p.i + j, Token::Operator});
if (j >= p.len())
if (p.i + j >= p.len())
return;
std::string delim;
bool interpolation = true;
@@ -358,15 +350,17 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
char q = p.peek(j++);
if (q == '\'')
interpolation = false;
while (j < p.len() && p.peek(j) != q)
while (j + p.i < p.len() && p.peek(j) != q)
delim += p.peek(j++);
} else {
if (j < p.len() && identifier_start_char(p.peek(j))) {
if (j + p.i < p.len() && identifier_start_char(p.peek(j))) {
delim += p.peek(j++);
while (j < p.len() && identifier_char(p.peek(j)))
while (j + p.i < p.len() && identifier_char(p.peek(j)))
delim += p.peek(j++);
}
}
if (p.peek() == '\0')
return;
p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
if (!delim.empty()) {
tokens->push_back({s, p.i + j, Token::Annotation});
@@ -762,7 +756,9 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
break;
}
tokens->push_back({p.i, p.i + prefix_len + 1, (is_regexp ? Token::Regexp : Token::String)});
p.current().state = is_regexp ? RubyState::RubyInternalState::REGEXP : RubyState::RubyInternalState::STRING;
p.current().state = is_regexp
? RubyState::RubyInternalState::REGEXP
: RubyState::RubyInternalState::STRING;
p.current().delim_start = delim_start;
p.current().delim_end = delim_end;
if (allow_interp)
@@ -775,12 +771,21 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
p.advance();
return;
case '\\':
if (p.peek(1) == '\0')
p.ending = false;
p.advance();
return;
case '\0':
if (p.ending)
p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
p.advance();
return;
default:
if ('0' <= p.peek() && p.peek() <= '9') {
p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
uint32_t start = p.i;
if (p.peek() == '0'
&& (p.peek(1) == 'X' || p.peek(1) == 'x' || p.peek(1) == 'b' || p.peek(1) == 'B' || p.peek(1) == 'o' || p.peek(1) == 'O')) {
if (p.peek() == '0') {
p.advance();
if (p.peek() == 'x' || p.peek() == 'X') {
p.advance();
@@ -802,8 +807,11 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
else
break;
}
} else if (p.peek() == 'o' || p.peek() == 'O') {
} else {
if (p.peek() == 'o' || p.peek() == 'O')
p.advance();
else if (p.peek() == '.')
goto decimal;
while (true) {
while (p.peek() >= '0' && p.peek() <= '7')
p.advance();
@@ -814,6 +822,7 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
}
}
} else {
decimal:
while (true) {
while (p.peek() >= '0' && p.peek() <= '9')
p.advance();
@@ -876,7 +885,7 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
p.advance(j);
return;
} else if (j == tries.types_trie.longest_match(p.peek_str(j))) {
p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
tokens->push_back({p.i, p.i + j, Token::Type});
p.advance(j);
return;
@@ -886,22 +895,21 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
p.advance(j);
return;
} else if (j == tries.expecting_end_keywords_trie.longest_match(p.peek_str(j))) {
events->push_back({p.peek_str(j), ParseEvent::Opening, 0});
events->push_back(ParseEvent::Opening);
p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
tokens->push_back({p.i, p.i + j, Token::Keyword});
p.advance(j);
return;
} else if (j == tries.conditional_keywords_trie.longest_match(p.peek_str(j))) {
if (p.current().flags & RubyState::RubyInternalState::EXPECTING_EXPRESSION)
events->push_back({p.peek_str(j), ParseEvent::Opening, 0});
events->push_back(ParseEvent::Opening);
p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
tokens->push_back({p.i, p.i + j, Token::Keyword});
p.advance(j);
return;
} else if (j == tries.end_keywords_trie.longest_match(p.peek_str(j))) {
p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION;
ScopeTypes kind = p.peek() == 'd' ? ScopeTypes::Block : ScopeTypes::None;
events->push_back({p.peek_str(j), ParseEvent::Opening, (uint8_t)kind});
p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION;
events->push_back(ParseEvent::Opening);
tokens->push_back({p.i, p.i + j, Token::Keyword});
p.advance(j);
return;
@@ -938,7 +946,7 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
return;
}
if (j == 3 && p.peek_str(3) == "end") {
events->push_back({p.peek_str(3), ParseEvent::Closing, 0});
events->push_back(ParseEvent::Closing);
tokens->push_back({p.i, p.i + j, Token::Keyword});
p.advance(3);
return;
@@ -973,7 +981,7 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
p.advance();
tokens->push_back({start, p.i, Token::Label});
return;
} else if (p.peek() == '!' || p.peek() == '?') {
} else if (p.peek(-1) == '!' || p.peek(-1) == '?') {
p.advance();
tokens->push_back({start, p.i, Token::Function});
return;
@@ -1019,6 +1027,7 @@ void handle_syntax(RubyParser &p, std::vector<Token> *tokens, std::vector<ParseE
return;
}
tokens->push_back({start, p.i, Token::Function});
return;
}
}
} else {
@@ -1042,14 +1051,14 @@ void ruby_parse(
std::vector<ParseEvent> *events
) {
RubyParser p(v_state, line);
while (p.i < p.len()) {
while (p.i <= p.len()) {
if (p.current().state == RubyState::RubyInternalState::END)
return;
if (p.current().state == RubyState::RubyInternalState::COMMENT) {
tokens->push_back({p.i, p.len(), Token::Comment});
if (p.i == 0 && p.peek_str(4) == "=end") {
p.current().state = RubyState::RubyInternalState::NONE;
events->push_back({{nullptr, 0}, ParseEvent::Closing, 0});
events->push_back(ParseEvent::Closing);
}
return;
}
@@ -1062,10 +1071,14 @@ void ruby_parse(
}
if (p.current().state == RubyState::RubyInternalState::STRING) {
handle_string(p, tokens);
if (p.current().state == RubyState::RubyInternalState::STRING)
return;
continue;
}
if (p.current().state == RubyState::RubyInternalState::REGEXP) {
handle_regex(p, tokens);
if (p.current().state == RubyState::RubyInternalState::REGEXP)
return;
continue;
}
if (!p.i && handle_line_markers(p, tokens, events))
src/internal/syntax/ruby/ruby.cc
+1 -1
View File
@@ -14,7 +14,7 @@ Language lang_ruby() {
.brace_level = 1,
.lit_brace_level = 0,
.state = RubyState::RubyInternalState::NONE,
.flags = 0,
.flags = RubyState::RubyInternalState::EXPECTING_EXPRESSION,
.delim_start = '\0',
.delim_end = '\0'
};