syedm/crib-old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve highlighting structure

Browse Source 
- switched to a sparse delta based map
- true lazy-loading to avoid any unneccessary allocations
- fixed windows management api
This commit is contained in:
Syed Daanish
2026-02-11 18:18:28 +00:00
parent d79ea4e75a
commit 5b66f503e4
24 changed files with 481 additions and 549 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/syntax/bash.cc
+17 -11
View File
@@ -10,7 +10,7 @@ struct BashFullState {
bool line_cont = false;
struct Lit {
std::string delim = ""; // Only 1 wide for strings
std::string delim = "";
bool allow_interp = false;
bool operator==(const BashFullState::Lit &other) const {
@@ -24,7 +24,7 @@ struct BashFullState {
}
};
struct BashState {
struct BashState : StateBase {
using full_state_type = BashFullState;
int interp_level = 0;
@@ -37,26 +37,32 @@ struct BashState {
((full_state && other.full_state &&
*full_state == *other.full_state));
}
std::unique_ptr<StateBase> clone() const override {
return std::make_unique<BashState>(*this);
}
};
bool bash_state_match(std::shared_ptr<void> state_1,
std::shared_ptr<void> state_2) {
bool bash_state_match(StateBase *state_1, StateBase *state_2) {
if (!state_1 || !state_2)
return false;
return *std::static_pointer_cast<BashState>(state_1) ==
*std::static_pointer_cast<BashState>(state_2);
return *static_cast<BashState *>(state_1) ==
*static_cast<BashState *>(state_2);
}
std::shared_ptr<void> bash_parse(std::vector<Token> *tokens,
std::shared_ptr<void> in_state,
const char *text, uint32_t len,
uint32_t line_num) {
std::unique_ptr<StateBase> bash_parse(std::vector<Token> *tokens,
StateBase *in_state, const char *text,
uint32_t len, uint32_t line_num) {
static bool keywords_trie_init = false;
if (!keywords_trie_init) {
keywords_trie_init = true;
}
tokens->clear();
auto state = ensure_state(std::static_pointer_cast<BashState>(in_state));
std::unique_ptr<BashState> state;
if (in_state)
state = static_unique_ptr_cast<BashState>(in_state->clone());
else
state = std::make_unique<BashState>();
uint32_t i = 0;
while (len > 0 && (text[len - 1] == '\n' || text[len - 1] == '\r' ||
text[len - 1] == '\t' || text[len - 1] == ' '))
src/syntax/parser.cc
+52 -59
View File
@@ -26,21 +26,19 @@ Parser::Parser(Editor *n_editor, std::string n_lang, uint32_t n_scroll_max) {
assert("unknown lang should be checked by caller" && 0);
}
}
edit(0, 0, editor->root->line_count + 1);
line_map.apply_edit(0, editor->root->line_count + 1);
}
void Parser::edit(uint32_t start_line, uint32_t removed_rows,
uint32_t inserted_rows) {
int64_t delta = (int64_t)inserted_rows - (int64_t)removed_rows;
if (delta < 0)
line_tree.erase(start_line, (uint32_t)(-delta));
else if (delta > 0)
line_tree.insert(start_line, (uint32_t)delta);
line_map.apply_edit(start_line, delta);
uint32_t span = MAX(removed_rows, inserted_rows);
uint32_t begin = (start_line > 0) ? start_line - 1 : 0;
uint32_t end = start_line + span;
for (uint32_t line = begin; line <= end + 1; ++line)
dirty_lines.push(line);
if (LineData *ld = line_map.at(line))
ld->out_state = nullptr;
}
void Parser::work() {
@@ -48,32 +46,30 @@ void Parser::work() {
return;
std::vector<uint32_t> batch;
uint32_t c_line;
while (dirty_lines.pop(c_line))
batch.push_back(c_line);
uint32_t i = MAX(0, (int64_t)scroll_max - 60);
LineData *l_iter = line_tree.start_iter(i);
while (l_iter && i < scroll_max + 10) {
if (!l_iter->out_state)
uint32_t line_count = editor->root->line_count + 1;
for (uint32_t i = MAX(0, (int64_t)scroll_max - MAX_LINES_LOOKBEHIND);
i <= scroll_max + 10 && i < line_count; ++i) {
auto l_opt = line_map.at(i);
if (!l_opt || (l_opt && !l_opt->out_state))
batch.push_back(i);
i++;
l_iter = line_tree.next();
}
line_tree.end_iter();
for (uint32_t c_line : batch) {
if (!running.load(std::memory_order_relaxed))
break;
uint32_t min_line = scroll_max > 60 ? scroll_max - 60 : 0;
uint32_t min_line = scroll_max > MAX_LINES_LOOKBEHIND
? scroll_max - MAX_LINES_LOOKBEHIND
: 0;
uint32_t max_line = scroll_max + 10;
if (c_line < min_line || c_line > max_line) {
dirty_lines.push(c_line);
if (c_line < min_line || c_line > max_line)
continue;
}
uint32_t scroll_snapshot = scroll_max;
std::shared_ptr<void> prev_state = nullptr;
uint32_t line_count;
line_count = line_tree.count();
if (c_line > 0 && c_line < line_count)
prev_state = line_tree.at(c_line - 1)->out_state;
std::unique_ptr<StateBase> prev_state = nullptr;
if (c_line > 0 && c_line < line_count) {
auto lm = line_map.at(c_line - 1);
if (lm && lm->out_state)
prev_state = lm->out_state ? lm->out_state->clone() : nullptr;
}
LineIterator *it = begin_l_iter(editor->root, c_line);
if (!it)
continue;
@@ -81,60 +77,57 @@ void Parser::work() {
while (cur_line < line_count) {
if (!running.load(std::memory_order_relaxed))
break;
if (scroll_snapshot != scroll_max) {
LineData *line_data = line_tree.at(cur_line);
if (line_data && !line_data->out_state)
dirty_lines.push(cur_line);
if (scroll_snapshot != scroll_max)
break;
}
if (cur_line < min_line || cur_line > max_line) {
dirty_lines.push(cur_line);
if (cur_line < min_line || cur_line > max_line)
break;
}
uint32_t len;
char *line = next_line(it, &len);
if (!line)
break;
LineData *line_data = line_tree.at(cur_line);
if (!line_data) {
cur_line++;
continue;
}
std::shared_ptr<void> new_state;
LineData *line_data = line_map.create_at(cur_line);
std::unique_ptr<StateBase> new_state;
if (is_custom) {
std::string state = "";
if (prev_state)
state = std::static_pointer_cast<std::string>(prev_state)->c_str();
std::string out_state = parse_custom(&line_data->tokens, parser_block,
line, len, state, cur_line);
new_state = std::make_shared<std::string>(out_state);
std::string prev_value = "";
if (prev_state) {
CustomState *prev_custom =
static_cast<CustomState *>(prev_state.get());
prev_value = prev_custom->value;
}
std::string out_value = parse_custom(&line_data->tokens, parser_block,
line, len, prev_value, cur_line);
new_state = std::make_unique<CustomState>(out_value);
} else {
new_state =
parse_func(&line_data->tokens, prev_state, line, len, cur_line);
new_state = parse_func(&line_data->tokens, prev_state.get(), line, len,
cur_line);
}
line_data->in_state = prev_state;
line_data->out_state = new_state;
line_data->in_state = std::move(prev_state);
line_data->out_state = std::move(new_state);
bool done = false;
if (cur_line + 1 < line_count) {
LineData *next_line_data = line_tree.at(cur_line + 1);
LineData *next_line_data = line_map.at(cur_line + 1);
if (next_line_data) {
if (is_custom) {
std::string a =
prev_state
? std::static_pointer_cast<std::string>(new_state)->c_str()
: "";
std::string b = next_line_data->in_state
? std::static_pointer_cast<std::string>(
next_line_data->in_state)
->c_str()
: "";
std::string a = "";
if (new_state) {
CustomState *cs = static_cast<CustomState *>(new_state.get());
a = cs->value;
}
std::string b = "";
if (next_line_data->in_state) {
CustomState *cs =
static_cast<CustomState *>(next_line_data->in_state.get());
b = cs->value;
}
done = custom_compare(match_block, a, b);
} else {
done = state_match_func(new_state, next_line_data->in_state);
done = state_match_func(new_state.get(),
next_line_data->in_state.get());
}
}
}
prev_state = new_state;
prev_state =
line_data->out_state ? line_data->out_state->clone() : nullptr;
cur_line++;
if (done)
break;
src/syntax/ruby.cc
+122 -118
View File
@@ -241,20 +241,22 @@ struct RubyFullState {
}
};
struct RubyState {
struct RubyState : StateBase {
using full_state_type = RubyFullState;
int interp_level = 0;
std::stack<std::shared_ptr<RubyFullState>> interp_stack;
std::shared_ptr<RubyFullState> full_state;
std::stack<RubyFullState> interp_stack;
RubyFullState full_state;
std::deque<HeredocInfo> heredocs;
bool operator==(const RubyState &other) const {
return interp_level == other.interp_level &&
interp_stack == other.interp_stack &&
((full_state && other.full_state &&
*full_state == *other.full_state)) &&
heredocs == other.heredocs;
full_state == other.full_state && heredocs == other.heredocs;
}
std::unique_ptr<StateBase> clone() const override {
return std::make_unique<RubyState>(*this);
}
};
@@ -278,18 +280,16 @@ inline static uint32_t get_next_word(const char *text, uint32_t i,
return width;
}
bool ruby_state_match(std::shared_ptr<void> state_1,
std::shared_ptr<void> state_2) {
bool ruby_state_match(StateBase *state_1, StateBase *state_2) {
if (!state_1 || !state_2)
return false;
return *std::static_pointer_cast<RubyState>(state_1) ==
*std::static_pointer_cast<RubyState>(state_2);
return *static_cast<RubyState *>(state_1) ==
*static_cast<RubyState *>(state_2);
}
std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
std::shared_ptr<void> in_state,
const char *text, uint32_t len,
uint32_t line_num) {
std::unique_ptr<StateBase> ruby_parse(std::vector<Token> *tokens,
StateBase *in_state, const char *text,
uint32_t len, uint32_t line_num) {
static bool keywords_trie_init = false;
static Trie<void> base_keywords_trie;
static Trie<void> expecting_keywords_trie;
@@ -313,7 +313,11 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
keywords_trie_init = true;
}
tokens->clear();
auto state = ensure_state(std::static_pointer_cast<RubyState>(in_state));
std::unique_ptr<RubyState> state;
if (in_state)
state = static_unique_ptr_cast<RubyState>(in_state->clone());
else
state = std::make_unique<RubyState>();
uint32_t i = 0;
while (len > 0 && (text[len - 1] == '\n' || text[len - 1] == '\r' ||
text[len - 1] == '\t' || text[len - 1] == ' '))
@@ -322,18 +326,18 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
return state;
bool heredoc_first = false;
while (i < len) {
if (state->full_state->in_state == RubyFullState::END)
if (state->full_state.in_state == RubyFullState::END)
return state;
if (state->full_state->in_state == RubyFullState::COMMENT) {
if (state->full_state.in_state == RubyFullState::COMMENT) {
tokens->push_back({i, len, TokenKind::K_COMMENT});
if (i == 0 && len == 4 && text[i] == '=' && text[i + 1] == 'e' &&
text[i + 2] == 'n' && text[i + 3] == 'd') {
state->full_state->in_state = RubyFullState::NONE;
state->full_state.in_state = RubyFullState::NONE;
}
return state;
}
if (!heredoc_first &&
state->full_state->in_state == RubyFullState::HEREDOC) {
state->full_state.in_state == RubyFullState::HEREDOC) {
if (i == 0) {
uint32_t start = 0;
if (state->heredocs.front().allow_indentation)
@@ -344,7 +348,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
state->heredocs.front().delim.length())) {
state->heredocs.pop_front();
if (state->heredocs.empty())
state->full_state->in_state = RubyFullState::NONE;
state->full_state.in_state = RubyFullState::NONE;
tokens->push_back({i, len, TokenKind::K_ANNOTATION});
return state;
}
@@ -421,7 +425,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
tokens->push_back({i, i + 2, TokenKind::K_INTERPOLATION});
i += 2;
state->interp_stack.push(state->full_state);
state->full_state = std::make_shared<RubyFullState>();
state->full_state = RubyFullState();
state->interp_level = 1;
break;
}
@@ -432,7 +436,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
continue;
}
}
if (state->full_state->in_state == RubyFullState::STRING) {
if (state->full_state.in_state == RubyFullState::STRING) {
uint32_t start = i;
while (i < len) {
if (text[i] == '\\') {
@@ -496,36 +500,36 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
tokens->push_back({start, i, TokenKind::K_ESCAPE});
continue;
}
if (state->full_state->lit.allow_interp && text[i] == '#' &&
if (state->full_state.lit.allow_interp && text[i] == '#' &&
i + 1 < len && text[i + 1] == '{') {
tokens->push_back({start, i, TokenKind::K_STRING});
tokens->push_back({i, i + 2, TokenKind::K_INTERPOLATION});
i += 2;
state->interp_stack.push(state->full_state);
state->full_state = std::make_shared<RubyFullState>();
state->full_state = RubyFullState();
state->interp_level = 1;
break;
}
if (text[i] == state->full_state->lit.delim_start &&
state->full_state->lit.delim_start !=
state->full_state->lit.delim_end) {
state->full_state->lit.brace_level++;
if (text[i] == state->full_state.lit.delim_start &&
state->full_state.lit.delim_start !=
state->full_state.lit.delim_end) {
state->full_state.lit.brace_level++;
}
if (text[i] == state->full_state->lit.delim_end) {
if (state->full_state->lit.delim_start ==
state->full_state->lit.delim_end) {
if (text[i] == state->full_state.lit.delim_end) {
if (state->full_state.lit.delim_start ==
state->full_state.lit.delim_end) {
i++;
tokens->push_back({start, i, TokenKind::K_STRING});
state->full_state->in_state = RubyFullState::NONE;
state->full_state->expecting_expr = false;
state->full_state.in_state = RubyFullState::NONE;
state->full_state.expecting_expr = false;
break;
} else {
state->full_state->lit.brace_level--;
if (state->full_state->lit.brace_level == 0) {
state->full_state.lit.brace_level--;
if (state->full_state.lit.brace_level == 0) {
i++;
tokens->push_back({start, i, TokenKind::K_STRING});
state->full_state->in_state = RubyFullState::NONE;
state->full_state->expecting_expr = false;
state->full_state.in_state = RubyFullState::NONE;
state->full_state.expecting_expr = false;
break;
}
}
@@ -536,7 +540,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
tokens->push_back({start, len, TokenKind::K_STRING});
continue;
}
if (state->full_state->in_state == RubyFullState::REGEXP) {
if (state->full_state.in_state == RubyFullState::REGEXP) {
uint32_t start = i;
while (i < len) {
if (text[i] == '\\') {
@@ -605,30 +609,30 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
tokens->push_back({i, i + 2, TokenKind::K_INTERPOLATION});
i += 2;
state->interp_stack.push(state->full_state);
state->full_state = std::make_shared<RubyFullState>();
state->full_state = RubyFullState();
state->interp_level = 1;
break;
}
if (text[i] == state->full_state->lit.delim_start &&
state->full_state->lit.delim_start !=
state->full_state->lit.delim_end) {
state->full_state->lit.brace_level++;
if (text[i] == state->full_state.lit.delim_start &&
state->full_state.lit.delim_start !=
state->full_state.lit.delim_end) {
state->full_state.lit.brace_level++;
}
if (text[i] == state->full_state->lit.delim_end) {
if (state->full_state->lit.delim_start ==
state->full_state->lit.delim_end) {
if (text[i] == state->full_state.lit.delim_end) {
if (state->full_state.lit.delim_start ==
state->full_state.lit.delim_end) {
i += 1;
tokens->push_back({start, i, TokenKind::K_REGEXP});
state->full_state->in_state = RubyFullState::NONE;
state->full_state->expecting_expr = false;
state->full_state.in_state = RubyFullState::NONE;
state->full_state.expecting_expr = false;
break;
} else {
state->full_state->lit.brace_level--;
if (state->full_state->lit.brace_level == 0) {
state->full_state.lit.brace_level--;
if (state->full_state.lit.brace_level == 0) {
i += 1;
tokens->push_back({start, i, TokenKind::K_REGEXP});
state->full_state->in_state = RubyFullState::NONE;
state->full_state->expecting_expr = false;
state->full_state.in_state = RubyFullState::NONE;
state->full_state.expecting_expr = false;
break;
}
}
@@ -642,8 +646,8 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
if (i == 0 && len == 6) {
if (text[i] == '=' && text[i + 1] == 'b' && text[i + 2] == 'e' &&
text[i + 3] == 'g' && text[i + 4] == 'i' && text[i + 5] == 'n') {
state->full_state->in_state = RubyFullState::COMMENT;
state->full_state->expecting_expr = false;
state->full_state.in_state = RubyFullState::COMMENT;
state->full_state.expecting_expr = false;
tokens->push_back({0, len, TokenKind::K_COMMENT});
return state;
}
@@ -653,8 +657,8 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
text[i + 3] == 'N' && text[i + 4] == 'D' && text[i + 5] == '_' &&
text[i + 6] == '_') {
tokens->clear();
state->full_state->in_state = RubyFullState::END;
state->full_state->expecting_expr = false;
state->full_state.in_state = RubyFullState::END;
state->full_state.expecting_expr = false;
return state;
}
}
@@ -684,33 +688,33 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
delim += text[j++];
}
}
state->full_state->expecting_expr = false;
state->full_state.expecting_expr = false;
if (!delim.empty()) {
tokens->push_back({s, j, TokenKind::K_ANNOTATION});
state->heredocs.push_back({delim, interpolation, indented});
state->full_state->in_state = RubyFullState::HEREDOC;
state->full_state.in_state = RubyFullState::HEREDOC;
heredoc_first = true;
}
i = j;
continue;
}
if (text[i] == '/' && state->full_state->expecting_expr) {
if (text[i] == '/' && state->full_state.expecting_expr) {
tokens->push_back({i, i + 1, TokenKind::K_REGEXP});
state->full_state->in_state = RubyFullState::REGEXP;
state->full_state->expecting_expr = false;
state->full_state->lit.delim_start = '/';
state->full_state->lit.delim_end = '/';
state->full_state->lit.allow_interp = true;
state->full_state.in_state = RubyFullState::REGEXP;
state->full_state.expecting_expr = false;
state->full_state.lit.delim_start = '/';
state->full_state.lit.delim_end = '/';
state->full_state.lit.allow_interp = true;
i++;
continue;
} else if (text[i] == '#') {
if (line_num == 0 && i == 0 && len > 4 && text[i + 1] == '!') {
state->full_state->expecting_expr = false;
state->full_state.expecting_expr = false;
tokens->push_back({0, len, TokenKind::K_SHEBANG});
return state;
}
tokens->push_back({i, len, TokenKind::K_COMMENT});
state->full_state->expecting_expr = false;
state->full_state.expecting_expr = false;
return state;
} else if (text[i] == '.') {
uint32_t start = i;
@@ -722,15 +726,15 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
}
}
tokens->push_back({start, i, TokenKind::K_OPERATOR});
state->full_state->expecting_expr = false;
state->full_state.expecting_expr = false;
continue;
} else if (text[i] == ':') {
state->full_state->expecting_expr = false;
state->full_state.expecting_expr = false;
uint32_t start = i;
i++;
if (i >= len) {
tokens->push_back({start, i, TokenKind::K_OPERATOR});
state->full_state->expecting_expr = true;
state->full_state.expecting_expr = true;
continue;
}
if (text[i] == ':') {
@@ -739,7 +743,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
}
if (text[i] == '\'' || text[i] == '"') {
tokens->push_back({start, i, TokenKind::K_LABEL});
state->full_state->expecting_expr = true;
state->full_state.expecting_expr = true;
continue;
}
if (text[i] == '$' || text[i] == '@') {
@@ -767,7 +771,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
tokens->push_back({start, i, TokenKind::K_OPERATOR});
continue;
} else if (text[i] == '@') {
state->full_state->expecting_expr = false;
state->full_state.expecting_expr = false;
uint32_t start = i;
i++;
if (i >= len)
@@ -783,7 +787,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
tokens->push_back({start, i, TokenKind::K_VARIABLEINSTANCE});
continue;
} else if (text[i] == '$') {
state->full_state->expecting_expr = false;
state->full_state.expecting_expr = false;
uint32_t start = i;
i++;
if (i >= len)
@@ -806,7 +810,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
tokens->push_back({start, i, TokenKind::K_VARIABLEGLOBAL});
continue;
} else if (text[i] == '?') {
state->full_state->expecting_expr = false;
state->full_state.expecting_expr = false;
uint32_t start = i;
i++;
if (i < len && text[i] == '\\') {
@@ -851,95 +855,95 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
tokens->push_back({start, i, TokenKind::K_CHAR});
continue;
} else {
state->full_state->expecting_expr = true;
state->full_state.expecting_expr = true;
tokens->push_back({start, i, TokenKind::K_OPERATOR});
continue;
}
} else if (text[i] == '{') {
state->full_state->expecting_expr = true;
state->full_state.expecting_expr = true;
uint8_t brace_color =
(uint8_t)TokenKind::K_BRACE1 + (state->full_state->brace_level % 5);
(uint8_t)TokenKind::K_BRACE1 + (state->full_state.brace_level % 5);
tokens->push_back({i, i + 1, (TokenKind)brace_color});
state->interp_level++;
state->full_state->brace_level++;
state->full_state.brace_level++;
i++;
continue;
} else if (text[i] == '}') {
state->full_state->expecting_expr = false;
state->full_state.expecting_expr = false;
state->interp_level--;
if (state->interp_level == 0 && !state->interp_stack.empty()) {
state->full_state = state->interp_stack.top();
state->interp_stack.pop();
tokens->push_back({i, i + 1, TokenKind::K_INTERPOLATION});
} else {
state->full_state->brace_level--;
state->full_state.brace_level--;
uint8_t brace_color =
(uint8_t)TokenKind::K_BRACE1 + (state->full_state->brace_level % 5);
(uint8_t)TokenKind::K_BRACE1 + (state->full_state.brace_level % 5);
tokens->push_back({i, i + 1, (TokenKind)brace_color});
}
i++;
continue;
} else if (text[i] == '(') {
state->full_state->expecting_expr = true;
state->full_state.expecting_expr = true;
uint8_t brace_color =
(uint8_t)TokenKind::K_BRACE1 + (state->full_state->brace_level % 5);
(uint8_t)TokenKind::K_BRACE1 + (state->full_state.brace_level % 5);
tokens->push_back({i, i + 1, (TokenKind)brace_color});
state->full_state->brace_level++;
state->full_state.brace_level++;
i++;
continue;
} else if (text[i] == ')') {
state->full_state->expecting_expr = false;
state->full_state->brace_level--;
state->full_state.expecting_expr = false;
state->full_state.brace_level--;
uint8_t brace_color =
(uint8_t)TokenKind::K_BRACE1 + (state->full_state->brace_level % 5);
(uint8_t)TokenKind::K_BRACE1 + (state->full_state.brace_level % 5);
tokens->push_back({i, i + 1, (TokenKind)brace_color});
i++;
continue;
} else if (text[i] == '[') {
state->full_state->expecting_expr = true;
state->full_state.expecting_expr = true;
uint8_t brace_color =
(uint8_t)TokenKind::K_BRACE1 + (state->full_state->brace_level % 5);
(uint8_t)TokenKind::K_BRACE1 + (state->full_state.brace_level % 5);
tokens->push_back({i, i + 1, (TokenKind)brace_color});
state->full_state->brace_level++;
state->full_state.brace_level++;
i++;
continue;
} else if (text[i] == ']') {
state->full_state->expecting_expr = false;
state->full_state->brace_level--;
state->full_state.expecting_expr = false;
state->full_state.brace_level--;
uint8_t brace_color =
(uint8_t)TokenKind::K_BRACE1 + (state->full_state->brace_level % 5);
(uint8_t)TokenKind::K_BRACE1 + (state->full_state.brace_level % 5);
tokens->push_back({i, i + 1, (TokenKind)brace_color});
i++;
continue;
} else if (text[i] == '\'') {
state->full_state->expecting_expr = false;
state->full_state.expecting_expr = false;
tokens->push_back({i, i + 1, TokenKind::K_STRING});
state->full_state->in_state = RubyFullState::STRING;
state->full_state->lit.delim_start = '\'';
state->full_state->lit.delim_end = '\'';
state->full_state->lit.allow_interp = false;
state->full_state.in_state = RubyFullState::STRING;
state->full_state.lit.delim_start = '\'';
state->full_state.lit.delim_end = '\'';
state->full_state.lit.allow_interp = false;
i++;
continue;
} else if (text[i] == '"') {
state->full_state->expecting_expr = false;
state->full_state.expecting_expr = false;
tokens->push_back({i, i + 1, TokenKind::K_STRING});
state->full_state->in_state = RubyFullState::STRING;
state->full_state->lit.delim_start = '"';
state->full_state->lit.delim_end = '"';
state->full_state->lit.allow_interp = true;
state->full_state.in_state = RubyFullState::STRING;
state->full_state.lit.delim_start = '"';
state->full_state.lit.delim_end = '"';
state->full_state.lit.allow_interp = true;
i++;
continue;
} else if (text[i] == '`') {
state->full_state->expecting_expr = false;
state->full_state.expecting_expr = false;
tokens->push_back({i, i + 1, TokenKind::K_STRING});
state->full_state->in_state = RubyFullState::STRING;
state->full_state->lit.delim_start = '`';
state->full_state->lit.delim_end = '`';
state->full_state->lit.allow_interp = true;
state->full_state.in_state = RubyFullState::STRING;
state->full_state.lit.delim_start = '`';
state->full_state.lit.delim_end = '`';
state->full_state.lit.allow_interp = true;
i++;
continue;
} else if (text[i] == '%') {
state->full_state->expecting_expr = false;
state->full_state.expecting_expr = false;
if (i + 1 >= len) {
i++;
continue;
@@ -1005,16 +1009,16 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
tokens->push_back(
{i, i + prefix_len + 1,
(is_regexp ? TokenKind::K_REGEXP : TokenKind::K_STRING)});
state->full_state->in_state =
state->full_state.in_state =
is_regexp ? RubyFullState::REGEXP : RubyFullState::STRING;
state->full_state->lit.delim_start = delim_start;
state->full_state->lit.delim_end = delim_end;
state->full_state->lit.allow_interp = allow_interp;
state->full_state->lit.brace_level = 1;
state->full_state.lit.delim_start = delim_start;
state->full_state.lit.delim_end = delim_end;
state->full_state.lit.allow_interp = allow_interp;
state->full_state.lit.brace_level = 1;
i += prefix_len + 1;
continue;
} else if (isdigit(text[i])) {
state->full_state->expecting_expr = false;
state->full_state.expecting_expr = false;
uint32_t start = i;
if (text[i] == '0') {
i++;
@@ -1115,7 +1119,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
tokens->push_back({start, i, TokenKind::K_NUMBER});
continue;
} else if (identifier_start_char(text[i])) {
state->full_state->expecting_expr = false;
state->full_state.expecting_expr = false;
uint32_t length;
if ((length = base_keywords_trie.match(text, i, len, identifier_char))) {
tokens->push_back({i, i + length, TokenKind::K_KEYWORD});
@@ -1123,7 +1127,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
continue;
} else if ((length = expecting_keywords_trie.match(text, i, len,
identifier_char))) {
state->full_state->expecting_expr = true;
state->full_state.expecting_expr = true;
tokens->push_back({i, i + length, TokenKind::K_KEYWORD});
i += length;
continue;
@@ -1134,7 +1138,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
continue;
} else if ((length = expecting_operators_trie.match(
text, i, len, identifier_char)) > 0) {
state->full_state->expecting_expr = true;
state->full_state.expecting_expr = true;
tokens->push_back({i, i + length, TokenKind::K_KEYWORDOPERATOR});
i += length;
continue;
@@ -1256,7 +1260,7 @@ std::shared_ptr<void> ruby_parse(std::vector<Token> *tokens,
operator_trie.match(text, i, len, [](char) { return false; }))) {
tokens->push_back({i, i + op_len, TokenKind::K_OPERATOR});
i += op_len;
state->full_state->expecting_expr = true;
state->full_state.expecting_expr = true;
continue;
} else {
i += utf8_codepoint_width(text[i]);