diff --git a/Makefile b/Makefile index 9f765f3..141851b 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ MRUBY_LIBS ?= -L./libs/mruby/build/host/lib -lmruby CFLAGS_DEBUG :=\ -std=c++23 -Wall -Wextra -fno-rtti \ - -Og -fno-inline -gsplit-dwarf \ + -Og -fno-inline -gsplit-dwarf -DDEBUG \ -g -fno-omit-frame-pointer -ffast-math \ -I./include -I./libs/unicode_width diff --git a/include/internal/syntax/decl.h b/include/internal/syntax/decl.h index 48adf05..654ea9f 100644 --- a/include/internal/syntax/decl.h +++ b/include/internal/syntax/decl.h @@ -79,7 +79,7 @@ struct ParseEvent { uint8_t type; // type. ParseEvent(T t) : ev_type(t) {} - ParseEvent(std::string_view s, T e_t, uint8_t type) : name(s), ev_type(e_t), type(type) {} + ParseEvent(T e_t, std::string_view s, uint8_t type) : name(s), ev_type(e_t), type(type) {} }; struct Language { @@ -119,12 +119,11 @@ struct ParseStateBranch : ParseState { struct ParseStateLeaf : ParseState { void *state; - uint64_t n; - static constexpr uint32_t IS_OPENING = 1ull << 31; - static constexpr uint32_t LINE_MASK = ~IS_OPENING; - // followed by n number of relative offsets - // stored as uint32_t with 1 bit for if it is start or end - // and rest as number. + uint32_t n; + uint32_t cap; + uint32_t *blocks; + static constexpr uint32_t IS_CLOSING = 1ull << 31; + static constexpr uint32_t LINE_MASK = ~IS_CLOSING; }; struct TreeCursor { diff --git a/include/internal/syntax/parser.h b/include/internal/syntax/parser.h index 0c79b32..246900f 100644 --- a/include/internal/syntax/parser.h +++ b/include/internal/syntax/parser.h @@ -5,8 +5,10 @@ #include "pch.h" namespace bed::internal::syntax { +void dump_events(ParseState *node); + struct Parser { - static constexpr uint64_t MAX_CHUNK = 256; + static constexpr uint64_t MAX_CHUNK = 512; ParseState *root; Language lang; @@ -21,6 +23,9 @@ struct Parser { void insert(vase::Vase &, uint64_t, uint64_t); void modify(vase::Vase &, uint64_t, uint64_t); + uint64_t next_closing(uint64_t line); + uint64_t prev_opening(uint64_t line); + std::pair split_tree(ParseState *node, uint64_t line); ParseState *join_tree(ParseState *a, ParseState *b); diff --git a/include/internal/syntax/ruby/decl.h b/include/internal/syntax/ruby/decl.h index a6ff7de..ef06e87 100644 --- a/include/internal/syntax/ruby/decl.h +++ b/include/internal/syntax/ruby/decl.h @@ -33,6 +33,7 @@ struct alignas(2) RubyState { DEF_NAME = 0b10, MODULE_NAME = 0b11 }; + static constexpr uint8_t NEWLINE = 1 << 5; static constexpr uint8_t ALLOW_INTERPOLATION = 1 << 6; static constexpr uint8_t EXPECTING_EXPRESSION = 1 << 7; uint8_t flags = 0; diff --git a/include/internal/syntax/ruby/parser.h b/include/internal/syntax/ruby/parser.h index 91d100a..b8c3f56 100644 --- a/include/internal/syntax/ruby/parser.h +++ b/include/internal/syntax/ruby/parser.h @@ -11,6 +11,8 @@ struct RubyParser { uint32_t i = 0; bool heredoc_start_line = false; bool ending = true; + bool op_last = false; + bool set_op_last = false; RubyParser(void **v_state, std::string_view line) : v_state(v_state), state((RubyState *)*v_state), line(line) {} @@ -58,7 +60,7 @@ struct RubyParser { .brace_level = 1, .lit_brace_level = 0, .state = RubyState::RubyInternalState::NONE, - .flags = RubyState::RubyInternalState::EXPECTING_EXPRESSION, + .flags = RubyState::RubyInternalState::EXPECTING_EXPRESSION | RubyState::RubyInternalState::NEWLINE, .delim_start = '\0', .delim_end = '\0' }; diff --git a/include/internal/syntax/ruby/tries.h b/include/internal/syntax/ruby/tries.h index 130341c..09fb336 100644 --- a/include/internal/syntax/ruby/tries.h +++ b/include/internal/syntax/ruby/tries.h @@ -189,7 +189,6 @@ const static std::vector methods = { "respond_to?", "equal?", "object_id", - "class", "singleton_class", "clone", "freeze", diff --git a/src/internal/address/address.cc b/src/internal/address/address.cc index 9813551..9c5183d 100644 --- a/src/internal/address/address.cc +++ b/src/internal/address/address.cc @@ -19,6 +19,14 @@ Address::Address(std::string &cmd, uint64_t &i) { base = Last(); i++; break; + case ']': + base = Block(Direction::Forward); + i++; + break; + case '[': + base = Block(Direction::Backward); + i++; + break; case '\'': { i++; if (i < cmd.size() && 'a' <= cmd[i] && cmd[i] <= 'z') diff --git a/src/internal/address/resolve.cc b/src/internal/address/resolve.cc index 28e14d7..aad659f 100644 --- a/src/internal/address/resolve.cc +++ b/src/internal/address/resolve.cc @@ -19,6 +19,18 @@ uint64_t Address::resolve(BEd &ctx) { line = ctx.active->vase.lines(); } else if constexpr (std::is_same_v) { line = addr.i; + } else if constexpr (std::is_same_v) { + if (addr.dir == Direction::Forward) { + if (ctx.active->parser) + line = ctx.active->parser->next_closing(ctx.active->line); + else + line = ctx.active->line + 10; + } else { + if (ctx.active->parser) + line = ctx.active->parser->prev_opening(ctx.active->line); + else + line = ctx.active->line - 10; + } } else { throw ed_error("Unhandled address given."); } diff --git a/src/internal/commands/commands.cc b/src/internal/commands/commands.cc index 01511f3..e19365d 100644 --- a/src/internal/commands/commands.cc +++ b/src/internal/commands/commands.cc @@ -101,6 +101,18 @@ void Command::register_posix(BEd &ctx) { } } ); + ctx.commands.insert( + "debug", + Command{ + .address_mode = Command::AddressMode::None, + .suffix = Command::SuffixKind::None, + .desc = "", + .accept_zero = false, + .handle = [](BEd &ctx, std::span, std::string_view) { + syntax::dump_events(ctx.active->parser->root); + } + } + ); ctx.commands.insert( "E", Command{ diff --git a/src/internal/syntax/parser.cc b/src/internal/syntax/parser.cc index 558be9c..a8223d3 100644 --- a/src/internal/syntax/parser.cc +++ b/src/internal/syntax/parser.cc @@ -1,6 +1,42 @@ #include "internal/syntax/parser.h" namespace bed::internal::syntax { +void dump_events(ParseState *root) { + if (!root) + return; + uint64_t offset = 0; + TreeCursor c(root, 0, &offset); + uint64_t line_offset = 0; + int depth = 0; + while (c.leaf) { + auto *leaf = c.leaf; + for (uint32_t i = 0; i < leaf->n; ++i) { + uint32_t block = leaf->blocks[i]; + uint32_t pos = block & ParseStateLeaf::LINE_MASK; + bool closing = block & ParseStateLeaf::IS_CLOSING; + if (closing) { + if (depth > 0) + --depth; + else + std::cout << "!! UNMATCHED CLOSE !! "; + } + std::cout << std::string(static_cast(depth) * 2, ' ') + << (closing ? "}" : "{") + << " line " << (line_offset + pos) + << '\n'; + if (!closing) + ++depth; + } + line_offset += leaf->lines(); + c.next(); + } + if (depth != 0) { + std::cout << "!! UNBALANCED: depth = " + << depth + << " !!\n"; + } +} + static void destroy_tree(ParseState *node, Language &lang) { if (!node) return; @@ -8,12 +44,14 @@ static void destroy_tree(ParseState *node, Language &lang) { auto *branch = (ParseStateBranch *)node; destroy_tree(branch->left, lang); destroy_tree(branch->right, lang); - delete branch; + free(branch); } else { auto *leaf = (ParseStateLeaf *)node; if (leaf->state) lang.destroy(leaf->state); - delete leaf; + if (leaf->blocks) + free(leaf->blocks); + free(leaf); } } @@ -22,9 +60,8 @@ static ParseState *make_branch(ParseState *left, ParseState *right) { return right; if (!right) return left; - auto *branch = new ParseStateBranch; - branch->header = - ParseState::BRANCH_BIT + left->lines() + right->lines(); + auto *branch = (ParseStateBranch *)malloc(sizeof(ParseStateBranch)); + branch->header = ParseState::BRANCH_BIT + left->lines() + right->lines(); branch->left = left; branch->right = right; return branch; @@ -57,32 +94,22 @@ void Parser::reset(vase::Vase &vase, uint64_t lines, Language lang_) { if (lines == 0) return; lang = std::move(lang_); - vase::Iterator it = vase.iterate(0, Direction::Forward); - it.next(); std::vector leaves; - std::vector tokens; - std::vector events; leaves.reserve((lines + MAX_CHUNK - 1) / MAX_CHUNK); - void *state = lang.none_state(); - uint32_t consumed = 0; + uint64_t consumed = 0; while (consumed < lines) { - ParseStateLeaf *leaf = new ParseStateLeaf; - leaf->header = 0; - leaf->state = lang.copy(state); - uint32_t chunk_lines = 0; - while (chunk_lines < MAX_CHUNK && consumed < lines) { - tokens.clear(); - lang.parse(&state, it.line, consumed == 0, &tokens, &events); - ++chunk_lines; - ++consumed; - if (!it.next() && consumed < lines) - break; - } - leaf->header = chunk_lines; + auto *leaf = (ParseStateLeaf *)malloc(sizeof(ParseStateLeaf)); + leaf->state = nullptr; + leaf->blocks = nullptr; + leaf->n = 0; + leaf->cap = 0; + uint64_t chunk = std::min(MAX_CHUNK, lines - consumed); + leaf->header = chunk; + consumed += chunk; leaves.push_back(leaf); } - lang.destroy(state); root = build_tree(leaves, 0, leaves.size()); + modify(vase, 0, lines); } std::pair Parser::split_tree(ParseState *node, uint64_t line) { @@ -98,26 +125,30 @@ std::pair Parser::split_tree(ParseState *node, uint6 if (line < left_lines) { auto [a, b] = split_tree(branch->left, line); ParseState *right = join_tree(b, branch->right); - delete branch; + free(branch); return {a, right}; } if (line == left_lines) { ParseState *left = branch->left; ParseState *right = branch->right; - delete branch; + free(branch); return {left, right}; } auto [a, b] = split_tree(branch->right, line - left_lines); ParseState *left = join_tree(branch->left, a); - delete branch; + free(branch); return {left, b}; } else { auto *leaf = (ParseStateLeaf *)node; uint64_t lines = leaf->lines(); - auto *right = new ParseStateLeaf; + auto *right = (ParseStateLeaf *)malloc(sizeof(ParseStateLeaf)); right->header = lines - line; right->state = nullptr; + right->blocks = nullptr; + right->n = 0; + right->cap = 0; leaf->header = line; + leaf->n = 0; return {leaf, right}; } } @@ -144,14 +175,14 @@ void Parser::insert(vase::Vase &vase, uint64_t start, uint64_t count) { leaves.reserve((count + MAX_CHUNK - 1) / MAX_CHUNK); uint64_t consumed = 0; while (consumed < count) { - auto *leaf = new ParseStateLeaf; + auto *leaf = (ParseStateLeaf *)malloc(sizeof(ParseStateLeaf)); leaf->state = nullptr; - uint64_t chunk = 0; - while (chunk < MAX_CHUNK && consumed < count) { - ++chunk; - ++consumed; - } + leaf->blocks = nullptr; + leaf->n = 0; + leaf->cap = 0; + uint64_t chunk = std::min(MAX_CHUNK, count - consumed); leaf->header = chunk; + consumed += chunk; leaves.push_back(leaf); } ParseState *subtree = build_tree(leaves, 0, leaves.size()); @@ -188,13 +219,17 @@ void Parser::modify(vase::Vase &vase, uint64_t target, uint64_t count) { } } vase::Iterator it = vase.iterate(at, Direction::Forward); + uint64_t chunk_start = at; uint64_t next_boundary = at + c.leaf->lines(); + c.leaf->n = 0; while (true) { it.next(); if (at == next_boundary) { c.next(); if (!c.leaf) break; + c.leaf->n = 0; + chunk_start = at; next_boundary += c.leaf->lines(); if (at >= target + count && c.leaf->state != nullptr @@ -205,12 +240,88 @@ void Parser::modify(vase::Vase &vase, uint64_t target, uint64_t count) { c.leaf->state = lang.copy(state); } tokens.clear(); + events.clear(); lang.parse(&state, it.line, at == 0, &tokens, &events); + for (const auto &ev : events) { + const auto t = uint8_t(ev.ev_type); + if (t > ParseEvent::Closing) + continue; + if (c.leaf->n == c.leaf->cap) { + uint32_t cap = c.leaf->cap ? c.leaf->cap * 2 : 8; + c.leaf->blocks = (uint32_t *)realloc(c.leaf->blocks, cap * sizeof(uint32_t)); + c.leaf->cap = cap; + } + c.leaf->blocks[c.leaf->n++] = t << 31 | (at - chunk_start); + } at++; } lang.destroy(state); } +uint64_t Parser::next_closing(uint64_t line) { + if (!root) + return line + 10; + uint64_t relative; + TreeCursor c(root, line, &relative); + uint64_t at = line - relative; + uint32_t from = (uint32_t)relative; + bool first = true; + int depth = 0; + while (c.leaf) { + auto *leaf = c.leaf; + for (uint32_t i = 0; i < leaf->n; ++i) { + uint32_t block = leaf->blocks[i]; + uint32_t pos = block & ParseStateLeaf::LINE_MASK; + if (first && pos <= from) + continue; + if (block & ParseStateLeaf::IS_CLOSING) { + if (depth == 0) + return at + pos; + --depth; + } else { + ++depth; + } + } + at += leaf->lines(); + first = false; + c.next(); + } + return line + 10; +} + +uint64_t Parser::prev_opening(uint64_t line) { + if (!root) + return line >= 10 ? line - 10 : 0; + uint64_t relative; + TreeCursor c(root, line, &relative); + uint64_t at = line - relative; + uint32_t from = (uint32_t)relative; + bool first = true; + int depth = 0; + while (c.leaf) { + auto *leaf = c.leaf; + for (uint32_t i = leaf->n; i-- > 0;) { + uint32_t block = leaf->blocks[i]; + uint32_t pos = block & ParseStateLeaf::LINE_MASK; + if (first && pos >= from) + continue; + if (!(block & ParseStateLeaf::IS_CLOSING)) { + if (depth == 0) + return at + pos; + --depth; + } else { + ++depth; + } + } + c.prev(); + if (!c.leaf) + break; + at -= c.leaf->lines(); + first = false; + } + return line >= 10 ? line - 10 : 0; +} + std::optional Parser::get_hl(vase::Vase &vase, uint64_t target) { if (!root) return std::nullopt; @@ -243,6 +354,7 @@ Parser::Iterator::Iterator(uint64_t target, Parser *p, vase::Vase &vase) : p(p) while (at < target) { it->next(); tokens.clear(); + events.clear(); p->lang.parse(&state, it->line, at == 0, &tokens, &events); at++; } @@ -277,6 +389,7 @@ Parser::Iterator &Parser::Iterator::operator=(Iterator &&other) { void Parser::Iterator::next() { it->next(); tokens.clear(); + events.clear(); p->lang.parse(&state, it->line, at++ == 0, &tokens, &events); } } // namespace bed::internal::syntax diff --git a/src/internal/syntax/ruby/parse.cc b/src/internal/syntax/ruby/parse.cc index 71ae7a2..9d3be86 100644 --- a/src/internal/syntax/ruby/parse.cc +++ b/src/internal/syntax/ruby/parse.cc @@ -250,15 +250,18 @@ bool handle_comment(RubyParser &p, std::vector *tokens, bool first_line) return false; } -void handle_syntax(RubyParser &p, std::vector *tokens, std::vector *events) { +bool handle_syntax(RubyParser &p, std::vector *tokens, std::vector *events) { static const RubyTries tries = RubyTries(); - while (p.peek() == ' ' || p.peek() == '\t') - p.advance(); + if (p.peek() == ' ' || p.peek() == '\t') { + while (p.peek() == ' ' || p.peek() == '\t') + p.advance(); + return true; + } if (p.current().flags & RubyState::RubyInternalState::NAME_MASK) { if (p.peek() == '\0') - return; + return false; if (p.peek() == '\\' && p.peek(1) == '\0') - return; + return false; switch (p.current().flags & RubyState::RubyInternalState::NAME_MASK) { case RubyState::RubyInternalState::CLASS_NAME: { if (p.peek() == '<' && p.peek(1) == '<') { @@ -266,7 +269,7 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vectorpush_back(ParseEvent::Opening); p.current().flags = (p.current().flags & ~RubyState::RubyInternalState::NAME_MASK); p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION; - return; + return false; } uint32_t j = 0; if (identifier_start_char(p.peek(j))) { @@ -279,12 +282,12 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vectorpush_back(ParseEvent::Opening); p.current().flags = (p.current().flags & ~RubyState::RubyInternalState::NAME_MASK); - return; + return false; } case RubyState::RubyInternalState::DEF_NAME: { uint32_t j = 0; @@ -305,12 +308,12 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vectorpush_back(ParseEvent::Opening); p.current().flags = (p.current().flags & ~RubyState::RubyInternalState::NAME_MASK); - return; + return false; } case RubyState::RubyInternalState::MODULE_NAME: { uint32_t j = 0; @@ -324,12 +327,12 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vectorpush_back(ParseEvent::Opening); p.current().flags = (p.current().flags & ~RubyState::RubyInternalState::NAME_MASK); - return; + return false; } } } @@ -342,7 +345,7 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vectorpush_back({p.i, p.i + j, Token::Operator}); if (p.i + j >= p.len()) - return; + return false; std::string delim; bool interpolation = true; uint32_t s = p.i + j; @@ -360,7 +363,7 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vectorpush_back({s, p.i + j, Token::Annotation}); @@ -374,7 +377,7 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vectorpush_back({p.i, p.i + 1, Token::Regexp}); @@ -384,7 +387,7 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vector *tokens, std::vectorpush_back({start, p.i, Token::Operator}); - return; + return false; } case ':': { p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION; @@ -406,17 +409,17 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vector= p.len()) { tokens->push_back({start, p.i, Token::Operator}); p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION; - return; + return false; } if (p.peek() == ':') { p.advance(); tokens->push_back({start, p.i, Token::Operator}); - return; + return false; } if (p.peek() == '\'' || p.peek() == '"') { tokens->push_back({start, p.i, Token::Label}); p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION; - return; + return false; } if (p.peek() == '$' || p.peek() == '@') { if (p.peek_str(2) == "@@") @@ -426,13 +429,13 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vectorpush_back({start, p.i, Token::Label}); - return; + return false; } uint32_t op_len = tries.operator_trie.longest_match(p.peek_str(p.len() - p.i)); if (op_len > 0) { tokens->push_back({start, p.i + op_len, Token::Label}); p.advance(op_len); - return; + return false; } if (identifier_start_char(p.peek())) { p.advance(); @@ -441,34 +444,34 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vectorpush_back({start, p.i, Token::Label}); - return; + return false; } tokens->push_back({start, p.i, Token::Operator}); - return; + return false; } case '@': { p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION; uint32_t start = p.i; p.advance(); if (p.i >= p.len()) - return; + return false; if (p.peek() == '@') p.advance(); if (identifier_start_char(p.peek())) p.advance(); else - return; + return false; while (identifier_char(p.peek())) p.advance(); tokens->push_back({start, p.i, Token::VariableInstance}); - return; + return false; } case '$': { p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION; uint32_t start = p.i; p.advance(); if (p.i >= p.len()) - return; + return false; if (identifier_start_char(p.peek())) { p.advance(); while (identifier_char(p.peek())) @@ -504,11 +507,11 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vectorpush_back({start, p.i, Token::VariableGlobal}); - return; + return false; } case '?': { p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION; @@ -524,7 +527,7 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vectorpush_back({start, p.i, Token::Char}); - return; + return false; } else if (p.peek() == 'u') { p.advance(); if (p.peek() == '{') { @@ -544,7 +547,7 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vectorpush_back({start, p.i, Token::Char}); - return; + return false; } else if ('0' <= p.peek() && p.peek() <= '7') { p.advance(); if ('0' <= p.peek() && p.peek() <= '7') @@ -552,7 +555,7 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vectorpush_back({start, p.i, Token::Char}); - return; + return false; } else if (p.peek() == 'c') { p.advance(); if (p.peek() != '\\') @@ -560,7 +563,7 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vectorpush_back({start, p.i, Token::Char}); - return; + return false; } else if (p.peek() == 'M' || p.peek() == 'C') { p.advance(); if (p.peek() == '-') { @@ -571,7 +574,7 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vectorpush_back({start, p.i, Token::Char}); - return; + return false; } else if (p.peek() == 'N') { p.advance(); if (p.peek() == '{') { @@ -582,20 +585,20 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vectorpush_back({start, p.i, Token::Char}); - return; + return false; } else { p.advance(); tokens->push_back({start, p.i, Token::Char}); - return; + return false; } } else if (p.peek() != '\0' && p.peek() != ' ' && p.peek() != '\t') { p.advance(); tokens->push_back({start, p.i, Token::Char}); - return; + return false; } else { p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION; tokens->push_back({start, p.i, Token::Operator}); - return; + return false; } } case '{': { @@ -605,7 +608,7 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vectorpush_back({p.i, p.i + 1, (Token::Kind)brace_color}); p.current().brace_level++; p.advance(); - return; + return false; } case '}': { p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION; @@ -618,7 +621,7 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vectorpush_back({p.i, p.i + 1, (Token::Kind)brace_color}); } p.advance(); - return; + return false; } case '(': { p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION; @@ -627,7 +630,7 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vectorpush_back({p.i, p.i + 1, (Token::Kind)brace_color}); p.current().brace_level++; p.advance(); - return; + return false; } case ')': { p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION; @@ -636,7 +639,7 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vectorpush_back({p.i, p.i + 1, (Token::Kind)brace_color}); p.advance(); - return; + return false; } case '[': { p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION; @@ -645,7 +648,7 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vectorpush_back({p.i, p.i + 1, (Token::Kind)brace_color}); p.current().brace_level++; p.advance(); - return; + return false; } case ']': { p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION; @@ -654,7 +657,7 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vectorpush_back({p.i, p.i + 1, (Token::Kind)brace_color}); p.advance(); - return; + return false; } case '\'': { p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION; @@ -664,7 +667,7 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vector *tokens, std::vector *tokens, std::vector= p.len()) { tokens->push_back({p.i, p.i + 1, Token::Operator}); p.advance(); p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION; - return; + return false; } p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION; char type = p.peek(1); @@ -729,14 +732,14 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vectorpush_back({p.i, p.i + 1, Token::Operator}); p.advance(prefix_len); p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION; - return; + return false; } delim_start = p.peek(prefix_len); if (identifier_char(delim_start) || delim_start == ' ') { tokens->push_back({p.i, p.i + 1, Token::Operator}); p.advance(prefix_len); p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION; - return; + return false; } switch (delim_start) { case '(': @@ -765,22 +768,23 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vector *tokens, std::vectorpush_back({start, p.i, Token::Number}); - return; + return false; } else if (identifier_start_char(p.peek())) { uint32_t j = 1; while (identifier_char(p.peek(j))) @@ -868,88 +872,88 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vectorpush_back({p.i, p.i + j, Token::Keyword}); p.advance(j); - return; + return false; } else if (j == tries.expecting_keywords_trie.longest_match(p.peek_str(j))) { p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION; tokens->push_back({p.i, p.i + j, Token::Keyword}); p.advance(j); - return; + return false; } else if (j == tries.operator_keywords_trie.longest_match(p.peek_str(j))) { p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION; tokens->push_back({p.i, p.i + j, Token::KeywordOperator}); p.advance(j); - return; + return false; } else if (j == tries.expecting_operators_trie.longest_match(p.peek_str(j))) { p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION; tokens->push_back({p.i, p.i + j, Token::KeywordOperator}); p.advance(j); - return; + return false; } else if (j == tries.types_trie.longest_match(p.peek_str(j))) { p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION; tokens->push_back({p.i, p.i + j, Token::Type}); p.advance(j); - return; + return false; } else if (j == tries.methods_trie.longest_match(p.peek_str(j))) { p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION; tokens->push_back({p.i, p.i + j, Token::Function}); p.advance(j); - return; + return false; } else if (j == tries.expecting_end_keywords_trie.longest_match(p.peek_str(j))) { events->push_back(ParseEvent::Opening); p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION; tokens->push_back({p.i, p.i + j, Token::Keyword}); p.advance(j); - return; + return false; } else if (j == tries.conditional_keywords_trie.longest_match(p.peek_str(j))) { - if (p.current().flags & RubyState::RubyInternalState::EXPECTING_EXPRESSION) + if (p.current().flags & RubyState::RubyInternalState::NEWLINE || p.op_last) events->push_back(ParseEvent::Opening); p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION; tokens->push_back({p.i, p.i + j, Token::Keyword}); p.advance(j); - return; + return false; } else if (j == tries.end_keywords_trie.longest_match(p.peek_str(j))) { p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION; events->push_back(ParseEvent::Opening); tokens->push_back({p.i, p.i + j, Token::Keyword}); p.advance(j); - return; + return false; } else if (j == tries.builtins_trie.longest_match(p.peek_str(j))) { p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION; tokens->push_back({p.i, p.i + j, Token::Constant}); p.advance(j); - return; + return false; } else if (j == tries.errors_trie.longest_match(p.peek_str(j))) { p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION; tokens->push_back({p.i, p.i + j, Token::Error}); p.advance(j); - return; + return false; } else if ('A' <= p.peek() && p.peek() <= 'Z' && !(p.peek(j) == '!' || p.peek(j) == '?')) { p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION; if (j >= 5 && p.peek_str(j).substr(j - 5) == "Error") { tokens->push_back({p.i, p.i + j, Token::Error}); p.advance(j); - return; + return false; } tokens->push_back({p.i, p.i + j, Token::Constant}); p.advance(j); - return; + return false; } else { p.current().flags &= ~RubyState::RubyInternalState::EXPECTING_EXPRESSION; if (j == 4 && p.peek_str(4) == "true") { tokens->push_back({p.i, p.i + j, Token::True}); p.advance(4); - return; + return false; } if (j == 5 && p.peek_str(5) == "false") { tokens->push_back({p.i, p.i + j, Token::False}); p.advance(5); - return; + return false; } if (j == 3 && p.peek_str(3) == "end") { events->push_back(ParseEvent::Closing); tokens->push_back({p.i, p.i + j, Token::Keyword}); p.advance(3); - return; + return false; } if (j == 5 && p.peek_str(5) == "class") { tokens->push_back({p.i, p.i + j, Token::Keyword}); @@ -957,7 +961,7 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vectorpush_back({p.i, p.i + j, Token::Keyword}); @@ -965,7 +969,7 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vectorpush_back({p.i, p.i + j, Token::Keyword}); @@ -973,32 +977,32 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vectorpush_back({start, p.i, Token::Label}); - return; + return false; } else if (p.peek(-1) == '!' || p.peek(-1) == '?') { p.advance(); tokens->push_back({start, p.i, Token::Function}); - return; + return false; } else { uint32_t j = 0; if (p.peek(j) == '(' || p.peek(j) == '{') { tokens->push_back({start, p.i, Token::Function}); - return; + return false; } else if (p.peek(j) == ' ' || p.peek(j) == '\t') { j++; } else { - return; + return false; } while (p.peek(j) == ' ' || p.peek(j) == '\t') j++; if (p.i + j >= p.len()) - return; + return false; if ( p.peek(j) == '-' || p.peek(j) == '&' @@ -1006,7 +1010,7 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vector') - return; + return false; } else if ( p.peek(j) == ']' || p.peek(j) == '}' @@ -1024,10 +1028,10 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vector' ) { - return; + return false; } tokens->push_back({start, p.i, Token::Function}); - return; + return false; } } } else { @@ -1036,11 +1040,13 @@ void handle_syntax(RubyParser &p, std::vector *tokens, std::vectorpush_back({p.i, p.i + op_len, Token::Operator}); p.advance(op_len); p.current().flags |= RubyState::RubyInternalState::EXPECTING_EXPRESSION; + p.set_op_last = true; } else { p.advance(utf8_codepoint_width(p.peek())); } } } + return false; } void ruby_parse( @@ -1052,6 +1058,8 @@ void ruby_parse( ) { RubyParser p(v_state, line); while (p.i <= p.len()) { + p.op_last = p.set_op_last; + p.set_op_last = false; if (p.current().state == RubyState::RubyInternalState::END) return; if (p.current().state == RubyState::RubyInternalState::COMMENT) { @@ -1085,8 +1093,11 @@ void ruby_parse( return; if (handle_comment(p, tokens, fl)) return; - handle_syntax(p, tokens, events); + if (!handle_syntax(p, tokens, events)) + p.current().flags &= ~RubyState::RubyInternalState::NEWLINE; continue; } + if (p.ending) + p.current().flags |= RubyState::RubyInternalState::NEWLINE; } } // namespace bed::internal::syntax::ruby diff --git a/src/internal/syntax/ruby/ruby.cc b/src/internal/syntax/ruby/ruby.cc index 567ea42..77a7c80 100644 --- a/src/internal/syntax/ruby/ruby.cc +++ b/src/internal/syntax/ruby/ruby.cc @@ -14,7 +14,7 @@ Language lang_ruby() { .brace_level = 1, .lit_brace_level = 0, .state = RubyState::RubyInternalState::NONE, - .flags = RubyState::RubyInternalState::EXPECTING_EXPRESSION, + .flags = RubyState::RubyInternalState::EXPECTING_EXPRESSION | RubyState::RubyInternalState::NEWLINE, .delim_start = '\0', .delim_end = '\0' }; diff --git a/src/main.cc b/src/main.cc index 0f1f2d6..ef69b23 100644 --- a/src/main.cc +++ b/src/main.cc @@ -10,9 +10,12 @@ int main(int argc, char *argv[]) { if (e.code) std::cout << "Fatal error: " << e.what() << std::endl; return e.code; - } catch (...) { + } +#ifndef DEBUG + catch (...) { std::cout << "Unexpected error." << std::endl; return 1; } +#endif return 0; }