Add escape in strings control to ruby parser.

include/internal/syntax/ruby/decl.h

@@ -24,7 +24,8 @@ struct alignas(2) RubyState {
       DEF_NAME = 0b10,
       MODULE_NAME = 0b11
     };
-    static constexpr uint8_t NEWLINE = 1 << 5;
+    static constexpr uint8_t NEWLINE = 1 << 4;
+    static constexpr uint8_t ALLOW_ESCAPE = 1 << 5;
     static constexpr uint8_t ALLOW_INTERPOLATION = 1 << 6;
     static constexpr uint8_t EXPECTING_EXPRESSION = 1 << 7;
     uint8_t flags = 0;

src/internal/syntax/ruby/parse.cc

@@ -35,72 +35,83 @@ inline uint8_t utf8_codepoint_width(unsigned char c) {
 }
 
 bool handle_escapes(RubyParser &p, std::vector<io::Token> *tokens, uint32_t &start, bool string = true) {
-  // TODO: properly handle escapes for %q ' etc.
-  if (p.peek() == '\\') {
-    if (string)
-      tokens->push_back({start, p.i, io::Token::String});
-    else
-      tokens->push_back({start, p.i, io::Token::Regexp});
-    start = p.i;
+  if (p.peek() != '\\')
+    return false;
+  if (!(p.current().flags & RubyState::RubyInternalState::ALLOW_ESCAPE)) {
     p.advance();
-    if (p.peek() == 'x') {
-      p.advance();
-      if (is_hex(p.peek()))
-        p.advance();
-      if (is_hex(p.peek()))
-        p.advance();
-    } else if (p.peek() == 'u') {
-      p.advance();
-      if (p.peek() == '{') {
-        p.advance();
-        while (p.peek() != '}' && p.peek() != '\0')
-          p.advance();
-        if (p.peek() == '}')
-          p.advance();
-      } else {
-        if (is_hex(p.peek()))
-          p.advance();
-        if (is_hex(p.peek()))
-          p.advance();
-        if (is_hex(p.peek()))
-          p.advance();
-        if (is_hex(p.peek()))
-          p.advance();
-      }
-    } else if ('0' <= p.peek() && p.peek() <= '7') {
-      p.advance();
-      if ('0' <= p.peek() && p.peek() <= '7')
-        p.advance();
-      if ('0' <= p.peek() && p.peek() <= '7')
-        p.advance();
-    } else if (p.peek() == 'c') {
-      p.advance();
-      if (p.peek() != '\\')
-        p.advance();
-    } else if (p.peek() == 'M' || p.peek() == 'C') {
-      p.advance();
-      if (p.peek() == '-') {
-        p.advance();
-        if (p.peek() != '\\')
-          p.advance();
-      }
-    } else if (p.peek() == 'N') {
-      p.advance();
-      if (p.peek() == '{') {
-        p.advance();
-        while (p.peek() != '}' && p.peek() != '\0')
-          p.advance();
-        if (p.peek() == '}')
-          p.advance();
-      }
-    } else {
-      p.advance();
-    }
-    tokens->push_back({start, p.i, io::Token::Escape});
+    if (p.peek() != '\'' && p.peek() != '\\')
+      return false;
+    if (string)
+      tokens->push_back({start, p.i - 1, io::Token::String});
+    else
+      tokens->push_back({start, p.i - 1, io::Token::Regexp});
+    p.advance();
+    tokens->push_back({p.i - 2, p.i, io::Token::Escape});
     start = p.i;
     return true;
   }
-  return false;
+  if (string)
+    tokens->push_back({start, p.i, io::Token::String});
+  else
+    tokens->push_back({start, p.i, io::Token::Regexp});
+  start = p.i;
+  p.advance();
+  if (p.peek() == 'x') {
+    p.advance();
+    if (is_hex(p.peek()))
+      p.advance();
+    if (is_hex(p.peek()))
+      p.advance();
+  } else if (p.peek() == 'u') {
+    p.advance();
+    if (p.peek() == '{') {
+      p.advance();
+      while (p.peek() != '}' && p.peek() != '\0')
+        p.advance();
+      if (p.peek() == '}')
+        p.advance();
+    } else {
+      if (is_hex(p.peek()))
+        p.advance();
+      if (is_hex(p.peek()))
+        p.advance();
+      if (is_hex(p.peek()))
+        p.advance();
+      if (is_hex(p.peek()))
+        p.advance();
+    }
+  } else if ('0' <= p.peek() && p.peek() <= '7') {
+    p.advance();
+    if ('0' <= p.peek() && p.peek() <= '7')
+      p.advance();
+    if ('0' <= p.peek() && p.peek() <= '7')
+      p.advance();
+  } else if (p.peek() == 'c') {
+    p.advance();
+    if (p.peek() != '\\')
+      p.advance();
+  } else if (p.peek() == 'M' || p.peek() == 'C') {
+    p.advance();
+    if (p.peek() == '-') {
+      p.advance();
+      if (p.peek() != '\\')
+        p.advance();
+    }
+  } else if (p.peek() == 'N') {
+    p.advance();
+    if (p.peek() == '{') {
+      p.advance();
+      while (p.peek() != '}' && p.peek() != '\0')
+        p.advance();
+      if (p.peek() == '}')
+        p.advance();
+    }
+  } else {
+    p.advance();
+  }
+  tokens->push_back({start, p.i, io::Token::Escape});
+  start = p.i;
+  return true;
 };
 
 bool handle_heredoc(RubyParser &p, std::vector<io::Token> *tokens, std::vector<ParseEvent> *events) {
@@ -669,6 +680,7 @@ bool handle_syntax(RubyParser &p, std::vector<io::Token> *tokens, std::vector<Pa
     p.current().delim_start = '\'';
     p.current().delim_end = '\'';
     p.current().flags &= ~RubyState::RubyInternalState::ALLOW_INTERPOLATION;
+    p.current().flags &= ~RubyState::RubyInternalState::ALLOW_ESCAPE;
     p.advance();
     return false;
   }
@@ -679,6 +691,7 @@ bool handle_syntax(RubyParser &p, std::vector<io::Token> *tokens, std::vector<Pa
     p.current().delim_start = '"';
     p.current().delim_end = '"';
     p.current().flags |= RubyState::RubyInternalState::ALLOW_INTERPOLATION;
+    p.current().flags |= RubyState::RubyInternalState::ALLOW_ESCAPE;
     p.advance();
     return false;
   }
@@ -704,6 +717,7 @@ bool handle_syntax(RubyParser &p, std::vector<io::Token> *tokens, std::vector<Pa
     char delim_start = '\0';
     char delim_end = '\0';
     bool allow_interp = true;
+    bool allow_escape = true;
     int prefix_len = 1;
     bool is_regexp = false;
     switch (type) {
@@ -717,6 +731,7 @@ bool handle_syntax(RubyParser &p, std::vector<io::Token> *tokens, std::vector<Pa
     case 'I':
     case 'W':
       allow_interp = true;
+      allow_escape = true;
       prefix_len = 2;
       break;
     case 'w':
@@ -724,10 +739,12 @@ bool handle_syntax(RubyParser &p, std::vector<io::Token> *tokens, std::vector<Pa
     case 'i':
     case 's':
       allow_interp = false;
+      allow_escape = false;
       prefix_len = 2;
       break;
     default:
       allow_interp = true;
+      allow_escape = true;
       prefix_len = 1;
       break;
     }
@@ -769,6 +786,8 @@ bool handle_syntax(RubyParser &p, std::vector<io::Token> *tokens, std::vector<Pa
     p.current().delim_end = delim_end;
     if (allow_interp)
       p.current().flags |= RubyState::RubyInternalState::ALLOW_INTERPOLATION;
+    if (allow_escape)
+      p.current().flags |= RubyState::RubyInternalState::ALLOW_ESCAPE;
     p.current().lit_brace_level = 1;
     p.advance(prefix_len + 1);
     return false;