Add indentation engine and other fixes

src/utils/text.cc

@@ -42,6 +42,39 @@ std::string percent_encode(const std::string &s) {
   return out;
 }
 
+std::string substitute_fence(const std::string &documentation,
+                             const std::string &lang) {
+  int errorcode;
+  PCRE2_SIZE erroroffset;
+  const char *pattern = "\n```\n(.*?)```\n";
+  pcre2_code *re =
+      pcre2_compile((PCRE2_SPTR)pattern, PCRE2_ZERO_TERMINATED, PCRE2_DOTALL,
+                    &errorcode, &erroroffset, nullptr);
+  if (!re)
+    return documentation;
+  PCRE2_SIZE outlen = 0;
+  std::string replacement = "```" + lang + "\n$1```";
+  int rc = pcre2_substitute(
+      re, (PCRE2_SPTR)documentation.c_str(), documentation.size(), 0,
+      PCRE2_SUBSTITUTE_GLOBAL | PCRE2_SUBSTITUTE_OVERFLOW_LENGTH, nullptr,
+      nullptr, (PCRE2_SPTR)replacement.c_str(), replacement.size(), nullptr,
+      &outlen);
+  if (rc < 0) {
+    pcre2_code_free(re);
+    return documentation;
+  }
+  std::string out(outlen, '\0');
+  rc = pcre2_substitute(re, (PCRE2_SPTR)documentation.c_str(),
+                        documentation.size(), 0, PCRE2_SUBSTITUTE_GLOBAL,
+                        nullptr, nullptr, (PCRE2_SPTR)replacement.c_str(),
+                        replacement.size(), (PCRE2_UCHAR *)out.data(), &outlen);
+  pcre2_code_free(re);
+  if (rc < 0)
+    return documentation;
+  out.resize(outlen);
+  return out;
+}
+
 std::string trim(const std::string &s) {
   size_t start = s.find_first_not_of(" \t\n\r");
   if (start == std::string::npos)

src/utils/unicode.cc

@@ -86,9 +86,9 @@ uint32_t count_clusters(const char *line, size_t len, size_t from, size_t to) {
   return count;
 }
 
-int utf8_byte_offset_to_utf16(const char *s, size_t byte_pos) {
-  int utf16_units = 0;
-  size_t i = 0;
+uint32_t utf8_byte_offset_to_utf16(const char *s, uint32_t byte_pos) {
+  uint32_t utf16_units = 0;
+  uint32_t i = 0;
   while (i < byte_pos) {
     unsigned char c = s[i];
     if ((c & 0x80) == 0x00) {
@@ -108,9 +108,9 @@ int utf8_byte_offset_to_utf16(const char *s, size_t byte_pos) {
   return utf16_units;
 }
 
-size_t utf16_offset_to_utf8(const char *s, int utf16_pos) {
-  int utf16_units = 0;
-  size_t i = 0;
+uint32_t utf16_offset_to_utf8(const char *s, uint32_t utf16_pos) {
+  uint32_t utf16_units = 0;
+  uint32_t i = 0;
   while (utf16_units < utf16_pos) {
     unsigned char c = s[i];
     if ((c & 0x80) == 0x00) {