Add file support for more file-types

- and syntax highlighting for bash

.gitmodules

@@ -54,3 +54,15 @@
 	path = libs/tree-sitter-bash
 	url = https://github.com/tree-sitter/tree-sitter-bash.git
   ignore = dirty
+[submodule "libs/tree-sitter-markdown"]
+	path = libs/tree-sitter-markdown
+	url = https://github.com/tree-sitter-grammars/tree-sitter-markdown
+[submodule "libs/tree-sitter-make"]
+	path = libs/tree-sitter-make
+	url = https://github.com/tree-sitter-grammars/tree-sitter-make
+[submodule "libs/tree-sitter-lua"]
+	path = libs/tree-sitter-lua
+	url = https://github.com/tree-sitter-grammars/tree-sitter-lua
+[submodule "libs/tree-sitter-fish"]
+	path = libs/tree-sitter-fish
+	url = https://github.com/ram02z/tree-sitter-fish

Makefile

@@ -23,11 +23,17 @@ UNICODE_SRC := $(wildcard libs/unicode_width/*.c)
 UNICODE_OBJ_DEBUG := $(patsubst libs/unicode_width/%.c,$(OBJ_DIR)/debug/unicode_width/%.o,$(UNICODE_SRC))
 UNICODE_OBJ_RELEASE := $(patsubst libs/unicode_width/%.c,$(OBJ_DIR)/release/unicode_width/%.o,$(UNICODE_SRC))
 
+TREE_SITTER_LIBS := $(wildcard libs/tree-sitter-*/libtree-sitter*.a)
+FISH_OBJ_PARSER := libs/tree-sitter-fish/build/Release/obj.target/tree_sitter_fish_binding/src/parser.o
+FISH_OBJ_SCANNER := libs/tree-sitter-fish/build/Release/obj.target/tree_sitter_fish_binding/src/scanner.o
+
 LIBS := \
 	libs/libgrapheme/libgrapheme.a \
 	libs/tree-sitter/libtree-sitter.a \
-	libs/tree-sitter-ruby/libtree-sitter-ruby.a \
+	$(TREE_SITTER_LIBS) \
-	-lpcre2-8
+	$(FISH_OBJ_PARSER) \
+	$(FISH_OBJ_SCANNER) \
+	-lpcre2-8 -lmagic
 
 SRC := $(wildcard $(SRC_DIR)/*.cc)
 OBJ_DEBUG := $(patsubst $(SRC_DIR)/%.cc,$(OBJ_DIR)/debug/%.o,$(SRC))

grammar/bash.scm

@@ -0,0 +1,288 @@
+[
+  (function_definition)
+  (if_statement)
+  (case_statement)
+  (for_statement)
+  (while_statement)
+  (c_style_for_statement)
+  (heredoc_redirect)
+] @fold
+
+;; #bd9ae6 #000000 0 0 0 1
+[
+  "("
+  ")"
+  "{"
+  "}"
+  "["
+  "]"
+  "[["
+  "]]"
+  "(("
+  "))"
+] @punctuation.bracket
+
+;; #bd9ae6 #000000 0 0 0 1
+[
+  ";"
+  ";;"
+  ";&"
+  ";;&"
+  "&"
+] @punctuation.delimiter
+
+;; #ffffff #000000 0 0 0 1
+[
+  ">"
+  ">>"
+  "<"
+  "<<"
+  "&&"
+  "|"
+  "|&"
+  "||"
+  "="
+  "+="
+  "=~"
+  "=="
+  "!="
+  "&>"
+  "&>>"
+  "<&"
+  ">&"
+  ">|"
+  "<&-"
+  ">&-"
+  "<<-"
+  "<<<"
+  ".."
+  "!"
+] @operator
+
+;; #aad84c #000000 0 0 0 1
+[
+  (string)
+  (raw_string)
+  (ansi_c_string)
+  (heredoc_body)
+] @string
+
+;; #fbb152 #000000 0 0 0 1
+[
+  (heredoc_start)
+  (heredoc_end)
+] @label
+
+(variable_assignment
+  (word) @string)
+
+(command
+  argument: "$" @string) ; bare dollar
+
+(concatenation
+  (word) @string)
+
+;; #fbb152 #000000 0 0 0 1
+[
+  "if"
+  "then"
+  "else"
+  "elif"
+  "fi"
+  "case"
+  "in"
+  "esac"
+] @keyword.conditional
+
+;; #fbb152 #000000 0 0 0 1
+[
+  "for"
+  "do"
+  "done"
+  "select"
+  "until"
+  "while"
+] @keyword.repeat
+
+;; #fbb152 #000000 0 0 0 1
+[
+  "declare"
+  "typeset"
+  "readonly"
+  "local"
+  "unset"
+  "unsetenv"
+] @keyword
+
+;; #fbb152 #000000 0 0 0 1
+"export" @keyword.import
+
+;; #fbb152 #000000 0 0 0 1
+"function" @keyword.function
+
+;; #ebda8c #000000 0 0 0 1
+(special_variable_name) @constant
+
+;; #ebda8c #000000 0 0 0 1
+((word) @constant.builtin
+  (#match? @constant.builtin "^(SIGHUP|SIGINT|SIGQUIT|SIGILL|SIGTRAP|SIGABRT|SIGBUS|SIGFPE|SIGKILL|SIGUSR1|SIGSEGV|SIGUSR2|SIGPIPE|SIGALRM|SIGTERM|SIGSTKFLT|SIGCHLD|SIGCONT|SIGSTOP|SIGTSTP|SIGTTIN|SIGTTOU|SIGURG|SIGXCPU|SIGXFSZ|SIGVTALRM|SIGPROF|SIGWINCH|SIGIO|SIGPWR|SIGSYS|SIGRTMIN|SIGRTMIN\+1|SIGRTMIN\+2|SIGRTMIN\+3|SIGRTMIN\+4|SIGRTMIN\+5|SIGRTMIN\+6|SIGRTMIN\+7|SIGRTMIN\+8|SIGRTMIN\+9|SIGRTMIN\+10|SIGRTMIN\+11|SIGRTMIN\+12|SIGRTMIN\+13|SIGRTMIN\+14|SIGRTMIN\+15|SIGRTMAX\-14|SIGRTMAX\-13|SIGRTMAX\-12|SIGRTMAX\-11|SIGRTMAX\-10|SIGRTMAX\-9|SIGRTMAX\-8|SIGRTMAX\-7|SIGRTMAX\-6|SIGRTMAX\-5|SIGRTMAX\-4|SIGRTMAX\-3|SIGRTMAX\-2|SIGRTMAX\-1|SIGRTMAX)$"))
+
+;; #51eeba #000000 0 0 0 1
+((word) @boolean.true
+  (#match? @boolean.true "^true$"))
+
+;; #ee513a #000000 0 0 0 1
+((word) @boolean.false
+  (#match? @boolean.false "^false$"))
+
+;; #AAAAAA #000000 0 1 0 1
+(comment) @comment @spell
+
+;; #ffffff #000000 0 0 0 1
+(test_operator) @operator
+
+;; #e6a24c #000000 0 0 0 2
+(command_substitution
+  "$(" @punctuation.special
+  ")" @punctuation.special)
+
+;; #e6a24c #000000 0 0 0 2
+(process_substitution
+  [
+    "<("
+    ">("
+  ] @punctuation.special
+  ")" @punctuation.special)
+
+;; #e6a24c #000000 0 0 0 2
+(arithmetic_expansion
+  [
+    "$(("
+    "(("
+  ] @punctuation.special
+  "))" @punctuation.special)
+
+;; #bd9ae6 #000000 0 0 0 1
+(arithmetic_expansion
+  "," @punctuation.delimiter)
+
+;; #ffffff #000000 0 0 0 1
+(ternary_expression
+  [
+    "?"
+    ":"
+  ] @keyword.conditional.ternary)
+
+;; #ffffff #000000 0 0 0 1
+(binary_expression
+  operator: _ @operator)
+
+;; #ffffff #000000 0 0 0 1
+(unary_expression
+  operator: _ @operator)
+
+;; #ffffff #000000 0 0 0 1
+(postfix_expression
+  operator: _ @operator)
+
+;; #aad84c #000000 0 0 0 3
+(function_definition
+  name: (word) @function)
+
+;; #aad84c #000000 0 0 0 3
+(command_name
+  (word) @function.call)
+
+;; #aad84c #000000 0 0 0 3
+(command_name
+  (word) @function.builtin
+  (#match? @function.builtin "^(\.|\:|alias|bg|bind|break|builtin|caller|cd|command|compgen|complete|compopt|continue|coproc|dirs|disown|echo|enable|eval|exec|exit|false|fc|fg|getopts|hash|help|history|jobs|kill|let|logout|mapfile|popd|printf|pushd|pwd|read|readarray|return|set|shift|shopt|source|suspend|test|time|times|trap|true|type|typeset|ulimit|umask|unalias|wait)$"))
+
+;; #ffffff #000000 0 0 0 1
+(command
+  argument: [
+    (word) @variable.parameter
+    (concatenation
+      (word) @variable.parameter)
+  ])
+
+;; #ffffff #000000 0 0 0 1
+(declaration_command
+  (word) @variable.parameter)
+
+;; #ffffff #000000 0 0 0 1
+(unset_command
+  (word) @variable.parameter)
+
+;; #ebda8c #000000 0 0 0 2
+(number) @number
+
+;; #ebda8c #000000 0 0 0 2
+((word) @number
+  (#match? @number "^[0-9]+$"))
+
+;; #aad84c #000000 0 0 0 1
+(file_redirect
+  (word) @string.special.path)
+
+;; #aad84c #000000 0 0 0 1
+(herestring_redirect
+  (word) @string)
+
+;; #ffffff #000000 0 0 0 1
+(file_descriptor) @operator
+
+;; #e6a24c #000000 0 0 0 2
+(simple_expansion
+  "$" @punctuation.special) @none
+
+;; #e6a24c #000000 0 0 0 2
+(expansion
+  "${" @punctuation.special
+  "}" @punctuation.special) @none
+
+;; #e6a24c #000000 0 0 0 2
+(expansion
+  operator: _ @punctuation.special)
+
+;; #e6a24c #000000 0 0 0 2
+(expansion
+  "@"
+  .
+  operator: _ @character.special)
+
+;; #e6a24c #000000 0 0 0 2
+((expansion
+  (subscript
+    index: (word) @character.special))
+  (#any-of? @character.special "@" "*"))
+
+;; #e6a24c #000000 0 0 0 2
+"``" @punctuation.special
+
+;; #ffffff #000000 0 0 0 1
+(variable_name) @variable
+
+;; #ebda8c #000000 0 0 0 1
+((variable_name) @constant
+  (#match? @constant "^[A-Z][A-Z_0-9]*$"))
+
+;; #ffffff #000000 0 0 0 1
+((variable_name) @variable.builtin
+  (#match? @variable.builtin "^(CDPATH|HOME|IFS|MAIL|MAILPATH|OPTARG|OPTIND|PATH|PS1|PS2|_|BASH|BASHOPTS|BASHPID|BASH_ALIASES|BASH_ARGC|BASH_ARGV|BASH_ARGV0|BASH_CMDS|BASH_COMMAND|BASH_COMPAT|BASH_ENV|BASH_EXECUTION_STRING|BASH_LINENO|BASH_LOADABLES_PATH|BASH_REMATCH|BASH_SOURCE|BASH_SUBSHELL|BASH_VERSINFO|BASH_VERSION|BASH_XTRACEFD|CHILD_MAX|COLUMNS|COMP_CWORD|COMP_LINE|COMP_POINT|COMP_TYPE|COMP_KEY|COMP_WORDBREAKS|COMP_WORDS|COMPREPLY|COPROC|DIRSTACK|EMACS|ENV|EPOCHREALTIME|EPOCHSECONDS|EUID|EXECIGNORE|FCEDIT|FIGNORE|FUNCNAME|FUNCNEST|GLOBIGNORE|GROUPS|histchars|HISTCMD|HISTCONTROL|HISTFILE|HISTFILESIZE|HISTIGNORE|HISTSIZE|HISTTIMEFORMAT|HOSTFILE|HOSTNAME|HOSTTYPE|IGNOREEOF|INPUTRC|INSIDE_EMACS|LANG|LC_ALL|LC_COLLATE|LC_CTYPE|LC_MESSAGES|LC_NUMERIC|LC_TIME|LINENO|LINES|MACHTYPE|MAILCHECK|MAPFILE|OLDPWD|OPTERR|OSTYPE|PIPESTATUS|POSIXLY_CORRECT|PPID|PROMPT_COMMAND|PROMPT_DIRTRIM|PS0|PS3|PS4|PWD|RANDOM|READLINE_ARGUMENT|READLINE_LINE|READLINE_MARK|READLINE_POINT|REPLY|SECONDS|SHELL|SHELLOPTS|SHLVL|SRANDOM|TIMEFORMAT|TMOUT|TMPDIR|UID)$"))
+
+;; #ffffff #000000 0 0 0 1
+(case_item
+  value: (word) @variable.parameter)
+
+;; #e6a24c #000000 0 0 0 2
+[
+  (regex)
+  (extglob_pattern)
+] @string.regexp
+
+;; #fbb152 #000000 0 0 0 3
+((program
+  .
+  (comment) @keyword.directive @nospell)
+  (#match? @keyword.directive "^#!/"))

include/ts_def.h

@@ -0,0 +1,24 @@
+#include "../libs/tree-sitter/lib/include/tree_sitter/api.h"
+#include <string>
+
+struct Language {
+  std::string name;
+  const TSLanguage *(*fn)();
+};
+
+extern "C" {
+const TSLanguage *tree_sitter_bash(void);
+const TSLanguage *tree_sitter_c(void);
+const TSLanguage *tree_sitter_cpp(void);
+const TSLanguage *tree_sitter_css(void);
+const TSLanguage *tree_sitter_fish(void);
+const TSLanguage *tree_sitter_go(void);
+const TSLanguage *tree_sitter_haskell(void);
+const TSLanguage *tree_sitter_html(void);
+const TSLanguage *tree_sitter_javascript(void);
+const TSLanguage *tree_sitter_json(void);
+const TSLanguage *tree_sitter_lua(void);
+const TSLanguage *tree_sitter_make(void);
+const TSLanguage *tree_sitter_python(void);
+const TSLanguage *tree_sitter_ruby(void);
+}

include/utils.h

@@ -1,6 +1,7 @@
 #ifndef UTILS_H
 #define UTILS_H
 
+#include "./ts_def.h"
 #include <mutex>
 #include <queue>
 #include <string>
@@ -33,5 +34,8 @@ uint32_t get_bytes_from_visual_col(const char *line,
                                    uint32_t target_visual_col);
 void log(const char *fmt, ...);
 std::string get_exe_dir();
+char *load_file(const char *path, uint32_t *out_len);
+char *detect_file_type(const char *filename);
+Language language_for_file(const char *filename);
 
 #endif

src/editor.cc

@@ -3,29 +3,8 @@ extern "C" {
 }
 #include "../include/editor.h"
 #include "../include/ts.h"
-#include "../libs/tree-sitter-ruby/bindings/c/tree-sitter-ruby.h"
+#include "../include/utils.h"
 #include <cstdint>
-#include <fstream>
-
-char *load_file(const char *path, uint32_t *out_len) {
-  std::ifstream file(path, std::ios::in | std::ios::binary | std::ios::ate);
-  if (!file.is_open())
-    return nullptr;
-  std::streamsize len = file.tellg();
-  if (len < 0 || (std::uint32_t)len > 0xFFFFFFFF)
-    return nullptr;
-  file.seekg(0, std::ios::beg);
-  char *buf = (char *)malloc(static_cast<std::uint32_t>(len));
-  if (!buf)
-    return nullptr;
-  if (file.read(buf, len)) {
-    *out_len = static_cast<uint32_t>(len);
-    return buf;
-  } else {
-    free(buf);
-    return nullptr;
-  }
-}
 
 Editor *new_editor(const char *filename, Coord position, Coord size) {
   Editor *editor = new Editor();
@@ -47,11 +26,12 @@ Editor *new_editor(const char *filename, Coord position, Coord size) {
   editor->root = load(str, len, optimal_chunk_size(len));
   free(str);
   editor->folded.resize(editor->root->line_count + 2);
-  std::string query = get_exe_dir() + "/../grammar/ruby.scm";
   if (len < (1024 * 64)) {
     editor->parser = ts_parser_new();
-    editor->language = tree_sitter_ruby();
+    Language language = language_for_file(filename);
+    editor->language = language.fn();
     ts_parser_set_language(editor->parser, editor->language);
+    std::string query = get_exe_dir() + "/../grammar/" + language.name + ".scm";
     editor->query = load_query(query.c_str(), editor);
   }
   return editor;

src/utils.cc

@@ -2,13 +2,18 @@ extern "C" {
 #include "../libs/libgrapheme/grapheme.h"
 }
 #include "../include/utils.h"
+#include <algorithm>
 #include <cstdarg>
 #include <cstdint>
 #include <cstdio>
+#include <cstring>
+#include <fstream>
 #include <limits.h>
+#include <magic.h>
 #include <string.h>
 #include <string>
 #include <unistd.h>
+#include <unordered_map>
 
 std::string get_exe_dir() {
   char exe_path[PATH_MAX];
@@ -76,12 +81,117 @@ void log(const char *fmt, ...) {
   FILE *fp = fopen("/tmp/log.txt", "a");
   if (!fp)
     return;
-
   va_list args;
   va_start(args, fmt);
   vfprintf(fp, fmt, args);
   va_end(args);
-
   fputc('\n', fp);
   fclose(fp);
 }
+
+char *load_file(const char *path, uint32_t *out_len) {
+  std::ifstream file(path, std::ios::in | std::ios::binary | std::ios::ate);
+  if (!file.is_open())
+    return nullptr;
+  std::streamsize len = file.tellg();
+  if (len < 0 || (std::uint32_t)len > 0xFFFFFFFF)
+    return nullptr;
+  file.seekg(0, std::ios::beg);
+  char *buf = (char *)malloc(static_cast<std::uint32_t>(len));
+  if (!buf)
+    return nullptr;
+  if (file.read(buf, len)) {
+    *out_len = static_cast<uint32_t>(len);
+    return buf;
+  } else {
+    free(buf);
+    return nullptr;
+  }
+}
+
+static std::string file_extension(const char *filename) {
+  std::string name(filename);
+  auto pos = name.find_last_of('.');
+  if (pos == std::string::npos)
+    return "";
+  std::string ext = name.substr(pos + 1);
+  std::transform(ext.begin(), ext.end(), ext.begin(), ::tolower);
+  return ext;
+}
+
+char *detect_file_type(const char *filename) {
+  magic_t magic = magic_open(MAGIC_MIME_TYPE);
+  if (!magic)
+    return nullptr;
+  if (magic_load(magic, nullptr) != 0) {
+    magic_close(magic);
+    return nullptr;
+  }
+  const char *type = magic_file(magic, filename);
+  if (!type) {
+    magic_close(magic);
+    return nullptr;
+  }
+  char *result = strdup(type);
+  magic_close(magic);
+  return result;
+}
+
+static const std::unordered_map<std::string, Language> ext_map = {
+    {"sh", {"bash", tree_sitter_bash}},
+    {"bash", {"bash", tree_sitter_bash}},
+    {"c", {"c", tree_sitter_c}},
+    {"cpp", {"cpp", tree_sitter_cpp}},
+    {"cxx", {"cpp", tree_sitter_cpp}},
+    {"cc", {"cpp", tree_sitter_cpp}},
+    {"hpp", {"cpp", tree_sitter_cpp}},
+    {"hh", {"cpp", tree_sitter_cpp}},
+    {"hxx", {"cpp", tree_sitter_cpp}},
+    {"h", {"cpp", tree_sitter_cpp}},
+    {"css", {"css", tree_sitter_css}},
+    {"fish", {"fish", tree_sitter_fish}},
+    {"go", {"go", tree_sitter_go}},
+    {"hs", {"haskell", tree_sitter_haskell}},
+    {"html", {"html", tree_sitter_html}},
+    {"htm", {"html", tree_sitter_html}},
+    {"js", {"javascript", tree_sitter_javascript}},
+    {"json", {"json", tree_sitter_json}},
+    {"lua", {"lua", tree_sitter_lua}},
+    {"mk", {"make", tree_sitter_make}},
+    {"makefile", {"make", tree_sitter_make}},
+    {"py", {"python", tree_sitter_python}},
+    {"rb", {"ruby", tree_sitter_ruby}},
+};
+
+static const std::unordered_map<std::string, Language> mime_map = {
+    {"text/x-c", {"c", tree_sitter_c}},
+    {"text/x-c++", {"cpp", tree_sitter_cpp}},
+    {"text/x-shellscript", {"bash", tree_sitter_bash}},
+    {"application/json", {"json", tree_sitter_json}},
+    {"text/javascript", {"javascript", tree_sitter_javascript}},
+    {"text/html", {"html", tree_sitter_html}},
+    {"text/css", {"css", tree_sitter_css}},
+    {"text/x-python", {"python", tree_sitter_python}},
+    {"text/x-ruby", {"ruby", tree_sitter_ruby}},
+    {"text/x-go", {"go", tree_sitter_go}},
+    {"text/x-haskell", {"haskell", tree_sitter_haskell}},
+    {"text/x-lua", {"lua", tree_sitter_lua}},
+};
+
+Language language_for_file(const char *filename) {
+  std::string ext = file_extension(filename);
+  if (!ext.empty()) {
+    auto it = ext_map.find(ext);
+    if (it != ext_map.end())
+      return it->second;
+  }
+  char *mime = detect_file_type(filename);
+  if (mime) {
+    std::string mime_type(mime);
+    free(mime);
+    auto it = mime_map.find(mime_type);
+    if (it != mime_map.end())
+      return it->second;
+  }
+  return {"unknown", nullptr};
+}