Allow ruby based configs and custom syntax parsers

Makefile

@@ -15,13 +15,13 @@ CXX := $(CCACHE) clang++
 CFLAGS_DEBUG :=\
 	-std=c++20 -Wall -Wextra \
 	-O0 -fno-inline -gsplit-dwarf\
-	-g -fsanitize=address -fno-omit-frame-pointer\
+	-g -fno-omit-frame-pointer\
 	-Wno-unused-command-line-argument \
 	-I./include -I./libs \
 	-I/usr/include/ruby-3.4.0 -I/usr/include/ruby-3.4.0/x86_64-linux
 CFLAGS_RELEASE :=\
 	-std=c++20 -O3 -march=native \
-	-fno-exceptions -fno-rtti -fstrict-aliasing \
+  -fno-rtti -fstrict-aliasing \
 	-ffast-math -flto=thin \
 	-fvisibility=hidden -fuse-ld=lld \
 	-fomit-frame-pointer -DNDEBUG -s \
@@ -41,7 +41,7 @@ UNICODE_OBJ_RELEASE := $(patsubst libs/unicode_width/%.c,$(OBJ_DIR)/release/unic
 
 LIBS := \
 	libs/libgrapheme/libgrapheme.a \
-	-lpcre2-8 -lmagic
+	-lpcre2-8 -lmagic -lruby
 
 SRC := $(wildcard $(SRC_DIR)/**/*.cc) $(wildcard $(SRC_DIR)/*.cc)
 OBJ_DEBUG := $(patsubst $(SRC_DIR)/%.cc,$(OBJ_DIR)/debug/%.o,$(SRC))

TODO.md

@@ -39,6 +39,8 @@ probably remove solargraph support and use ruby-lsp (or sorbet?) instead
 
 move lsp configs to json and also allow configs for windows-style vs unix-style line endings and utf-8 vs utf-16
 
+* the ruby should have an api to be able to draw windows and add mappings to them
+
 * finish bash then do all the directive-like ones like jsonc (first to help with theme files) / toml / yaml / ini / nginx
 * then markdown / html
 * then gitignore / gitattributes

config/config.json

@@ -1,8 +0,0 @@
-{
-  "startup": "bash $CRIB_CONFIG_DIR/startup.sh",
-  "shutdown": "bash $CRIB_CONFIG_DIR/shutdown.sh",
-
-  "theme": "theme.json",
-
-  "lsp_config": "lsp_config.json"
-}

config/libcrib.rb

@@ -0,0 +1,419 @@
+module C
+  K_DATA               = 0
+  K_SHEBANG            = 1
+  K_COMMENT            = 2
+  K_ERROR              = 3
+  K_STRING             = 4
+  K_ESCAPE             = 5
+  K_INTERPOLATION      = 6
+  K_REGEXP             = 7
+  K_NUMBER             = 8
+  K_TRUE               = 9
+  K_FALSE              = 10
+  K_CHAR               = 11
+  K_KEYWORD            = 12
+  K_KEYWORDOPERATOR    = 13
+  K_OPERATOR           = 14
+  K_FUNCTION           = 15
+  K_TYPE               = 16
+  K_CONSTANT           = 17
+  K_VARIABLEINSTANCE   = 18
+  K_VARIABLEGLOBAL     = 19
+  K_ANNOTATION         = 20
+  K_DIRECTIVE          = 21
+  K_LABEL              = 22
+  K_BRACE1             = 23
+  K_BRACE2             = 24
+  K_BRACE3             = 25
+  K_BRACE4             = 26
+  K_BRACE5             = 27
+  K_HEADING1           = 28
+  K_HEADING2           = 29
+  K_HEADING3           = 30
+  K_HEADING4           = 31
+  K_HEADING5           = 32
+  K_HEADING6           = 33
+  K_BLOCKQUOTE         = 34
+  K_LIST               = 35
+  K_LISTITEM           = 36
+  K_CODE               = 37
+  K_LANGUAGENAME       = 38
+  K_LINKLABEL          = 39
+  K_IMAGELABEL         = 40
+  K_LINK               = 41
+  K_TABLE              = 42
+  K_TABLEHEADER        = 43
+  K_ITALIC             = 44
+  K_BOLD               = 45
+  K_UNDERLINE          = 46
+  K_STRIKETHROUGH      = 47
+  K_HORIXONTALRULE     = 48
+  K_TAG                = 49
+  K_ATTRIBUTE          = 50
+  K_CHECKDONE          = 51
+  K_CHECKNOTDONE       = 52
+  @lsp_config = {
+    "clangd" => [
+      "--background-index",
+      "--clang-tidy",
+      "--completion-style=detailed",
+      "--header-insertion=never",
+      "--pch-storage=memory",
+      "--limit-results=50",
+      "--log=error"
+    ],
+    "ruby-lsp" => [],
+    "solargraph" => ["stdio"],
+    "bash-language-server" => ["start"],
+    "vscode-css-language-server" => ["--stdio"],
+    "vscode-json-language-server" => ["--stdio"],
+    "fish-lsp" => ["start"],
+    "gopls" => ["serve"],
+    "haskell-language-server" => ["lsp"],
+    "emmet-language-server" => ["--stdio"],
+    "typescript-language-server" => ["--stdio"],
+    "lua-language-server" => [],
+    "pyright-langserver" => ["--stdio"],
+    "rust-analyzer" => [],
+    "intelephense" => ["--stdio"],
+    "marksman" => ["server"],
+    "nginx-language-server" => [],
+    "taplo" => ["lsp", "stdio"],
+    "yaml-language-server" => ["--stdio"],
+    "sqls" => ["serve"],
+    "make-language-server" => [],
+    "sql-language-server" => ["up", "--method", "stdio"]
+  }
+  @languages = {
+    c: {
+      color: 0x555555,
+      symbol: " ",
+      extensions: ["c"],
+      filenames: [],
+      mimetypes: ["text/x-c"],
+      lsp: "clangd"
+    },
+    cpp: {
+      color: 0x00599C,
+      symbol: " ",
+      extensions: ["cpp", "cc", "cxx"],
+      filenames: [],
+      mimetypes: ["text/x-cpp"],
+      lsp: "clangd"
+    },
+    h: {
+      color: 0xA8B9CC,
+      symbol: " ",
+      extensions: ["h", "hpp"],
+      filenames: [],
+      mimetypes: ["text/x-c-header"],
+      lsp: "clangd"
+    },
+    css: {
+      color: 0x36a3d9,
+      symbol: " ",
+      extensions: ["css"],
+      filenames: [],
+      mimetypes: ["text/css"],
+      lsp: "vscode-css-language-server"
+    },
+    fish: {
+      color: 0x4d5a5e,
+      symbol: " ",
+      extensions: ["fish"],
+      filenames: [],
+      mimetypes: ["application/x-fish"],
+      lsp: "fish-lsp"
+    },
+    go: {
+      color: 0x00add8,
+      symbol: " ",
+      extensions: ["go"],
+      filenames: [],
+      mimetypes: ["text/x-go"],
+      lsp: "gopls"
+    },
+    gomod: {
+      color: 0x00add8,
+      symbol: " ",
+      extensions: ["mod"],
+      filenames: [],
+      mimetypes: ["text/x-go-mod"],
+      lsp: "gopls"
+    },
+    haskell: {
+      color: 0xa074c4,
+      symbol: " ",
+      extensions: ["hs", "lhs"],
+      filenames: [],
+      mimetypes: ["text/x-haskell"],
+      lsp: "haskell-language-server"
+    },
+    html: {
+      color: 0xef8a91,
+      symbol: " ",
+      extensions: ["html", "htm"],
+      filenames: [],
+      mimetypes: ["text/html"],
+      lsp: "emmet-language-server"
+    },
+    javascript: {
+      color: 0xf0df8a,
+      symbol: " ",
+      extensions: ["js"],
+      filenames: [],
+      mimetypes: ["application/javascript"],
+      lsp: "typescript-language-server"
+    },
+    typescript: {
+      color: 0x36a3d9,
+      symbol: " ",
+      extensions: ["ts"],
+      filenames: [],
+      mimetypes: ["application/typescript"],
+      lsp: "typescript-language-server"
+    },
+    json: {
+      color: 0xcbcb41,
+      symbol: "{}",
+      extensions: ["json"],
+      filenames: [],
+      mimetypes: ["application/json"],
+      lsp: "vscode-json-language-server"
+    },
+    jsonc: {
+      color: 0xcbcb41,
+      symbol: "{}",
+      extensions: ["jsonc"],
+      filenames: [],
+      mimetypes: ["application/json"],
+      lsp: "vscode-json-language-server"
+    },
+    erb: {
+      color: 0x6e1516,
+      symbol: " ",
+      extensions: ["erb"],
+      filenames: [],
+      mimetypes: ["text/x-erb"],
+      lsp: "emmet-language-server"
+    },
+    lua: {
+      color: 0x36a3d9,
+      symbol: "󰢱 ",
+      extensions: ["lua"],
+      filenames: [],
+      mimetypes: ["text/x-lua"],
+      lsp: "lua-language-server"
+    },
+    python: {
+      color: 0x95e6cb,
+      symbol: "󰌠 ",
+      extensions: ["py"],
+      filenames: [],
+      mimetypes: ["text/x-python"],
+      lsp: "pyright"
+    },
+    rust: {
+      color: 0xdea584,
+      symbol: "󱘗 ",
+      extensions: ["rs"],
+      filenames: [],
+      mimetypes: ["text/x-rust"],
+      lsp: "rust-analyzer"
+    },
+    php: {
+      color: 0xa074c4,
+      symbol: "󰌟 ",
+      extensions: ["php"],
+      filenames: [],
+      mimetypes: ["application/x-php"],
+      lsp: "intelephense"
+    },
+    markdown: {
+      color: 0x36a3d9,
+      symbol: " ",
+      extensions: ["md", "markdown"],
+      filenames: [],
+      mimetypes: ["text/markdown"],
+      lsp: "marksman"
+    },
+    nginx: {
+      color: 0x6d8086,
+      symbol: " ",
+      extensions: ["conf"],
+      filenames: [],
+      mimetypes: ["text/nginx"],
+      lsp: "nginx-language-server"
+    },
+    toml: {
+      color: 0x36a3d9,
+      symbol: " ",
+      extensions: ["toml"],
+      filenames: [],
+      mimetypes: ["application/toml"],
+      lsp: "taplo"
+    },
+    yaml: {
+      color: 0x6d8086,
+      symbol: " ",
+      extensions: ["yml", "yaml"],
+      filenames: [],
+      mimetypes: ["text/yaml"],
+      lsp: "yaml-language-server"
+    },
+    sql: {
+      color: 0xdad8d8,
+      symbol: " ",
+      extensions: ["sql"],
+      filenames: [],
+      mimetypes: ["text/x-sql"],
+      lsp: "sqls"
+    },
+    make: {
+      color: 0x4e5c61,
+      symbol: " ",
+      extensions: ["Makefile", "makefile"],
+      filenames: [],
+      mimetypes: ["text/x-makefile"],
+      lsp: "make-language-server"
+    },
+    gdscript: {
+      color: 0x6d8086,
+      symbol: " ",
+      extensions: ["gd"],
+      filenames: [],
+      mimetypes: ["text/x-gdscript"],
+      lsp: nil
+    },
+    man: {
+      color: 0xdad8d8,
+      symbol: " ",
+      extensions: ["man"],
+      filenames: [],
+      mimetypes: ["application/x-troff-man"],
+      lsp: nil
+    },
+    diff: {
+      color: 0xDD4C35,
+      symbol: " ",
+      extensions: ["diff", "patch"],
+      filenames: [],
+      mimetypes: ["text/x-diff"],
+      lsp: nil
+    },
+    gitattributes: {
+      color: 0xF05032,
+      symbol: " ",
+      extensions: ["gitattributes"],
+      filenames: [],
+      mimetypes: [],
+      lsp: nil
+    },
+    gitignore: {
+      color: 0xF05032,
+      symbol: " ",
+      extensions: ["gitignore"],
+      filenames: [],
+      mimetypes: [],
+      lsp: nil
+    },
+    regex: {
+      color: 0x9E9E9E,
+      symbol: ".*",
+      extensions: ["regex"],
+      filenames: [],
+      mimetypes: [],
+      lsp: nil
+    },
+    ini: {
+      color: 0x6d8086,
+      symbol: " ",
+      extensions: ["ini"],
+      filenames: [],
+      mimetypes: [],
+      lsp: nil
+    },
+    ruby: {
+      color: 0xff8087,
+      symbol: "󰴭 ",
+      extensions: ["rb"],
+      filenames: ["Gemfile"],
+      mimetypes: ["text/x-ruby"],
+      lsp: "solargraph"
+    },
+    bash: {
+      color: 0x4d5a5e,
+      symbol: " ",
+      extensions: ["sh"],
+      filenames: ["bash_profile", "bashrc"],
+      mimetypes: ["text/x-sh"],
+      lsp: "bash-language-server"
+    }
+  }
+  @key_handlers = {}
+  @key_binds = {}
+  @highlighters = {}
+  @log_queue = []
+  @line_endings = :unix
+
+  class << self
+    attr_accessor :theme, :lsp_config, :languages,
+                  :line_endings, :highlighters
+    attr_reader :b_startup, :b_shutdown, :b_extra_highlights
+
+    def startup(&block)
+      @b_startup = block
+    end
+
+    def shutdown(&block)
+      @b_shutdown = block
+    end
+
+    def queue_log(msg)
+      @log_queue << msg
+    end
+
+    def log_all
+      @log_queue.each do |msg|
+        puts msg
+      end
+      @log_queue = []
+    end
+
+    def extra_highlights(&block)
+      @b_extra_highlights = block
+    end
+
+    def bind(modes, keys = nil, action = nil, &block)
+      modes = [modes] unless modes.is_a?(Array)
+      if keys.nil?
+        app = self
+        dsl = Object.new
+        dsl.define_singleton_method(:set) do |k, act = nil, &blk|
+          app.bind(modes, k, act, &blk)
+        end
+        dsl.instance_exec(&block) if block_given?
+      elsif block_given?
+        keys = [keys] unless keys.is_a?(Array)
+        modes.each do |mode|
+          keys.each do |key|
+            @key_handlers[mode] ||= {}
+            @key_handlers[mode][key] ||= []
+            @key_handlers[mode][key] << block
+          end
+        end
+      elsif action.is_a?(String)
+        keys = [keys] unless keys.is_a?(Array)
+        modes.each do |mode|
+          keys.each do |key|
+            @key_binds[mode] ||= {}
+            @key_binds[mode][key] ||= []
+            @key_binds[mode][key] << action
+          end
+        end
+      end
+    end
+  end
+end
+
+at_exit { C.log_all }

config/main.rb

@@ -1,151 +1,127 @@
-module C
+require_relative "libcrib"
-  attr_accessor :theme, :lsp_config, :languages,
-                :line_endings, :utf_mode, :highlighters
-  attr_reader :b_startup, :b_shutdown, :b_extra_highlights
-
-  @lsp_config = {}
-  @languages = {}
-  @key_handlers = {}
-  @key_binds = {}
-
-  def startup(&block)
-    @b_startup = block
-  end
-
-  def shutdown(&block)
-    @b_shutdown = block
-  end
-
-  def extra_highlights(&block)
-    @b_extra_highlights = block
-  end
-
-  def bind(modes, keys = nil, action = nil, &block)
-    modes = [modes] unless modes.is_a?(Array)
-    if keys.nil?
-      app = self
-      dsl = Object.new
-      dsl.define_singleton_method(:set) do |k, act = nil, &blk|
-        app.bind(modes, k, act, &blk)
-      end
-      dsl.instance_exec(&handler)
-    elsif block_given?
-      keys = [keys] unless keys.is_a?(Array)
-      modes.each do |mode|
-        keys.each do |key|
-          @key_handlers[mode] ||= {}
-          @key_handlers[mode][key] ||= []
-          @key_handlers[mode][key] << block
-        end
-      end
-    elsif action.is_a?(String)
-      keys = [keys] unless keys.is_a?(Array)
-      modes.each do |mode|
-        keys.each do |key|
-          @key_binds[mode] ||= {}
-          @key_binds[mode][key] ||= []
-          @key_binds[mode][key] << action
-        end
-      end
-    else
-      raise ArgumentError("invalid arguments")
-    end
-  end
-end
 
 # basic configuration
 
 C.startup do
-  do_something_random_here!
+  puts "Starting crib..."
 end
 
 C.shutdown do
-  do_something_random_here!
+  puts "Exiting crib..."
 end
 
 # this can be modified by the user during runtime through keybindings
-# But i need to know how to ever read this value only when needed .
+# But i need to know how to ever read this value only when needed.
 # maybe i can write a function that notifies if theme maybe changed then reload
+# It can also be scripted to load different theme formats into a hash usable by crib
 C.theme = {
-  # i have a predefined list of keys that can be used here
+  :default => { fg: 0xEEEEEE },
-  :default => {
+  :shebang => { fg: 0x7DCFFF },
-    # here fg bg and style are all optional and have default values
+  :error => { fg: 0xEF5168 },
-    # if not specified
+  :comment => { fg: 0xAAAAAA, italic: true },
-    fg: 0xEEEEEE,
+  :string => { fg: 0xAAD94C },
-    bg: 0x000000,
+  :escape => { fg: 0x7DCFFF },
-    italic: false,
+  :interpolation => { fg: 0x7DCFFF },
-    bold: false,
+  :regexp => { fg: 0xD2A6FF },
-    underline: false,
+  :number => { fg: 0xE6C08A },
-    strikethrough: false
+  # rubocop:disable Lint/BooleanSymbol
-  }
+  :true => { fg: 0x7AE93C },
+  :false => { fg: 0xEF5168 },
+  # rubocop:enable Lint/BooleanSymbol
+  :char => { fg: 0xFFAF70 },
+  :keyword => { fg: 0xFF8F40 },
+  :keywordoperator => { fg: 0xF07178 },
+  :operator => { fg: 0xFFFFFF, italic: true },
+  :function => { fg: 0xFFAF70 },
+  :type => { fg: 0xF07178 },
+  :constant => { fg: 0x7DCFFF },
+  :variableinstance => { fg: 0x95E6CB },
+  :variableglobal => { fg: 0xF07178 },
+  :annotation => { fg: 0x7DCFFF },
+  :directive => { fg: 0xFF8F40 },
+  :label => { fg: 0xD2A6FF },
+  :brace1 => { fg: 0xD2A6FF },
+  :brace2 => { fg: 0xFFAFAF },
+  :brace3 => { fg: 0xFFFF00 },
+  :brace4 => { fg: 0x0FFF0F },
+  :brace5 => { fg: 0xFF0F0F }
 }
 
-# this part uses dsl bindings to define the bind function
+# # TODO: to be done once a proper api for binding and window drawing is made
-# Hopefully extend to give more context/power to bindings
+# #       The binds will be connected to either `editor` or windows where editor can
-# but try to keep simple for performance
+# #       only use a preset set of stuff to bind while teh windows are purely custom
-# for default keybindings
+# # # this part uses dsl bindings to define the bind function
-C.bind [:normal, :select], :a => "insert_mode"
+# # # Hopefully extend to give more context/power to bindings
-# for custom keybindings
+# # # but try to keep simple for performance
-C.bind :select, [:x, :c] do
+# # # for default keybindings
-  puts "cut"
+# # C.bind [:normal, :select], :a => "insert_mode"
-end
+# # # for custom keybindings
-C.bind :jumper do
+# # C.bind :select, [:x, :c] do
-  set [:x, :c] do
+# #   puts "cut"
-    puts "jump to first bookmark"
+# # end
-  end
+# # C.bind :jumper do
-end
+# #   set [:x, :c] do
-# they can also be defined conditionally
+# #     puts "jump to first bookmark"
-# This code is just an example and doesnt actually work
+# #   end
-if using_tmux?
+# # end
-  bind :C-p do
+# # # they can also be defined conditionally
-    system("tmux select-pane -U")
+# # # This code is just an example and doesnt actually work
-  end
+# # if using_tmux?
-end
+# #   bind :C-p do
+# #     system("tmux select-pane -U")
+# #   end
+# # end
 
-# This can for example be modified by user bindings during runtime
+# This can, for example, be modified by user bindings during runtime
-C.lsp_config[:solargraph] = {
+# TODO: dynamic registration to actually be implemented once keybinds and extentions are implemented
-  command: "solargraph",
+# A predefined list already exists and can be found in libcrib.rb
-  args: ["stdio"],
+# C.lsp_config["solargraph"] = ["stdio"]
-  languages: [:ruby]
+#
-}
+# C.languages[:ruby] = {
+#   color: 0xff8087,
+#   symbol: "󰴭 ",
+#   extensions: ["rb"],
+#   filenames: ["Gemfile"],
+#   mimetypes: ["text/x-ruby"],
+#   lsp: "solargraph"
+# }
 
-# these are actually cached into cpp by the editor upon setting
+C.line_endings = :unix # or :windows
-C.languages[:ruby] = {
-  color: 0xff8087,
-  symbol: "󰴭 ",
-  extensions: ["rb"],
-  filenames: ["Gemfile"],
-  mimetypes: ["text/x-ruby"]
-}
 
-C.line_endings = :auto_unix # or :auto_windows or :unix or :windows to force
+# C.extra_highlights do |_line, _idx|
-C.utf_mode = :auto_utf8 # or :auto_utf16 or :utf8 or :utf16 to force
+#   # the return can be an array of
+#   # [fg, bg. flags, start, end]
+#   # where fg and bg are integers (using 24 bit color)
+#   # and flags is a bitmask of bold/underline/italic etc
+#   # and start and end are integers strictly inside the line
+#   return []
+# end
 
-C.extra_highlights do |_line, _idx|
+# The highlighter will be aplied to the language as long as the langauge is defined in C.languages
-  # the return can be an array of
+C.highlighters[:ruby_n] = {
-  # [fg, bg. flags, start, end]
+  parser: ->(line, state) {
-  # where fg and bg are integers (using 24 bit color)
+    # the return value is a hash
-  # and flags is a bitmask of bold/underline/italic etc
+    # it contains the state and the highlights
-  # and start and end are integers strictly inside the line
-  return []
-end
-
-C.highlighters[:language_name] = {
-  parser: ->(_state, _line) {
-    # the return value is an array of
-    # [state, highlights]
     # state can be of any type but will be consistent between calls
-    # initially nil is sent for uninitialized state
+    # initially nil is sent for uninitialized state the returned must be anything but nil
+    # the same state can be used for multiple lines
     # the highlights can be an array of
-    # [fg, bg. flags, start, end]
+    # [K_type, start, end]
-    # where fg and bg are integers (using 24 bit color)
+    # K_type is a constant from the constants defined in libcrib.rb
-    # and flags is a bitmask of bold/underline/italic etc
+    # for ex: for strings it would be C::K_STRING or for numbers C::K_NUMBER etc.
     # and start and end are integers strictly inside the line
-    return []
+    return {
+      state: "",
+      tokens: [
+        # This will highlight the entire line as a string
+        # Any wrong format will not be handled and lead to crashes
+        { type: C::K_STRING, start: 0, end: line.length }
+      ]
+    }
   },
-  matcher: ->(_state1, _state2) {
+  matcher: ->(state1, state2) {
     # returns true if the states are equal
     # And so would not need recomputation for further lines
-    return false
+    return state1 == state2
   }
 }

config/scripts/shutdown.sh

@@ -1,11 +0,0 @@
-#!/usr/bin/env sh
-
-# This file can be used to execute commands right after the editor session ends.
-#
-# The location of this file is defined in the editor's config.json and can be changed
-#
-# It can for example be used to unset environment variables for any lsp(s) used
-# Or like this example to set kitty padding back higher
-#   kitty @ --to="$KITTY_LISTEN_ON" set-spacing padding=8 margin=0 2>/dev/null || true
-
-echo "Exiting crib editor..."

config/scripts/startup.sh

@@ -1,11 +0,0 @@
-#!/usr/bin/env sh
-
-# This file can be used to execute commands before the editor session starts.
-#
-# The location of this file is defined in the editor's config.json and can be changed
-#
-# It can for example be used to set environment variables for any lsp(s) used
-# Or like this example to set kitty padding to 0
-#   kitty @ --to="$KITTY_LISTEN_ON" set-spacing padding=0 margin=0 2>/dev/null || true
-
-echo "Starting crib editor..."

config/theme.json

@@ -1,88 +0,0 @@
-{
-  "Default": {
-    "fg": "#EEEEEE"
-  },
-  "Shebang": {
-    "fg": "#7dcfff"
-  },
-  "Error": {
-    "fg": "#EF5168"
-  },
-  "Comment": {
-    "fg": "#AAAAAA",
-    "italic": true
-  },
-  "String": {
-    "fg": "#AAD94C"
-  },
-  "Escape": {
-    "fg": "#7dcfff"
-  },
-  "Interpolation": {
-    "fg": "#7dcfff"
-  },
-  "Regexp": {
-    "fg": "#D2A6FF"
-  },
-  "Number": {
-    "fg": "#E6C08A"
-  },
-  "True": {
-    "fg": "#7AE93C"
-  },
-  "False": {
-    "fg": "#EF5168"
-  },
-  "Char": {
-    "fg": "#FFAF70"
-  },
-  "Keyword": {
-    "fg": "#FF8F40"
-  },
-  "KeywordOperator": {
-    "fg": "#F07178"
-  },
-  "Operator": {
-    "fg": "#FFFFFF",
-    "italic": true
-  },
-  "Function": {
-    "fg": "#FFAF70"
-  },
-  "Type": {
-    "fg": "#F07178"
-  },
-  "Constant": {
-    "fg": "#7dcfff"
-  },
-  "VariableInstance": {
-    "fg": "#95E6CB"
-  },
-  "VariableGlobal": {
-    "fg": "#F07178"
-  },
-  "Annotation": {
-    "fg": "#7dcfff"
-  },
-  "Directive": {
-    "fg": "#FF8F40"
-  },
-  "Label": {
-    "fg": "#D2A6FF"
-  },
-  "Brace1": {
-    "fg": "#D2A6FF"
-  },
-  "Brace2": {
-    "fg": "#FFAFAF"
-  },
-  "Brace3": {
-    "fg": "#FFFF00"
-  },
-  "Brace4": {
-    "fg": "#0FFF0F"
-  },
-  "Brace5": {
-    "fg": "#FF0F0F"
-  }
-}

include/config.h

@@ -1,291 +0,0 @@
-#ifndef CONFIG_H
-#define CONFIG_H
-
-#include "lsp/lsp.h"
-#include "pch.h"
-
-static const std::unordered_map<uint8_t, LSP> kLsps = {
-    {1,
-     {"clangd",
-      {
-          "clangd",
-          "--background-index",
-          "--clang-tidy",
-          "--completion-style=detailed",
-          "--header-insertion=never",
-          "--pch-storage=memory",
-          "--limit-results=50",
-          "--log=error",
-          nullptr,
-      }}},
-    {2,
-     {"ruby-lsp",
-      {
-          "ruby-lsp",
-          nullptr,
-      }}},
-    {3,
-     {"solargraph",
-      {
-          "solargraph",
-          "stdio",
-          nullptr,
-      }}},
-    {4,
-     {"bash-language-server",
-      {
-          "bash-language-server",
-          "start",
-          nullptr,
-      }}},
-    {5,
-     {"vscode-css-language-server",
-      {
-          "vscode-css-language-server",
-          "--stdio",
-          nullptr,
-      }}},
-    {6,
-     {"vscode-json-language-server",
-      {
-          "vscode-json-language-server",
-          "--stdio",
-          nullptr,
-      }}},
-    {7,
-     {"fish-lsp",
-      {
-          "fish-lsp",
-          "start",
-          nullptr,
-      }}},
-    {8,
-     {"gopls",
-      {
-          "gopls",
-          "serve",
-          nullptr,
-      }}},
-    {9,
-     {"haskell-language-server",
-      {
-          "haskell-language-server",
-          "lsp",
-          nullptr,
-      }}},
-    {10,
-     {"emmet-language-server",
-      {
-          "emmet-language-server",
-          "--stdio",
-          nullptr,
-      }}},
-    {11,
-     {"typescript-language-server",
-      {
-          "typescript-language-server",
-          "--stdio",
-          nullptr,
-      }}},
-#define LUA_LS 12
-    {12,
-     {"lua-language-server",
-      {
-          "lua-language-server",
-          nullptr,
-      }}},
-    {13,
-     {"pyright-langserver",
-      {
-          "pyright-langserver",
-          "--stdio",
-          nullptr,
-      }}},
-    {14,
-     {"rust-analyzer",
-      {
-          "rust-analyzer",
-          nullptr,
-      }}},
-    {15,
-     {"intelephense",
-      {
-          "intelephense",
-          "--stdio",
-          nullptr,
-      }}},
-    {16,
-     {"marksman",
-      {
-          "marksman",
-          "server",
-          nullptr,
-      }}},
-    {17,
-     {"nginx-language-server",
-      {
-          "nginx-language-server",
-          nullptr,
-      }}},
-    {18,
-     {"taplo",
-      {
-          "taplo",
-          "lsp",
-          "stdio",
-          nullptr,
-      }}},
-    {19,
-     {"yaml-language-server",
-      {
-          "yaml-language-server",
-          "--stdio",
-          nullptr,
-      }}},
-    {20,
-     {"sqls",
-      {
-          "sqls",
-          "serve",
-          nullptr,
-      }}},
-    {21,
-     {"make-language-server",
-      {
-          "make-language-server",
-          nullptr,
-      }}},
-    {22,
-     {"sql-language-server",
-      {
-          "sql-language-server",
-          "up",
-          "--method",
-          "stdio",
-          nullptr,
-      }}},
-};
-
-static const std::unordered_map<std::string, Language> kLanguages = {
-    {"bash", {"bash", 4, 0x4d5a5e, " "}},
-    {"c", {"c", 1, 0x555555, " "}},
-    {"cpp", {"cpp", 1, 0x00599C, " "}},
-    {"h", {"h", 1, 0xA8B9CC, " "}},
-    {"css", {"css", 5, 0x36a3d9, " "}},
-    {"fish", {"fish", 7, 0x4d5a5e, " "}},
-    {"go", {"go", 8, 0x00add8, " "}},
-    {"gomod", {"gomod", 8, 0x00add8, " "}},
-    {"haskell", {"haskell", 9, 0xa074c4, " "}},
-    {"html", {"html", 10, 0xef8a91, " "}},
-    {"javascript", {"javascript", 11, 0xf0df8a, " "}},
-    {"typescript", {"typescript", 11, 0x36a3d9, " "}},
-    {"json", {"json", 6, 0xcbcb41, "{}"}},
-    {"jsonc", {"jsonc", 6, 0xcbcb41, "{}"}},
-    {"erb", {"erb", 10, 0x6e1516, " "}},
-    {"ruby", {"ruby", 3, 0xff8087, "󰴭 "}},
-    {"lua", {"lua", 12, 0x36a3d9, "󰢱 "}},
-    {"python", {"python", 13, 0x95e6cb, "󰌠 "}},
-    {"rust", {"rust", 14, 0xdea584, "󱘗 "}},
-    {"php", {"php", 15, 0xa074c4, "󰌟 "}},
-    {"markdown", {"markdown", 16, 0x36a3d9, " "}},
-    {"markdown_inline", {"markdown_inline", 16, 0x36a3d9, " "}},
-    {"nginx", {"nginx", 17, 0x6d8086, " "}},
-    {"toml", {"toml", 18, 0x36a3d9, " "}},
-    {"yaml", {"yaml", 19, 0x6d8086, " "}},
-    {"sql", {"sql", 20, 0xdad8d8, " "}},
-    {"make", {"make", 21, 0x4e5c61, " "}},
-    {"gdscript", {"gdscript", 0, 0x6d8086, " "}},
-    {"man", {"man", 0, 0xdad8d8, " "}},
-    {"diff", {"diff", 0, 0xDD4C35, " "}},
-    {"gitattributes", {"gitattributes", 0, 0xF05032, " "}},
-    {"gitignore", {"gitignore", 0, 0xF05032, " "}},
-    {"query", {"query", 0, 0x7E57C2, " "}},
-    {"regex", {"regex", 0, 0x9E9E9E, ".*"}},
-    {"ini", {"ini", 0, 0x6d8086, " "}},
-};
-
-static const std::unordered_map<std::string, std::string> kExtToLang = {
-    {"sh", "bash"},
-    {"bash", "bash"},
-    {"c", "c"},
-    {"cpp", "cpp"},
-    {"cxx", "cpp"},
-    {"cc", "cpp"},
-    {"hpp", "h"},
-    {"hh", "h"},
-    {"hxx", "h"},
-    {"h", "h"},
-    {"css", "css"},
-    {"fish", "fish"},
-    {"go", "go"},
-    {"hs", "haskell"},
-    {"html", "html"},
-    {"htm", "html"},
-    {"js", "javascript"},
-    {"jsx", "javascript"},
-    {"ts", "typescript"},
-    {"tsx", "typescript"},
-    {"json", "json"},
-    {"jsonc", "jsonc"},
-    {"lua", "lua"},
-    {"make", "make"},
-    {"mk", "make"},
-    {"makefile", "make"},
-    {"man", "man"},
-    {"py", "python"},
-    {"rb", "ruby"},
-    {"rs", "rust"},
-    {"diff", "diff"},
-    {"patch", "diff"},
-    {"erb", "erb"},
-    {"gd", "gdscript"},
-    {"gitattributes", "gitattributes"},
-    {"gitignore", "gitignore"},
-    {"mod", "gomod"},
-    {"ini", "ini"},
-    {"gitmodules", "ini"},
-    {"md", "markdown"},
-    {"markdown", "markdown"},
-    {"conf", "nginx"},
-    {"php", "php"},
-    {"scm", "query"},
-    {"regex", "regex"},
-    {"sql", "sql"},
-    {"toml", "toml"},
-    {"yaml", "yaml"},
-    {"yml", "yaml"},
-    {"clangd", "yaml"},
-};
-
-static const std::unordered_map<std::string, std::string> kMimeToLang = {
-    {"text/x-c", "c"},
-    {"text/x-c++", "cpp"},
-    {"text/x-shellscript", "bash"},
-    {"application/json", "json"},
-    {"text/javascript", "javascript"},
-    {"text/html", "html"},
-    {"text/css", "css"},
-    {"text/x-python", "python"},
-    {"text/x-ruby", "ruby"},
-    {"text/x-go", "go"},
-    {"text/x-haskell", "haskell"},
-    {"text/x-rust", "rust"},
-    {"text/x-lua", "lua"},
-    {"text/x-diff", "diff"},
-    {"text/x-gdscript", "gdscript"},
-    {"text/x-gitattributes", "gitattributes"},
-    {"text/x-gitignore", "gitignore"},
-    {"text/x-gomod", "gomod"},
-    {"text/x-ini", "ini"},
-    {"text/markdown", "markdown"},
-    {"text/x-nginx-conf", "nginx"},
-    {"application/x-php", "php"},
-    {"text/x-tree-sitter-query", "query"},
-    {"text/x-regex", "regex"},
-    {"text/x-sql", "sql"},
-    {"text/x-toml", "toml"},
-    {"text/x-yaml", "yaml"},
-    {"text/x-man", "man"},
-};
-
-#endif

include/editor/editor.h

@@ -30,6 +30,7 @@ struct Editor {
   uint32_t cursor_preffered;
   Coord selection;
   bool selection_active;
+  bool unix_eol; // false for windows
   int selection_type;
   Coord position;
   Coord size;
@@ -55,7 +56,8 @@ struct Editor {
   bool is_css_color;
 };
 
-Editor *new_editor(const char *filename_arg, Coord position, Coord size);
+Editor *new_editor(const char *filename_arg, Coord position, Coord size,
+                   bool unix_eol);
 void save_file(Editor *editor);
 void free_editor(Editor *editor);
 void render_editor(Editor *editor);

include/lsp/lsp.h

@@ -5,11 +5,6 @@
 #include "pch.h"
 #include "utils/utils.h"
 
-struct LSP {
-  const char *command;
-  std::vector<const char *> args;
-};
-
 struct LSPPending {
   std::string method;
   Editor *editor = nullptr;
@@ -50,7 +45,8 @@ struct LSPInstance {
 };
 
 extern std::shared_mutex active_lsps_mtx;
-extern std::unordered_map<uint8_t, std::shared_ptr<LSPInstance>> active_lsps;
+extern std::unordered_map<std::string, std::shared_ptr<LSPInstance>>
+    active_lsps;
 extern Queue<LSPOpenRequest> lsp_open_queue;
 
 static json client_capabilities = {
@@ -81,9 +77,9 @@ void lsp_send(std::shared_ptr<LSPInstance> lsp, json message,
               LSPPending *pending);
 void lsp_worker();
 
-std::shared_ptr<LSPInstance> get_or_init_lsp(uint8_t lsp_id);
+std::shared_ptr<LSPInstance> get_or_init_lsp(std::string lsp_id);
-void clean_lsp(std::shared_ptr<LSPInstance> lsp, uint8_t lsp_id);
+void clean_lsp(std::shared_ptr<LSPInstance> lsp, std::string lsp_id);
-void close_lsp(uint8_t lsp_id);
+void close_lsp(std::string lsp_id);
 
 void open_editor(std::shared_ptr<LSPInstance> lsp,
                  std::pair<Language, Editor *> entry);

include/pch.h

@@ -4,7 +4,10 @@
 #define PCRE2_CODE_UNIT_WIDTH 8
 #define PCRE_WORKSPACE_SIZE 512
 
-#include "ruby.h"
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wunused-parameter"
+#include <ruby.h>
+#pragma clang diagnostic pop
 #include <magic.h>
 #include <nlohmann/json.hpp>
 #include <pcre2.h>

include/scripting/decl.h

@@ -1,12 +1,21 @@
 #ifndef SCRIPTING_DECL_H
 #define SCRIPTING_DECL_H
 
+#include "syntax/decl.h"
 #include "utils/utils.h"
 
+extern std::unordered_map<std::string, std::pair<VALUE, VALUE>>
+    custom_highlighters;
+
 void ruby_start(const char *main_file);
 void ruby_shutdown();
+void ruby_log(std::string msg);
 void load_theme();
-std::string read_line_endings();
+void load_languages_info();
-std::string read_utf_mode();
+bool read_line_endings();
+void load_custom_highlighters();
+VALUE parse_custom(std::vector<Token> *tokens, VALUE parser_block,
+                   const char *line, uint32_t len, VALUE state);
+bool custom_compare(VALUE match_block, VALUE state1, VALUE state2);
 
 #endif

include/syntax/decl.h

@@ -40,4 +40,10 @@ struct LineData {
   std::shared_ptr<void> out_state{nullptr};
 };
 
+struct CustomState {
+  VALUE state;
+  CustomState(VALUE s) : state(s) { rb_gc_register_address(&state); }
+  ~CustomState() { rb_gc_unregister_address(&state); }
+};
+
 #endif

include/syntax/parser.h

@@ -1,6 +1,7 @@
 #ifndef SYNTAX_PARSER_H
 #define SYNTAX_PARSER_H
 
+#include "scripting/decl.h"
 #include "syntax/decl.h"
 #include "syntax/line_tree.h"
 
@@ -12,11 +13,15 @@ struct Parser {
                                       const char *text, uint32_t len);
   bool (*state_match_func)(std::shared_ptr<void> state_1,
                            std::shared_ptr<void> state_2);
+  VALUE parser_block = Qnil;
+  VALUE match_block = Qnil;
+  bool is_custom{false};
   std::atomic<uint32_t> scroll_max{UINT32_MAX - 2048};
+  std::atomic<bool> scroll_dirty{false};
   std::mutex mutex;
   std::mutex data_mutex;
   LineTree line_tree;
-  std::set<uint32_t> dirty_lines;
+  UniqueQueue<uint32_t> dirty_lines;
 
   Parser(Editor *editor, std::string n_lang, uint32_t n_scroll_max);
   void edit(uint32_t start_line, uint32_t old_end_line, uint32_t inserted_rows);

include/syntax/tokens.def

@@ -1,3 +1,4 @@
+// When updating this file with new types do not forget to update libcrib.rb to parallel changes
 ADD(K_DATA)
 ADD(K_SHEBANG)
 ADD(K_COMMENT)

include/utils/utils.h

@@ -4,9 +4,6 @@
 #include "pch.h"
 
 template <typename T> struct Queue {
-  std::queue<T> q;
-  std::mutex m;
-
   void push(T val) {
     std::lock_guard<std::mutex> lock(m);
     q.push(val);
@@ -32,6 +29,49 @@ template <typename T> struct Queue {
     std::lock_guard<std::mutex> lock(m);
     return q.empty();
   }
+
+private:
+  std::queue<T> q;
+  std::mutex m;
+};
+
+template <typename T> struct UniqueQueue {
+  bool push(const T &value) {
+    std::lock_guard<std::mutex> lock(m);
+    if (set.contains(value))
+      return false;
+
+    dq.push_back(value);
+    set.insert(value);
+    return true;
+  }
+  bool pop(T &out) {
+    std::lock_guard<std::mutex> lock(m);
+    if (dq.empty())
+      return false;
+    out = dq.front();
+    dq.pop_front();
+    set.erase(out);
+    return true;
+  }
+  bool empty() const {
+    std::lock_guard<std::mutex> lock(m);
+    return dq.empty();
+  }
+  void clear() {
+    std::lock_guard<std::mutex> lock(m);
+    dq.clear();
+    set.clear();
+  }
+  size_t size() const {
+    std::lock_guard<std::mutex> lock(m);
+    return dq.size();
+  }
+
+private:
+  std::deque<T> dq;
+  std::set<T> set;
+  mutable std::mutex m;
 };
 
 struct Coord {
@@ -60,11 +100,21 @@ struct Match {
 
 struct Language {
   std::string name;
-  uint8_t lsp_id;
+  std::string lsp_name;
   uint32_t color;
-  const char *symbol;
+  std::string symbol;
 };
 
+struct LSP {
+  std::string command;
+  std::vector<std::string> args;
+};
+
+extern std::unordered_map<std::string, Language> languages;
+extern std::unordered_map<std::string, std::string> language_extensions;
+extern std::unordered_map<std::string, std::string> language_mimetypes;
+extern std::unordered_map<std::string, LSP> lsps;
+
 #define MAX(a, b) ((a) > (b) ? (a) : (b))
 #define MIN(a, b) ((a) < (b) ? (a) : (b))
 #define ABS(x) ((x) < 0 ? -(x) : (x))

src/editor/editor.cc

@@ -4,7 +4,8 @@
 #include "syntax/langs.h"
 #include "utils/utils.h"
 
-Editor *new_editor(const char *filename_arg, Coord position, Coord size) {
+Editor *new_editor(const char *filename_arg, Coord position, Coord size,
+                   bool unix_eol) {
   Editor *editor = new Editor();
   if (!editor)
     return nullptr;
@@ -15,6 +16,7 @@ Editor *new_editor(const char *filename_arg, Coord position, Coord size) {
     free_editor(editor);
     return nullptr;
   }
+  editor->unix_eol = unix_eol;
   editor->filename = filename;
   editor->uri = path_to_file_uri(filename);
   editor->position = position;
@@ -61,7 +63,15 @@ void save_file(Editor *editor) {
     return;
   lock.unlock();
   std::ofstream out(editor->filename);
-  out.write(str, char_count);
+  if (!editor->unix_eol) {
+    for (uint32_t i = 0; i < char_count; ++i) {
+      if (str[i] == '\n')
+        out.put('\r');
+      out.put(str[i]);
+    }
+  } else {
+    out.write(str, char_count);
+  }
   out.close();
   free(str);
   if (editor->lsp) {
@@ -109,7 +119,15 @@ void save_file(Editor *editor) {
             return;
           lock.unlock();
           std::ofstream out(editor->filename);
-          out.write(str, char_count);
+          if (!editor->unix_eol) {
+            for (uint32_t i = 0; i < char_count; ++i) {
+              if (str[i] == '\n')
+                out.put('\r');
+              out.put(str[i]);
+            }
+          } else {
+            out.write(str, char_count);
+          }
           out.close();
           free(str);
           lsp_send(editor->lsp, save_msg, nullptr);

src/editor/events.cc

@@ -186,8 +186,6 @@ void handle_editor_event(Editor *editor, KeyEvent event) {
     break;
   case SELECT:
     if (event.key_type == KEY_CHAR && event.len == 1) {
-      uint32_t len;
-      char *text;
       switch (event.c[0]) {
       case 0x1B:
       case 's':

src/editor/helpers.cc

@@ -357,7 +357,6 @@ void handle_mouse(Editor *editor, KeyEvent event) {
       std::chrono::steady_clock::now();
   static uint32_t click_count = 0;
   static Coord last_click_pos = {UINT32_MAX, UINT32_MAX};
-  uint8_t old_mode = mode;
   if (event.key_type == KEY_MOUSE) {
     auto now = std::chrono::steady_clock::now();
     auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(

src/lsp/handlers.cc

@@ -7,7 +7,7 @@ void request_add_to_lsp(Language language, Editor *editor) {
 }
 
 void add_to_lsp(Language language, Editor *editor) {
-  std::shared_ptr<LSPInstance> lsp = get_or_init_lsp(language.lsp_id);
+  std::shared_ptr<LSPInstance> lsp = get_or_init_lsp(language.lsp_name);
   if (!lsp)
     return;
   std::unique_lock lock(lsp->mtx);
@@ -41,11 +41,11 @@ void open_editor(std::shared_ptr<LSPInstance> lsp,
   lsp_send(lsp, message, nullptr);
 }
 
-static uint8_t find_lsp_id(std::shared_ptr<LSPInstance> needle) {
+static std::string find_lsp_id(std::shared_ptr<LSPInstance> needle) {
   for (const auto &[id, lsp] : active_lsps)
     if (lsp == needle)
       return id;
-  return 0;
+  return "";
 }
 
 void remove_from_lsp(Editor *editor) {
@@ -64,8 +64,8 @@ void remove_from_lsp(Editor *editor) {
                   {"method", "textDocument/didClose"},
                   {"params", {{"textDocument", {{"uri", editor->uri}}}}}};
   lsp_send(lsp, message, nullptr);
-  uint8_t lsp_id = find_lsp_id(lsp);
+  std::string lsp_id = find_lsp_id(lsp);
-  if (lsp_id && lsp->editors.empty())
+  if (!lsp_id.empty() && lsp->editors.empty())
     close_lsp(lsp_id);
 }
 

src/lsp/process.cc

@@ -1,4 +1,3 @@
-#include "config.h"
 #include "lsp/lsp.h"
 
 static bool init_lsp(std::shared_ptr<LSPInstance> lsp) {
@@ -25,7 +24,12 @@ static bool init_lsp(std::shared_ptr<LSPInstance> lsp) {
     close(in_pipe[1]);
     close(out_pipe[0]);
     close(out_pipe[1]);
-    execvp(lsp->lsp->command, (char *const *)(lsp->lsp->args.data()));
+    std::vector<char *> argv;
+    argv.push_back(const_cast<char *>(lsp->lsp->command.c_str()));
+    for (auto &arg : lsp->lsp->args)
+      argv.push_back(const_cast<char *>(arg.c_str()));
+    argv.push_back(nullptr);
+    execvp(lsp->lsp->command.c_str(), argv.data());
     perror("execvp");
     _exit(127);
   }
@@ -37,12 +41,12 @@ static bool init_lsp(std::shared_ptr<LSPInstance> lsp) {
   return true;
 }
 
-std::shared_ptr<LSPInstance> get_or_init_lsp(uint8_t lsp_id) {
+std::shared_ptr<LSPInstance> get_or_init_lsp(std::string lsp_id) {
   std::unique_lock lock(active_lsps_mtx);
   auto it = active_lsps.find(lsp_id);
   if (it == active_lsps.end()) {
-    auto map_it = kLsps.find(lsp_id);
+    auto map_it = lsps.find(lsp_id);
-    if (map_it == kLsps.end())
+    if (map_it == lsps.end())
       return nullptr;
     std::shared_ptr<LSPInstance> lsp = std::make_shared<LSPInstance>();
     lsp->lsp = &map_it->second;
@@ -54,12 +58,13 @@ std::shared_ptr<LSPInstance> get_or_init_lsp(uint8_t lsp_id) {
     pending->callback = [lsp, lsp_id](Editor *, std::string, json msg) {
       if (msg.contains("result") && msg["result"].contains("capabilities")) {
         auto &caps = msg["result"]["capabilities"];
-        if (caps.contains("positionEncoding")) {
+        // if (caps.contains("positionEncoding")) {
-          std::string s = caps["positionEncoding"].get<std::string>();
+        //   std::string s = caps["positionEncoding"].get<std::string>();
-          if (s == "utf-8")
+        //   if (s == "utf-8")
-            lsp->is_utf8 = true;
+        //     lsp->is_utf8 = true;
-          log("Lsp name: %s, supports: %s", lsp->lsp->command, s.c_str());
+        //   log("Lsp name: %s, supports: %s", lsp->lsp->command.c_str(),
-        }
+        //       s.c_str());
+        // }
         if (caps.contains("textDocumentSync")) {
           auto &sync = caps["textDocumentSync"];
           if (sync.is_number()) {
@@ -71,8 +76,8 @@ std::shared_ptr<LSPInstance> get_or_init_lsp(uint8_t lsp_id) {
           }
         }
         lsp->allow_formatting = caps.value("documentFormattingProvider", false);
-        if (lsp_id !=
+        if (lsp_id != "lua-language-server" /* Lua ls gives terrible ontype
-                LUA_LS /* Lua ls gives terrible ontype formatting so disable */
+                                               formatting so disable */
             && caps.contains("documentOnTypeFormattingProvider")) {
           auto &fmt = caps["documentOnTypeFormattingProvider"];
           if (fmt.is_object()) {
@@ -149,7 +154,7 @@ std::shared_ptr<LSPInstance> get_or_init_lsp(uint8_t lsp_id) {
   return it->second;
 }
 
-void close_lsp(uint8_t lsp_id) {
+void close_lsp(std::string lsp_id) {
   std::shared_lock active_lsps_lock(active_lsps_mtx);
   auto it = active_lsps.find(lsp_id);
   if (it == active_lsps.end())
@@ -186,7 +191,7 @@ void close_lsp(uint8_t lsp_id) {
   t.detach();
 }
 
-void clean_lsp(std::shared_ptr<LSPInstance> lsp, uint8_t lsp_id) {
+void clean_lsp(std::shared_ptr<LSPInstance> lsp, std::string lsp_id) {
   for (auto &kv : lsp->pending)
     delete kv.second;
   lsp->pid = -1;

src/lsp/workers.cc

@@ -1,7 +1,7 @@
 #include "lsp/lsp.h"
 
 std::shared_mutex active_lsps_mtx;
-std::unordered_map<uint8_t, std::shared_ptr<LSPInstance>> active_lsps;
+std::unordered_map<std::string, std::shared_ptr<LSPInstance>> active_lsps;
 
 void lsp_send(std::shared_ptr<LSPInstance> lsp, json message,
               LSPPending *pending) {

src/main.cc

@@ -2,6 +2,7 @@
 #include "editor/editor.h"
 #include "io/sysio.h"
 #include "lsp/lsp.h"
+#include "scripting/decl.h"
 #include "ui/bar.h"
 #include "utils/utils.h"
 
@@ -12,32 +13,12 @@ std::vector<Editor *> editors;
 uint8_t current_editor = 0;
 std::atomic<uint8_t> mode = NORMAL;
 
-void background_worker() {
-  while (running)
-    throttle(16ms, editor_worker, editors[current_editor]);
-}
-
 void background_lsp() {
   while (running)
     throttle(8ms, lsp_worker);
 }
 
-void input_listener() {
+inline Editor *editor_at(uint8_t x, uint8_t y) {
-  while (running) {
-    KeyEvent event = throttle(1ms, read_key);
-    if (event.key_type == KEY_NONE)
-      continue;
-    if (event.key_type == KEY_CHAR && event.len == 1 &&
-        event.c[0] == CTRL('q')) {
-      free(event.c);
-      running = false;
-      return;
-    }
-    event_queue.push(event);
-  }
-}
-
-Editor *editor_at(uint8_t x, uint8_t y) {
   for (Editor *ed : editors) {
     Coord pos = ed->position;
     Coord size = ed->size;
@@ -48,30 +29,73 @@ Editor *editor_at(uint8_t x, uint8_t y) {
   return nullptr;
 }
 
-uint8_t index_of(Editor *ed) {
+inline uint8_t index_of(Editor *ed) {
   for (uint8_t i = 0; i < editors.size(); i++)
     if (editors[i] == ed)
       return i;
   return 0;
 }
 
+void input_listener(Bar bar) {
+  while (running) {
+    KeyEvent event = throttle(1ms, read_key);
+    if (event.key_type == KEY_NONE)
+      goto render;
+    if (event.key_type == KEY_CHAR && event.len == 1 &&
+        event.c[0] == CTRL('q')) {
+      free(event.c);
+      running = false;
+      break;
+    }
+    if (mode != RUNNER) {
+      if (event.key_type == KEY_MOUSE) {
+        Editor *target = editor_at(event.mouse_x, event.mouse_y);
+        if (target) {
+          if (event.mouse_state == PRESS)
+            current_editor = index_of(target);
+
+          event.mouse_x -= target->position.col;
+          event.mouse_y -= target->position.row;
+          handle_editor_event(target, event);
+        }
+      } else {
+        handle_editor_event(editors[current_editor], event);
+      }
+    } else {
+      bar.handle(event);
+    }
+  render:
+    render_editor(editors[current_editor]);
+    bar.render();
+    throttle(4ms, render);
+  }
+}
+
 int main(int argc, char *argv[]) {
+  auto start = std::chrono::high_resolution_clock::now();
+
+  ruby_init();
+
+  ruby_start((get_exe_dir() + "/../config/main.rb").c_str());
+  load_theme();
+  load_languages_info();
+  load_custom_highlighters();
+
   Coord screen = start_screen();
   const char *filename = (argc > 1) ? argv[1] : "";
 
-  int state;
+  bool unix_eol = read_line_endings();
-  VALUE result;
-  result = rb_eval_string_protect("puts 'Hello, world!'", &state);
 
-  if (state) {
+  Editor *editor =
-    /* handle exception */
+      new_editor(filename, {0, 0}, {screen.row - 2, screen.col}, unix_eol);
-  }
-
-  load_theme();
-
-  Editor *editor = new_editor(filename, {0, 0}, {screen.row - 2, screen.col});
   Bar bar(screen);
 
+  auto end = std::chrono::high_resolution_clock::now();
+  auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(end - start)
+                .count();
+  ruby_log("[LOG] STARTUP_TIME: " + std::to_string(static_cast<long long>(ms)) +
+           "ms");
+
   if (!editor) {
     end_screen();
     fprintf(stderr, "Failed to load editor\n");
@@ -81,48 +105,22 @@ int main(int argc, char *argv[]) {
   editors.push_back(editor);
   current_editor = editors.size() - 1;
 
-  std::thread input_thread(input_listener);
+  std::thread input_thread(input_listener, bar);
-  std::thread work_thread(background_worker);
   std::thread lsp_thread(background_lsp);
 
-  while (running) {
+  while (running)
-    KeyEvent event;
+    throttle(16ms, editor_worker, editors[current_editor]);
-    while (event_queue.pop(event)) {
-      if (mode != RUNNER) {
-        if (event.key_type == KEY_MOUSE) {
-          Editor *target = editor_at(event.mouse_x, event.mouse_y);
-          if (!target)
-            continue;
-          if (event.mouse_state == PRESS)
-            current_editor = index_of(target);
-          event.mouse_x -= target->position.col;
-          event.mouse_y -= target->position.row;
-          handle_editor_event(target, event);
-        } else {
-          handle_editor_event(editors[current_editor], event);
-        }
-      } else {
-        bar.handle(event);
-      }
-    }
-    render_editor(editors[current_editor]);
-    bar.render();
-    throttle(4ms, render);
-  }
 
   if (input_thread.joinable())
     input_thread.join();
 
-  if (work_thread.joinable())
-    work_thread.join();
-
   if (lsp_thread.joinable())
     lsp_thread.join();
 
-  system(("bash " + get_exe_dir() + "/../scripts/exit.sh").c_str());
-
   end_screen();
 
+  ruby_shutdown();
+
   for (auto editor : editors)
     free_editor(editor);
 
@@ -136,5 +134,7 @@ int main(int argc, char *argv[]) {
     throttle(16ms, lsp_worker);
   }
 
-  return 0;
+  rb_gc_start();
+
+  return ruby_cleanup(0);
 }

src/scripting/process.cc

@@ -1,8 +1,11 @@
+#include "ruby/internal/gc.h"
+#include "ruby/internal/value.h"
 #include "scripting/decl.h"
-#include "syntax/decl.h"
 #include "utils/utils.h"
 
-struct ThemeEntry {
+std::unordered_map<std::string, std::pair<VALUE, VALUE>> custom_highlighters;
+
+struct R_ThemeEntry {
   std::string key;
   uint32_t fg = 0xFFFFFF;
   uint32_t bg = 0x000000;
@@ -12,18 +15,29 @@ struct ThemeEntry {
   bool strikethrough = false;
 };
 
+struct R_Language {
+  std::string name;
+  uint32_t color = 0xFFFFFF;
+  std::string symbol;
+  std::vector<std::string> extensions;
+  std::vector<std::string> filenames;
+  std::vector<std::string> mimetypes;
+  std::string lsp_command; // link to LSP by name
+};
+
 VALUE C_module = Qnil;
+std::mutex ruby_mutex;
 
 void ruby_start(const char *main_file) {
-  USING(Language);
+  std::lock_guard lock(ruby_mutex);
-  ruby_init();
   ruby_init_loadpath();
   int state = 0;
   rb_load_protect(rb_str_new_cstr(main_file), 0, &state);
   if (state) {
-    VALUE err = rb_errinfo();
+    rb_errinfo();
     rb_set_errinfo(Qnil);
-    fprintf(stderr, "Failed to load Ruby file\n");
+    fprintf(stderr, "%d: Failed to load Ruby file\n", state);
+    return;
   }
   C_module = rb_const_get(rb_cObject, rb_intern("C"));
   if (C_module == Qnil)
@@ -34,16 +48,86 @@ void ruby_start(const char *main_file) {
 }
 
 void ruby_shutdown() {
+  std::lock_guard lock(ruby_mutex);
   if (C_module == Qnil)
     return;
   VALUE block = rb_funcall(C_module, rb_intern("b_shutdown"), 0);
   if (block != Qnil)
     rb_funcall(block, rb_intern("call"), 0);
-  ruby_finalize();
 }
 
-static std::vector<ThemeEntry> read_theme() {
+inline std::vector<std::string> ruby_array_to_vector(VALUE rb_array) {
-  std::vector<ThemeEntry> result;
+  std::vector<std::string> result;
+  if (NIL_P(rb_array) || !RB_TYPE_P(rb_array, T_ARRAY))
+    return result;
+  for (long i = 0; i < RARRAY_LEN(rb_array); ++i) {
+    VALUE item = rb_ary_entry(rb_array, i);
+    if (RB_TYPE_P(item, T_STRING))
+      result.push_back(StringValueCStr(item));
+  }
+  return result;
+}
+
+void ruby_log(std::string msg) {
+  std::lock_guard lock(ruby_mutex);
+  VALUE str = rb_str_new(msg.c_str(), msg.size());
+  rb_funcall(C_module, rb_intern("queue_log"), 1, str);
+}
+
+void load_custom_highlighters() {
+  std::lock_guard lock(ruby_mutex);
+  if (C_module == Qnil)
+    return;
+  VALUE hashmap = rb_funcall(C_module, rb_intern("highlighters"), 0);
+  if (NIL_P(hashmap))
+    return;
+  VALUE keys = rb_funcall(hashmap, rb_intern("keys"), 0);
+  for (long i = 0; i < RARRAY_LEN(keys); ++i) {
+    VALUE key_sym = rb_ary_entry(keys, i);
+    std::string key = rb_id2name(SYM2ID(key_sym));
+    VALUE val_hash = rb_hash_aref(hashmap, key_sym);
+    if (NIL_P(val_hash))
+      continue;
+    VALUE parse_block = rb_hash_aref(val_hash, ID2SYM(rb_intern("parser")));
+    VALUE match_block = rb_hash_aref(val_hash, ID2SYM(rb_intern("matcher")));
+    rb_gc_register_address(&match_block);
+    rb_gc_register_address(&parse_block);
+    custom_highlighters[key] = {parse_block, match_block};
+  }
+}
+
+bool custom_compare(VALUE match_block, VALUE state1, VALUE state2) {
+  std::lock_guard lock(ruby_mutex);
+  return RTEST(rb_funcall(match_block, rb_intern("call"), 2, state1, state2));
+}
+
+VALUE parse_custom(std::vector<Token> *tokens, VALUE parser_block,
+                   const char *line, uint32_t len, VALUE state) {
+  std::lock_guard lock(ruby_mutex);
+  tokens->clear();
+  if (NIL_P(parser_block))
+    return {};
+  VALUE ruby_line = rb_str_new(line, len);
+  VALUE tokens_and_state_hash =
+      rb_funcall(parser_block, rb_intern("call"), 2, ruby_line, state);
+  VALUE tokens_rb =
+      rb_hash_aref(tokens_and_state_hash, ID2SYM(rb_intern("tokens")));
+  for (long i = 0; i < RARRAY_LEN(tokens_rb); ++i) {
+    VALUE token = rb_ary_entry(tokens_rb, i);
+    Token tok;
+    tok.type =
+        (TokenKind)NUM2INT(rb_hash_aref(token, ID2SYM(rb_intern("type"))));
+    tok.start = NUM2UINT(rb_hash_aref(token, ID2SYM(rb_intern("start"))));
+    tok.end = NUM2UINT(rb_hash_aref(token, ID2SYM(rb_intern("end"))));
+    if (tok.type < TokenKind::Count && tok.end > tok.start && tok.end <= len)
+      tokens->push_back(tok);
+  }
+  return rb_hash_aref(tokens_and_state_hash, ID2SYM(rb_intern("state")));
+}
+
+static std::vector<R_ThemeEntry> read_theme() {
+  std::lock_guard lock(ruby_mutex);
+  std::vector<R_ThemeEntry> result;
   if (C_module == Qnil)
     return result;
   VALUE theme_hash = rb_funcall(C_module, rb_intern("theme"), 0);
@@ -56,7 +140,7 @@ static std::vector<ThemeEntry> read_theme() {
     VALUE val_hash = rb_hash_aref(theme_hash, key_sym);
     if (NIL_P(val_hash))
       continue;
-    ThemeEntry entry;
+    R_ThemeEntry entry;
     entry.key = key;
     VALUE fg = rb_hash_aref(val_hash, ID2SYM(rb_intern("fg")));
     VALUE bg = rb_hash_aref(val_hash, ID2SYM(rb_intern("bg")));
@@ -83,7 +167,7 @@ static std::vector<ThemeEntry> read_theme() {
 }
 
 void load_theme() {
-  std::vector<ThemeEntry> entries = read_theme();
+  std::vector<R_ThemeEntry> entries = read_theme();
   Highlight default_hl = {0xFFFFFF, 0, 0};
   for (auto &entry : entries) {
     if (entry.key == "default") {
@@ -126,20 +210,89 @@ void load_theme() {
   }
 }
 
-std::string read_line_endings() {
+std::vector<LSP> read_lsps() {
+  std::vector<LSP> result;
   if (C_module == Qnil)
-    return "";
+    return result;
-  VALUE le = rb_funcall(C_module, rb_intern("line_endings"), 0);
+  VALUE lsp_hash = rb_funcall(C_module, rb_intern("lsp_config"), 0);
-  if (SYMBOL_P(le))
+  if (NIL_P(lsp_hash))
-    return rb_id2name(SYM2ID(le));
+    return result;
-  return "";
+  VALUE keys = rb_funcall(lsp_hash, rb_intern("keys"), 0);
+  for (long i = 0; i < RARRAY_LEN(keys); ++i) {
+    VALUE key = rb_ary_entry(keys, i);
+    std::string cmd = StringValueCStr(key);
+    VALUE args_array = rb_hash_aref(lsp_hash, key);
+    std::vector<std::string> args = ruby_array_to_vector(args_array);
+    result.push_back({cmd, args});
+  }
+  return result;
 }
 
-std::string read_utf_mode() {
+std::vector<R_Language> read_languages() {
+  std::vector<R_Language> result;
   if (C_module == Qnil)
-    return "";
+    return result;
-  VALUE utf = rb_funcall(C_module, rb_intern("utf_mode"), 0);
+  VALUE lang_hash = rb_funcall(C_module, rb_intern("languages"), 0);
-  if (SYMBOL_P(utf))
+  if (NIL_P(lang_hash))
-    return rb_id2name(SYM2ID(utf));
+    return result;
-  return "";
+  VALUE keys = rb_funcall(lang_hash, rb_intern("keys"), 0);
+  for (long i = 0; i < RARRAY_LEN(keys); ++i) {
+    VALUE key = rb_ary_entry(keys, i);
+    VALUE val_hash = rb_hash_aref(lang_hash, key);
+    if (NIL_P(val_hash))
+      continue;
+    R_Language lang;
+    lang.name = rb_id2name(SYM2ID(key));
+    VALUE fg = rb_hash_aref(val_hash, ID2SYM(rb_intern("color")));
+    VALUE symbol = rb_hash_aref(val_hash, ID2SYM(rb_intern("symbol")));
+    VALUE extensions = rb_hash_aref(val_hash, ID2SYM(rb_intern("extensions")));
+    VALUE filenames = rb_hash_aref(val_hash, ID2SYM(rb_intern("filenames")));
+    VALUE mimetypes = rb_hash_aref(val_hash, ID2SYM(rb_intern("mimetypes")));
+    VALUE lsp = rb_hash_aref(val_hash, ID2SYM(rb_intern("lsp")));
+    if (!NIL_P(fg))
+      lang.color = NUM2UINT(fg);
+    if (!NIL_P(symbol))
+      lang.symbol = StringValueCStr(symbol);
+    lang.extensions = ruby_array_to_vector(extensions);
+    lang.filenames = ruby_array_to_vector(filenames);
+    lang.mimetypes = ruby_array_to_vector(mimetypes);
+    if (!NIL_P(lsp))
+      lang.lsp_command = StringValueCStr(lsp);
+    result.push_back(lang);
+  }
+  return result;
+}
+
+void load_languages_info() {
+  std::lock_guard lock(ruby_mutex);
+  auto langs = read_languages();
+  auto lsps_t = read_lsps();
+  languages.clear();
+  for (auto &lang : langs) {
+    Language l;
+    l.name = lang.name;
+    l.color = lang.color;
+    l.lsp_name = lang.lsp_command;
+    l.symbol = lang.symbol;
+    languages[lang.name] = l;
+    for (auto &ext : lang.extensions)
+      language_extensions[ext] = lang.name;
+    // TODO: seperate extensions and filenames
+    for (auto &filename : lang.filenames)
+      language_extensions[filename] = lang.name;
+    for (auto &mimetype : lang.mimetypes)
+      language_mimetypes[mimetype] = lang.name;
+  }
+  for (auto &lsp : lsps_t)
+    lsps[lsp.command] = lsp;
+}
+
+bool read_line_endings() {
+  std::lock_guard lock(ruby_mutex);
+  if (C_module == Qnil)
+    return true;
+  VALUE le = rb_funcall(C_module, rb_intern("line_endings"), 0);
+  if (SYMBOL_P(le))
+    return std::string(rb_id2name(SYM2ID(le))) == "unix";
+  return true;
 }

src/syntax/bash.cc

@@ -68,28 +68,28 @@ std::shared_ptr<void> bash_parse(std::vector<Token> *tokens,
       uint32_t start = i;
       while (i < len) {
         if (text[i] == state->full_state->lit.delim[0]) {
-          tokens->push_back({start, i, TokenKind::String});
+          tokens->push_back({start, i, TokenKind::K_STRING});
           state->full_state->in_state = BashFullState::NONE;
           break;
         }
         i++;
       }
       if (i == len)
-        tokens->push_back({start, i, TokenKind::String});
+        tokens->push_back({start, i, TokenKind::K_STRING});
       continue;
     }
     if (text[i] == '#') {
       if (i == 0 && len > 4 && text[i + 1] == '!') {
-        tokens->push_back({0, len, TokenKind::Shebang});
+        tokens->push_back({0, len, TokenKind::K_SHEBANG});
         return state;
       }
-      tokens->push_back({i, len, TokenKind::Comment});
+      tokens->push_back({i, len, TokenKind::K_COMMENT});
       return state;
     } else if (text[i] == '\'') {
       state->full_state->in_state = BashFullState::STRING;
       state->full_state->lit.delim = "'";
       state->full_state->lit.allow_interp = false;
-      tokens->push_back({i, ++i, TokenKind::String});
+      tokens->push_back({i, ++i, TokenKind::K_STRING});
       continue;
     }
     i++;

src/syntax/parser.cc

@@ -2,6 +2,7 @@
 #include "editor/editor.h"
 #include "io/knot.h"
 #include "main.h"
+#include "ruby/internal/special_consts.h"
 #include "syntax/decl.h"
 #include "syntax/langs.h"
 
@@ -11,12 +12,20 @@ Parser::Parser(Editor *n_editor, std::string n_lang, uint32_t n_scroll_max) {
   editor = n_editor;
   scroll_max = n_scroll_max;
   lang = n_lang;
-  auto pair = parsers.find(n_lang);
+  auto custom_parser = custom_highlighters.find(n_lang);
-  if (pair != parsers.end()) {
+  if (custom_parser != custom_highlighters.end()) {
-    parse_func = std::get<0>(pair->second);
+    parser_block = custom_parser->second.first;
-    state_match_func = std::get<1>(pair->second);
+    match_block = custom_parser->second.second;
+    is_custom = true;
   } else {
-    assert("unknown lang should be checked by caller" && 0);
+    auto pair = parsers.find(n_lang);
+    if (pair != parsers.end()) {
+      parse_func = std::get<0>(pair->second);
+      state_match_func = std::get<1>(pair->second);
+      is_custom = false;
+    } else {
+      assert("unknown lang should be checked by caller" && 0);
+    }
   }
   edit(0, 0, editor->root->line_count);
 }
@@ -29,9 +38,9 @@ void Parser::edit(uint32_t start_line, uint32_t old_end_line,
   if (inserted_rows > 0)
     line_tree.insert(start_line, inserted_rows);
   if (start_line > 0)
-    dirty_lines.insert(start_line - 1);
+    dirty_lines.push(start_line - 1);
-  dirty_lines.insert(start_line);
+  dirty_lines.push(start_line);
-  dirty_lines.insert(start_line + 1);
+  dirty_lines.push(start_line + 1);
 }
 
 void Parser::work() {
@@ -41,16 +50,18 @@ void Parser::work() {
   k_lock.unlock();
   uint32_t capacity = 256;
   char *text = (char *)calloc((capacity + 1), sizeof(char));
-  std::set<uint32_t> tmp_dirty;
   std::unique_lock lock_data(data_mutex);
-  tmp_dirty.swap(dirty_lines);
   lock_data.unlock();
-  std::set<uint32_t> remaining_dirty;
   std::unique_lock lock(mutex);
   lock.unlock();
-  for (uint32_t c_line : tmp_dirty) {
+  uint32_t c_line;
+  while (dirty_lines.pop(c_line)) {
+    if (!running.load(std::memory_order_relaxed)) {
+      free(text);
+      return;
+    }
     if (c_line > scroll_max + 40) {
-      remaining_dirty.insert(c_line);
+      dirty_lines.push(c_line);
       continue;
     }
     uint32_t line_count = line_tree.count();
@@ -65,6 +76,10 @@ void Parser::work() {
         free(text);
         return;
       }
+      if (scroll_dirty.exchange(false, std::memory_order_acq_rel)) {
+        dirty_lines.push(c_line);
+        c_line = scroll_max < 50 ? 0 : scroll_max - 50;
+      }
       k_lock.lock();
       if (c_line > editor->root->line_count) {
         k_lock.unlock();
@@ -95,8 +110,22 @@ void Parser::work() {
           lock.unlock();
         continue;
       }
-      std::shared_ptr<void> new_state =
+      std::shared_ptr<void> new_state{nullptr};
-          parse_func(&line_data->tokens, prev_state, text, r_len);
+      if (is_custom) {
+        VALUE state = Qnil;
+        if (prev_state) {
+          std::shared_ptr<CustomState> state_ptr =
+              std::static_pointer_cast<CustomState>(prev_state);
+          state = state_ptr->state;
+        }
+        VALUE out_state =
+            parse_custom(&line_data->tokens, parser_block, text, r_len, state);
+        std::shared_ptr<CustomState> out_state_ptr =
+            std::make_shared<CustomState>(out_state);
+        new_state = out_state_ptr;
+      } else {
+        new_state = parse_func(&line_data->tokens, prev_state, text, r_len);
+      }
       line_data->in_state = prev_state;
       line_data->out_state = new_state;
       if (!running.load(std::memory_order_relaxed)) {
@@ -110,16 +139,32 @@ void Parser::work() {
         if (lock.owns_lock())
           lock.unlock();
         if (c_line > 0)
-          remaining_dirty.insert(c_line - 1);
+          dirty_lines.push(c_line - 1);
-        remaining_dirty.insert(c_line);
+        dirty_lines.push(c_line);
         break;
       }
-      if (c_line < line_count && (line_data = line_tree.at(c_line)) &&
+      if (c_line < line_count && (line_data = line_tree.at(c_line))) {
-          state_match_func(prev_state, line_data->in_state)) {
+        bool done = false;
-        lock_data.unlock();
+        if (is_custom) {
-        if (lock.owns_lock())
+          VALUE in_state_v = Qnil;
-          lock.unlock();
+          if (prev_state)
-        break;
+            in_state_v =
+                std::static_pointer_cast<CustomState>(prev_state)->state;
+          VALUE out_state_v = Qnil;
+          if (line_data->in_state)
+            out_state_v =
+                std::static_pointer_cast<CustomState>(line_data->in_state)
+                    ->state;
+          done = custom_compare(match_block, in_state_v, out_state_v);
+        } else {
+          done = state_match_func(prev_state, line_data->in_state);
+        }
+        if (done) {
+          lock_data.unlock();
+          if (lock.owns_lock())
+            lock.unlock();
+          break;
+        }
       }
       lock_data.unlock();
       if (lock.owns_lock())
@@ -132,80 +177,19 @@ void Parser::work() {
   }
   free(text);
   lock_data.lock();
-  dirty_lines = std::move(remaining_dirty);
 }
 
 void Parser::scroll(uint32_t line) {
   if (line != scroll_max) {
     scroll_max = line;
-    uint32_t c_line = line > 100 ? line - 100 : 0;
+    uint32_t c_line = line > 50 ? line - 50 : 0;
-    if (line_tree.count() < c_line)
+    if (c_line >= line_tree.count())
       return;
     std::unique_lock lock_data(data_mutex);
     if (line_tree.at(c_line)->in_state || line_tree.at(c_line)->out_state)
       return;
-    lock_data.unlock();
+    scroll_dirty = true;
-    std::shared_lock k_lock(editor->knot_mtx);
+    dirty_lines.push(c_line);
-    k_lock.unlock();
-    uint32_t capacity = 256;
-    char *text = (char *)calloc((capacity + 1), sizeof(char));
-    uint32_t line_count = line_tree.count();
-    std::unique_lock lock(mutex);
-    std::shared_ptr<void> prev_state =
-        (c_line > 0) ? line_tree.at(c_line - 1)->out_state : nullptr;
-    lock.unlock();
-    while (c_line < line_count) {
-      if (!running.load(std::memory_order_relaxed)) {
-        free(text);
-        return;
-      }
-      k_lock.lock();
-      if (c_line > editor->root->line_count) {
-        k_lock.unlock();
-        continue;
-      }
-      uint32_t r_offset, r_len;
-      r_offset = line_to_byte(editor->root, c_line, &r_len);
-      if (r_len > capacity) {
-        capacity = r_len;
-        text = (char *)realloc(text, capacity + 1);
-        memset(text, 0, capacity + 1);
-      }
-      read_into(editor->root, r_offset, r_len, text);
-      k_lock.unlock();
-      if (c_line < scroll_max &&
-          ((scroll_max > 100 && c_line > scroll_max - 100) || c_line < 100))
-        lock.lock();
-      if (line_tree.count() < c_line) {
-        if (lock.owns_lock())
-          lock.unlock();
-        continue;
-      }
-      lock_data.lock();
-      LineData *line_data = line_tree.at(c_line);
-      if (!line_data) {
-        lock_data.unlock();
-        if (lock.owns_lock())
-          lock.unlock();
-        continue;
-      }
-      std::shared_ptr<void> new_state =
-          parse_func(&line_data->tokens, prev_state, text, r_len);
-      line_data->in_state = nullptr;
-      line_data->out_state = new_state;
-      lock_data.unlock();
-      if (lock.owns_lock())
-        lock.unlock();
-      if (!running.load(std::memory_order_relaxed)) {
-        free(text);
-        return;
-      }
-      prev_state = new_state;
-      c_line++;
-      if (c_line < line_count && c_line > scroll_max + 50)
-        break;
-    }
-    free(text);
   } else {
     scroll_max = line;
   }

src/utils/system.cc

@@ -1,6 +1,10 @@
-#include "config.h"
 #include "utils/utils.h"
 
+std::unordered_map<std::string, Language> languages;
+std::unordered_map<std::string, std::string> language_extensions;
+std::unordered_map<std::string, std::string> language_mimetypes;
+std::unordered_map<std::string, LSP> lsps;
+
 void log(const char *fmt, ...) {
   FILE *fp = fopen("/tmp/log.txt", "a");
   if (!fp)
@@ -41,33 +45,39 @@ std::string get_exe_dir() {
 }
 
 char *load_file(const char *path, uint32_t *out_len) {
-  std::ifstream file(path, std::ios::in | std::ios::binary | std::ios::ate);
+  std::ifstream file(path, std::ios::binary | std::ios::ate);
   if (!file.is_open())
     return nullptr;
   std::streamsize len = file.tellg();
-  if (len < 0 || static_cast<uint32_t>(len) > 0xFFFFFFFF)
+  if (len < 0 || static_cast<uint64_t>(len) > 0xFFFFFFFF)
     return nullptr;
   file.seekg(0, std::ios::beg);
-  bool add_newline = false;
+  unsigned char bom[3] = {0};
-  if (len > 0) {
+  file.read(reinterpret_cast<char *>(bom), 3);
-    file.seekg(-1, std::ios::end);
+  if ((bom[0] == 0xFF && bom[1] == 0xFE) || (bom[0] == 0xFE && bom[1] == 0xFF))
-    char last_char;
+    return nullptr;
-    file.read(&last_char, 1);
+  bool has_utf8_bom = (bom[0] == 0xEF && bom[1] == 0xBB && bom[2] == 0xBF);
-    if (last_char != '\n')
+  uint32_t skip = has_utf8_bom ? 3 : 0;
-      add_newline = true;
+  uint32_t data_len = static_cast<uint32_t>(len) - skip;
-  }
+  file.seekg(skip, std::ios::beg);
-  file.seekg(0, std::ios::beg);
+  char *buf = (char *)malloc(data_len + 1);
-  uint32_t alloc_size = static_cast<uint32_t>(len) + (add_newline ? 1 : 0);
-  char *buf = (char *)malloc(alloc_size);
   if (!buf)
     return nullptr;
-  if (!file.read(buf, len)) {
+  file.read(buf, data_len);
-    free(buf);
+  if (memchr(buf, '\r', data_len) == nullptr) {
-    return nullptr;
+    uint32_t write = data_len;
+    if (write == 0 || buf[write - 1] != '\n')
+      buf[write++] = '\n';
+    *out_len = write;
+    return buf;
   }
-  if (add_newline)
+  uint32_t write = 0;
-    buf[len++] = '\n';
+  for (uint32_t i = 0; i < data_len; ++i)
-  *out_len = static_cast<uint32_t>(len);
+    if (buf[i] != '\r')
+      buf[write++] = buf[i];
+  if (write == 0 || buf[write - 1] != '\n')
+    buf[write++] = '\n';
+  *out_len = write;
   return buf;
 }
 
@@ -108,17 +118,17 @@ Language language_for_file(const char *filename) {
   std::string ext = file_extension(filename);
   std::string lang_name;
   if (!ext.empty()) {
-    auto it = kExtToLang.find(ext);
+    auto it = language_extensions.find(ext);
-    if (it != kExtToLang.end())
+    if (it != language_extensions.end())
-      return kLanguages.find(it->second)->second;
+      return languages.find(it->second)->second;
   }
   char *mime = detect_file_type(filename);
   if (mime) {
     std::string mime_type(mime);
     free(mime);
-    auto it = kMimeToLang.find(mime_type);
+    auto it = language_mimetypes.find(mime_type);
-    if (it != kMimeToLang.end())
+    if (it != language_mimetypes.end())
-      return kLanguages.find(it->second)->second;
+      return languages.find(it->second)->second;
   }
   return Language{};
 }