Fix lsp bugs

- Fix: Incorrect setting of incremental edits for lsp and more

README.md

@@ -18,7 +18,6 @@ A TUI IDE.
         - `textDocument/foldingRange` - i will never use this for folding but it might be useful for other things.
         - `textDocument/rename` & `textDocument/prepareRename` - probably useful
         - And a lot more (just go through each for `clangd` and then expand to say `solargraph`).
-    - Make incremental edits apply. // make a bool field in LSP qhich says if it supports incremental and based on it apply edits
     - Make a universal plug for lsp. So focus more on making a general purpose solid communication interface. Instead of something specific.
         - With a 4ish pass system. (more like each returned value from the lsp is used in 4 ways)
             1. One for stuff like jump to x position. or rename symbol x to y. (stuff that explicitly requires user request to do something)

grammar/jsonc.scm

@@ -0,0 +1,22 @@
+;; #D2A6FF #000000 0 0 0 2
+(pair
+  key: (_) @string.special.key)
+
+;; #AAD94C #000000 0 0 0 1
+(string) @string
+
+;; #7dcfff #000000 0 0 0 2
+(number) @number
+
+;; #F07178 #000000 0 0 0 1
+[
+  (null)
+  (true)
+  (false)
+] @constant.builtin
+
+;; #7dcfff #000000 0 0 0 2
+(escape_sequence) @escape
+
+;; #99ADBF #000000 0 1 0 1
+(comment) @comment

include/editor.h

@@ -156,7 +156,7 @@ struct Editor {
   std::vector<VWarn> warnings;
   VAI ai;
   std::shared_mutex lsp_mtx;
-  struct LSPInstance *lsp;
+  std::shared_ptr<struct LSPInstance> lsp;
   int lsp_version = 1;
 };
 

include/lsp.h

@@ -29,28 +29,33 @@ struct LSPInstance {
   int pid{-1};
   int stdin_fd{-1};
   int stdout_fd{-1};
-  bool initialized = false;
+  std::atomic<bool> initialized = false;
-  bool incremental_sync = true;
+  std::atomic<bool> exited = false;
+  bool incremental_sync = false;
   uint32_t last_id = 0;
   Queue<json> inbox;
   Queue<json> outbox;
+  Queue<std::pair<Language, Editor *>> open_queue;
   std::unordered_map<uint32_t, LSPPending *> pending;
   std::vector<Editor *> editors;
 };
 
 extern std::shared_mutex active_lsps_mtx;
-extern std::unordered_map<uint8_t, LSPInstance *> active_lsps;
+extern std::unordered_map<uint8_t, std::shared_ptr<LSPInstance>> active_lsps;
 
 void lsp_worker();
-void lsp_handle(LSPInstance *lsp, json message);
+void lsp_handle(std::shared_ptr<LSPInstance> lsp, json message);
 
-LSPInstance *get_or_init_lsp(uint8_t lsp_id);
+std::shared_ptr<LSPInstance> get_or_init_lsp(uint8_t lsp_id);
 void close_lsp(uint8_t lsp_id);
 
 void request_add_to_lsp(Language language, Editor *editor);
+void open_editor(std::shared_ptr<LSPInstance> lsp,
+                 std::pair<Language, Editor *> entry);
 void add_to_lsp(Language language, Editor *editor);
 void remove_from_lsp(Editor *editor);
 
-void lsp_send(LSPInstance *lsp, json message, LSPPending *pending);
+void lsp_send(std::shared_ptr<LSPInstance> lsp, json message,
+              LSPPending *pending);
 
 #endif

include/maps.h

@@ -155,6 +155,15 @@ static const std::unordered_map<uint8_t, LSP> kLsps = {
           "make-language-server",
           nullptr,
       }}},
+    {22,
+     {"sql-language-server",
+      {
+          "sql-language-server",
+          "up",
+          "--method",
+          "stdio",
+          nullptr,
+      }}},
 };
 
 static const std::unordered_map<std::string, Language> kLanguages = {
@@ -170,6 +179,7 @@ static const std::unordered_map<std::string, Language> kLanguages = {
     {"html", {"html", LANG(html), 10}},
     {"javascript", {"javascript", LANG(javascript), 11}},
     {"json", {"json", LANG(json), 6}},
+    {"jsonc", {"jsonc", LANG(json), 6}},
     {"erb", {"erb", LANG(embedded_template), 10}},
     {"ruby", {"ruby", LANG(ruby), 3}},
     {"lua", {"lua", LANG(lua), 12}},
@@ -181,7 +191,8 @@ static const std::unordered_map<std::string, Language> kLanguages = {
     {"nginx", {"nginx", LANG(nginx), 17}},
     {"toml", {"toml", LANG(toml), 18}},
     {"yaml", {"yaml", LANG(yaml), 19}},
-    {"sql", {"sql", LANG(sql), 20}},
+    {"sql", {"sql", LANG(sql), 20}}, // Can use `22` for more accuracy but need
+                                     // config to connect to database
     {"make", {"make", LANG(make), 21}},
     {"gdscript", {"gdscript", LANG(gdscript)}}, // TODO: connect to godot
     {"diff", {"diff", LANG(diff)}},
@@ -212,7 +223,7 @@ static const std::unordered_map<std::string, std::string> kExtToLang = {
     {"js", "javascript"},
     {"jsx", "javascript"},
     {"json", "json"},
-    {"jsonc", "json"},
+    {"jsonc", "jsonc"},
     {"lua", "lua"},
     {"mk", "make"},
     {"makefile", "make"},

include/utils.h

@@ -12,8 +12,9 @@ template <typename T> struct Queue {
     std::lock_guard<std::mutex> lock(m);
     q.push(val);
   }
-  T front() {
+  std::optional<T> front() {
-    std::lock_guard<std::mutex> lock(m);
+    if (q.empty())
+      return std::nullopt;
     return q.front();
   }
   bool pop(T &val) {

src/lsp.cc

@@ -1,5 +1,6 @@
 #include "../include/lsp.h"
 #include "../include/maps.h"
+#include <cmath>
 #include <fcntl.h>
 #include <signal.h>
 #include <sys/poll.h>
@@ -8,11 +9,11 @@
 #include <unistd.h>
 
 std::shared_mutex active_lsps_mtx;
-std::unordered_map<uint8_t, LSPInstance *> active_lsps;
+std::unordered_map<uint8_t, std::shared_ptr<LSPInstance>> active_lsps;
 
 Queue<LSPOpenRequest> lsp_open_queue;
 
-static bool init_lsp(LSPInstance *lsp) {
+static bool init_lsp(std::shared_ptr<LSPInstance> lsp) {
   log("initializing %s\n", lsp->lsp->command);
   int in_pipe[2];
   int out_pipe[2];
@@ -47,19 +48,17 @@ static bool init_lsp(LSPInstance *lsp) {
   return true;
 }
 
-LSPInstance *get_or_init_lsp(uint8_t lsp_id) {
+std::shared_ptr<LSPInstance> get_or_init_lsp(uint8_t lsp_id) {
   std::unique_lock lock(active_lsps_mtx);
   auto it = active_lsps.find(lsp_id);
   if (it == active_lsps.end()) {
     auto map_it = kLsps.find(lsp_id);
     if (map_it == kLsps.end())
       return nullptr;
-    LSPInstance *lsp = new LSPInstance();
+    std::shared_ptr<LSPInstance> lsp = std::make_shared<LSPInstance>();
     lsp->lsp = &map_it->second;
-    if (!init_lsp(lsp)) {
+    if (!init_lsp(lsp))
-      delete lsp;
       return nullptr;
-    }
     log("starting %s\n", lsp->lsp->command);
     LSPPending *pending = new LSPPending();
     pending->method = "initialize";
@@ -67,18 +66,29 @@ LSPInstance *get_or_init_lsp(uint8_t lsp_id) {
     pending->callback = [lsp](Editor *, std::string, json msg) {
       if (msg.contains("result") && msg["result"].contains("capabilities")) {
         auto &caps = msg["result"]["capabilities"];
-        if (caps.contains("textDocumentSync") &&
+        if (caps.contains("textDocumentSync")) {
-            caps["textDocumentSync"].contains("change")) {
+          auto &sync = caps["textDocumentSync"];
-          int change_type = caps["textDocumentSync"]["change"];
+          if (sync.is_number()) {
+            int change_type = sync.get<int>();
+            lsp->incremental_sync = (change_type == 2);
+          } else if (sync.is_object() && sync.contains("change")) {
+            int change_type = sync["change"].get<int>();
             lsp->incremental_sync = (change_type == 2);
           }
         }
+      }
+      log("incremental_sync %d\n", lsp->incremental_sync);
       lsp->initialized = true;
       json initialized = {{"jsonrpc", "2.0"},
                           {"method", "initialized"},
                           {"params", json::object()}};
       lsp_send(lsp, initialized, nullptr);
       log("initialized %s\n", lsp->lsp->command);
+      while (!lsp->open_queue.empty()) {
+        std::pair<Language, Editor *> request;
+        lsp->open_queue.pop(request);
+        open_editor(lsp, request);
+      }
     };
     json init_message = {
         {"jsonrpc", "2.0"},
@@ -96,7 +106,8 @@ LSPInstance *get_or_init_lsp(uint8_t lsp_id) {
   return it->second;
 }
 
-void lsp_send(LSPInstance *lsp, json message, LSPPending *pending) {
+void lsp_send(std::shared_ptr<LSPInstance> lsp, json message,
+              LSPPending *pending) {
   if (!lsp || lsp->stdin_fd == -1)
     return;
   std::unique_lock lock(lsp->mtx);
@@ -113,8 +124,10 @@ void close_lsp(uint8_t lsp_id) {
   auto it = active_lsps.find(lsp_id);
   if (it == active_lsps.end())
     return;
-  LSPInstance *lsp = it->second;
+  std::shared_ptr<LSPInstance> lsp = it->second;
   active_lsps_lock.unlock();
+  lsp->exited = true;
+  lsp->initialized = false;
   LSPPending *shutdown_pending = new LSPPending();
   shutdown_pending->method = "shutdown";
   shutdown_pending->callback = [lsp, lsp_id](Editor *, std::string, json) {
@@ -127,18 +140,17 @@ void close_lsp(uint8_t lsp_id) {
     std::this_thread::sleep_for(100ms);
     std::unique_lock active_lsps_lock(active_lsps_mtx);
     std::unique_lock lock(lsp->mtx);
-    if (kill(lsp->pid, 0) == 0)
+    if (lsp->pid != -1 && kill(lsp->pid, 0) == 0)
       kill(lsp->pid, SIGKILL);
     waitpid(lsp->pid, nullptr, 0);
     close(lsp->stdin_fd);
     close(lsp->stdout_fd);
-    while (!lsp->outbox.empty())
-      lsp->outbox.pop();
-    while (!lsp->inbox.empty())
-      lsp->inbox.pop();
     for (auto &kv : lsp->pending)
       delete kv.second;
-    delete lsp;
+    for (auto &editor : lsp->editors) {
+      std::unique_lock editor_lock(editor->lsp_mtx);
+      editor->lsp = nullptr;
+    }
     active_lsps.erase(lsp_id);
   });
   t.detach();
@@ -177,7 +189,8 @@ static std::optional<json> read_lsp_message(int fd) {
   return json::parse(body);
 }
 
-static Editor *editor_for_uri(LSPInstance *lsp, std::string uri) {
+static Editor *editor_for_uri(std::shared_ptr<LSPInstance> lsp,
+                              std::string uri) {
   if (uri.empty())
     return nullptr;
   for (auto &editor : lsp->editors)
@@ -186,20 +199,45 @@ static Editor *editor_for_uri(LSPInstance *lsp, std::string uri) {
   return nullptr;
 }
 
+static void clean_lsp(std::shared_ptr<LSPInstance> lsp, uint8_t lsp_id) {
+  log("cleaning up lsp %d\n", lsp_id);
+  for (auto &kv : lsp->pending)
+    delete kv.second;
+  lsp->pid = -1;
+  close(lsp->stdin_fd);
+  close(lsp->stdout_fd);
+  for (auto &editor : lsp->editors) {
+    std::unique_lock editor_lock(editor->lsp_mtx);
+    editor->lsp = nullptr;
+  }
+  active_lsps.erase(lsp_id);
+}
+
 void lsp_worker() {
   LSPOpenRequest request;
   while (lsp_open_queue.pop(request))
     add_to_lsp(request.language, request.editor);
-  std::shared_lock active_lsps_lock(active_lsps_mtx);
+  std::unique_lock active_lsps_lock(active_lsps_mtx);
   for (auto &kv : active_lsps) {
-    LSPInstance *lsp = kv.second;
+    std::shared_ptr<LSPInstance> lsp = kv.second;
     std::unique_lock lock(lsp->mtx);
+    int status;
+    pid_t res = waitpid(lsp->pid, &status, WNOHANG);
+    if (res == lsp->pid) {
+      clean_lsp(lsp, kv.first);
+      return;
+    }
     while (!lsp->outbox.empty()) {
-      json message;
+      json message = lsp->outbox.front();
-      message = lsp->outbox.front();
-      if (!lsp->initialized) {
       std::string m = message.value("method", "");
-        if (m != "initialize")
+      if (lsp->exited) {
+        if (m != "exit" && m != "shutdown") {
+          lsp->outbox.pop(message);
+          continue;
+        }
+      }
+      if (!lsp->initialized) {
+        if (m != "initialize" && m != "exit" && m != "shutdown")
           break;
       }
       lsp->outbox.pop(message);
@@ -210,6 +248,12 @@ void lsp_worker() {
       const char *ptr = out.data();
       size_t remaining = out.size();
       while (remaining > 0) {
+        int status;
+        pid_t res = waitpid(lsp->pid, &status, WNOHANG);
+        if (res == lsp->pid) {
+          clean_lsp(lsp, kv.first);
+          return;
+        }
         ssize_t written = write(lsp->stdin_fd, ptr, remaining);
         if (written == 0)
           break;
@@ -217,15 +261,25 @@ void lsp_worker() {
           if (errno == EINTR)
             continue;
           perror("write");
-          break;
+          clean_lsp(lsp, kv.first);
+          return;
         } else {
           ptr += written;
           remaining -= written;
         }
       }
     }
-    pollfd pfd{lsp->stdout_fd, POLLIN, 0};
+    pollfd pfd{lsp->stdout_fd, POLLIN | POLLHUP | POLLERR, 0};
-    while (poll(&pfd, 1, 0) > 0) {
+    int r = poll(&pfd, 1, 0);
+    if (r > 0 && pfd.revents & (POLLHUP | POLLERR)) {
+      clean_lsp(lsp, kv.first);
+      return;
+    }
+    while ((r = poll(&pfd, 1, 0) > 0)) {
+      if (r > 0 && pfd.revents & (POLLHUP | POLLERR)) {
+        clean_lsp(lsp, kv.first);
+        return;
+      }
       auto msg = read_lsp_message(lsp->stdout_fd);
       if (!msg)
         break;
@@ -268,14 +322,21 @@ void request_add_to_lsp(Language language, Editor *editor) {
 }
 
 void add_to_lsp(Language language, Editor *editor) {
-  LSPInstance *lsp = get_or_init_lsp(language.lsp_id);
+  std::shared_ptr<LSPInstance> lsp = get_or_init_lsp(language.lsp_id);
   if (!lsp)
     return;
   std::unique_lock lock(lsp->mtx);
   if (editor->lsp == lsp)
     return;
   lsp->editors.push_back(editor);
+  lsp->open_queue.push({language, editor});
   lock.unlock();
+}
+
+void open_editor(std::shared_ptr<LSPInstance> lsp,
+                 std::pair<Language, Editor *> entry) {
+  Language language = entry.first;
+  Editor *editor = entry.second;
   std::unique_lock lock2(editor->lsp_mtx);
   editor->lsp = lsp;
   lock2.unlock();
@@ -295,7 +356,7 @@ void add_to_lsp(Language language, Editor *editor) {
   lsp_send(lsp, message, nullptr);
 }
 
-static uint8_t find_lsp_id(LSPInstance *needle) {
+static uint8_t find_lsp_id(std::shared_ptr<LSPInstance> needle) {
   for (const auto &[id, lsp] : active_lsps)
     if (lsp == needle)
       return id;
@@ -323,7 +384,7 @@ void remove_from_lsp(Editor *editor) {
     close_lsp(lsp_id);
 }
 
-void lsp_handle(LSPInstance *, json message) {
+void lsp_handle(std::shared_ptr<LSPInstance>, json message) {
   std::string method = message.value("method", "");
   if (method == "window/showMessage") {
     if (message.contains("params")) {

src/utils.cc

@@ -186,6 +186,7 @@ uint32_t count_clusters(const char *line, size_t len, size_t from, size_t to) {
 }
 
 void log(const char *fmt, ...) {
+#if defined(__GNUC__) && !defined(__clang__)
   FILE *fp = fopen("/tmp/log.txt", "a");
   if (!fp)
     return;
@@ -195,6 +196,7 @@ void log(const char *fmt, ...) {
   va_end(args);
   fputc('\n', fp);
   fclose(fp);
+#endif
 }
 
 char *load_file(const char *path, uint32_t *out_len) {