patch 8.2.4648: handling LSP messages is a bit slow

Problem:    Handling LSP messages is a bit slow.
Solution:   Included support for LSP messages. (Yegappan Lakshmanan,
            closes #10025)

runtime/doc/channel.txt

@@ -53,6 +53,7 @@ RAW	nothing known, Vim cannot tell where a message ends
 NL	every message ends in a NL (newline) character
 JSON	JSON encoding |json_encode()|
 JS	JavaScript style JSON-like encoding |js_encode()|
+LSP	Language Server Protocol encoding |language-server-protocol|
 
 Common combination are:
 - Using a job connected through pipes in NL mode.  E.g., to run a style
@@ -130,6 +131,7 @@ When using an IPv6 address, enclose it within square brackets.  E.g.,
 	"js"   - Use JS (JavaScript) encoding, more efficient than JSON.
 	"nl"   - Use messages that end in a NL character
 	"raw"  - Use raw messages
+	"lsp"  - Use language server protocol encoding
 						*channel-callback* *E921*
 "callback"	A function that is called when a message is received that is
 		not handled otherwise (e.g. a JSON message with ID zero).  It
@@ -140,8 +142,8 @@ When using an IPv6 address, enclose it within square brackets.  E.g.,
 	endfunc
 	let channel = ch_open("localhost:8765", {"callback": "Handle"})
 <
-		When "mode" is "json" or "js" the "msg" argument is the body
-		of the received message, converted to Vim types.
+		When "mode" is "json" or "js" or "lsp" the "msg" argument is
+		the body of the received message, converted to Vim types.
 		When "mode" is "nl" the "msg" argument is one message,
 		excluding the NL.
 		When "mode" is "raw" the "msg" argument is the whole message
@@ -165,7 +167,19 @@ When using an IPv6 address, enclose it within square brackets.  E.g.,
 		to check for messages, the close_cb may be invoked while still
 		in the callback.  The plugin must handle this somehow, it can
 		be useful to know that no more data is coming.
-							*channel-drop*
+		If it is not known if there is a message to be read, use a
+		try/catch block: >
+			try
+			  let msg = ch_readraw(a:channel)
+			catch
+			  let msg = 'no message'
+			endtry
+			try
+			  let err = ch_readraw(a:channel, #{part: 'err'})
+			catch
+			  let err = 'no error'
+			endtry
+<							*channel-drop*
 "drop"		Specifies when to drop messages:
 		    "auto"	When there is no callback to handle a message.
 				The "close_cb" is also considered for this.
@@ -443,7 +457,7 @@ to check if there is something to read.
 Note that when there is no callback, messages are dropped.  To avoid that add
 a close callback to the channel.
 
-To read all output from a RAW channel that is available: >
+To read all normal output from a RAW channel that is available: >
 	let output = ch_readraw(channel)
 To read the error output: >
 	let output = ch_readraw(channel, {"part": "err"})
@@ -503,6 +517,7 @@ ch_evalexpr({handle}, {expr} [, {options}])			*ch_evalexpr()*
 		according to the type of channel.  The function cannot be used
 		with a raw channel.  See |channel-use|.
 		{handle} can be a Channel or a Job that has a Channel.
+		When using the "lsp" channel mode, {expr} must be a |Dict|.
 								*E917*
 		{options} must be a Dictionary.  It must not have a "callback"
 		entry.  It can have a "timeout" entry to specify the timeout
@@ -578,7 +593,7 @@ ch_info({handle})						*ch_info()*
 		   "err_io"	  "out", "null", "pipe", "file" or "buffer"
 		   "err_timeout"  timeout in msec
 		   "in_status"	  "open" or "closed"
-		   "in_mode"	  "NL", "RAW", "JSON" or "JS"
+		   "in_mode"	  "NL", "RAW", "JSON", "JS" or "LSP"
 		   "in_io"	  "null", "pipe", "file" or "buffer"
 		   "in_timeout"	  timeout in msec
 
@@ -674,6 +689,7 @@ ch_sendexpr({handle}, {expr} [, {options}])			*ch_sendexpr()*
 		with a raw channel.
 		See |channel-use|.				*E912*
 		{handle} can be a Channel or a Job that has a Channel.
+		When using the "lsp" channel mode, {expr} must be a |Dict|.
 
 		Can also be used as a |method|: >
 			GetChannel()->ch_sendexpr(expr)
@@ -1361,5 +1377,76 @@ The same in |Vim9| script: >
 	# start accepting shell commands
 	startinsert
 
+==============================================================================
+14. Language Server Protocol			*language-server-protocol*
+
+The language server protocol specification is available at:
+
+    https://microsoft.github.io/language-server-protocol/specification
+
+Each LSP protocol message starts with a simple HTTP header followed by the
+payload encoded in JSON-RPC format.  This is described in:
+
+    https://www.jsonrpc.org/specification
+
+For messages received on a channel with mode set to "lsp", Vim will process
+the HTTP header and decode the payload into a Vim |Dict| type and call the
+channel callback or the specified callback function.  When sending messages on
+a channel using |ch_evalexpr()| or |ch_sendexpr()|, Vim will add the HTTP
+header and encode the Vim expression into JSON-RPC.
+
+To open a channel using the 'lsp' mode, set the 'mode' item in the |ch_open()|
+{options} argument to 'lsp'.  Example: >
+
+    let ch = ch_open(..., #{mode: 'lsp'})
+
+To open a channel using the 'lsp' mode with a job, set the 'in_mode' and
+'out_mode' items in the |job_start()| {options} argument to 'lsp'. Example: >
+
+    let job = job_start(...., #{in_mode: 'lsp', out_mode: 'lsp'})
+
+To synchronously send a JSON-RPC request to the server, use the |ch_evalexpr()|
+function. This function will return the response from the server. You can use
+the 'timeout' field in the {options} argument to control the response wait
+time. Example: >
+
+    let req = {}
+    let req.method = 'textDocument/definition'
+    let req.params = {}
+    let req.params.textDocument = #{uri: 'a.c'}
+    let req.params.position = #{line: 10, character: 3}
+    let resp = ch_evalexpr(ch, req, #{timeout: 100})
+
+Note that in the request message the 'id' field should not be specified. If it
+is specified, then Vim will overwrite the value with an internally generated
+identifier.  Vim currently supports only a number type for the 'id' field.
+
+To send a JSON-RPC request to the server and asynchronously process the
+response, use the |ch_sendexpr()| function and supply a callback function.
+Example: >
+
+    let req = {}
+    let req.method = 'textDocument/hover'
+    let req.params = {}
+    let req.params.textDocument = #{uri: 'a.c'}
+    let req.params.position = #{line: 10, character: 3}
+    let resp = ch_sendexpr(ch, req, #{callback: 'MyFn'})
+
+To send a JSON-RPC notification message to the server, use the |ch_sendexpr()|
+function. Example: >
+
+    call ch_sendexpr(ch, #{method: 'initialized'})
+
+To respond to a JSON-RPC request message from the server, use the
+|ch_sendexpr()| function.  In the response message, copy the 'id' field value
+from the server request message. Example: >
+
+    let resp = {}
+    let resp.id = req.id
+    let resp.result = 1
+    call ch_sendexpr(ch, resp)
+
+The JSON-RPC notification messages from the server are delivered through the
+|channel-callback| function.
 
  vim:tw=78:ts=8:noet:ft=help:norl:

src/channel.c

@@ -2111,6 +2111,83 @@ channel_fill(js_read_T *reader)
     return TRUE;
 }
 
+/*
+ * Process the HTTP header in a Language Server Protocol (LSP) message.
+ *
+ * The message format is described in the LSP specification:
+ * https://microsoft.github.io/language-server-protocol/specification
+ *
+ * It has the following two fields:
+ *
+ *	Content-Length: ...
+ *	Content-Type: application/vscode-jsonrpc; charset=utf-8
+ *
+ * Each field ends with "\r\n". The header ends with an additional "\r\n".
+ *
+ * Returns OK if a valid header is received and FAIL if some fields in the
+ * header are not correct. Returns MAYBE if a partial header is received and
+ * need to wait for more data to arrive.
+ */
+    static int
+channel_process_lsp_http_hdr(js_read_T *reader)
+{
+    char_u	*line_start;
+    char_u	*p;
+    int_u	hdr_len;
+    int		payload_len = -1;
+    int_u	jsbuf_len;
+
+    // We find the end once, to avoid calling strlen() many times.
+    jsbuf_len = (int_u)STRLEN(reader->js_buf);
+    reader->js_end = reader->js_buf + jsbuf_len;
+
+    p = reader->js_buf;
+
+    // Process each line in the header till an empty line is read (header
+    // separator).
+    while (TRUE)
+    {
+	line_start = p;
+	while (*p != NUL && *p != '\n')
+	    p++;
+	if (*p == NUL)			// partial header
+	    return MAYBE;
+	p++;
+
+	// process the content length field (if present)
+	if ((p - line_start > 16)
+		&& STRNICMP(line_start, "Content-Length: ", 16) == 0)
+	{
+	    errno = 0;
+	    payload_len = strtol((char *)line_start + 16, NULL, 10);
+	    if (errno == ERANGE || payload_len < 0)
+		// invalid length, discard the payload
+		return FAIL;
+	}
+
+	if ((p - line_start) == 2 && line_start[0] == '\r' &&
+		line_start[1] == '\n')
+	    // reached the empty line
+	    break;
+    }
+
+    if (payload_len == -1)
+	// Content-Length field is not present in the header
+	return FAIL;
+
+    hdr_len = p - reader->js_buf;
+
+    // if the entire payload is not received, wait for more data to arrive
+    if (jsbuf_len < hdr_len + payload_len)
+	return MAYBE;
+
+    reader->js_used += hdr_len;
+    // recalculate the end based on the length read from the header.
+    reader->js_end = reader->js_buf + hdr_len + payload_len;
+
+    return OK;
+}
+
 /*
  * Use the read buffer of "channel"/"part" and parse a JSON message that is
  * complete.  The messages are added to the queue.
@@ -2124,7 +2201,7 @@ channel_parse_json(channel_T *channel, ch_part_T part)
     jsonq_T	*item;
     chanpart_T	*chanpart = &channel->ch_part[part];
     jsonq_T	*head = &chanpart->ch_json_head;
-    int		status;
+    int		status = OK;
     int		ret;
 
     if (channel_peek(channel, part) == NULL)
@@ -2136,19 +2213,31 @@ channel_parse_json(channel_T *channel, ch_part_T part)
     reader.js_cookie = channel;
     reader.js_cookie_arg = part;
 
+    if (chanpart->ch_mode == MODE_LSP)
+	status = channel_process_lsp_http_hdr(&reader);
+
     // When a message is incomplete we wait for a short while for more to
     // arrive.  After the delay drop the input, otherwise a truncated string
     // or list will make us hang.
     // Do not generate error messages, they will be written in a channel log.
-    ++emsg_silent;
-    status = json_decode(&reader, &listtv,
-				  chanpart->ch_mode == MODE_JS ? JSON_JS : 0);
-    --emsg_silent;
+    if (status == OK)
+    {
+	++emsg_silent;
+	status = json_decode(&reader, &listtv,
+				chanpart->ch_mode == MODE_JS ? JSON_JS : 0);
+	--emsg_silent;
+    }
     if (status == OK)
     {
 	// Only accept the response when it is a list with at least two
 	// items.
-	if (listtv.v_type != VAR_LIST || listtv.vval.v_list->lv_len < 2)
+	if (chanpart->ch_mode == MODE_LSP && listtv.v_type != VAR_DICT)
+	{
+	    ch_error(channel, "Did not receive a LSP dict, discarding");
+	    clear_tv(&listtv);
+	}
+	else if (chanpart->ch_mode != MODE_LSP &&
+		(listtv.v_type != VAR_LIST || listtv.vval.v_list->lv_len < 2))
 	{
 	    if (listtv.v_type != VAR_LIST)
 		ch_error(channel, "Did not receive a list, discarding");
@@ -2375,11 +2464,38 @@ channel_get_json(
 
     while (item != NULL)
     {
-	list_T	    *l = item->jq_value->vval.v_list;
+	list_T	    *l;
 	typval_T    *tv;
 
-	CHECK_LIST_MATERIALIZE(l);
-	tv = &l->lv_first->li_tv;
+	if (channel->ch_part[part].ch_mode != MODE_LSP)
+	{
+	    l = item->jq_value->vval.v_list;
+	    CHECK_LIST_MATERIALIZE(l);
+	    tv = &l->lv_first->li_tv;
+	}
+	else
+	{
+	    dict_T	*d;
+	    dictitem_T	*di;
+
+	    // LSP message payload is a JSON-RPC dict.
+	    // For RPC requests and responses, the 'id' item will be present.
+	    // For notifications, it will not be present.
+	    if (id > 0)
+	    {
+		if (item->jq_value->v_type != VAR_DICT)
+		    goto nextitem;
+		d = item->jq_value->vval.v_dict;
+		if (d == NULL)
+		    goto nextitem;
+		di = dict_find(d, (char_u *)"id", -1);
+		if (di == NULL)
+		    goto nextitem;
+		tv = &di->di_tv;
+	    }
+	    else
+		tv = item->jq_value;
+	}
 
 	if ((without_callback || !item->jq_no_callback)
 	    && ((id > 0 && tv->v_type == VAR_NUMBER && tv->vval.v_number == id)
@@ -2395,6 +2511,7 @@ channel_get_json(
 	    remove_json_node(head, item);
 	    return OK;
 	}
+nextitem:
 	item = item->jq_next;
     }
     return FAIL;
@@ -2762,6 +2879,7 @@ may_invoke_callback(channel_T *channel, ch_part_T part)
     callback_T	*callback = NULL;
     buf_T	*buffer = NULL;
     char_u	*p;
+    int		called_otc;		// one time callbackup
 
     if (channel->ch_nb_close_cb != NULL)
 	// this channel is handled elsewhere (netbeans)
@@ -2788,7 +2906,7 @@ may_invoke_callback(channel_T *channel, ch_part_T part)
 	buffer = NULL;
     }
 
-    if (ch_mode == MODE_JSON || ch_mode == MODE_JS)
+    if (ch_mode == MODE_JSON || ch_mode == MODE_JS || ch_mode == MODE_LSP)
     {
 	listitem_T	*item;
 	int		argc = 0;
@@ -2802,29 +2920,47 @@ may_invoke_callback(channel_T *channel, ch_part_T part)
 		return FALSE;
 	}
 
-	for (item = listtv->vval.v_list->lv_first;
-			    item != NULL && argc < CH_JSON_MAX_ARGS;
-						    item = item->li_next)
-	    argv[argc++] = item->li_tv;
-	while (argc < CH_JSON_MAX_ARGS)
-	    argv[argc++].v_type = VAR_UNKNOWN;
-
-	if (argv[0].v_type == VAR_STRING)
+	if (ch_mode == MODE_LSP)
 	{
-	    // ["cmd", arg] or ["cmd", arg, arg] or ["cmd", arg, arg, arg]
-	    channel_exe_cmd(channel, part, argv);
-	    free_tv(listtv);
-	    return TRUE;
-	}
+	    dict_T	*d = listtv->vval.v_dict;
+	    dictitem_T	*di;
 
-	if (argv[0].v_type != VAR_NUMBER)
-	{
-	    ch_error(channel,
-		      "Dropping message with invalid sequence number type");
-	    free_tv(listtv);
-	    return FALSE;
+	    seq_nr = 0;
+	    if (d != NULL)
+	    {
+		di = dict_find(d, (char_u *)"id", -1);
+		if (di != NULL && di->di_tv.v_type == VAR_NUMBER)
+		    seq_nr = di->di_tv.vval.v_number;
+	    }
+
+	    argv[1] = *listtv;
+	}
+	else
+	{
+	    for (item = listtv->vval.v_list->lv_first;
+		    item != NULL && argc < CH_JSON_MAX_ARGS;
+		    item = item->li_next)
+		argv[argc++] = item->li_tv;
+	    while (argc < CH_JSON_MAX_ARGS)
+		argv[argc++].v_type = VAR_UNKNOWN;
+
+	    if (argv[0].v_type == VAR_STRING)
+	    {
+		// ["cmd", arg] or ["cmd", arg, arg] or ["cmd", arg, arg, arg]
+		channel_exe_cmd(channel, part, argv);
+		free_tv(listtv);
+		return TRUE;
+	    }
+
+	    if (argv[0].v_type != VAR_NUMBER)
+	    {
+		ch_error(channel,
+			"Dropping message with invalid sequence number type");
+		free_tv(listtv);
+		return FALSE;
+	    }
+	    seq_nr = argv[0].vval.v_number;
 	}
-	seq_nr = argv[0].vval.v_number;
     }
     else if (channel_peek(channel, part) == NULL)
     {
@@ -2906,24 +3042,35 @@ may_invoke_callback(channel_T *channel, ch_part_T part)
 	argv[1].vval.v_string = msg;
     }
 
+    called_otc = FALSE;
     if (seq_nr > 0)
     {
-	int	done = FALSE;
-
-	// JSON or JS mode: invoke the one-time callback with the matching nr
+	// JSON or JS or LSP mode: invoke the one-time callback with the
+	// matching nr
 	for (cbitem = cbhead->cq_next; cbitem != NULL; cbitem = cbitem->cq_next)
 	    if (cbitem->cq_seq_nr == seq_nr)
 	    {
 		invoke_one_time_callback(channel, cbhead, cbitem, argv);
-		done = TRUE;
+		called_otc = TRUE;
 		break;
 	    }
-	if (!done)
+    }
+
+    if (seq_nr > 0 && (ch_mode != MODE_LSP || called_otc))
+    {
+	if (!called_otc)
 	{
+	    // If the 'drop' channel attribute is set to 'never' or if
+	    // ch_evalexpr() is waiting for this response message, then don't
+	    // drop this message.
 	    if (channel->ch_drop_never)
 	    {
 		// message must be read with ch_read()
 		channel_push_json(channel, part, listtv);
+
+		// Change the type to avoid the value being freed.
+		listtv->v_type = VAR_NUMBER;
+		free_tv(listtv);
 		listtv = NULL;
 	    }
 	    else
@@ -3006,7 +3153,7 @@ channel_has_readahead(channel_T *channel, ch_part_T part)
 {
     ch_mode_T	ch_mode = channel->ch_part[part].ch_mode;
 
-    if (ch_mode == MODE_JSON || ch_mode == MODE_JS)
+    if (ch_mode == MODE_JSON || ch_mode == MODE_JS || ch_mode == MODE_LSP)
     {
 	jsonq_T   *head = &channel->ch_part[part].ch_json_head;
 
@@ -3092,6 +3239,7 @@ channel_part_info(channel_T *channel, dict_T *dict, char *name, ch_part_T part)
 	case MODE_RAW: s = "RAW"; break;
 	case MODE_JSON: s = "JSON"; break;
 	case MODE_JS: s = "JS"; break;
+	case MODE_LSP: s = "LSP"; break;
     }
     dict_add_string(dict, namebuf, (char_u *)s);
 
@@ -4291,9 +4439,59 @@ ch_expr_common(typval_T *argvars, typval_T *rettv, int eval)
 	return;
     }
 
-    id = ++channel->ch_last_msg_id;
-    text = json_encode_nr_expr(id, &argvars[1],
-				 (ch_mode == MODE_JS ? JSON_JS : 0) | JSON_NL);
+    if (ch_mode == MODE_LSP)
+    {
+	dict_T		*d;
+	dictitem_T	*di;
+	int		callback_present = FALSE;
+
+	if (argvars[1].v_type != VAR_DICT)
+	{
+	    semsg(_(e_dict_required_for_argument_nr), 2);
+	    return;
+	}
+	d = argvars[1].vval.v_dict;
+	di = dict_find(d, (char_u *)"id", -1);
+	if (di != NULL && di->di_tv.v_type != VAR_NUMBER)
+	{
+	    // only number type is supported for the 'id' item
+	    semsg(_(e_invalid_value_for_argument_str), "id");
+	    return;
+	}
+
+	if (argvars[2].v_type == VAR_DICT)
+	    if (dict_find(argvars[2].vval.v_dict, (char_u *)"callback", -1)
+									!= NULL)
+		callback_present = TRUE;
+
+	if (eval || callback_present)
+	{
+	    // When evaluating an expression or sending an expression with a
+	    // callback, always assign a generated ID
+	    id = ++channel->ch_last_msg_id;
+	    if (di == NULL)
+		dict_add_number(d, "id", id);
+	    else
+		di->di_tv.vval.v_number = id;
+	}
+	else
+	{
+	    // When sending an expression, if the message has an 'id' item,
+	    // then use it.
+	    id = 0;
+	    if (di != NULL)
+		id = di->di_tv.vval.v_number;
+	}
+	if (dict_find(d, (char_u *)"jsonrpc", -1) == NULL)
+	    dict_add_string(d, "jsonrpc", (char_u *)"2.0");
+	text = json_encode_lsp_msg(&argvars[1]);
+    }
+    else
+    {
+	id = ++channel->ch_last_msg_id;
+	text = json_encode_nr_expr(id, &argvars[1],
+				(ch_mode == MODE_JS ? JSON_JS : 0) | JSON_NL);
+    }
     if (text == NULL)
 	return;
 
@@ -4309,13 +4507,23 @@ ch_expr_common(typval_T *argvars, typval_T *rettv, int eval)
 	if (channel_read_json_block(channel, part_read, timeout, id, &listtv)
 									== OK)
 	{
-	    list_T *list = listtv->vval.v_list;
+	    if (ch_mode == MODE_LSP)
+	    {
+		*rettv = *listtv;
+		// Change the type to avoid the value being freed.
+		listtv->v_type = VAR_NUMBER;
+		free_tv(listtv);
+	    }
+	    else
+	    {
+		list_T *list = listtv->vval.v_list;
 
-	    // Move the item from the list and then change the type to
-	    // avoid the value being freed.
-	    *rettv = list->lv_u.mat.lv_last->li_tv;
-	    list->lv_u.mat.lv_last->li_tv.v_type = VAR_NUMBER;
-	    free_tv(listtv);
+		// Move the item from the list and then change the type to
+		// avoid the value being freed.
+		*rettv = list->lv_u.mat.lv_last->li_tv;
+		list->lv_u.mat.lv_last->li_tv.v_type = VAR_NUMBER;
+		free_tv(listtv);
+	    }
 	}
     }
     free_job_options(&opt);

src/job.c

@@ -31,6 +31,8 @@ handle_mode(typval_T *item, jobopt_T *opt, ch_mode_T *modep, int jo)
 	*modep = MODE_JS;
     else if (STRCMP(val, "json") == 0)
 	*modep = MODE_JSON;
+    else if (STRCMP(val, "lsp") == 0)
+	*modep = MODE_LSP;
     else
     {
 	semsg(_(e_invalid_argument_str), val);

src/json.c

@@ -86,6 +86,32 @@ json_encode_nr_expr(int nr, typval_T *val, int options)
     ga_append(&ga, NUL);
     return ga.ga_data;
 }
+
+/*
+ * Encode "val" into a JSON format string prefixed by the LSP HTTP header.
+ * Returns NULL when out of memory.
+ */
+    char_u *
+json_encode_lsp_msg(typval_T *val)
+{
+    garray_T	ga;
+    garray_T	lspga;
+
+    ga_init2(&ga, 1, 4000);
+    if (json_encode_gap(&ga, val, 0) == FAIL)
+	return NULL;
+    ga_append(&ga, NUL);
+
+    ga_init2(&lspga, 1, 4000);
+    vim_snprintf((char *)IObuff, IOSIZE,
+	    "Content-Length: %u\r\n"
+	    "Content-Type: application/vim-jsonrpc; charset=utf-8\r\n\r\n",
+	    ga.ga_len - 1);
+    ga_concat(&lspga, IObuff);
+    ga_concat_len(&lspga, ga.ga_data, ga.ga_len);
+    ga_clear(&ga);
+    return lspga.ga_data;
+}
 #endif
 
     static void

src/proto/json.pro

@@ -1,6 +1,7 @@
 /* json.c */
 char_u *json_encode(typval_T *val, int options);
 char_u *json_encode_nr_expr(int nr, typval_T *val, int options);
+char_u *json_encode_lsp_msg(typval_T *val);
 int json_decode(js_read_T *reader, typval_T *res, int options);
 int json_find_end(js_read_T *reader, int options);
 void f_js_decode(typval_T *argvars, typval_T *rettv);

src/structs.h

@@ -2193,6 +2193,7 @@ typedef enum
     MODE_RAW,
     MODE_JSON,
     MODE_JS,
+    MODE_LSP			// Language Server Protocol (http + json)
 } ch_mode_T;
 
 typedef enum {

src/testdir/test_channel.vim

@@ -2378,5 +2378,214 @@ func Test_job_start_with_invalid_argument()
   call assert_fails('call job_start([0zff])', 'E976:')
 endfunc
 
+" Test for the 'lsp' channel mode
+func LspCb(chan, msg)
+  call add(g:lspNotif, a:msg)
+endfunc
+
+func LspOtCb(chan, msg)
+  call add(g:lspOtMsgs, a:msg)
+endfunc
+
+func LspTests(port)
+  " call ch_logfile('Xlsprpc.log', 'w')
+  let ch = ch_open(s:localhost .. a:port, #{mode: 'lsp', callback: 'LspCb'})
+  if ch_status(ch) == "fail"
+    call assert_report("Can't open the lsp channel")
+    return
+  endif
+
+  " check for channel information
+  let info = ch_info(ch)
+  call assert_equal('LSP', info.sock_mode)
+
+  " Evaluate an expression
+  let resp = ch_evalexpr(ch, #{method: 'simple-rpc', params: [10, 20]})
+  call assert_false(empty(resp))
+  call assert_equal(#{id: 1, jsonrpc: '2.0', result: 'simple-rpc'}, resp)
+
+  " Evaluate an expression. While waiting for the response, a notification
+  " message is delivered.
+  let g:lspNotif = []
+  let resp = ch_evalexpr(ch, #{method: 'rpc-with-notif', params: {'v': 10}})
+  call assert_false(empty(resp))
+  call assert_equal(#{id: 2, jsonrpc: '2.0', result: 'rpc-with-notif-resp'},
+        \ resp)
+  call assert_equal([#{jsonrpc: '2.0', result: 'rpc-with-notif-notif'}],
+        \ g:lspNotif)
+
+  " Wrong payload notification test
+  let g:lspNotif = []
+  call ch_sendexpr(ch, #{method: 'wrong-payload', params: {}})
+  " Send a ping to wait for all the notification messages to arrive
+  call ch_evalexpr(ch, #{method: 'ping'})
+  call assert_equal([#{jsonrpc: '2.0', result: 'wrong-payload'}], g:lspNotif)
+
+  " Test for receiving a response with incorrect 'id' and additional
+  " notification messages while evaluating an expression.
+  let g:lspNotif = []
+  let resp = ch_evalexpr(ch, #{method: 'rpc-resp-incorrect-id',
+        \ params: {'a': [1, 2]}})
+  call assert_false(empty(resp))
+  call assert_equal(#{id: 4, jsonrpc: '2.0',
+        \ result: 'rpc-resp-incorrect-id-4'}, resp)
+  call assert_equal([#{jsonrpc: '2.0', result: 'rpc-resp-incorrect-id-1'},
+        \ #{jsonrpc: '2.0', result: 'rpc-resp-incorrect-id-2'},
+        \ #{jsonrpc: '2.0', id: 1, result: 'rpc-resp-incorrect-id-3'}],
+        \ g:lspNotif)
+
+  " simple notification test
+  let g:lspNotif = []
+  call ch_sendexpr(ch, #{method: 'simple-notif', params: [#{a: 10, b: []}]})
+  " Send a ping to wait for all the notification messages to arrive
+  call ch_evalexpr(ch, #{method: 'ping'})
+  call assert_equal([#{jsonrpc: '2.0', result: 'simple-notif'}], g:lspNotif)
+
+  " multiple notifications test
+  let g:lspNotif = []
+  call ch_sendexpr(ch, #{method: 'multi-notif', params: [#{a: {}, b: {}}]})
+  " Send a ping to wait for all the notification messages to arrive
+  call ch_evalexpr(ch, #{method: 'ping'})
+  call assert_equal([#{jsonrpc: '2.0', result: 'multi-notif1'},
+        \ #{jsonrpc: '2.0', result: 'multi-notif2'}], g:lspNotif)
+
+  " Test for sending a message with an identifier.
+  let g:lspNotif = []
+  call ch_sendexpr(ch, #{method: 'msg-with-id', id: 93, params: #{s: 'str'}})
+  " Send a ping to wait for all the notification messages to arrive
+  call ch_evalexpr(ch, #{method: 'ping'})
+  call assert_equal([#{jsonrpc: '2.0', id: 93, result: 'msg-with-id'}],
+        \ g:lspNotif)
+
+  " Test for setting the 'id' value in a request message
+  let resp = ch_evalexpr(ch, #{method: 'ping', id: 1, params: {}})
+  call assert_equal(#{id: 8, jsonrpc: '2.0', result: 'alive'}, resp)
+
+  " Test for using a one time callback function to process a response
+  let g:lspOtMsgs = []
+  call ch_sendexpr(ch, #{method: 'msg-specifc-cb', params: {}},
+        \ #{callback: 'LspOtCb'})
+  call ch_evalexpr(ch, #{method: 'ping'})
+  call assert_equal([#{id: 9, jsonrpc: '2.0', result: 'msg-specifc-cb'}],
+        \ g:lspOtMsgs)
+
+  " Test for generating a request message from the other end (server)
+  let g:lspNotif = []
+  call ch_sendexpr(ch, #{method: 'server-req', params: #{}})
+  call ch_evalexpr(ch, #{method: 'ping'})
+  call assert_equal([{'id': 201, 'jsonrpc': '2.0',
+        \ 'result': {'method': 'checkhealth', 'params': {'a': 20}}}],
+        \ g:lspNotif)
+
+  " Test for sending a message without an id
+  let g:lspNotif = []
+  call ch_sendexpr(ch, #{method: 'echo', params: #{s: 'msg-without-id'}})
+  " Send a ping to wait for all the notification messages to arrive
+  call ch_evalexpr(ch, #{method: 'ping'})
+  call assert_equal([#{jsonrpc: '2.0', result:
+        \ #{method: 'echo', jsonrpc: '2.0', params: #{s: 'msg-without-id'}}}],
+        \ g:lspNotif)
+
+  " Test for sending a notification message with an id
+  let g:lspNotif = []
+  call ch_sendexpr(ch, #{method: 'echo', id: 110, params: #{s: 'msg-with-id'}})
+  " Send a ping to wait for all the notification messages to arrive
+  call ch_evalexpr(ch, #{method: 'ping'})
+  call assert_equal([#{jsonrpc: '2.0', result:
+        \ #{method: 'echo', jsonrpc: '2.0', id: 110,
+        \ params: #{s: 'msg-with-id'}}}], g:lspNotif)
+
+  " Test for processing the extra fields in the HTTP header
+  let resp = ch_evalexpr(ch, #{method: 'extra-hdr-fields', params: {}})
+  call assert_equal({'id': 14, 'jsonrpc': '2.0', 'result': 'extra-hdr-fields'},
+        \ resp)
+
+  " Test for processing a HTTP header without the Content-Length field
+  let resp = ch_evalexpr(ch, #{method: 'hdr-without-len', params: {}},
+        \ #{timeout: 200})
+  call assert_equal('', resp)
+  " send a ping to make sure communication still works
+  let resp = ch_evalexpr(ch, #{method: 'ping'})
+  call assert_equal({'id': 16, 'jsonrpc': '2.0', 'result': 'alive'}, resp)
+
+  " Test for processing a HTTP header with wrong length
+  let resp = ch_evalexpr(ch, #{method: 'hdr-with-wrong-len', params: {}},
+        \ #{timeout: 200})
+  call assert_equal('', resp)
+  " send a ping to make sure communication still works
+  let resp = ch_evalexpr(ch, #{method: 'ping'})
+  call assert_equal({'id': 18, 'jsonrpc': '2.0', 'result': 'alive'}, resp)
+
+  " Test for processing a HTTP header with negative length
+  let resp = ch_evalexpr(ch, #{method: 'hdr-with-negative-len', params: {}},
+        \ #{timeout: 200})
+  call assert_equal('', resp)
+  " send a ping to make sure communication still works
+  let resp = ch_evalexpr(ch, #{method: 'ping'})
+  call assert_equal({'id': 20, 'jsonrpc': '2.0', 'result': 'alive'}, resp)
+
+  " Test for an empty header
+  let resp = ch_evalexpr(ch, #{method: 'empty-header', params: {}},
+        \ #{timeout: 200})
+  call assert_equal('', resp)
+  " send a ping to make sure communication still works
+  let resp = ch_evalexpr(ch, #{method: 'ping'})
+  call assert_equal({'id': 22, 'jsonrpc': '2.0', 'result': 'alive'}, resp)
+
+  " Test for an empty payload
+  let resp = ch_evalexpr(ch, #{method: 'empty-payload', params: {}},
+        \ #{timeout: 200})
+  call assert_equal('', resp)
+  " send a ping to make sure communication still works
+  let resp = ch_evalexpr(ch, #{method: 'ping'})
+  call assert_equal({'id': 24, 'jsonrpc': '2.0', 'result': 'alive'}, resp)
+
+  " Test for invoking an unsupported method
+  let resp = ch_evalexpr(ch, #{method: 'xyz', params: {}}, #{timeout: 200})
+  call assert_equal('', resp)
+
+  " Test for sending a message without a callback function. Notification
+  " message should be dropped but RPC response should not be dropped.
+  call ch_setoptions(ch, #{callback: ''})
+  let g:lspNotif = []
+  call ch_sendexpr(ch, #{method: 'echo', params: #{s: 'no-callback'}})
+  " Send a ping to wait for all the notification messages to arrive
+  call ch_evalexpr(ch, #{method: 'ping'})
+  call assert_equal([], g:lspNotif)
+  " Restore the callback function
+  call ch_setoptions(ch, #{callback: 'LspCb'})
+  let g:lspNotif = []
+  call ch_sendexpr(ch, #{method: 'echo', params: #{s: 'no-callback'}})
+  " Send a ping to wait for all the notification messages to arrive
+  call ch_evalexpr(ch, #{method: 'ping'})
+  call assert_equal([#{jsonrpc: '2.0', result:
+        \ #{method: 'echo', jsonrpc: '2.0', params: #{s: 'no-callback'}}}],
+        \ g:lspNotif)
+
+  " " Test for sending a raw message
+  " let g:lspNotif = []
+  " let s = "Content-Length: 62\r\n"
+  " let s ..= "Content-Type: application/vim-jsonrpc; charset=utf-8\r\n"
+  " let s ..= "\r\n"
+  " let s ..= '{"method":"echo","jsonrpc":"2.0","params":{"m":"raw-message"}}'
+  " call ch_sendraw(ch, s)
+  " call ch_evalexpr(ch, #{method: 'ping'})
+  " call assert_equal([{'jsonrpc': '2.0',
+  "       \ 'result': {'method': 'echo', 'jsonrpc': '2.0',
+  "       \ 'params': {'m': 'raw-message'}}}], g:lspNotif)
+
+  " Invalid arguments to ch_evalexpr() and ch_sendexpr()
+  call assert_fails('call ch_sendexpr(ch, #{method: "cookie", id: "cookie"})',
+        \ 'E475:')
+  call assert_fails('call ch_evalexpr(ch, #{method: "ping", id: [{}]})', 'E475:')
+  call assert_fails('call ch_evalexpr(ch, [1, 2, 3])', 'E1206:')
+  call assert_fails('call ch_sendexpr(ch, "abc")', 'E1206:')
+  call assert_fails('call ch_evalexpr(ch, #{method: "ping"}, #{callback: "LspOtCb"})', 'E917:')
+  " call ch_logfile('', 'w')
+endfunc
+
+func Test_channel_lsp_mode()
+  call RunServer('test_channel_lsp.py', 'LspTests', [])
+endfunc
 
 " vim: shiftwidth=2 sts=2 expandtab

src/testdir/test_channel_lsp.py

@@ -0,0 +1,299 @@
+#!/usr/bin/env python
+#
+# Server that will accept connections from a Vim channel.
+# Used by test_channel.vim to test LSP functionality.
+#
+# This requires Python 2.6 or later.
+
+from __future__ import print_function
+import json
+import socket
+import sys
+import time
+import threading
+
+try:
+    # Python 3
+    import socketserver
+except ImportError:
+    # Python 2
+    import SocketServer as socketserver
+
+class ThreadedTCPRequestHandler(socketserver.BaseRequestHandler):
+
+    def setup(self):
+        self.request.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+
+    def send_lsp_msg(self, msgid, resp_dict):
+        v = {'jsonrpc': '2.0', 'result': resp_dict}
+        if msgid != -1:
+            v['id'] = msgid
+        s = json.dumps(v)
+        resp = "Content-Length: " + str(len(s)) + "\r\n"
+        resp += "Content-Type: application/vim-jsonrpc; charset=utf-8\r\n"
+        resp += "\r\n"
+        resp += s
+        if self.debug:
+            with open("Xlspdebug.log", "a") as myfile:
+                myfile.write("\n=> send\n" + resp)
+        self.request.sendall(resp.encode('utf-8'))
+
+    def send_wrong_payload(self):
+        v = 'wrong-payload'
+        s = json.dumps(v)
+        resp = "Content-Length: " + str(len(s)) + "\r\n"
+        resp += "Content-Type: application/vim-jsonrpc; charset=utf-8\r\n"
+        resp += "\r\n"
+        resp += s
+        self.request.sendall(resp.encode('utf-8'))
+
+    def send_empty_header(self, msgid, resp_dict):
+        v = {'jsonrpc': '2.0', 'id': msgid, 'result': resp_dict}
+        s = json.dumps(v)
+        resp = "\r\n"
+        resp += s
+        self.request.sendall(resp.encode('utf-8'))
+
+    def send_empty_payload(self):
+        resp = "Content-Length: 0\r\n"
+        resp += "Content-Type: application/vim-jsonrpc; charset=utf-8\r\n"
+        resp += "\r\n"
+        self.request.sendall(resp.encode('utf-8'))
+
+    def send_extra_hdr_fields(self, msgid, resp_dict):
+        # test for sending extra fields in the http header
+        v = {'jsonrpc': '2.0', 'id': msgid, 'result': resp_dict}
+        s = json.dumps(v)
+        resp = "Host: abc.vim.org\r\n"
+        resp += "User-Agent: Python\r\n"
+        resp += "Accept-Language: en-US,en\r\n"
+        resp += "Content-Type: application/vim-jsonrpc; charset=utf-8\r\n"
+        resp += "Content-Length: " + str(len(s)) + "\r\n"
+        resp += "\r\n"
+        resp += s
+        self.request.sendall(resp.encode('utf-8'))
+
+    def send_hdr_without_len(self, msgid, resp_dict):
+        # test for sending the http header without length
+        v = {'jsonrpc': '2.0', 'id': msgid, 'result': resp_dict}
+        s = json.dumps(v)
+        resp = "Content-Type: application/vim-jsonrpc; charset=utf-8\r\n"
+        resp += "\r\n"
+        resp += s
+        self.request.sendall(resp.encode('utf-8'))
+
+    def send_hdr_with_wrong_len(self, msgid, resp_dict):
+        # test for sending the http header with wrong length
+        v = {'jsonrpc': '2.0', 'id': msgid, 'result': resp_dict}
+        s = json.dumps(v)
+        resp = "Content-Length: 1000\r\n"
+        resp += "\r\n"
+        resp += s
+        self.request.sendall(resp.encode('utf-8'))
+
+    def send_hdr_with_negative_len(self, msgid, resp_dict):
+        # test for sending the http header with negative length
+        v = {'jsonrpc': '2.0', 'id': msgid, 'result': resp_dict}
+        s = json.dumps(v)
+        resp = "Content-Length: -1\r\n"
+        resp += "\r\n"
+        resp += s
+        self.request.sendall(resp.encode('utf-8'))
+
+    def do_ping(self, payload):
+        time.sleep(0.2)
+        self.send_lsp_msg(payload['id'], 'alive')
+
+    def do_echo(self, payload):
+        self.send_lsp_msg(-1, payload)
+
+    def do_simple_rpc(self, payload):
+        # test for a simple RPC request
+        self.send_lsp_msg(payload['id'], 'simple-rpc')
+
+    def do_rpc_with_notif(self, payload):
+        # test for sending a notification before replying to a request message
+        self.send_lsp_msg(-1, 'rpc-with-notif-notif')
+        # sleep for some time to make sure the notification is delivered
+        time.sleep(0.2)
+        self.send_lsp_msg(payload['id'], 'rpc-with-notif-resp')
+
+    def do_wrong_payload(self, payload):
+        # test for sending a non dict payload
+        self.send_wrong_payload()
+        time.sleep(0.2)
+        self.send_lsp_msg(-1, 'wrong-payload')
+
+    def do_rpc_resp_incorrect_id(self, payload):
+        self.send_lsp_msg(-1, 'rpc-resp-incorrect-id-1')
+        self.send_lsp_msg(-1, 'rpc-resp-incorrect-id-2')
+        self.send_lsp_msg(1, 'rpc-resp-incorrect-id-3')
+        time.sleep(0.2)
+        self.send_lsp_msg(payload['id'], 'rpc-resp-incorrect-id-4')
+
+    def do_simple_notif(self, payload):
+        # notification message test
+        self.send_lsp_msg(-1, 'simple-notif')
+
+    def do_multi_notif(self, payload):
+        # send multiple notifications
+        self.send_lsp_msg(-1, 'multi-notif1')
+        self.send_lsp_msg(-1, 'multi-notif2')
+
+    def do_msg_with_id(self, payload):
+        self.send_lsp_msg(payload['id'], 'msg-with-id')
+
+    def do_msg_specific_cb(self, payload):
+        self.send_lsp_msg(payload['id'], 'msg-specifc-cb')
+
+    def do_server_req(self, payload):
+        self.send_lsp_msg(201, {'method': 'checkhealth', 'params': {'a': 20}})
+
+    def do_extra_hdr_fields(self, payload):
+        self.send_extra_hdr_fields(payload['id'], 'extra-hdr-fields')
+
+    def do_hdr_without_len(self, payload):
+        self.send_hdr_without_len(payload['id'], 'hdr-without-len')
+
+    def do_hdr_with_wrong_len(self, payload):
+        self.send_hdr_with_wrong_len(payload['id'], 'hdr-with-wrong-len')
+
+    def do_hdr_with_negative_len(self, payload):
+        self.send_hdr_with_negative_len(payload['id'], 'hdr-with-negative-len')
+
+    def do_empty_header(self, payload):
+        self.send_empty_header(payload['id'], 'empty-header')
+
+    def do_empty_payload(self, payload):
+        self.send_empty_payload()
+
+    def process_msg(self, msg):
+        try:
+            decoded = json.loads(msg)
+            print("Decoded:")
+            print(str(decoded))
+            if 'method' in decoded:
+                test_map = {
+                        'ping': self.do_ping,
+                        'echo': self.do_echo,
+                        'simple-rpc': self.do_simple_rpc,
+                        'rpc-with-notif': self.do_rpc_with_notif,
+                        'wrong-payload': self.do_wrong_payload,
+                        'rpc-resp-incorrect-id': self.do_rpc_resp_incorrect_id,
+                        'simple-notif': self.do_simple_notif,
+                        'multi-notif': self.do_multi_notif,
+                        'msg-with-id': self.do_msg_with_id,
+                        'msg-specifc-cb': self.do_msg_specific_cb,
+                        'server-req': self.do_server_req,
+                        'extra-hdr-fields': self.do_extra_hdr_fields,
+                        'hdr-without-len': self.do_hdr_without_len,
+                        'hdr-with-wrong-len': self.do_hdr_with_wrong_len,
+                        'hdr-with-negative-len': self.do_hdr_with_negative_len,
+                        'empty-header': self.do_empty_header,
+                        'empty-payload': self.do_empty_payload
+                        }
+                if decoded['method'] in test_map:
+                    test_map[decoded['method']](decoded)
+                else:
+                    print("Error: Unsupported method: " + decoded['method'])
+            else:
+                print("Error: 'method' field is not found")
+
+        except ValueError:
+            print("json decoding failed")
+
+    def process_msgs(self, msgbuf):
+        while True:
+            sidx = msgbuf.find('Content-Length: ')
+            if sidx == -1:
+                return msgbuf
+            sidx += 16
+            eidx = msgbuf.find('\r\n')
+            if eidx == -1:
+                return msgbuf
+            msglen = int(msgbuf[sidx:eidx])
+
+            hdrend = msgbuf.find('\r\n\r\n')
+            if hdrend == -1:
+                return msgbuf
+
+            # Remove the header
+            msgbuf = msgbuf[hdrend + 4:]
+            payload = msgbuf[:msglen]
+
+            self.process_msg(payload)
+
+            # Remove the processed message
+            msgbuf = msgbuf[msglen:]
+
+    def handle(self):
+        print("=== socket opened ===")
+        self.debug = False
+        msgbuf = ''
+        while True:
+            try:
+                received = self.request.recv(4096).decode('utf-8')
+            except socket.error:
+                print("=== socket error ===")
+                break
+            except IOError:
+                print("=== socket closed ===")
+                break
+            if received == '':
+                print("=== socket closed ===")
+                break
+            print("\nReceived:\n{0}".format(received))
+
+            # Write the received lines into the file for debugging
+            if self.debug:
+                with open("Xlspdebug.log", "a") as myfile:
+                    myfile.write("\n<= recv\n" + received)
+
+            # Can receive more than one line in a response or a partial line.
+            # Accumulate all the received characters and process one line at
+            # a time.
+            msgbuf += received
+            msgbuf = self.process_msgs(msgbuf)
+
+class ThreadedTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
+    pass
+
+def writePortInFile(port):
+    # Write the port number in Xportnr, so that the test knows it.
+    f = open("Xportnr", "w")
+    f.write("{0}".format(port))
+    f.close()
+
+def main(host, port, server_class=ThreadedTCPServer):
+    # Wait half a second before opening the port to test waittime in ch_open().
+    # We do want to get the port number, get that first.  We cannot open the
+    # socket, guess a port is free.
+    if len(sys.argv) >= 2 and sys.argv[1] == 'delay':
+        port = 13684
+        writePortInFile(port)
+
+        print("Wait for it...")
+        time.sleep(0.5)
+
+    server = server_class((host, port), ThreadedTCPRequestHandler)
+    ip, port = server.server_address[0:2]
+
+    # Start a thread with the server.  That thread will then start a new thread
+    # for each connection.
+    server_thread = threading.Thread(target=server.serve_forever)
+    server_thread.start()
+
+    writePortInFile(port)
+
+    print("Listening on port {0}".format(port))
+
+    # Main thread terminates, but the server continues running
+    # until server.shutdown() is called.
+    try:
+        while server_thread.is_alive():
+            server_thread.join(1)
+    except (KeyboardInterrupt, SystemExit):
+        server.shutdown()
+
+if __name__ == "__main__":
+    main("localhost", 0)

src/version.c

@@ -750,6 +750,8 @@ static char *(features[]) =
 
 static int included_patches[] =
 {   /* Add new patch number below this line */
+/**/
+    4648,
 /**/
     4647,
 /**/