patch 8.2.0936: some terminals misinterpret the code for getting cursor style

Problem:    Some terminals misinterpret the code for getting cursor style.
Solution:   Send a sequence to the terminal and check the result. (IWAMOTO
            Kouichi, closes #2126)  Merged with current code.

src/main.c

@@ -793,7 +793,7 @@ vim_main2(void)
 
 #if defined(FEAT_TERMRESPONSE)
     // Must be done before redrawing, puts a few characters on the screen.
-    may_req_ambiguous_char_width();
+    check_terminal_behavior();
 #endif
 
     RedrawingDisabled = 0;

src/proto/term.pro

@@ -47,7 +47,7 @@ void settmode(tmode_T tmode);
 void starttermcap(void);
 void stoptermcap(void);
 void may_req_termresponse(void);
-void may_req_ambiguous_char_width(void);
+void check_terminal_behavior(void);
 void may_req_bg_color(void);
 int swapping_screen(void);
 void scroll_start(void);

src/term.c

@@ -126,6 +126,9 @@ static termrequest_T crv_status = TERMREQUEST_INIT;
 // Request Cursor position report:
 static termrequest_T u7_status = TERMREQUEST_INIT;
 
+// Request xterm compatibility check:
+static termrequest_T xcc_status = TERMREQUEST_INIT;
+
 #  ifdef FEAT_TERMINAL
 // Request foreground color report:
 static termrequest_T rfg_status = TERMREQUEST_INIT;
@@ -152,6 +155,7 @@ static termrequest_T winpos_status = TERMREQUEST_INIT;
 static termrequest_T *all_termrequests[] = {
     &crv_status,
     &u7_status,
+    &xcc_status,
 #  ifdef FEAT_TERMINAL
     &rfg_status,
 #  endif
@@ -1428,6 +1432,7 @@ static char_u	termleader[256 + 1];	    // for check_termcode()
 #ifdef FEAT_TERMRESPONSE
 static int	check_for_codes = FALSE;    // check for key code response
 static int	is_not_xterm = FALSE;	    // recognized not-really-xterm
+static int	xcc_test_failed = FALSE;    // xcc_status check failed
 #endif
 
     static struct builtin_term *
@@ -3605,40 +3610,80 @@ may_req_termresponse(void)
 }
 
 /*
- * Check how the terminal treats ambiguous character width (UAX #11).
+ * Send sequences to the terminal and check with t_u7 how the cursor moves, to
- * First, we move the cursor to (1, 0) and print a test ambiguous character
+ * find out properties of the terminal.
- * \u25bd (WHITE DOWN-POINTING TRIANGLE) and query current cursor position.
- * If the terminal treats \u25bd as single width, the position is (1, 1),
- * or if it is treated as double width, that will be (1, 2).
- * This function has the side effect that changes cursor position, so
- * it must be called immediately after entering termcap mode.
  */
     void
-may_req_ambiguous_char_width(void)
+check_terminal_behavior(void)
 {
+    int	    did_send = FALSE;
+
+    if (!can_get_termresponse() || starting != 0 || *T_U7 == NUL)
+	return;
+
     if (u7_status.tr_progress == STATUS_GET
-	    && can_get_termresponse()
-	    && starting == 0
-	    && *T_U7 != NUL
 	    && !option_was_set((char_u *)"ambiwidth"))
     {
 	char_u	buf[16];
 
-	LOG_TR(("Sending U7 request"));
+	// Ambiguous width check.
+	// Check how the terminal treats ambiguous character width (UAX #11).
+	// First, we move the cursor to (1, 0) and print a test ambiguous
+	// character \u25bd (WHITE DOWN-POINTING TRIANGLE) and then query
+	// the current cursor position.  If the terminal treats \u25bd as
+	// single width, the position is (1, 1), or if it is treated as double
+	// width, that will be (1, 2).  This function has the side effect that
+	// changes cursor position, so it must be called immediately after
+	// entering termcap mode.
+	LOG_TR(("Sending request for ambiwidth check"));
 	// Do this in the second row.  In the first row the returned sequence
 	// may be CSI 1;2R, which is the same as <S-F3>.
 	term_windgoto(1, 0);
-	buf[mb_char2bytes(0x25bd, buf)] = 0;
+	buf[mb_char2bytes(0x25bd, buf)] = NUL;
 	out_str(buf);
 	out_str(T_U7);
 	termrequest_sent(&u7_status);
 	out_flush();
+	did_send = TRUE;
 
 	// This overwrites a few characters on the screen, a redraw is needed
 	// after this. Clear them out for now.
 	screen_stop_highlight();
 	term_windgoto(1, 0);
 	out_str((char_u *)"  ");
+    }
+
+    if (xcc_status.tr_progress == STATUS_GET)
+    {
+	// 2. Check compatibility with xterm.
+	// We move the cursor to (2, 0), print a test sequence and then query
+	// the current cursor position.  If the terminal properly handles
+	// unknown DCS string and CSI sequence with intermediate byte, the test
+	// sequence is ignored and the cursor does not move.  If the terminal
+	// handles test sequence incorrectly, a garbage string is displayed and
+	// the cursor does move.
+	LOG_TR(("Sending xterm compatibility test sequence."));
+	// Do this in the third row.  Second row is used by ambiguous
+	// chararacter width check.
+	term_windgoto(2, 0);
+	// send the test DCS string.
+	out_str((char_u *)"\033Pzz\033\\");
+	// send the test CSI sequence with intermediate byte.
+	out_str((char_u *)"\033[0%m");
+	out_str(T_U7);
+	termrequest_sent(&xcc_status);
+	out_flush();
+	did_send = TRUE;
+
+	// If the terminal handles test sequence incorrectly, garbage text is
+	// displayed. Clear them out for now.
+	screen_stop_highlight();
+	term_windgoto(2, 0);
+	out_str((char_u *)"           ");
+    }
+
+    if (did_send)
+    {
 	term_windgoto(0, 0);
 
 	// Need to reset the known cursor position.
@@ -4680,6 +4725,16 @@ not_enough:
 # endif
 			}
 		    }
+		    else if (arg[0] == 3)
+		    {
+			// Third row: xterm compatibility test.
+			// If the cursor is not on the first column then the
+			// terminal is not xterm compatible.
+			if (arg[1] != 1)
+			    xcc_test_failed = TRUE;
+			xcc_status.tr_progress = STATUS_GOT;
+		    }
+
 		    key_name[0] = (int)KS_EXTRA;
 		    key_name[1] = (int)KE_IGNORE;
 		    slen = csi_len;
@@ -4814,6 +4869,14 @@ not_enough:
 			else if (version == 115 && arg[0] == 0 && arg[2] == 0)
 			    is_not_xterm = TRUE;
 
+			// GNU screen sends 83;30600;0, 83;40500;0, etc.
+			// 30600/40500 is a version number of GNU screen. DA2
+			// support is added on 3.6.  DCS string has a special
+			// meaning to GNU screen, but xterm compatibility
+			// checking does not detect GNU screen.
+			if (version >= 30600 && arg[0] == 83)
+			    xcc_test_failed = TRUE;
+
 			// Xterm first responded to this request at patch level
 			// 95, so assume anything below 95 is not xterm.
 			if (version < 95)
@@ -4827,11 +4890,14 @@ not_enough:
 			// Only request the cursor style if t_SH and t_RS are
 			// set. Only supported properly by xterm since version
 			// 279 (otherwise it returns 0x18).
+			// Only when the xcc_status was set, the test finished,
+			// and xcc_test_failed is FALSE;
 			// Not for Terminal.app, it can't handle t_RS, it
 			// echoes the characters to the screen.
 			if (rcs_status.tr_progress == STATUS_GET
+				&& xcc_status.tr_progress == STATUS_GOT
+				&& !xcc_test_failed
 				&& version >= 279
-				&& !is_not_xterm
 				&& *T_CSH != NUL
 				&& *T_CRS != NUL)
 			{
@@ -4844,8 +4910,11 @@ not_enough:
 			// Only request the cursor blink mode if t_RC set. Not
 			// for Gnome terminal, it can't handle t_RC, it
 			// echoes the characters to the screen.
+			// Only when the xcc_status was set, the test finished,
+			// and xcc_test_failed is FALSE;
 			if (rbm_status.tr_progress == STATUS_GET
-				&& !is_not_xterm
+				&& xcc_status.tr_progress == STATUS_GOT
+				&& !xcc_test_failed
 				&& *T_CRC != NUL)
 			{
 			    LOG_TR(("Sending cursor blink mode request"));

src/testdir/dumps/Test_balloon_eval_term_01.dump

@@ -1,5 +1,5 @@
 |o+0&#ffffff0|n>e| |o|n|e| |o|n|e| @38
-@2|o| |t|X|o| |t|w|o| @38
+|t|w|o| |t|X|o| |t|w|o| @38
 |t|h|r|e| +0#0000001#ffd7ff255@17| +0#0000000#ffffff0@27
 |~+0#4040ff13&| @2| +0#0000001#ffd7ff255|l|i|n|e| |2| |c|o|l|u|m|n| |6|:| | +0#4040ff13#ffffff0@27
 |~| @2| +0#0000001#ffd7ff255|t|X|o| @13| +0#4040ff13#ffffff0@27

src/testdir/dumps/Test_balloon_eval_term_01a.dump

@@ -1,5 +1,5 @@
 |o+0&#ffffff0|n>e| |o|n|e| |o|n|e| @38
-@2|o| |t|X|o| |t|w|o| @38
+|t|w|o| |t|X|o| |t|w|o| @38
 |t|h|r|e| +0#0000001#ffd7ff255@17| +0#0000000#ffffff0@27
 |~+0#4040ff13&| @2| +0#0000001#ffd7ff255|l|i|n|e| |2| |c|o|l|u|m|n| |6|:| | +0#4040ff13#ffffff0@27
 |~| @2| +0#0000001#ffd7ff255|t|X|o| @13| +0#4040ff13#ffffff0@27

src/testdir/dumps/Test_balloon_eval_term_02.dump

@@ -1,5 +1,5 @@
 |o+0&#ffffff0|n|e| |o|n|e| |o|n|e| @38
-@2|o| |t|X|o| |t|w|o| @38
+|t|w|o| |t|X|o| |t|w|o| @38
 |t|h|r|e|e+0&#e0e0e08| |t|h>r+0&#ffffff0|e@1| |t|h|r|e@1| @32
 |~+0#4040ff13&| @2| +0#0000001#ffd7ff255@17| +0#4040ff13#ffffff0@27
 |~| @2| +0#0000001#ffd7ff255|l|i|n|e| |3| |c|o|l|u|m|n| |5|:| | +0#4040ff13#ffffff0@27

src/testdir/dumps/Test_terminal_all_ansi_colors.dump

@@ -1,5 +1,5 @@
 >A+0#0000001#8080809@1|B+0#e000002#ff404010@1|C+0#00e0003#40ff4011@1|D+0#e0e0004#ffff4012@1|E+0#0000e05#4040ff13@1|F+0#e000e06#ff40ff14@1|G+0#00e0e07#40ffff15@1|H+0#e0e0e08#ffffff16@1|I+0#8080809#0000001@1|J+0#ff404010#e000002@1|K+0#40ff4011#00e0003@1|L+0#ffff4012#e0e0004@1|M+0#4040ff13#0000e05@1|N+0#ff40ff14#e000e06@1|O+0#40ffff15#00e0e07@1|P+0#ffffff16#e0e0e08@1| +0#0000000#ffffff0|X+2#e000002&@1|Y+2#40ff4011&@1|Z+2#ff40ff14#e000e06@1| +0#0000000#ffffff0@35
-@2| +0#4040ff13&@72
+|~+0#4040ff13&| @73
 |~| @73
 |~| @73
 |~| @73

src/testdir/term_util.vim

@@ -44,6 +44,8 @@ endfunc
 " Returns the buffer number of the terminal.
 "
 " Options is a dictionary, these items are recognized:
+" "keep_t_u7" - when 1 do not make t_u7 empty (resetting t_u7 avoids clearing
+"               parts of line 2 and 3 on the display)
 " "rows" - height of the terminal window (max. 20)
 " "cols" - width of the terminal window (max. 78)
 " "statusoff" - number of lines the status is offset from default
@@ -74,7 +76,13 @@ func RunVimInTerminal(arguments, options)
   let cols = get(a:options, 'cols', 75)
   let statusoff = get(a:options, 'statusoff', 1)
 
-  let cmd = GetVimCommandCleanTerm() .. a:arguments
+  if get(a:options, 'keep_t_u7', 0)
+    let reset_u7 = ''
+  else
+    let reset_u7 = ' --cmd "set t_u7=" '
+  endif
+
+  let cmd = GetVimCommandCleanTerm() .. reset_u7 .. a:arguments
 
   let options = {
 	\ 'curwin': 1,

src/testdir/test_quickfix.vim

@@ -2710,7 +2710,6 @@ func Test_cwindow_highlight()
   CheckScreendump
 
   let lines =<< trim END
-	set t_u7=
 	call setline(1, ['some', 'text', 'with', 'matches'])
 	write XCwindow
 	vimgrep e XCwindow

src/testdir/test_startup_utf8.vim

@@ -1,5 +1,6 @@
 " Tests for startup using utf-8.
 
+source check.vim
 source shared.vim
 source screendump.vim
 
@@ -71,7 +72,7 @@ func Test_detect_ambiwidth()
 	\ 'call test_option_not_set("ambiwidth")',
 	\ 'redraw',
 	\ ], 'Xscript')
-  let buf = RunVimInTerminal('-S Xscript', {})
+  let buf = RunVimInTerminal('-S Xscript', #{keep_t_u7: 1})
   call TermWait(buf)
   call term_sendkeys(buf, "S\<C-R>=&ambiwidth\<CR>\<Esc>")
   call WaitForAssert({-> assert_match('single', term_getline(buf, 1))})

src/testdir/test_terminal.vim

@@ -2465,7 +2465,6 @@ func Test_terminal_in_popup()
   call writefile(text, 'Xtext')
   let cmd = GetVimCommandCleanTerm()
   let lines = [
-	\ 'set t_u7=',
 	\ 'call setline(1, range(20))',
 	\ 'hi PopTerm ctermbg=grey',
 	\ 'func OpenTerm(setColor)',
@@ -2545,7 +2544,6 @@ func Test_terminal_in_popup_min_size()
   END
   call writefile(text, 'Xtext')
   let lines = [
-	\ 'set t_u7=',
 	\ 'call setline(1, range(20))',
 	\ 'func OpenTerm()',
 	\ "  let s:buf = term_start('cat Xtext', #{hidden: 1})",
@@ -2575,7 +2573,6 @@ func Terminal_in_popup_colored(group_name, highlight_cmd, highlight_opt)
   CheckUnix
 
   let lines = [
-	\ 'set t_u7=',
 	\ 'call setline(1, range(20))',
 	\ 'func OpenTerm()',
 	\ "  let s:buf = term_start('cat', #{hidden: 1, "

src/version.c

@@ -754,6 +754,8 @@ static char *(features[]) =
 
 static int included_patches[] =
 {   /* Add new patch number below this line */
+/**/
+    936,
 /**/
     935,
 /**/