patch 8.1.1580: cannot make part of a popup transparent

Problem:    Cannot make part of a popup transparent.
Solution:   Add the "mask" option.

runtime/doc/popup.txt

@@ -112,7 +112,6 @@ TODO:
   How to add highlighting?
 - Implement:
 	flip option
-	transparent area, to minimize covering text.  Define rectangles?
 
 ==============================================================================
 2. Functions						*popup-functions*
@@ -372,6 +371,7 @@ popup_setoptions({id}, {options})			*popup_setoptions()*
 			borderhighlight
 			borderchars
 			zindex
+			mask
 			time
 			moved
 			filter
@@ -527,6 +527,8 @@ The second argument of |popup_create()| is a dictionary with options:
 			otherwise ASCII characters are used.
 	zindex		Priority for the popup, default 50.  Minimum value is
 			1, maximum value is 32000.
+	mask		A list of lists with coordinates, defining parts of
+			the popup that are transparent.  See |popup-mask|.
 	time		Time in milliseconds after which the popup will close.
 			When omitted |popup_close()| must be used.
 	moved		Specifies to close the popup if the cursor moved:
@@ -557,7 +559,6 @@ These are similar to the third argument of |prop_add()| except:
 - "lnum" is always the current line in the list
 - "bufnr" is always the buffer of the popup
 - "col" is in the Dict instead of a separate argument
-- "transparent" is extra
 So we get:
 	col		starting column, counted in bytes, use one for the
 			first column.
@@ -570,10 +571,6 @@ So we get:
 			used
 	type		name of the text property type, as added with
 			|prop_type_add()|
-	transparent	do not show these characters, show the text under it;
-			if there is a border character to the right or below
-			it will be made transparent as well
-			{not implemented yet}
 
 
 POPUP FILTER						*popup-filter*
@@ -632,6 +629,26 @@ the second argument of `popup_close()`.
 If the popup is force-closed, e.g. because the cursor moved or CTRL-C was
 pressed, the number -1 is passed to the callback.
 
+
+POPUP MASK						*popup-mask*
+
+To minimize the text that the popup covers, parts of it can be made
+transparent.  This is defined by a "mask" which is a list of lists, where each
+list has four numbers:
+    col		start column, positive for counting from the left, 1 for
+		leftmost, negative for counting from the right, -1 for
+		rightmost
+    endcol	last column, like "col"
+    line	start line, positive for conting from the top, 1 for top,
+		negative for counting from the bottom, -1 for bottom
+    endline	end line, like "line"
+
+For example, to make the last 10 columns of the last line transparent:
+	[[-10, -1, -1, -1]]
+
+To make the four corners transparent:
+	[[1, 1, 1, 1], [-1, -1, 1, 1], [1, 1, -1, -1], [-1, -1, -1, -1]]
+
 ==============================================================================
 3. Examples						*popup-examples*
 

src/globals.h

@@ -74,6 +74,8 @@ EXTERN short	*TabPageIdxs INIT(= NULL);
 // Array with size Rows x Columns containing zindex of popups.
 EXTERN short	*popup_mask INIT(= NULL);
 EXTERN short	*popup_mask_next INIT(= NULL);
+// Array with flags for tansparent cells of current popup.
+EXTERN char	*popup_transparent INIT(= NULL);
 
 // Flag set to TRUE when popup_mask needs to be updated.
 EXTERN int	popup_mask_refresh INIT(= TRUE);
@@ -1681,5 +1683,7 @@ EXTERN HINSTANCE g_hinst INIT(= NULL);
 
 #ifdef FEAT_TEXT_PROP
 EXTERN int text_prop_frozen INIT(= 0);
+
+// Set to TRUE if there is any visible popup.
 EXTERN int popup_visible INIT(= FALSE);
 #endif

src/popupwin.c

@@ -381,6 +381,38 @@ apply_general_options(win_T *wp, dict_T *dict)
 	    wp->w_zindex = 32000;
     }
 
+    di = dict_find(dict, (char_u *)"mask", -1);
+    if (di != NULL)
+    {
+	int ok = TRUE;
+
+	if (di->di_tv.v_type != VAR_LIST)
+	    ok = FALSE;
+	else if (di->di_tv.vval.v_list != NULL)
+	{
+	    listitem_T *li;
+
+	    for (li = di->di_tv.vval.v_list->lv_first; li != NULL;
+							      li = li->li_next)
+	    {
+		if (li->li_tv.v_type != VAR_LIST
+			|| li->li_tv.vval.v_list == NULL
+			|| li->li_tv.vval.v_list->lv_len != 4)
+		{
+		    ok = FALSE;
+		    break;
+		}
+	    }
+	}
+	if (ok)
+	{
+	    wp->w_popup_mask = di->di_tv.vval.v_list;
+	    ++wp->w_popup_mask->lv_refcount;
+	}
+	else
+	    semsg(_(e_invargval), "mask");
+    }
+
 #if defined(FEAT_TIMERS)
     // Add timer to close the popup after some time.
     nr = dict_get_number(dict, (char_u *)"time");
@@ -1825,6 +1857,101 @@ popup_check_cursor_pos()
 	}
 }
 
+/*
+ * Return TRUE if "col" / "line" matches with an entry in w_popup_mask.
+ * "col" and "line" are screen coordinates.
+ */
+    static int
+popup_masked(win_T *wp, int screencol, int screenline)
+{
+    int		col = screencol - wp->w_wincol + 1;
+    int		line = screenline - wp->w_winrow + 1;
+    listitem_T	*lio, *li;
+    int		width, height;
+
+    if (wp->w_popup_mask == NULL)
+	return FALSE;
+    width = popup_width(wp);
+    height = popup_height(wp);
+
+    for (lio = wp->w_popup_mask->lv_first; lio != NULL; lio = lio->li_next)
+    {
+	int cols, cole;
+	int lines, linee;
+
+	li = lio->li_tv.vval.v_list->lv_first;
+	cols = tv_get_number(&li->li_tv);
+	if (cols < 0)
+	    cols = width + cols + 1;
+	if (col < cols)
+	    continue;
+	li = li->li_next;
+	cole = tv_get_number(&li->li_tv);
+	if (cole < 0)
+	    cole = width + cole + 1;
+	if (col > cole)
+	    continue;
+	li = li->li_next;
+	lines = tv_get_number(&li->li_tv);
+	if (lines < 0)
+	    lines = height + lines + 1;
+	if (line < lines)
+	    continue;
+	li = li->li_next;
+	linee = tv_get_number(&li->li_tv);
+	if (linee < 0)
+	    linee = height + linee + 1;
+	if (line > linee)
+	    continue;
+	return TRUE;
+    }
+    return FALSE;
+}
+
+/*
+ * Set flags in popup_transparent[] for window "wp" to "val".
+ */
+    static void
+update_popup_transparent(win_T *wp, int val)
+{
+    if (wp->w_popup_mask != NULL)
+    {
+	int		width = popup_width(wp);
+	int		height = popup_height(wp);
+	listitem_T	*lio, *li;
+	int		cols, cole;
+	int		lines, linee;
+	int		col, line;
+
+	for (lio = wp->w_popup_mask->lv_first; lio != NULL; lio = lio->li_next)
+	{
+	    li = lio->li_tv.vval.v_list->lv_first;
+	    cols = tv_get_number(&li->li_tv);
+	    if (cols < 0)
+		cols = width + cols + 1;
+	    li = li->li_next;
+	    cole = tv_get_number(&li->li_tv);
+	    if (cole < 0)
+		cole = width + cole + 1;
+	    li = li->li_next;
+	    lines = tv_get_number(&li->li_tv);
+	    if (lines < 0)
+		lines = height + lines + 1;
+	    li = li->li_next;
+	    linee = tv_get_number(&li->li_tv);
+	    if (linee < 0)
+		linee = height + linee + 1;
+
+	    --cols;
+	    --lines;
+	    for (line = lines; line < linee && line < screen_Rows; ++line)
+		for (col = cols; col < cole && col < screen_Columns; ++col)
+		    popup_transparent[(line + wp->w_winrow) * screen_Columns
+						   + col + wp->w_wincol] = val;
+	}
+    }
+}
+
 /*
  * Update "popup_mask" if needed.
  * Also recomputes the popup size and positions.
@@ -1880,6 +2007,9 @@ may_update_popup_mask(int type)
     popup_reset_handled();
     while ((wp = find_next_popup(TRUE)) != NULL)
     {
+	int height = popup_height(wp);
+	int width = popup_width(wp);
+
 	popup_visible = TRUE;
 
 	// Recompute the position if the text changed.
@@ -1888,11 +2018,10 @@ may_update_popup_mask(int type)
 	    popup_adjust_position(wp);
 
 	for (line = wp->w_winrow;
-		line < wp->w_winrow + popup_height(wp)
+		line < wp->w_winrow + height && line < screen_Rows; ++line)
-						 && line < screen_Rows; ++line)
 	    for (col = wp->w_wincol;
-		 col < wp->w_wincol + popup_width(wp)
+		 col < wp->w_wincol + width && col < screen_Columns; ++col)
-						&& col < screen_Columns; ++col)
+		if (!popup_masked(wp, col, line))
 		    mask[line * screen_Columns + col] = wp->w_zindex;
     }
 
@@ -1995,6 +2124,9 @@ update_popups(void (*win_update)(win_T *wp))
 	// zindex is on top of the character.
 	screen_zindex = wp->w_zindex;
 
+	// Set flags in popup_transparent[] for masked cells.
+	update_popup_transparent(wp, 1);
+
 	// adjust w_winrow and w_wincol for border and padding, since
 	// win_update() doesn't handle them.
 	top_off = popup_top_extra(wp);
@@ -2135,6 +2267,8 @@ update_popups(void (*win_update)(win_T *wp))
 	    }
 	}
 
+	update_popup_transparent(wp, 0);
+
 	// Back to the normal zindex.
 	screen_zindex = 0;
     }
@@ -2161,6 +2295,11 @@ set_ref_in_one_popup(win_T *wp, int copyID)
 	tv.vval.v_partial = wp->w_filter_cb.cb_partial;
 	abort = abort || set_ref_in_item(&tv, copyID, NULL, NULL);
     }
+    if (wp->w_popup_mask != NULL && wp->w_popup_mask->lv_copyID != copyID)
+    {
+	wp->w_popup_mask->lv_copyID = copyID;
+	abort = abort || set_ref_in_list(wp->w_popup_mask, copyID, NULL);
+    }
     return abort;
 }
 

src/screen.c

@@ -6265,6 +6265,23 @@ screen_get_current_line_off()
 }
 #endif
 
+#ifdef FEAT_TEXT_PROP
+/*
+ * Return TRUE if this position has a higher level popup or this cell is
+ * transparent in the current popup.
+ */
+    static int
+blocked_by_popup(int row, int col)
+{
+    int off;
+
+    if (!popup_visible)
+	return FALSE;
+    off = row * screen_Columns + col;
+    return popup_mask[off] > screen_zindex || popup_transparent[off];
+}
+#endif
+
 /*
  * Move one "cooked" screen line to the screen, but only the characters that
  * have actually changed.  Handle insert/delete character.
@@ -6371,11 +6388,9 @@ screen_line(
 	}
 #endif
 #ifdef FEAT_TEXT_PROP
-	// Skip if under a(nother) popup.
+	if (blocked_by_popup(row, col + coloff))
-	if (popup_mask[row * screen_Columns + col + coloff] > screen_zindex)
 	    redraw_this = FALSE;
 #endif
-
 	if (redraw_this)
 	{
 	    /*
@@ -6627,8 +6642,7 @@ screen_line(
 	if (coloff + col < Columns)
 	{
 #ifdef FEAT_TEXT_PROP
-	    if (popup_mask[row * screen_Columns + col + coloff]
+	    if (!blocked_by_popup(row, col + coloff))
-							     <= screen_zindex)
 #endif
 	    {
 		int c;
@@ -7721,7 +7735,7 @@ screen_puts_len(
 
 	if ((need_redraw || force_redraw_this)
 #ifdef FEAT_TEXT_PROP
-		&& popup_mask[row * screen_Columns + col] <= screen_zindex
+		&& !blocked_by_popup(row, col)
 #endif
 	   )
 	{
@@ -8490,8 +8504,7 @@ screen_char(unsigned off, int row, int col)
 	return;
 #endif
 #ifdef FEAT_TEXT_PROP
-    // Skip if under a(nother) popup.
+    if (blocked_by_popup(row, col))
-    if (popup_mask[row * screen_Columns + col] > screen_zindex)
 	return;
 #endif
 
@@ -8794,7 +8807,7 @@ screen_fill(
 		    )
 #ifdef FEAT_TEXT_PROP
 		    // Skip if under a(nother) popup.
-		    && popup_mask[row * screen_Columns + col] <= screen_zindex
+		    && !blocked_by_popup(row, col)
 #endif
 	       )
 	    {
@@ -8936,6 +8949,7 @@ screenalloc(int doclear)
 #ifdef FEAT_TEXT_PROP
     short	    *new_popup_mask;
     short	    *new_popup_mask_next;
+    char	    *new_popup_transparent;
 #endif
     tabpage_T	    *tp;
     static int	    entered = FALSE;		/* avoid recursiveness */
@@ -9021,6 +9035,7 @@ retry:
 #ifdef FEAT_TEXT_PROP
     new_popup_mask = LALLOC_MULT(short, Rows * Columns);
     new_popup_mask_next = LALLOC_MULT(short, Rows * Columns);
+    new_popup_transparent = LALLOC_MULT(char, Rows * Columns);
 #endif
 
     FOR_ALL_TAB_WINDOWS(tp, wp)
@@ -9067,6 +9082,7 @@ give_up:
 #ifdef FEAT_TEXT_PROP
 	    || new_popup_mask == NULL
 	    || new_popup_mask_next == NULL
+	    || new_popup_transparent == NULL
 #endif
 	    || outofmem)
     {
@@ -9091,6 +9107,7 @@ give_up:
 #ifdef FEAT_TEXT_PROP
 	VIM_CLEAR(new_popup_mask);
 	VIM_CLEAR(new_popup_mask_next);
+	VIM_CLEAR(new_popup_transparent);
 #endif
     }
     else
@@ -9180,8 +9197,10 @@ give_up:
     TabPageIdxs = new_TabPageIdxs;
 #ifdef FEAT_TEXT_PROP
     popup_mask = new_popup_mask;
-    popup_mask_next = new_popup_mask_next;
     vim_memset(popup_mask, 0, Rows * Columns * sizeof(short));
+    popup_mask_next = new_popup_mask_next;
+    popup_transparent = new_popup_transparent;
+    vim_memset(popup_transparent, 0, Rows * Columns * sizeof(char));
     popup_mask_refresh = TRUE;
 #endif
 
@@ -9250,6 +9269,7 @@ free_screenlines(void)
 #ifdef FEAT_TEXT_PROP
     VIM_CLEAR(popup_mask);
     VIM_CLEAR(popup_mask_next);
+    VIM_CLEAR(popup_transparent);
 #endif
 }
 

src/structs.h

@@ -2916,6 +2916,7 @@ struct window_S
     colnr_T	w_popup_mincol;	    // close popup if cursor before this col
     colnr_T	w_popup_maxcol;	    // close popup if cursor after this col
     int		w_popup_drag;	    // allow moving the popup with the mouse
+    list_T	*w_popup_mask;	    // list of lists for "mask"
 
 # if defined(FEAT_TIMERS)
     timer_T	*w_popup_timer;	    // timer for closing popup window

src/testdir/dumps/Test_popupwin_mask_1.dump

@@ -0,0 +1,10 @@
+>1+0&#ffffff0|2|3|4|5|6|7|8|9|1|0|1@2|2|1|3|1|4|1|5|1|6|1|7|1|8|1|9|2|0|2|1|2@2|3|2|4|2|5|2|6|2|7|2|8|2|9|3|0|3|1|3|2|3@2|4|3|5|3|6|3|7|3|8|3|9|4|0| @3
+|1|2|3|4|5|6|7|8|9|1| +0&#e0e0e08@12|1+0&#ffffff0|7|1|8|1|9|2|0|2|1|2@2|3|2|4|2|5|2|6|2|7|2|8|2|9|3|0|3|1|3|2|3@2|4|3|5|3|6|3|7|3|8|3|9|4|0| @3
+|1|2|3|4|5|6|7|8|9| +0&#e0e0e08|s|o|m|e| |1+0&#ffffff0|3|x+0#0000001#ffd7ff255|t+0#0000000#e0e0e08| @3|x+0#0000001#ffd7ff255@2|8+0#0000000#ffffff0|1|9|2|0|2|1|2@2|3|2|4|2|5|2|6|2|7|2|8|2|9|3|0|3|1|3|2|3@2|4|3|5|3|6|3|7|3|8|3|9|4|0| @3
+|1|2|3|4|5|6|7|8|9| +0&#e0e0e08|0+0&#ffffff0|1@1|t+0&#e0e0e08|h|1+0&#ffffff0|3|y+0#0000001#ffd7ff255|l+0#0000000#e0e0e08|i|n|e| |y+0#0000001#ffd7ff255@2|8+0#0000000#ffffff0|1|9|2|0|2|1|2@2|3|2|4|2|5|2|6|2|7|2|8|2|9|3|0|3|1|3|2|3@2|4|3|5|3|6|3|7|3|8|3|9|4|0| @3
+|1|2|3|4|5|6|7|8|9| +0&#e0e0e08@8|4+0&#ffffff0|1|5|1|6|1|7|1|8|1|9|2|0|2|1|2@2|3|2|4|2|5|2|6|2|7|2|8|2|9|3|0|3|1|3|2|3@2|4|3|5|3|6|3|7|3|8|3|9|4|0| @3
+|1|2|3|4|5|6|7|8|9|1|0|1@2|2|1|3|1|4|1|5|1|6|1|7|1|8|1|9|2|0|2|1|2@2|3|2|4|2|5|2|6|2|7|2|8|2|9|3|0|3|1|3|2|3@2|4|3|5|3|6|3|7|3|8|3|9|4|0| @3
+|1|2|3|4|5|6|7|8|9|1|0|1@2|2|1|3|1|4|1|5|1|6|1|7|1|8|1|9|2|0|2|1|2@2|3|2|4|2|5|2|6|2|7|2|8|2|9|3|0|3|1|3|2|3@2|4|3|5|3|6|3|7|3|8|3|9|4|0| @3
+|1|2|3|4|5|6|7|8|9|1|0|1@2|2|1|3|1|4|1|5|1|6|1|7|1|8|1|9|2|0|2|1|2@2|3|2|4|2|5|2|6|2|7|2|8|2|9|3|0|3|1|3|2|3@2|4|3|5|3|6|3|7|3|8|3|9|4|0| @3
+|1|2|3|4|5|6|7|8|9|1|0|1@2|2|1|3|1|4|1|5|1|6|1|7|1|8|1|9|2|0|2|1|2@2|3|2|4|2|5|2|6|2|7|2|8|2|9|3|0|3|1|3|2|3@2|4|3|5|3|6|3|7|3|8|3|9|4|0| @3
+@57|1|,|1| @10|T|o|p| 

src/testdir/dumps/Test_popupwin_mask_2.dump

@@ -0,0 +1,10 @@
+>1+0&#ffffff0|2|3|4|5|6|7|8|9|1|0|1@2|2|1|3|1|4|1|5|1|6|1|7|1|8|1|9|2|0|2|1|2@2|3|2|4|2|5|2|6|2|7|2|8|2|9|3|0|3|1|3|2|3@2|4|3|5|3|6|3|7|3|8|3|9|4|0| @3
+|1|2|3|4|5|6|7|8|9|1|0|1@2|2|1|3|1|4|1|5|1|6|1|7|1|8|1|9|2|0|2|1|2@2|3|2|4|2|5|2|6|2|7|2|8|2|9|3|0|3|1|3|2|3@2|4|3|5|3|6|3|7|3|8|3|9|4|0| @3
+|1|2|3|4|5|6|7|8|9|1|0| +0&#e0e0e08@12|x+0#0000001#ffd7ff255@1|8+0#0000000#ffffff0|1|9|2|0|2|1|2@2|3|2|4|2|5|2|6|2|7|2|8|2|9|3|0|3|1|3|2|3@2|4|3|5|3|6|3|7|3|8|3|9|4|0| @3
+|1|2|3|4|5|6|7|8|9|1| +0&#e0e0e08|s|o|m|e| |3+0&#ffffff0|y+0#0000001#ffd7ff255@1|t+0#0000000#e0e0e08| @3|y+0#0000001#ffd7ff255@1|8+0#0000000#ffffff0|1|9|2|0|2|1|2@2|3|2|4|2|5|2|6|2|7|2|8|2|9|3|0|3|1|3|2|3@2|4|3|5|3|6|3|7|3|8|3|9|4|0| @3
+|1|2|3|4|5|6|7|8|9|1| +0&#e0e0e08|1+0&#ffffff0@2|t+0&#e0e0e08|h|3+0&#ffffff0|1|4|l+0&#e0e0e08|i|n|e| |7+0&#ffffff0|1|8|1|9|2|0|2|1|2@2|3|2|4|2|5|2|6|2|7|2|8|2|9|3|0|3|1|3|2|3@2|4|3|5|3|6|3|7|3|8|3|9|4|0| @3
+|1|2|3|4|5|6|7|8|9|1| +0&#e0e0e08@8|1+0&#ffffff0|5|1|6|1|7|1|8|1|9|2|0|2|1|2@2|3|2|4|2|5|2|6|2|7|2|8|2|9|3|0|3|1|3|2|3@2|4|3|5|3|6|3|7|3|8|3|9|4|0| @3
+|1|2|3|4|5|6|7|8|9|1|0|1@2|2|1|3|1|4|1|5|1|6|1|7|1|8|1|9|2|0|2|1|2@2|3|2|4|2|5|2|6|2|7|2|8|2|9|3|0|3|1|3|2|3@2|4|3|5|3|6|3|7|3|8|3|9|4|0| @3
+|1|2|3|4|5|6|7|8|9|1|0|1@2|2|1|3|1|4|1|5|1|6|1|7|1|8|1|9|2|0|2|1|2@2|3|2|4|2|5|2|6|2|7|2|8|2|9|3|0|3|1|3|2|3@2|4|3|5|3|6|3|7|3|8|3|9|4|0| @3
+|1|2|3|4|5|6|7|8|9|1|0|1@2|2|1|3|1|4|1|5|1|6|1|7|1|8|1|9|2|0|2|1|2@2|3|2|4|2|5|2|6|2|7|2|8|2|9|3|0|3|1|3|2|3@2|4|3|5|3|6|3|7|3|8|3|9|4|0| @3
+|:|c|a|l@1| |p|o|p|u|p|_|m|o|v|e|(|w|i|n|i|d|,| |{|'|c|o|l|'|:| |1@1|,| |'|l|i|n|e|'|:| |3|}|)| @9|1|,|1| @10|T|o|p| 

src/ui.c

@@ -1498,7 +1498,7 @@ clip_invert_rectangle(
 
 #ifdef FEAT_TEXT_PROP
     // this goes on top of all popup windows
-    screen_zindex = 32000;
+    screen_zindex = CLIP_ZINDEX;
 
     if (col < cbd->min_col)
     {

src/version.c

@@ -777,6 +777,8 @@ static char *(features[]) =
 
 static int included_patches[] =
 {   /* Add new patch number below this line */
+/**/
+    1580,
 /**/
     1579,
 /**/

src/vim.h

@@ -2602,4 +2602,6 @@ long elapsed(DWORD start_tick);
 #define APC_SAVE_FOR_UNDO	1   // call u_savesub() before making changes
 #define APC_SUBSTITUTE		2   // text is replaced, not inserted
 
+#define CLIP_ZINDEX 32000
+
 #endif /* VIM__H */

src/window.c

@@ -4858,6 +4858,7 @@ win_free(
     for (i = 0; i < 4; ++i)
 	VIM_CLEAR(wp->w_border_highlight[i]);
     vim_free(wp->w_popup_title);
+    list_unref(wp->w_popup_mask);
 #endif
 
 #ifdef FEAT_SYN_HL