patch 8.1.1228: not possible to process tags with a function

Problem:    Not possible to process tags with a function.
Solution:   Add tagfunc() (Christian Brabandt, Andy Massimino, closes #4010)

runtime/doc/options.txt

@@ -7458,6 +7458,16 @@ A jump table for the options with a short description can be found at |Q_op|.
 	NOTE: This option is set to the Vi default value when 'compatible' is
 	set and to the Vim default value when 'compatible' is reset.
 
+							*'tagfunc'* *'tfu'*
+'tagfunc' 'tfu'		string	(default: empty)
+			local to buffer
+			{not available when compiled without the |+eval|
+			feature}
+	This option specifies a function to be used to perform tag searches.
+	The function gets the tag pattern and should return a List of matching
+	tags.  See |tag-function| for an explanation of how to write the
+	function and an example.
+
 						*'taglength'* *'tl'*
 'taglength' 'tl'	number	(default 0)
 			global

runtime/doc/tagsrch.txt

@@ -1,4 +1,4 @@
-*tagsrch.txt*   For Vim version 8.1.  Last change: 2019 Mar 30
+*tagsrch.txt*   For Vim version 8.1.  Last change: 2019 Apr 28
 
 
 		  VIM REFERENCE MANUAL    by Bram Moolenaar
@@ -14,6 +14,7 @@ See section |29.1| of the user manual for an introduction.
 4. Tags details			|tag-details|
 5. Tags file format		|tags-file-format|
 6. Include file searches	|include-search|
+7. Using 'tagfunc'		|tag-function|
 
 ==============================================================================
 1. Jump to a tag					*tag-commands*
@@ -871,4 +872,70 @@ Common arguments for the commands above:
 <	For a ":djump", ":dsplit", ":dlist" and ":dsearch" command the pattern
 	is used as a literal string, not as a search pattern.
 
+==============================================================================
+7. Using 'tagfunc'						*tag-function*
+
+It is possible to provide Vim with a function which will generate a list of
+tags used for commands like |:tag|, |:tselect| and Normal mode tag commands
+like |CTRL-]|.
+
+The function used for generating the taglist is specified by setting the
+'tagfunc' option.  The function will be called with three arguments:
+   a:pattern	The tag identifier used during the tag search.
+   a:flags	List of flags to control the function behavior.
+   a:info	Dict containing the following entries:
+		    buf_ffname	  Full filename which can be used for priority.
+		    user_data	  Custom data String, if stored in the tag
+				  stack previously by tagfunc.
+
+Currently two flags may be passed to the tag function:
+  'c'		The function was invoked by a normal command being processed
+	        (mnemonic: the tag function may use the context around the
+		cursor to perform a better job of generating the tag list.)
+  'i'		In Insert mode, the user was completing a tag (with
+		|i_CTRL-X_CTRL-]|).
+
+Note that when 'tagfunc' is set, the priority of the tags described in
+|tag-priority| does not apply.  Instead, the priority is exactly as the
+ordering of the elements in the list returned by the function.
+								*E987*
+The function should return a List of Dict entries.  Each Dict must at least
+include the following entries and each value must be a string:
+	name		Name of the tag.
+	filename	Name of the file where the tag is defined.  It is
+			either relative to the current directory or a full path.
+	cmd		Ex command used to locate the tag in the file.  This
+			can be either an Ex search pattern or a line number.
+Note that the format is similar to that of |taglist()|, which makes it possible
+to use its output to generate the result.
+The following fields are optional:
+	kind		Type of the tag.
+	user_data	String of custom data stored in the tag stack which
+			can be used to disambiguate tags between operations.
+
+If the function returns |v:null| instead of a List, a standard tag lookup will
+be performed instead.
+
+It is not allowed to change the tagstack from inside 'tagfunc'.  *E986* 
+
+The following is a hypothetical example of a function used for 'tagfunc'.  It
+uses the output of |taglist()| to generate the result: a list of tags in the
+inverse order of file names.
+>
+	function! TagFunc(pattern, flags, info)
+	  function! CompareFilenames(item1, item2)
+	    let f1 = a:item1['filename']
+	    let f2 = a:item2['filename']
+	    return f1 >=# f2 ?
+			\ -1 : f1 <=# f2 ? 1 : 0
+	  endfunction
+
+	  let result = taglist(a:pattern)
+	  call sort(result, "CompareFilenames")
+
+	  return result
+	endfunc
+	set tagfunc=TagFunc
+<
+
  vim:tw=78:ts=8:noet:ft=help:norl:

runtime/optwin.vim

@@ -300,6 +300,11 @@ call append("$", "tagstack\ta :tag command will use the tagstack")
 call <SID>BinOptionG("tgst", &tgst)
 call append("$", "showfulltag\twhen completing tags in Insert mode show more info")
 call <SID>BinOptionG("sft", &sft)
+if has("eval")
+  call append("$", "tagfunc\ta function to be used to perform tag searches")
+  call append("$", "\t(local to buffer)")
+  call <SID>OptionL("tfu")
+endif
 if has("cscope")
   call append("$", "cscopeprg\tcommand for executing cscope")
   call <SID>OptionG("csprg", &csprg)

src/buffer.c

@@ -2219,6 +2219,9 @@ free_buf_options(
     clear_string_option(&buf->b_p_path);
     clear_string_option(&buf->b_p_tags);
     clear_string_option(&buf->b_p_tc);
+#ifdef FEAT_EVAL
+    clear_string_option(&buf->b_p_tfu);
+#endif
 #ifdef FEAT_INS_EXPAND
     clear_string_option(&buf->b_p_dict);
     clear_string_option(&buf->b_p_tsr);

src/dict.c

@@ -448,6 +448,55 @@ dict_add_list(dict_T *d, char *key, list_T *list)
     return OK;
 }
 
+/*
+ * Initializes "iter" for iterating over dictionary items with
+ * dict_iterate_next().
+ * If "var" is not a Dict or an empty Dict then there will be nothing to
+ * iterate over, no error is given.
+ * NOTE: The dictionary must not change until iterating is finished!
+ */
+    void
+dict_iterate_start(typval_T *var, dict_iterator_T *iter)
+{
+    if (var->v_type != VAR_DICT || var->vval.v_dict == NULL)
+	iter->dit_todo = 0;
+    else
+    {
+	dict_T	*d = var->vval.v_dict;
+
+	iter->dit_todo = d->dv_hashtab.ht_used;
+	iter->dit_hi = d->dv_hashtab.ht_array;
+    }
+}
+
+/*
+ * Iterate over the items referred to by "iter".  It should be initialized with
+ * dict_iterate_start().
+ * Returns a pointer to the key.
+ * "*tv_result" is set to point to the value for that key.
+ * If there are no more items, NULL is returned.
+ */
+    char_u *
+dict_iterate_next(dict_iterator_T *iter, typval_T **tv_result)
+{
+    dictitem_T	*di;
+    char_u      *result;
+
+    if (iter->dit_todo == 0)
+	return NULL;
+
+    while (HASHITEM_EMPTY(iter->dit_hi))
+	++iter->dit_hi;
+
+    di = HI2DI(iter->dit_hi);
+    result = di->di_key;
+    *tv_result = &di->di_tv;
+
+    --iter->dit_todo;
+    ++iter->dit_hi;
+    return result;
+}
+
 /*
  * Add a dict entry to dictionary "d".
  * Returns FAIL when out of memory and when key already exists.

src/ex_cmds.c

@@ -6813,7 +6813,7 @@ find_help_tags(
 
     *matches = (char_u **)"";
     *num_matches = 0;
-    flags = TAG_HELP | TAG_REGEXP | TAG_NAMES | TAG_VERBOSE;
+    flags = TAG_HELP | TAG_REGEXP | TAG_NAMES | TAG_VERBOSE | TAG_NO_TAGFUNC;
     if (keep_lang)
 	flags |= TAG_KEEP_LANG;
     if (find_tags(IObuff, num_matches, matches, flags, (int)MAXCOL, NULL) == OK

src/globals.h

@@ -1067,9 +1067,13 @@ EXTERN int	postponed_split INIT(= 0);  /* for CTRL-W CTRL-] command */
 EXTERN int	postponed_split_flags INIT(= 0);  /* args for win_split() */
 EXTERN int	postponed_split_tab INIT(= 0);  /* cmdmod.tab */
 #ifdef FEAT_QUICKFIX
-EXTERN int	g_do_tagpreview INIT(= 0);  /* for tag preview commands:
+EXTERN int	g_do_tagpreview INIT(= 0);  // for tag preview commands:
-					       height of preview window */
+					    // height of preview window
 #endif
+EXTERN int	g_tag_at_cursor INIT(= FALSE); // whether the tag command comes
+					    // from the command line (0) or was
+					    // invoked as a normal command (1)
+
 EXTERN int	replace_offset INIT(= 0);   /* offset for replace_push() */
 
 EXTERN char_u	*escape_chars INIT(= (char_u *)" \t\\\"|");

src/insexpand.c

@@ -2654,11 +2654,13 @@ ins_compl_get_exp(pos_T *ini)
 
 	    // Find up to TAG_MANY matches.  Avoids that an enormous number
 	    // of matches is found when compl_pattern is empty
+	    g_tag_at_cursor = TRUE;
 	    if (find_tags(compl_pattern, &num_matches, &matches,
 		    TAG_REGEXP | TAG_NAMES | TAG_NOIC | TAG_INS_COMP
 		    | (ctrl_x_mode != CTRL_X_NORMAL ? TAG_VERBOSE : 0),
 		    TAG_MANY, curbuf->b_ffname) == OK && num_matches > 0)
 		ins_compl_add_matches(num_matches, matches, p_ic);
+	    g_tag_at_cursor = FALSE;
 	    p_ic = save_p_ic;
 	    break;
 

src/normal.c

@@ -5724,7 +5724,11 @@ nv_ident(cmdarg_T *cap)
 	(void)normal_search(cap, cmdchar == '*' ? '/' : '?', buf, 0);
     }
     else
+    {
+	g_tag_at_cursor = TRUE;
 	do_cmdline_cmd(buf);
+	g_tag_at_cursor = FALSE;
+    }
 
     vim_free(buf);
 }

src/option.c

@@ -167,6 +167,9 @@
 #endif
 #define PV_SW		OPT_BUF(BV_SW)
 #define PV_SWF		OPT_BUF(BV_SWF)
+#ifdef FEAT_EVAL
+# define PV_TFU		OPT_BUF(BV_TFU)
+#endif
 #define PV_TAGS		OPT_BOTH(OPT_BUF(BV_TAGS))
 #define PV_TC		OPT_BOTH(OPT_BUF(BV_TC))
 #define PV_TS		OPT_BUF(BV_TS)
@@ -303,6 +306,9 @@ static char_u	*p_cpt;
 static char_u	*p_cfu;
 static char_u	*p_ofu;
 #endif
+#ifdef FEAT_EVAL
+static char_u	*p_tfu;
+#endif
 static int	p_eol;
 static int	p_fixeol;
 static int	p_et;
@@ -2642,6 +2648,15 @@ static struct vimoption options[] =
     {"tagcase",	    "tc",   P_STRING|P_VIM,
 			    (char_u *)&p_tc, PV_TC,
 			    {(char_u *)"followic", (char_u *)"followic"} SCTX_INIT},
+    {"tagfunc",    "tfu",   P_STRING|P_ALLOCED|P_VI_DEF|P_SECURE,
+#ifdef FEAT_EVAL
+			    (char_u *)&p_tfu, PV_TFU,
+			    {(char_u *)"", (char_u *)0L}
+#else
+			    (char_u *)NULL, PV_NONE,
+			    {(char_u *)0L, (char_u *)0L}
+#endif
+			    SCTX_INIT},
     {"taglength",   "tl",   P_NUM|P_VI_DEF,
 			    (char_u *)&p_tl, PV_NONE,
 			    {(char_u *)0L, (char_u *)0L} SCTX_INIT},
@@ -5689,6 +5704,9 @@ check_buf_options(buf_T *buf)
     check_string_option(&buf->b_p_cfu);
     check_string_option(&buf->b_p_ofu);
 #endif
+#ifdef FEAT_EVAL
+    check_string_option(&buf->b_p_tfu);
+#endif
 #ifdef FEAT_KEYMAP
     check_string_option(&buf->b_p_keymap);
 #endif
@@ -10943,6 +10961,9 @@ get_varp(struct vimoption *p)
 #ifdef FEAT_COMPL_FUNC
 	case PV_CFU:	return (char_u *)&(curbuf->b_p_cfu);
 	case PV_OFU:	return (char_u *)&(curbuf->b_p_ofu);
+#endif
+#ifdef FEAT_EVAL
+	case PV_TFU:	return (char_u *)&(curbuf->b_p_tfu);
 #endif
 	case PV_EOL:	return (char_u *)&(curbuf->b_p_eol);
 	case PV_FIXEOL:	return (char_u *)&(curbuf->b_p_fixeol);
@@ -11331,6 +11352,9 @@ buf_copy_options(buf_T *buf, int flags)
 #ifdef FEAT_COMPL_FUNC
 	    buf->b_p_cfu = vim_strsave(p_cfu);
 	    buf->b_p_ofu = vim_strsave(p_ofu);
+#endif
+#ifdef FEAT_EVAL
+	    buf->b_p_tfu = vim_strsave(p_tfu);
 #endif
 	    buf->b_p_sts = p_sts;
 	    buf->b_p_sts_nopaste = p_sts_nopaste;

src/option.h

@@ -1068,6 +1068,9 @@ enum
 #endif
     , BV_SW
     , BV_SWF
+#ifdef FEAT_EVAL
+    , BV_TFU
+#endif
     , BV_TAGS
     , BV_TC
     , BV_TS

src/proto/dict.pro

@@ -18,6 +18,8 @@ int dict_add_special(dict_T *d, char *key, varnumber_T nr);
 int dict_add_string(dict_T *d, char *key, char_u *str);
 int dict_add_string_len(dict_T *d, char *key, char_u *str, int len);
 int dict_add_list(dict_T *d, char *key, list_T *list);
+void dict_iterate_start(typval_T *var, dict_iterator_T *iter);
+char_u *dict_iterate_next(dict_iterator_T *iter, typval_T **tv_result);
 int dict_add_dict(dict_T *d, char *key, dict_T *dict);
 long dict_len(dict_T *d);
 dictitem_T *dict_find(dict_T *d, char_u *key, int len);

src/structs.h

@@ -147,10 +147,11 @@ typedef struct xfilemark
  */
 typedef struct taggy
 {
-    char_u	*tagname;	/* tag name */
+    char_u	*tagname;	// tag name
-    fmark_T	fmark;		/* cursor position BEFORE ":tag" */
+    fmark_T	fmark;		// cursor position BEFORE ":tag"
-    int		cur_match;	/* match number */
+    int		cur_match;	// match number
-    int		cur_fnum;	/* buffer number used for cur_match */
+    int		cur_fnum;	// buffer number used for cur_match
+    char_u	*user_data;	// used with tagfunc
 } taggy_T;
 
 /*
@@ -1885,6 +1886,16 @@ typedef struct list_stack_S
     struct list_stack_S	*prev;
 } list_stack_T;
 
+/*
+ * Structure used for iterating over dictionary items.
+ * Initialize with dict_iterate_start().
+ */
+typedef struct
+{
+    long_u	dit_todo;
+    hashitem_T	*dit_hi;
+} dict_iterator_T;
+
 /* values for b_syn_spell: what to do with toplevel text */
 #define SYNSPL_DEFAULT	0	/* spell check if @Spell not defined */
 #define SYNSPL_TOP	1	/* spell check toplevel text */
@@ -2244,6 +2255,9 @@ struct file_buffer
 #ifdef FEAT_COMPL_FUNC
     char_u	*b_p_cfu;	/* 'completefunc' */
     char_u	*b_p_ofu;	/* 'omnifunc' */
+#endif
+#ifdef FEAT_EVAL
+    char_u	*b_p_tfu;	/* 'tagfunc' */
 #endif
     int		b_p_eol;	/* 'endofline' */
     int		b_p_fixeol;	/* 'fixendofline' */

src/tag.c

@@ -18,20 +18,23 @@
  */
 typedef struct tag_pointers
 {
-    /* filled in by parse_tag_line(): */
+    // filled in by parse_tag_line():
-    char_u	*tagname;	/* start of tag name (skip "file:") */
+    char_u	*tagname;	// start of tag name (skip "file:")
-    char_u	*tagname_end;	/* char after tag name */
+    char_u	*tagname_end;	// char after tag name
-    char_u	*fname;		/* first char of file name */
+    char_u	*fname;		// first char of file name
-    char_u	*fname_end;	/* char after file name */
+    char_u	*fname_end;	// char after file name
-    char_u	*command;	/* first char of command */
+    char_u	*command;	// first char of command
-    /* filled in by parse_match(): */
+    // filled in by parse_match():
-    char_u	*command_end;	/* first char after command */
+    char_u	*command_end;	// first char after command
-    char_u	*tag_fname;	/* file name of the tags file */
+    char_u	*tag_fname;	// file name of the tags file. This is used
+				// when 'tr' is set.
 #ifdef FEAT_EMACS_TAGS
-    int		is_etag;	/* TRUE for emacs tag */
+    int		is_etag;	// TRUE for emacs tag
 #endif
-    char_u	*tagkind;	/* "kind:" value */
+    char_u	*tagkind;	// "kind:" value
-    char_u	*tagkind_end;	/* end of tagkind */
+    char_u	*tagkind_end;	// end of tagkind
+    char_u	*user_data;	// user_data string
+    char_u	*user_data_end;	// end of user_data
 } tagptrs_T;
 
 /*
@@ -78,9 +81,14 @@ static void print_tag_list(int new_tag, int use_tagstack, int num_matches, char_
 #if defined(FEAT_QUICKFIX) && defined(FEAT_EVAL)
 static int add_llist_tags(char_u *tag, int num_matches, char_u **matches);
 #endif
+static void tagstack_clear_entry(taggy_T *item);
 
 static char_u *bottommsg = (char_u *)N_("E555: at bottom of tag stack");
 static char_u *topmsg = (char_u *)N_("E556: at top of tag stack");
+#ifdef FEAT_EVAL
+static char_u *recurmsg = (char_u *)N_("E986: cannot modify the tag stack within tagfunc");
+static char_u *tfu_inv_ret_msg = (char_u *)N_("E987: invalid return value from tagfunc");
+#endif
 
 static char_u	*tagmatchname = NULL;	/* name of last used tag */
 
@@ -89,9 +97,16 @@ static char_u	*tagmatchname = NULL;	/* name of last used tag */
  * Tag for preview window is remembered separately, to avoid messing up the
  * normal tagstack.
  */
-static taggy_T ptag_entry = {NULL, {{0, 0, 0}, 0}, 0, 0};
+static taggy_T ptag_entry = {NULL, {{0, 0, 0}, 0}, 0, 0, NULL};
 #endif
 
+#ifdef FEAT_EVAL
+static int  tfu_in_use = FALSE;	    // disallow recursive call of tagfunc
+#endif
+
+// Used instead of NUL to separate tag fields in the growarrays.
+#define TAG_SEP 0x02
+
 /*
  * Jump to tag; handling of tag commands and tag stack
  *
@@ -144,6 +159,7 @@ do_tag(
     int		skip_msg = FALSE;
     char_u	*buf_ffname = curbuf->b_ffname;	    /* name to use for
 						       priority computation */
+    int         use_tfu = 1;
 
     /* remember the matches for the last used tag */
     static int		num_matches = 0;
@@ -151,6 +167,14 @@ do_tag(
     static char_u	**matches = NULL;
     static int		flags;
 
+#ifdef FEAT_EVAL
+    if (tfu_in_use)
+    {
+	emsg(_(recurmsg));
+	return FALSE;
+    }
+#endif
+
 #ifdef EXITFREE
     if (type == DT_FREE)
     {
@@ -168,6 +192,7 @@ do_tag(
     {
 	type = DT_TAG;
 	no_regexp = TRUE;
+	use_tfu = 0;
     }
 
     prev_num_matches = num_matches;
@@ -187,7 +212,7 @@ do_tag(
 #if defined(FEAT_QUICKFIX)
 	if (g_do_tagpreview != 0)
 	{
-	    vim_free(ptag_entry.tagname);
+	    tagstack_clear_entry(&ptag_entry);
 	    if ((ptag_entry.tagname = vim_strsave(tag)) == NULL)
 		goto end_do_tag;
 	}
@@ -226,7 +251,7 @@ do_tag(
 		}
 		else
 		{
-		    vim_free(ptag_entry.tagname);
+		    tagstack_clear_entry(&ptag_entry);
 		    if ((ptag_entry.tagname = vim_strsave(tag)) == NULL)
 			goto end_do_tag;
 		}
@@ -239,13 +264,13 @@ do_tag(
 		 * stack entries above it.
 		 */
 		while (tagstackidx < tagstacklen)
-		    vim_free(tagstack[--tagstacklen].tagname);
+		    tagstack_clear_entry(&tagstack[--tagstacklen]);
 
 		/* if the tagstack is full: remove oldest entry */
 		if (++tagstacklen > TAGSTACKSIZE)
 		{
 		    tagstacklen = TAGSTACKSIZE;
-		    vim_free(tagstack[0].tagname);
+		    tagstack_clear_entry(&tagstack[0]);
 		    for (i = 1; i < tagstacklen; ++i)
 			tagstack[i - 1] = tagstack[i];
 		    --tagstackidx;
@@ -529,6 +554,10 @@ do_tag(
 #endif
 	    if (verbose)
 		flags |= TAG_VERBOSE;
+
+	    if (!use_tfu)
+		flags |= TAG_NO_TAGFUNC;
+
 	    if (find_tags(name, &new_num_matches, &new_matches, flags,
 					    max_num_matches, buf_ffname) == OK
 		    && new_num_matches < max_num_matches)
@@ -647,8 +676,20 @@ do_tag(
 	    }
 	    if (use_tagstack)
 	    {
+		tagptrs_T   tagp;
+
 		tagstack[tagstackidx].cur_match = cur_match;
 		tagstack[tagstackidx].cur_fnum = cur_fnum;
+
+		// store user-provided data originating from tagfunc
+		if (use_tfu && parse_match(matches[cur_match], &tagp) == OK
+			&& tagp.user_data)
+		{
+		    VIM_CLEAR(tagstack[tagstackidx].user_data);
+		    tagstack[tagstackidx].user_data = vim_strnsave(
+			    tagp.user_data, tagp.user_data_end - tagp.user_data);
+		}
+
 		++tagstackidx;
 	    }
 #if defined(FEAT_QUICKFIX)
@@ -1243,6 +1284,237 @@ prepare_pats(pat_T *pats, int has_re)
 	pats->regmatch.regprog = NULL;
 }
 
+#ifdef FEAT_EVAL
+/*
+ * Call the user-defined function to generate a list of tags used by
+ * find_tags().
+ *
+ * Return OK if at least 1 tag has been successfully found,
+ * NOTDONE if the function returns v:null, and FAIL otherwise.
+ */
+    static int
+find_tagfunc_tags(
+    char_u	*pat,		// pattern supplied to the user-defined function
+    garray_T	*ga,		// the tags will be placed here
+    int		*match_count,	// here the number of tags found will be placed
+    int		flags,		// flags from find_tags (TAG_*)
+    char_u	*buf_ffname)	// name of buffer for priority
+{
+    pos_T       save_pos;
+    list_T      *taglist;
+    listitem_T  *item;
+    int         ntags = 0;
+    int         result = FAIL;
+    typval_T	args[4];
+    typval_T	rettv;
+    char_u      flagString[3];
+    dict_T	*d;
+    taggy_T	*tag = &curwin->w_tagstack[curwin->w_tagstackidx];
+
+    if (*curbuf->b_p_tfu == NUL)
+	return FAIL;
+
+    args[0].v_type = VAR_STRING;
+    args[0].vval.v_string = pat;
+    args[1].v_type = VAR_STRING;
+    args[1].vval.v_string = flagString;
+
+    // create 'info' dict argument
+    if ((d = dict_alloc_lock(VAR_FIXED)) == NULL)
+	return FAIL;
+    if (tag->user_data != NULL)
+	dict_add_string(d, "user_data", tag->user_data);
+    if (buf_ffname != NULL)
+	dict_add_string(d, "buf_ffname", buf_ffname);
+
+    ++d->dv_refcount;
+    args[2].v_type = VAR_DICT;
+    args[2].vval.v_dict = d;
+
+    args[3].v_type = VAR_UNKNOWN;
+
+    vim_snprintf((char *)flagString, sizeof(flagString),
+		 "%s%s",
+		 g_tag_at_cursor      ? "c": "",
+		 flags & TAG_INS_COMP ? "i": "");
+
+    save_pos = curwin->w_cursor;
+    result = call_vim_function(curbuf->b_p_tfu, 3, args, &rettv);
+    curwin->w_cursor = save_pos;	// restore the cursor position
+    --d->dv_refcount;
+
+    if (result == FAIL)
+	return FAIL;
+    if (rettv.v_type == VAR_SPECIAL && rettv.vval.v_number == VVAL_NULL)
+    {
+	clear_tv(&rettv);
+	return NOTDONE;
+    }
+    if (rettv.v_type != VAR_LIST || !rettv.vval.v_list)
+    {
+	clear_tv(&rettv);
+	emsg(_(tfu_inv_ret_msg));
+	return FAIL;
+    }
+    taglist = rettv.vval.v_list;
+
+    for (item = taglist->lv_first; item != NULL; item = item->li_next)
+    {
+	char_u		*mfp;
+	char_u		*res_name, *res_fname, *res_cmd, *res_kind;
+	int		len;
+	dict_iterator_T	iter;
+	char_u		*dict_key;
+	typval_T	*tv;
+	int		has_extra = 0;
+	int		name_only = flags & TAG_NAMES;
+
+	if (item->li_tv.v_type != VAR_DICT)
+	{
+	    emsg(_(tfu_inv_ret_msg));
+	    break;
+	}
+
+#ifdef FEAT_EMACS_TAGS
+	len = 3;
+#else
+	len = 2;
+#endif
+	res_name = NULL;
+	res_fname = NULL;
+	res_cmd = NULL;
+	res_kind = NULL;
+
+	dict_iterate_start(&item->li_tv, &iter);
+	while (NULL != (dict_key = dict_iterate_next(&iter, &tv)))
+	{
+	    if (tv->v_type != VAR_STRING || tv->vval.v_string == NULL)
+		continue;
+
+	    len += STRLEN(tv->vval.v_string) + 1;   // Space for "\tVALUE"
+	    if (!STRCMP(dict_key, "name"))
+	    {
+		res_name = tv->vval.v_string;
+		continue;
+	    }
+	    if (!STRCMP(dict_key, "filename"))
+	    {
+		res_fname = tv->vval.v_string;
+		continue;
+	    }
+	    if (!STRCMP(dict_key, "cmd"))
+	    {
+		res_cmd = tv->vval.v_string;
+		continue;
+	    }
+	    has_extra = 1;
+	    if (!STRCMP(dict_key, "kind"))
+	    {
+		res_kind = tv->vval.v_string;
+		continue;
+	    }
+	    // Other elements will be stored as "\tKEY:VALUE"
+	    // Allocate space for the key and the colon
+	    len += STRLEN(dict_key) + 1;
+	}
+
+	if (has_extra)
+	    len += 2;	// need space for ;"
+
+	if (!res_name || !res_fname || !res_cmd)
+	{
+	    emsg(_(tfu_inv_ret_msg));
+	    break;
+	}
+
+	if (name_only)
+	    mfp = vim_strsave(res_name);
+	else
+	    mfp = (char_u *)alloc((int)sizeof(char_u) + len + 1);
+
+	if (mfp == NULL)
+	    continue;
+
+	if (!name_only)
+	{
+	    char_u *p = mfp;
+
+	    *p++ = MT_GL_OTH + 1;   // mtt
+	    *p++ = TAG_SEP;	    // no tag file name
+#ifdef FEAT_EMACS_TAGS
+	    *p++ = TAG_SEP;
+#endif
+
+	    STRCPY(p, res_name);
+	    p += STRLEN(p);
+
+	    *p++ = TAB;
+	    STRCPY(p, res_fname);
+	    p += STRLEN(p);
+
+	    *p++ = TAB;
+	    STRCPY(p, res_cmd);
+	    p += STRLEN(p);
+
+	    if (has_extra)
+	    {
+		STRCPY(p, ";\"");
+		p += STRLEN(p);
+
+		if (res_kind)
+		{
+		    *p++ = TAB;
+		    STRCPY(p, res_kind);
+		    p += STRLEN(p);
+		}
+
+		dict_iterate_start(&item->li_tv, &iter);
+		while (NULL != (dict_key = dict_iterate_next(&iter, &tv)))
+		{
+		    if (tv->v_type != VAR_STRING || tv->vval.v_string == NULL)
+			continue;
+
+		    if (!STRCMP(dict_key, "name"))
+			continue;
+		    if (!STRCMP(dict_key, "filename"))
+			continue;
+		    if (!STRCMP(dict_key, "cmd"))
+			continue;
+		    if (!STRCMP(dict_key, "kind"))
+			continue;
+
+		    *p++ = TAB;
+		    STRCPY(p, dict_key);
+		    p += STRLEN(p);
+		    STRCPY(p, ":");
+		    p += STRLEN(p);
+		    STRCPY(p, tv->vval.v_string);
+		    p += STRLEN(p);
+		}
+	    }
+	}
+
+	// Add all matches because tagfunc should do filtering.
+	if (ga_grow(ga, 1) == OK)
+	{
+	    ((char_u **)(ga->ga_data))[ga->ga_len++] = mfp;
+	    ++ntags;
+	    result = OK;
+	}
+	else
+	{
+	    vim_free(mfp);
+	    break;
+	}
+    }
+
+    clear_tv(&rettv);
+
+    *match_count = ntags;
+    return result;
+}
+#endif
+
 /*
  * find_tags() - search for tags in tags files
  *
@@ -1268,6 +1540,7 @@ prepare_pats(pat_T *pats, int has_re)
  * TAG_NOIC	  don't always ignore case
  * TAG_KEEP_LANG  keep language
  * TAG_CSCOPE	  use cscope results for tags
+ * TAG_NO_TAGFUNC do not call the 'tagfunc' function
  */
     int
 find_tags(
@@ -1385,6 +1658,9 @@ find_tags(
     int		use_cscope = (flags & TAG_CSCOPE);
 #endif
     int		verbose = (flags & TAG_VERBOSE);
+#ifdef FEAT_EVAL
+    int         use_tfu = ((flags & TAG_NO_TAGFUNC) == 0);
+#endif
     int		save_p_ic = p_ic;
 
     /*
@@ -1480,6 +1756,18 @@ find_tags(
     vim_memset(&search_info, 0, (size_t)1);
 #endif
 
+#ifdef FEAT_EVAL
+    if (*curbuf->b_p_tfu != NUL && use_tfu && !tfu_in_use)
+    {
+	tfu_in_use = TRUE;
+	retval = find_tagfunc_tags(pat, &ga_match[0], &match_count,
+							   flags, buf_ffname);
+	tfu_in_use = FALSE;
+	if (retval != NOTDONE)
+	    goto findtag_end;
+    }
+#endif
+
     /*
      * When finding a specified number of matches, first try with matching
      * case, so binary search can be used, and try ignore-case matches in a
@@ -2308,7 +2596,6 @@ parse_line:
 		}
 		else
 		{
-#define TAG_SEP 0x02
 		    size_t tag_fname_len = STRLEN(tag_fname);
 #ifdef FEAT_EMACS_TAGS
 		    size_t ebuf_len = 0;
@@ -2577,8 +2864,7 @@ free_tag_stuff(void)
     tag_freematch();
 
 # if defined(FEAT_QUICKFIX)
-    if (ptag_entry.tagname)
+    tagstack_clear_entry(&ptag_entry);
-	VIM_CLEAR(ptag_entry.tagname);
 # endif
 }
 #endif
@@ -2940,6 +3226,7 @@ parse_match(
 			tagp);
 
     tagp->tagkind = NULL;
+    tagp->user_data = NULL;
     tagp->command_end = NULL;
 
     if (retval == OK)
@@ -2957,17 +3244,15 @@ parse_match(
 		while (ASCII_ISALPHA(*p))
 		{
 		    if (STRNCMP(p, "kind:", 5) == 0)
-		    {
 			tagp->tagkind = p + 5;
+		    else if (STRNCMP(p, "user_data:", 10) == 0)
+			tagp->user_data = p + 10;
+		    if (tagp->tagkind != NULL && tagp->user_data != NULL)
 			break;
-		    }
 		    pc = vim_strchr(p, ':');
 		    pt = vim_strchr(p, '\t');
 		    if (pc == NULL || (pt != NULL && pc > pt))
-		    {
 			tagp->tagkind = p;
-			break;
-		    }
 		    if (pt == NULL)
 			break;
 		    p = pt + 1;
@@ -2980,6 +3265,13 @@ parse_match(
 		;
 	    tagp->tagkind_end = p;
 	}
+	if (tagp->user_data != NULL)
+	{
+	    for (p = tagp->user_data;
+			    *p && *p != '\t' && *p != '\r' && *p != '\n'; ++p)
+		;
+	    tagp->user_data_end = p;
+	}
     }
     return retval;
 }
@@ -3547,6 +3839,16 @@ find_extra(char_u **pp)
     return FAIL;
 }
 
+/*
+ * Free a single entry in a tag stack
+ */
+    static void
+tagstack_clear_entry(taggy_T *item)
+{
+    VIM_CLEAR(item->tagname);
+    VIM_CLEAR(item->user_data);
+}
+
 #if defined(FEAT_CMDL_COMPL) || defined(PROTO)
     int
 expand_tags(
@@ -3568,11 +3870,11 @@ expand_tags(
 	tagnmflag = 0;
     if (pat[0] == '/')
 	ret = find_tags(pat + 1, num_file, file,
-		TAG_REGEXP | tagnmflag | TAG_VERBOSE,
+		TAG_REGEXP | tagnmflag | TAG_VERBOSE | TAG_NO_TAGFUNC,
 		TAG_MANY, curbuf->b_ffname);
     else
 	ret = find_tags(pat, num_file, file,
-		TAG_REGEXP | tagnmflag | TAG_VERBOSE | TAG_NOIC,
+	      TAG_REGEXP | tagnmflag | TAG_VERBOSE | TAG_NO_TAGFUNC | TAG_NOIC,
 		TAG_MANY, curbuf->b_ffname);
     if (ret == OK && !tagnames)
     {
@@ -3753,6 +4055,8 @@ get_tag_details(taggy_T *tag, dict_T *retdict)
     dict_add_string(retdict, "tagname", tag->tagname);
     dict_add_number(retdict, "matchnr", tag->cur_match + 1);
     dict_add_number(retdict, "bufnr", tag->cur_fnum);
+    if (tag->user_data)
+	dict_add_string(retdict, "user_data", tag->user_data);
 
     if ((pos = list_alloc_id(aid_tagstack_from)) == NULL)
 	return;
@@ -3805,7 +4109,7 @@ tagstack_clear(win_T *wp)
 
     // Free the current tag stack
     for (i = 0; i < wp->w_tagstacklen; ++i)
-	vim_free(wp->w_tagstack[i].tagname);
+	tagstack_clear_entry(&wp->w_tagstack[i]);
     wp->w_tagstacklen = 0;
     wp->w_tagstackidx = 0;
 }
@@ -3820,7 +4124,7 @@ tagstack_shift(win_T *wp)
     taggy_T	*tagstack = wp->w_tagstack;
     int		i;
 
-    vim_free(tagstack[0].tagname);
+    tagstack_clear_entry(&tagstack[0]);
     for (i = 1; i < wp->w_tagstacklen; ++i)
 	tagstack[i - 1] = tagstack[i];
     wp->w_tagstacklen--;
@@ -3836,7 +4140,8 @@ tagstack_push_item(
 	int	cur_fnum,
 	int	cur_match,
 	pos_T	mark,
-	int	fnum)
+	int	fnum,
+	char_u  *user_data)
 {
     taggy_T	*tagstack = wp->w_tagstack;
     int		idx = wp->w_tagstacklen;	// top of the stack
@@ -3856,6 +4161,7 @@ tagstack_push_item(
 	tagstack[idx].cur_match = 0;
     tagstack[idx].fmark.mark = mark;
     tagstack[idx].fmark.fnum = fnum;
+    tagstack[idx].user_data = user_data;
 }
 
 /*
@@ -3892,7 +4198,8 @@ tagstack_push_items(win_T *wp, list_T *l)
 	tagstack_push_item(wp, tagname,
 		(int)dict_get_number(itemdict, (char_u *)"bufnr"),
 		(int)dict_get_number(itemdict, (char_u *)"matchnr") - 1,
-		mark, fnum);
+		mark, fnum,
+		dict_get_string(itemdict, (char_u *)"user_data", TRUE));
     }
 }
 
@@ -3920,6 +4227,15 @@ set_tagstack(win_T *wp, dict_T *d, int action)
     dictitem_T	*di;
     list_T	*l;
 
+#ifdef FEAT_EVAL
+    // not allowed to alter the tag stack entries from inside tagfunc
+    if (tfu_in_use)
+    {
+	emsg(_(recurmsg));
+	return FAIL;
+    }
+#endif
+
     if ((di = dict_find(d, (char_u *)"items", -1)) != NULL)
     {
 	if (di->di_tv.v_type != VAR_LIST)

src/testdir/Make_all.mak

@@ -244,6 +244,7 @@ NEW_TESTS = \
 	test_tabline \
 	test_tabpage \
 	test_tagcase \
+	test_tagfunc \
 	test_tagjump \
 	test_taglist \
 	test_tcl \

src/testdir/test_alot.vim

@@ -60,6 +60,7 @@ source test_syn_attr.vim
 source test_tabline.vim
 source test_tabpage.vim
 source test_tagcase.vim
+source test_tagfunc.vim
 source test_tagjump.vim
 source test_taglist.vim
 source test_timers.vim

src/testdir/test_tagfunc.vim

@@ -0,0 +1,84 @@
+" Test 'tagfunc'
+
+func TagFunc(pat, flag, info)
+  let g:tagfunc_args = [a:pat, a:flag, a:info]
+  let tags = []
+  for num in range(1,10)
+    let tags += [{
+          \ 'cmd': '2', 'name': 'nothing'.num, 'kind': 'm',
+          \ 'filename': 'Xfile1', 'user_data': 'somedata'.num,
+          \}]
+  endfor
+  return tags
+endfunc
+
+func Test_tagfunc()
+  set tagfunc=TagFunc
+  new Xfile1
+  call setline(1, ['empty', 'one()', 'empty'])
+  write
+
+  call assert_equal({'cmd': '2', 'static': 0,
+        \ 'name': 'nothing2', 'user_data': 'somedata2',
+        \ 'kind': 'm', 'filename': 'Xfile1'}, taglist('.')[1])
+
+  call settagstack(win_getid(), {'items': []})
+
+  tag arbitrary
+  call assert_equal('arbitrary', g:tagfunc_args[0])
+  call assert_equal('', g:tagfunc_args[1])
+  call assert_equal('somedata1', gettagstack().items[0].user_data)
+  5tag arbitrary
+  call assert_equal('arbitrary', g:tagfunc_args[0])
+  call assert_equal('', g:tagfunc_args[1])
+  call assert_equal('somedata5', gettagstack().items[1].user_data)
+  pop
+  tag
+  call assert_equal('arbitrary', g:tagfunc_args[0])
+  call assert_equal('', g:tagfunc_args[1])
+  call assert_equal('somedata5', gettagstack().items[1].user_data)
+
+  let g:tagfunc_args=[]
+  execute "normal! \<c-]>"
+  call assert_equal('one', g:tagfunc_args[0])
+  call assert_equal('c', g:tagfunc_args[1])
+
+  set cpt=t
+  let g:tagfunc_args=[]
+  execute "normal! i\<c-n>\<c-y>"
+  call assert_equal('ci', g:tagfunc_args[1])
+  call assert_equal('nothing1', getline('.')[0:7])
+
+  func BadTagFunc1(...)
+    return 0
+  endfunc
+  func BadTagFunc2(...)
+    return [1]
+  endfunc
+  func BadTagFunc3(...)
+    return [{'name': 'foo'}]
+  endfunc
+
+  for &tagfunc in ['BadTagFunc1', 'BadTagFunc2', 'BadTagFunc3']
+    try
+      tag nothing
+      call assert_false(1, 'tag command should have failed')
+    catch
+      call assert_exception('E987:')
+    endtry
+    exe 'delf' &tagfunc
+  endfor
+
+  func NullTagFunc(...)
+    return v:null
+  endfunc
+  set tags= tfu=NullTagFunc
+  call assert_fails('tag nothing', 'E426')
+  delf NullTagFunc
+
+  bwipe!
+  set tags& tfu& cpt& 
+  call delete('Xfile1')
+endfunc
+
+" vim: shiftwidth=2 sts=2 expandtab

src/version.c

@@ -767,6 +767,8 @@ static char *(features[]) =
 
 static int included_patches[] =
 {   /* Add new patch number below this line */
+/**/
+    1228,
 /**/
     1227,
 /**/

src/vim.h

@@ -1133,19 +1133,20 @@ typedef struct {
 /*
  * flags for find_tags().
  */
-#define TAG_HELP	1	/* only search for help tags */
+#define TAG_HELP	1	// only search for help tags
-#define TAG_NAMES	2	/* only return name of tag */
+#define TAG_NAMES	2	// only return name of tag
-#define	TAG_REGEXP	4	/* use tag pattern as regexp */
+#define	TAG_REGEXP	4	// use tag pattern as regexp
-#define	TAG_NOIC	8	/* don't always ignore case */
+#define	TAG_NOIC	8	// don't always ignore case
 #ifdef FEAT_CSCOPE
-# define TAG_CSCOPE	16	/* cscope tag */
+# define TAG_CSCOPE	16	// cscope tag
 #endif
-#define TAG_VERBOSE	32	/* message verbosity */
+#define TAG_VERBOSE	32	// message verbosity
-#define TAG_INS_COMP	64	/* Currently doing insert completion */
+#define TAG_INS_COMP	64	// Currently doing insert completion
-#define TAG_KEEP_LANG	128	/* keep current language */
+#define TAG_KEEP_LANG	128	// keep current language
+#define TAG_NO_TAGFUNC	256	// do not use 'tagfunc'
 
-#define TAG_MANY	300	/* When finding many tags (for completion),
+#define TAG_MANY	300	// When finding many tags (for completion),
-				   find up to this many tags */
+				// find up to this many tags
 
 /*
  * Types of dialogs passed to do_vim_dialog().

src/window.c

@@ -1326,10 +1326,12 @@ win_init(win_T *newp, win_T *oldp, int flags UNUSED)
     /* copy tagstack and folds */
     for (i = 0; i < oldp->w_tagstacklen; i++)
     {
-	newp->w_tagstack[i] = oldp->w_tagstack[i];
+	taggy_T	*tag = &newp->w_tagstack[i];
-	if (newp->w_tagstack[i].tagname != NULL)
+	*tag = oldp->w_tagstack[i];
-	    newp->w_tagstack[i].tagname =
+	if (tag->tagname != NULL)
-				   vim_strsave(newp->w_tagstack[i].tagname);
+	    tag->tagname = vim_strsave(tag->tagname);
+	if (tag->user_data != NULL)
+	    tag->user_data = vim_strsave(tag->user_data);
     }
     newp->w_tagstackidx = oldp->w_tagstackidx;
     newp->w_tagstacklen = oldp->w_tagstacklen;