patch 8.2.0677: Vim9: no support for closures

Problem:    Vim9: no support for closures.
Solution:   Find variables in the outer function scope, so long as the scope
            exists.

src/proto/vim9compile.pro

@@ -9,7 +9,7 @@ imported_T *find_imported(char_u *name, size_t len, cctx_T *cctx);
 char_u *to_name_const_end(char_u *arg);
 int assignment_len(char_u *p, int *heredoc);
 int check_vim9_unlet(char_u *name);
-void compile_def_function(ufunc_T *ufunc, int set_return_type);
+void compile_def_function(ufunc_T *ufunc, int set_return_type, cctx_T *outer_cctx);
 void delete_instr(isn_T *isn);
 void delete_def_function(ufunc_T *ufunc);
 void free_def_functions(void);

src/structs.h

@@ -1561,7 +1561,11 @@ typedef struct
     sctx_T	uf_script_ctx;	// SCTX where function was defined,
 				// used for s: variables
     int		uf_refcount;	// reference count, see func_name_refcount()
+
     funccall_T	*uf_scoped;	// l: local variables for closure
+    garray_T	*uf_ectx_stack;	// where compiled closure finds local vars
+    int		uf_ectx_frame;	// index of function frame in uf_ectx_stack
+
     char_u	*uf_name_exp;	// if "uf_name[]" starts with SNR the name with
 				// "<SNR>" as a string, otherwise NULL
     char_u	uf_name[1];	// name of function (actually longer); can
@@ -1569,6 +1573,19 @@ typedef struct
 				// KS_EXTRA KE_SNR)
 } ufunc_T;
 
+// flags used in uf_flags
+#define FC_ABORT    0x01	// abort function on error
+#define FC_RANGE    0x02	// function accepts range
+#define FC_DICT	    0x04	// Dict function, uses "self"
+#define FC_CLOSURE  0x08	// closure, uses outer scope variables
+#define FC_DELETED  0x10	// :delfunction used while uf_refcount > 0
+#define FC_REMOVED  0x20	// function redefined while uf_refcount > 0
+#define FC_SANDBOX  0x40	// function defined in the sandbox
+#define FC_DEAD	    0x80	// function kept only for reference to dfunc
+#define FC_EXPORT   0x100	// "export def Func()"
+#define FC_NOARGS   0x200	// no a: variables in lambda
+#define FC_VIM9	    0x400	// defined in vim9 script file
+
 #define MAX_FUNC_ARGS	20	// maximum number of function arguments
 #define VAR_SHORT_LEN	20	// short variable name length
 #define FIXVAR_CNT	12	// number of fixed variables

src/testdir/test_vim9_func.vim

@@ -641,4 +641,13 @@ func Test_E1056_1059()
   call assert_equal(1, caught_1059)
 endfunc
 
+def RefFunc(Ref: func(string): string): string
+  return Ref('more')
+enddef
+
+def Test_closure_simple()
+  let local = 'some '
+  assert_equal('some more', RefFunc({s -> local .. s}))
+enddef
+
 " vim: ts=8 sw=2 sts=2 expandtab tw=80 fdm=marker

src/userfunc.c

@@ -14,19 +14,6 @@
 #include "vim.h"
 
 #if defined(FEAT_EVAL) || defined(PROTO)
-// flags used in uf_flags
-#define FC_ABORT    0x01	// abort function on error
-#define FC_RANGE    0x02	// function accepts range
-#define FC_DICT	    0x04	// Dict function, uses "self"
-#define FC_CLOSURE  0x08	// closure, uses outer scope variables
-#define FC_DELETED  0x10	// :delfunction used while uf_refcount > 0
-#define FC_REMOVED  0x20	// function redefined while uf_refcount > 0
-#define FC_SANDBOX  0x40	// function defined in the sandbox
-#define FC_DEAD	    0x80	// function kept only for reference to dfunc
-#define FC_EXPORT   0x100	// "export def Func()"
-#define FC_NOARGS   0x200	// no a: variables in lambda
-#define FC_VIM9	    0x400	// defined in vim9 script file
-
 /*
  * All user-defined functions are found in this hashtable.
  */
@@ -3267,7 +3254,7 @@ ex_function(exarg_T *eap)
 
     // ":def Func()" needs to be compiled
     if (eap->cmdidx == CMD_def)
-	compile_def_function(fp, FALSE);
+	compile_def_function(fp, FALSE, NULL);
 
     goto ret_free;
 

src/version.c

@@ -746,6 +746,8 @@ static char *(features[]) =
 
 static int included_patches[] =
 {   /* Add new patch number below this line */
+/**/
+    677,
 /**/
     676,
 /**/

src/vim9.h

@@ -27,6 +27,7 @@ typedef enum {
     ISN_LOADW,	    // push w: variable isn_arg.string
     ISN_LOADT,	    // push t: variable isn_arg.string
     ISN_LOADS,	    // push s: variable isn_arg.loadstore
+    ISN_LOADOUTER,  // push variable from outer scope isn_arg.number
     ISN_LOADSCRIPT, // push script-local variable isn_arg.script.
     ISN_LOADOPT,    // push option isn_arg.string
     ISN_LOADENV,    // push environment variable isn_arg.string

src/vim9compile.c

@@ -98,6 +98,7 @@ typedef struct {
     char_u	*lv_name;
     type_T	*lv_type;
     int		lv_idx;		// index of the variable on the stack
+    int		lv_from_outer;	// when TRUE using ctx_outer scope
     int		lv_const;	// when TRUE cannot be assigned to
     int		lv_arg;		// when TRUE this is an argument
 } lvar_T;
@@ -123,6 +124,7 @@ struct cctx_S {
 
     cctx_T	*ctx_outer;	    // outer scope for lambda or nested
 				    // function
+    int		ctx_outer_used;	    // var in ctx_outer was used
 
     garray_T	ctx_type_stack;	    // type of each item on the stack
     garray_T	*ctx_type_list;	    // list of pointers to allocated types
@@ -146,17 +148,37 @@ static int check_type(type_T *expected, type_T *actual, int give_msg);
 lookup_local(char_u *name, size_t len, cctx_T *cctx)
 {
     int	    idx;
+    lvar_T  *lvar;
 
     if (len == 0)
 	return NULL;
+
+    // Find local in current function scope.
     for (idx = 0; idx < cctx->ctx_locals.ga_len; ++idx)
     {
-	lvar_T *lvar = ((lvar_T *)cctx->ctx_locals.ga_data) + idx;
+	lvar = ((lvar_T *)cctx->ctx_locals.ga_data) + idx;
-
 	if (STRNCMP(name, lvar->lv_name, len) == 0
 					       && STRLEN(lvar->lv_name) == len)
+	{
+	    lvar->lv_from_outer = FALSE;
 	    return lvar;
 	}
+    }
+
+    // Find local in outer function scope.
+    if (cctx->ctx_outer != NULL)
+    {
+	lvar = lookup_local(name, len, cctx->ctx_outer);
+	if (lvar != NULL)
+	{
+	    // TODO: are there situations we should not mark the outer scope as
+	    // used?
+	    cctx->ctx_outer_used = TRUE;
+	    lvar->lv_from_outer = TRUE;
+	    return lvar;
+	}
+    }
+
     return NULL;
 }
 
@@ -417,6 +439,71 @@ typval2type(typval_T *tv)
     return &t_any;  // not used
 }
 
+    static void
+type_mismatch(type_T *expected, type_T *actual)
+{
+    char *tofree1, *tofree2;
+
+    semsg(_("E1013: type mismatch, expected %s but got %s"),
+		   type_name(expected, &tofree1), type_name(actual, &tofree2));
+    vim_free(tofree1);
+    vim_free(tofree2);
+}
+
+    static void
+arg_type_mismatch(type_T *expected, type_T *actual, int argidx)
+{
+    char *tofree1, *tofree2;
+
+    semsg(_("E1013: argument %d: type mismatch, expected %s but got %s"),
+	    argidx,
+	    type_name(expected, &tofree1), type_name(actual, &tofree2));
+    vim_free(tofree1);
+    vim_free(tofree2);
+}
+
+/*
+ * Check if the expected and actual types match.
+ * Does not allow for assigning "any" to a specific type.
+ */
+    static int
+check_type(type_T *expected, type_T *actual, int give_msg)
+{
+    int ret = OK;
+
+    // When expected is "unknown" we accept any actual type.
+    // When expected is "any" we accept any actual type except "void".
+    if (expected->tt_type != VAR_UNKNOWN
+	    && !(expected->tt_type == VAR_ANY && actual->tt_type != VAR_VOID))
+
+    {
+	if (expected->tt_type != actual->tt_type)
+	{
+	    if (give_msg)
+		type_mismatch(expected, actual);
+	    return FAIL;
+	}
+	if (expected->tt_type == VAR_DICT || expected->tt_type == VAR_LIST)
+	{
+	    // "unknown" is used for an empty list or dict
+	    if (actual->tt_member != &t_unknown)
+		ret = check_type(expected->tt_member, actual->tt_member, FALSE);
+	}
+	else if (expected->tt_type == VAR_FUNC)
+	{
+	    if (expected->tt_member != &t_unknown)
+		ret = check_type(expected->tt_member, actual->tt_member, FALSE);
+	    if (ret == OK && expected->tt_argcount != -1
+		    && (actual->tt_argcount < expected->tt_min_argcount
+			|| actual->tt_argcount > expected->tt_argcount))
+		    ret = FAIL;
+	}
+	if (ret == FAIL && give_msg)
+	    type_mismatch(expected, actual);
+    }
+    return ret;
+}
+
 /////////////////////////////////////////////////////////////////////
 // Following generate_ functions expect the caller to call ga_grow().
 
@@ -739,6 +826,29 @@ generate_TYPECHECK(cctx_T *cctx, type_T *vartype, int offset)
     return OK;
 }
 
+/*
+ * Check that
+ * - "actual" is "expected" type or
+ * - "actual" is a type that can be "expected" type: add a runtime check; or
+ * - return FAIL.
+ */
+    static int
+need_type(type_T *actual, type_T *expected, int offset, cctx_T *cctx)
+{
+    if (check_type(expected, actual, FALSE) == OK)
+	return OK;
+    if (actual->tt_type != VAR_ANY
+	    && actual->tt_type != VAR_UNKNOWN
+	    && !(actual->tt_type == VAR_FUNC
+		&& (actual->tt_member == &t_any || actual->tt_argcount < 0)))
+    {
+	type_mismatch(expected, actual);
+	return FAIL;
+    }
+    generate_TYPECHECK(cctx, expected, offset);
+    return OK;
+}
+
 /*
  * Generate an ISN_PUSHNR instruction.
  */
@@ -1272,7 +1382,7 @@ generate_CALL(cctx_T *cctx, ufunc_T *ufunc, int pushed_argcount)
 	    else
 		expected = ufunc->uf_va_type->tt_member;
 	    actual = ((type_T **)stack->ga_data)[stack->ga_len - argcount + i];
-	    if (check_type(expected, actual, FALSE) == FAIL)
+	    if (need_type(actual, expected, -argcount + i, cctx) == FAIL)
 	    {
 		arg_type_mismatch(expected, actual, i + 1);
 		return FAIL;
@@ -1543,6 +1653,20 @@ skip_type(char_u *start)
 	if (*p == '>')
 	    ++p;
     }
+    else if (*p == '(' && STRNCMP("func", start, 4) == 0)
+    {
+	// handle func(args): type
+	++p;
+	while (*p != ')' && *p != NUL)
+	{
+	    p = skip_type(p);
+	    if (*p == ',')
+		p = skipwhite(p + 1);
+	}
+	if (*p == ')' && p[1] == ':')
+	    p = skip_type(skipwhite(p + 2));
+    }
+
     return p;
 }
 
@@ -2309,6 +2433,7 @@ compile_load(char_u **arg, char_u *end_arg, cctx_T *cctx, int error)
 	size_t	    len = end - *arg;
 	int	    idx;
 	int	    gen_load = FALSE;
+	int	    gen_load_outer = FALSE;
 
 	name = vim_strnsave(*arg, end - *arg);
 	if (name == NULL)
@@ -2343,6 +2468,9 @@ compile_load(char_u **arg, char_u *end_arg, cctx_T *cctx, int error)
 	    {
 		type = lvar->lv_type;
 		idx = lvar->lv_idx;
+		if (lvar->lv_from_outer)
+		    gen_load_outer = TRUE;
+		else
 		    gen_load = TRUE;
 	    }
 	    else
@@ -2370,6 +2498,8 @@ compile_load(char_u **arg, char_u *end_arg, cctx_T *cctx, int error)
 	}
 	if (gen_load)
 	    res = generate_LOAD(cctx, ISN_LOAD, idx, NULL, type);
+	if (gen_load_outer)
+	    res = generate_LOAD(cctx, ISN_LOADOUTER, idx, NULL, type);
     }
 
     *arg = end;
@@ -2578,94 +2708,6 @@ to_name_const_end(char_u *arg)
     return p;
 }
 
-    static void
-type_mismatch(type_T *expected, type_T *actual)
-{
-    char *tofree1, *tofree2;
-
-    semsg(_("E1013: type mismatch, expected %s but got %s"),
-		   type_name(expected, &tofree1), type_name(actual, &tofree2));
-    vim_free(tofree1);
-    vim_free(tofree2);
-}
-
-    static void
-arg_type_mismatch(type_T *expected, type_T *actual, int argidx)
-{
-    char *tofree1, *tofree2;
-
-    semsg(_("E1013: argument %d: type mismatch, expected %s but got %s"),
-	    argidx,
-	    type_name(expected, &tofree1), type_name(actual, &tofree2));
-    vim_free(tofree1);
-    vim_free(tofree2);
-}
-
-/*
- * Check if the expected and actual types match.
- * Does not allow for assigning "any" to a specific type.
- */
-    static int
-check_type(type_T *expected, type_T *actual, int give_msg)
-{
-    int ret = OK;
-
-    // When expected is "unknown" we accept any actual type.
-    // When expected is "any" we accept any actual type except "void".
-    if (expected->tt_type != VAR_UNKNOWN
-	    && !(expected->tt_type == VAR_ANY && actual->tt_type != VAR_VOID))
-
-    {
-	if (expected->tt_type != actual->tt_type)
-	{
-	    if (give_msg)
-		type_mismatch(expected, actual);
-	    return FAIL;
-	}
-	if (expected->tt_type == VAR_DICT || expected->tt_type == VAR_LIST)
-	{
-	    // "unknown" is used for an empty list or dict
-	    if (actual->tt_member != &t_unknown)
-		ret = check_type(expected->tt_member, actual->tt_member, FALSE);
-	}
-	else if (expected->tt_type == VAR_FUNC)
-	{
-	    if (expected->tt_member != &t_unknown)
-		ret = check_type(expected->tt_member, actual->tt_member, FALSE);
-	    if (ret == OK && expected->tt_argcount != -1
-		    && (actual->tt_argcount < expected->tt_min_argcount
-			|| actual->tt_argcount > expected->tt_argcount))
-		    ret = FAIL;
-	}
-	if (ret == FAIL && give_msg)
-	    type_mismatch(expected, actual);
-    }
-    return ret;
-}
-
-/*
- * Check that
- * - "actual" is "expected" type or
- * - "actual" is a type that can be "expected" type: add a runtime check; or
- * - return FAIL.
- */
-    static int
-need_type(type_T *actual, type_T *expected, int offset, cctx_T *cctx)
-{
-    if (check_type(expected, actual, FALSE) == OK)
-	return OK;
-    if (actual->tt_type != VAR_ANY
-	    && actual->tt_type != VAR_UNKNOWN
-	    && !(actual->tt_type == VAR_FUNC
-		&& (actual->tt_member == &t_any || actual->tt_argcount < 0)))
-    {
-	type_mismatch(expected, actual);
-	return FAIL;
-    }
-    generate_TYPECHECK(cctx, expected, offset);
-    return OK;
-}
-
 /*
  * parse a list: [expr, expr]
  * "*arg" points to the '['.
@@ -2734,7 +2776,7 @@ compile_lambda(char_u **arg, cctx_T *cctx)
 
     // The function will have one line: "return {expr}".
     // Compile it into instructions.
-    compile_def_function(ufunc, TRUE);
+    compile_def_function(ufunc, TRUE, cctx);
 
     if (ufunc->uf_dfunc_idx >= 0)
     {
@@ -2779,7 +2821,7 @@ compile_lambda_call(char_u **arg, cctx_T *cctx)
 
     // The function will have one line: "return {expr}".
     // Compile it into instructions.
-    compile_def_function(ufunc, TRUE);
+    compile_def_function(ufunc, TRUE, cctx);
 
     // compile the arguments
     *arg = skipwhite(*arg + 1);
@@ -4227,16 +4269,12 @@ compile_assignment(char_u *arg, exarg_T *eap, cmdidx_T cmdidx, cctx_T *cctx)
 		    semsg(_("E1017: Variable already declared: %s"), name);
 		    goto theend;
 		}
-		else
+		else if (lvar->lv_const)
 		{
-		    if (lvar->lv_const)
+		    semsg(_("E1018: Cannot assign to a constant: %s"), name);
-		    {
-			semsg(_("E1018: Cannot assign to a constant: %s"),
-									 name);
 		    goto theend;
 		}
 	    }
-	    }
 	    else if (STRNCMP(arg, "s:", 2) == 0
 		    || lookup_script(arg, varlen) == OK
 		    || find_imported(arg, varlen, cctx) != NULL)
@@ -5931,11 +5969,12 @@ theend:
  * Adds the function to "def_functions".
  * When "set_return_type" is set then set ufunc->uf_ret_type to the type of the
  * return statement (used for lambda).
+ * "outer_cctx" is set for a nested function.
  * This can be used recursively through compile_lambda(), which may reallocate
  * "def_functions".
  */
     void
-compile_def_function(ufunc_T *ufunc, int set_return_type)
+compile_def_function(ufunc_T *ufunc, int set_return_type, cctx_T *outer_cctx)
 {
     char_u	*line = NULL;
     char_u	*p;
@@ -5976,6 +6015,7 @@ compile_def_function(ufunc_T *ufunc, int set_return_type)
     CLEAR_FIELD(cctx);
     cctx.ctx_ufunc = ufunc;
     cctx.ctx_lnum = -1;
+    cctx.ctx_outer = outer_cctx;
     ga_init2(&cctx.ctx_locals, sizeof(lvar_T), 10);
     ga_init2(&cctx.ctx_type_stack, sizeof(type_T *), 50);
     ga_init2(&cctx.ctx_imports, sizeof(imported_T), 10);
@@ -6355,6 +6395,8 @@ compile_def_function(ufunc_T *ufunc, int set_return_type)
 	dfunc->df_instr = instr->ga_data;
 	dfunc->df_instr_count = instr->ga_len;
 	dfunc->df_varcount = cctx.ctx_locals_count;
+	if (cctx.ctx_outer_used)
+	    ufunc->uf_flags |= FC_CLOSURE;
     }
 
     {
@@ -6533,6 +6575,7 @@ delete_instr(isn_T *isn)
 	case ISN_INDEX:
 	case ISN_JUMP:
 	case ISN_LOAD:
+	case ISN_LOADOUTER:
 	case ISN_LOADSCRIPT:
 	case ISN_LOADREG:
 	case ISN_LOADV:

src/vim9execute.c

@@ -58,6 +58,9 @@ typedef struct {
     garray_T	ec_stack;	// stack of typval_T values
     int		ec_frame;	// index in ec_stack: context of ec_dfunc_idx
 
+    garray_T	*ec_outer_stack;    // stack used for closures
+    int		ec_outer_frame;	    // stack frame in ec_outer_stack
+
     garray_T	ec_trystack;	// stack of trycmd_T values
     int		ec_in_catch;	// when TRUE in catch or finally block
 
@@ -229,6 +232,10 @@ call_dfunc(int cdf_idx, int argcount_arg, ectx_T *ectx)
     ectx->ec_instr = dfunc->df_instr;
     estack_push_ufunc(ETYPE_UFUNC, dfunc->df_ufunc, 1);
 
+    // used for closures
+    ectx->ec_outer_stack = ufunc->uf_ectx_stack;
+    ectx->ec_outer_frame = ufunc->uf_ectx_frame;
+
     // Decide where to start execution, handles optional arguments.
     init_instr_idx(ufunc, argcount, ectx);
 
@@ -508,6 +515,9 @@ call_def_function(
 // Get pointer to a local variable on the stack.  Negative for arguments.
 #define STACK_TV_VAR(idx) (((typval_T *)ectx.ec_stack.ga_data) + ectx.ec_frame + STACK_FRAME_SIZE + idx)
 
+// Like STACK_TV_VAR but use the outer scope
+#define STACK_OUT_TV_VAR(idx) (((typval_T *)ectx.ec_outer_stack->ga_data) + ectx.ec_outer_frame + STACK_FRAME_SIZE + idx)
+
     CLEAR_FIELD(ectx);
     ga_init2(&ectx.ec_stack, sizeof(typval_T), 500);
     if (ga_grow(&ectx.ec_stack, 20) == FAIL)
@@ -786,6 +796,15 @@ call_def_function(
 		++ectx.ec_stack.ga_len;
 		break;
 
+	    // load variable or argument from outer scope
+	    case ISN_LOADOUTER:
+		if (ga_grow(&ectx.ec_stack, 1) == FAIL)
+		    goto failed;
+		copy_tv(STACK_OUT_TV_VAR(iptr->isn_arg.number),
+							      STACK_TV_BOT(0));
+		++ectx.ec_stack.ga_len;
+		break;
+
 	    // load v: variable
 	    case ISN_LOADV:
 		if (ga_grow(&ectx.ec_stack, 1) == FAIL)
@@ -1304,6 +1323,14 @@ call_def_function(
 		    pt->pt_refcount = 1;
 		    ++dfunc->df_ufunc->uf_refcount;
 
+		    if (dfunc->df_ufunc->uf_flags & FC_CLOSURE)
+		    {
+			// Closure needs to find local variables in the current
+			// stack.
+			dfunc->df_ufunc->uf_ectx_stack = &ectx.ec_stack;
+			dfunc->df_ufunc->uf_ectx_frame = ectx.ec_frame;
+		    }
+
 		    if (ga_grow(&ectx.ec_stack, 1) == FAIL)
 			goto failed;
 		    tv = STACK_TV_BOT(0);
@@ -1862,7 +1889,12 @@ call_def_function(
 		    checktype_T *ct = &iptr->isn_arg.type;
 
 		    tv = STACK_TV_BOT(ct->ct_off);
-		    if (tv->v_type != ct->ct_type)
+		    // TODO: better type comparison
+		    if (tv->v_type != ct->ct_type
+			    && !((tv->v_type == VAR_PARTIAL
+						   && ct->ct_type == VAR_FUNC)
+				|| (tv->v_type == VAR_FUNC
+					       && ct->ct_type == VAR_PARTIAL)))
 		    {
 			semsg(_("E1029: Expected %s but got %s"),
 				    vartype_name(ct->ct_type),
@@ -2029,12 +2061,18 @@ ex_disassemble(exarg_T *eap)
 					    (long long)(iptr->isn_arg.number));
 		break;
 	    case ISN_LOAD:
+	    case ISN_LOADOUTER:
+		{
+		    char *add = iptr->isn_type == ISN_LOAD ? "" : "OUTER";
+
 		    if (iptr->isn_arg.number < 0)
-		    smsg("%4d LOAD arg[%lld]", current,
+			smsg("%4d LOAD%s arg[%lld]", current, add,
-			 (long long)(iptr->isn_arg.number + STACK_FRAME_SIZE));
+				(long long)(iptr->isn_arg.number
+							  + STACK_FRAME_SIZE));
 		    else
-		    smsg("%4d LOAD $%lld", current,
+			smsg("%4d LOAD%s $%lld", current, add,
 					    (long long)(iptr->isn_arg.number));
+		}
 		break;
 	    case ISN_LOADV:
 		smsg("%4d LOADV v:%s", current,