0
0
mirror of https://github.com/vim/vim.git synced 2025-09-01 21:03:39 -04:00

patch 8.0.0420: text garbled when the system encoding differs from 'encoding'

Problem:    When running :make the output may be in the system encoding,
            different from 'encoding'.
Solution:   Add the 'makeencoding' option. (Ken Takata)
This commit is contained in:
Bram Moolenaar 2017-03-05 17:43:31 +01:00
parent 214641f77d
commit 2c7292dc5b
16 changed files with 328 additions and 16 deletions

View File

@ -4994,6 +4994,25 @@ A jump table for the options with a short description can be found at |Q_op|.
This option cannot be set from a |modeline| or in the |sandbox|, for
security reasons.
*'makeencoding'* *'menc'*
'makeencoding' 'menc' string (default "")
global or local to buffer |global-local|
{only available when compiled with the |+multi_byte|
feature}
{not in Vi}
Encoding used for reading the output of external commands. When empty,
encoding is not converted.
This is used for `:make`, `:lmake`, `:grep`, `:lgrep`, `:grepadd`,
`:lgrepadd`, `:cfile`, `:cgetfile`, `:caddfile`, `:lfile`, `:lgetfile`,
and `:laddfile`.
This would be mostly useful when you use MS-Windows and set 'encoding'
to "utf-8". If |+iconv| is enabled and GNU libiconv is used, setting
'makeencoding' to "char" has the same effect as setting to the system
locale encoding. Example: >
:set encoding=utf-8
:set makeencoding=char " system locale is used
<
*'makeprg'* *'mp'*
'makeprg' 'mp' string (default "make", VMS: "MMS")
global or local to buffer |global-local|

View File

@ -164,6 +164,9 @@ processing a quickfix or location list command, it will be aborted.
keep Vim running while compiling. If you give the
name of the errorfile, the 'errorfile' option will
be set to [errorfile]. See |:cc| for [!].
If the encoding of the error file differs from the
'encoding' option, you can use the 'makeencoding'
option to specify the encoding.
*:lf* *:lfile*
:lf[ile][!] [errorfile] Same as ":cfile", except the location list for the
@ -175,6 +178,9 @@ processing a quickfix or location list command, it will be aborted.
:cg[etfile] [errorfile] *:cg* *:cgetfile*
Read the error file. Just like ":cfile" but don't
jump to the first error.
If the encoding of the error file differs from the
'encoding' option, you can use the 'makeencoding'
option to specify the encoding.
:lg[etfile] [errorfile] *:lg* *:lgetfile*
@ -185,6 +191,9 @@ processing a quickfix or location list command, it will be aborted.
:caddf[ile] [errorfile] Read the error file and add the errors from the
errorfile to the current quickfix list. If a quickfix
list is not present, then a new list is created.
If the encoding of the error file differs from the
'encoding' option, you can use the 'makeencoding'
option to specify the encoding.
*:laddf* *:laddfile*
:laddf[ile] [errorfile] Same as ":caddfile", except the location list for the
@ -320,6 +329,7 @@ use this code: >
endfunction
au QuickfixCmdPost make call QfMakeConv()
Another option is using 'makeencoding'.
EXECUTE A COMMAND IN ALL THE BUFFERS IN QUICKFIX OR LOCATION LIST:
@ -586,6 +596,9 @@ lists, use ":cnewer 99" first.
like |:cnext| and |:cprevious|, see above.
This command does not accept a comment, any "
characters are considered part of the arguments.
If the encoding of the program output differs from the
'encoding' option, you can use the 'makeencoding'
option to specify the encoding.
*:lmak* *:lmake*
:lmak[e][!] [arguments]
@ -645,6 +658,7 @@ read the error messages: >
au QuickfixCmdPost make call QfMakeConv()
(Example by Faque Cheng)
Another option is using 'makeencoding'.
==============================================================================
5. Using :vimgrep and :grep *grep* *lid*
@ -759,6 +773,9 @@ id-utils) in a similar way to its compiler integration (see |:make| above).
When 'grepprg' is "internal" this works like
|:vimgrep|. Note that the pattern needs to be
enclosed in separator characters then.
If the encoding of the program output differs from the
'encoding' option, you can use the 'makeencoding'
option to specify the encoding.
*:lgr* *:lgrep*
:lgr[ep][!] [arguments] Same as ":grep", except the location list for the
@ -783,6 +800,10 @@ id-utils) in a similar way to its compiler integration (see |:make| above).
\ | catch /E480:/
\ | endtry"
<
If the encoding of the program output differs from the
'encoding' option, you can use the 'makeencoding'
option to specify the encoding.
*:lgrepa* *:lgrepadd*
:lgrepa[dd][!] [arguments]
Same as ":grepadd", except the location list for the

View File

@ -782,6 +782,7 @@ Short explanation of each option: *option-list*
'macatsui' Mac GUI: use ATSUI text drawing
'magic' changes special characters in search patterns
'makeef' 'mef' name of the errorfile for ":make"
'makeencoding' 'menc' encoding of external make/grep commands
'makeprg' 'mp' program to use for the ":make" command
'matchpairs' 'mps' pairs of characters that "%" can match
'matchtime' 'mat' tenths of a second to show matching paren

View File

@ -2157,6 +2157,7 @@ test_arglist \
test_listlbr \
test_listlbr_utf8 \
test_lua \
test_makeencoding \
test_man \
test_mapping \
test_marks \

View File

@ -2255,6 +2255,9 @@ free_buf_options(
clear_string_option(&buf->b_p_lw);
#endif
clear_string_option(&buf->b_p_bkc);
#ifdef FEAT_MBYTE
clear_string_option(&buf->b_p_menc);
#endif
}
/*

View File

@ -1242,7 +1242,7 @@ cs_find_common(
wp = curwin;
/* '-' starts a new error list */
if (qf_init(wp, tmp, (char_u *)"%f%*\\t%l%*\\t%m",
*qfpos == '-', cmdline) > 0)
*qfpos == '-', cmdline, NULL) > 0)
{
# ifdef FEAT_WINDOWS
if (postponed_split != 0)

View File

@ -559,11 +559,16 @@ vim_main2(void)
*/
if (params.edit_type == EDIT_QF)
{
char_u *enc = NULL;
# ifdef FEAT_MBYTE
enc = p_menc;
# endif
if (params.use_ef != NULL)
set_string_option_direct((char_u *)"ef", -1,
params.use_ef, OPT_FREE, SID_CARG);
vim_snprintf((char *)IObuff, IOSIZE, "cfile %s", p_ef);
if (qf_init(NULL, p_ef, p_efm, TRUE, IObuff) < 0)
if (qf_init(NULL, p_ef, p_efm, TRUE, IObuff, enc) < 0)
{
out_char('\n');
mch_exit(3);

View File

@ -139,6 +139,9 @@
# define PV_LISP OPT_BUF(BV_LISP)
# define PV_LW OPT_BOTH(OPT_BUF(BV_LW))
#endif
#ifdef FEAT_MBYTE
# define PV_MENC OPT_BOTH(OPT_BUF(BV_MENC))
#endif
#define PV_MA OPT_BUF(BV_MA)
#define PV_ML OPT_BUF(BV_ML)
#define PV_MOD OPT_BUF(BV_MOD)
@ -1898,6 +1901,15 @@ static struct vimoption options[] =
#else
(char_u *)NULL, PV_NONE,
{(char_u *)NULL, (char_u *)0L}
#endif
SCRIPTID_INIT},
{"makeencoding","menc", P_STRING|P_VI_DEF,
#ifdef FEAT_MBYTE
(char_u *)&p_menc, PV_MENC,
{(char_u *)"", (char_u *)0L}
#else
(char_u *)NULL, PV_NONE,
{(char_u *)0L, (char_u *)0L}
#endif
SCRIPTID_INIT},
{"makeprg", "mp", P_STRING|P_EXPAND|P_VI_DEF|P_SECURE,
@ -5686,6 +5698,9 @@ check_buf_options(buf_T *buf)
check_string_option(&buf->b_p_lw);
#endif
check_string_option(&buf->b_p_bkc);
#ifdef FEAT_MBYTE
check_string_option(&buf->b_p_menc);
#endif
}
/*
@ -6289,8 +6304,9 @@ did_set_string_option(
#endif
#ifdef FEAT_MBYTE
/* 'encoding' and 'fileencoding' */
else if (varp == &p_enc || gvarp == &p_fenc || varp == &p_tenc)
/* 'encoding', 'fileencoding', 'termencoding' and 'makeencoding' */
else if (varp == &p_enc || gvarp == &p_fenc || varp == &p_tenc
|| gvarp == &p_menc)
{
if (gvarp == &p_fenc)
{
@ -10424,6 +10440,11 @@ unset_global_local_option(char_u *name, void *from)
case PV_LW:
clear_string_option(&buf->b_p_lw);
break;
#endif
#ifdef FEAT_MBYTE
case PV_MENC:
clear_string_option(&buf->b_p_menc);
break;
#endif
}
}
@ -10478,6 +10499,9 @@ get_varp_scope(struct vimoption *p, int opt_flags)
case PV_LW: return (char_u *)&(curbuf->b_p_lw);
#endif
case PV_BKC: return (char_u *)&(curbuf->b_p_bkc);
#ifdef FEAT_MBYTE
case PV_MENC: return (char_u *)&(curbuf->b_p_menc);
#endif
}
return NULL; /* "cannot happen" */
}
@ -10553,6 +10577,10 @@ get_varp(struct vimoption *p)
case PV_LW: return *curbuf->b_p_lw != NUL
? (char_u *)&(curbuf->b_p_lw) : p->var;
#endif
#ifdef FEAT_MBYTE
case PV_MENC: return *curbuf->b_p_menc != NUL
? (char_u *)&(curbuf->b_p_menc) : p->var;
#endif
#ifdef FEAT_ARABIC
case PV_ARAB: return (char_u *)&(curwin->w_p_arab);
@ -11154,6 +11182,9 @@ buf_copy_options(buf_T *buf, int flags)
#ifdef FEAT_LISP
buf->b_p_lw = empty_option;
#endif
#ifdef FEAT_MBYTE
buf->b_p_menc = empty_option;
#endif
/*
* Don't copy the options set by ex_help(), use the saved values,

View File

@ -630,6 +630,9 @@ EXTERN char_u *p_luadll; /* 'luadll' */
EXTERN int p_macatsui; /* 'macatsui' */
#endif
EXTERN int p_magic; /* 'magic' */
#ifdef FEAT_MBYTE
EXTERN char_u *p_menc; /* 'makeencoding' */
#endif
#ifdef FEAT_QUICKFIX
EXTERN char_u *p_mef; /* 'makeef' */
EXTERN char_u *p_mp; /* 'makeprg' */
@ -1064,6 +1067,9 @@ enum
#ifdef FEAT_LISP
, BV_LISP
, BV_LW
#endif
#ifdef FEAT_MBYTE
, BV_MENC
#endif
, BV_MA
, BV_ML

View File

@ -1,5 +1,5 @@
/* quickfix.c */
int qf_init(win_T *wp, char_u *efile, char_u *errorformat, int newlist, char_u *qf_title);
int qf_init(win_T *wp, char_u *efile, char_u *errorformat, int newlist, char_u *qf_title, char_u *enc);
void qf_free_all(win_T *wp);
void copy_loclist(win_T *from, win_T *to);
void qf_jump(qf_info_T *qi, int dir, int errornr, int forceit);

View File

@ -116,7 +116,7 @@ struct efm_S
static efm_T *fmt_start = NULL; /* cached across qf_parse_line() calls */
static int qf_init_ext(qf_info_T *qi, char_u *efile, buf_T *buf, typval_T *tv, char_u *errorformat, int newlist, linenr_T lnumfirst, linenr_T lnumlast, char_u *qf_title);
static int qf_init_ext(qf_info_T *qi, char_u *efile, buf_T *buf, typval_T *tv, char_u *errorformat, int newlist, linenr_T lnumfirst, linenr_T lnumlast, char_u *qf_title, char_u *enc);
static void qf_store_title(qf_info_T *qi, char_u *title);
static void qf_new_list(qf_info_T *qi, char_u *qf_title);
static void ll_free_all(qf_info_T **pqi);
@ -167,7 +167,8 @@ qf_init(
char_u *efile,
char_u *errorformat,
int newlist, /* TRUE: start a new error list */
char_u *qf_title)
char_u *qf_title,
char_u *enc)
{
qf_info_T *qi = &ql_info;
@ -180,7 +181,7 @@ qf_init(
return qf_init_ext(qi, efile, curbuf, NULL, errorformat, newlist,
(linenr_T)0, (linenr_T)0,
qf_title);
qf_title, enc);
}
/*
@ -498,6 +499,7 @@ typedef struct {
buf_T *buf;
linenr_T buflnum;
linenr_T lnumlast;
vimconv_T vc;
} qfstate_T;
static char_u *
@ -713,6 +715,30 @@ qf_get_next_file_line(qfstate_T *state)
else
state->linebuf = IObuff;
#ifdef FEAT_MBYTE
/* Convert a line if it contains a non-ASCII character. */
if (state->vc.vc_type != CONV_NONE && has_non_ascii(state->linebuf)) {
char_u *line;
line = string_convert(&state->vc, state->linebuf, &state->linelen);
if (line != NULL)
{
if (state->linelen < IOSIZE)
{
STRCPY(state->linebuf, line);
vim_free(line);
}
else
{
vim_free(state->growbuf);
state->linebuf = state->growbuf = line;
state->growbufsiz = state->linelen < LINE_MAXLEN
? state->linelen : LINE_MAXLEN;
}
}
}
#endif
return QF_OK;
}
@ -1105,11 +1131,11 @@ qf_init_ext(
int newlist, /* TRUE: start a new error list */
linenr_T lnumfirst, /* first line number to use */
linenr_T lnumlast, /* last line number to use */
char_u *qf_title)
char_u *qf_title,
char_u *enc)
{
qfstate_T state = {NULL, 0, NULL, 0, NULL, NULL, NULL, NULL,
NULL, 0, 0};
qffields_T fields = {NULL, NULL, 0, 0L, 0, FALSE, NULL, 0, 0, 0};
qfstate_T state;
qffields_T fields;
#ifdef FEAT_WINDOWS
qfline_T *old_last = NULL;
int adding = FALSE;
@ -1120,6 +1146,13 @@ qf_init_ext(
int retval = -1; /* default: return error flag */
int status;
vim_memset(&state, 0, sizeof(state));
vim_memset(&fields, 0, sizeof(fields));
#ifdef FEAT_MBYTE
state.vc.vc_type = CONV_NONE;
if (enc != NULL && *enc != NUL)
convert_setup(&state.vc, enc, p_enc);
#endif
fields.namebuf = alloc_id(CMDBUFFSIZE + 1, aid_qf_namebuf);
fields.errmsglen = CMDBUFFSIZE + 1;
fields.errmsg = alloc_id(fields.errmsglen, aid_qf_errmsg);
@ -1286,6 +1319,10 @@ qf_init_end:
#ifdef FEAT_WINDOWS
qf_update_buffer(qi, old_last);
#endif
#ifdef FEAT_MBYTE
if (state.vc.vc_type != CONV_NONE)
convert_setup(&state.vc, NULL, NULL);
#endif
return retval;
}
@ -3431,6 +3468,7 @@ ex_make(exarg_T *eap)
{
char_u *fname;
char_u *cmd;
char_u *enc = NULL;
unsigned len;
win_T *wp = NULL;
qf_info_T *qi = &ql_info;
@ -3464,6 +3502,9 @@ ex_make(exarg_T *eap)
# endif
}
#endif
#ifdef FEAT_MBYTE
enc = (*curbuf->b_p_menc != NUL) ? curbuf->b_p_menc : p_menc;
#endif
if (eap->cmdidx == CMD_lmake || eap->cmdidx == CMD_lgrep
|| eap->cmdidx == CMD_lgrepadd)
@ -3511,7 +3552,7 @@ ex_make(exarg_T *eap)
&& eap->cmdidx != CMD_lmake) ? p_gefm : p_efm,
(eap->cmdidx != CMD_grepadd
&& eap->cmdidx != CMD_lgrepadd),
*eap->cmdlinep);
*eap->cmdlinep, enc);
if (wp != NULL)
qi = GET_LOC_LIST(wp);
#ifdef FEAT_AUTOCMD
@ -3850,6 +3891,7 @@ ex_cnext(exarg_T *eap)
void
ex_cfile(exarg_T *eap)
{
char_u *enc = NULL;
win_T *wp = NULL;
qf_info_T *qi = &ql_info;
#ifdef FEAT_AUTOCMD
@ -3874,6 +3916,9 @@ ex_cfile(exarg_T *eap)
if (au_name != NULL)
apply_autocmds(EVENT_QUICKFIXCMDPRE, au_name, NULL, FALSE, curbuf);
#endif
#ifdef FEAT_MBYTE
enc = (*curbuf->b_p_menc != NUL) ? curbuf->b_p_menc : p_menc;
#endif
#ifdef FEAT_BROWSE
if (cmdmod.browse)
{
@ -3901,7 +3946,7 @@ ex_cfile(exarg_T *eap)
*/
if (qf_init(wp, p_ef, p_efm, (eap->cmdidx != CMD_caddfile
&& eap->cmdidx != CMD_laddfile),
*eap->cmdlinep) > 0
*eap->cmdlinep, enc) > 0
&& (eap->cmdidx == CMD_cfile
|| eap->cmdidx == CMD_lfile))
{
@ -4927,7 +4972,7 @@ ex_cbuffer(exarg_T *eap)
(eap->cmdidx != CMD_caddbuffer
&& eap->cmdidx != CMD_laddbuffer),
eap->line1, eap->line2,
qf_title) > 0)
qf_title, NULL) > 0)
{
#ifdef FEAT_AUTOCMD
if (au_name != NULL)
@ -4995,7 +5040,8 @@ ex_cexpr(exarg_T *eap)
if (qf_init_ext(qi, NULL, NULL, tv, p_efm,
(eap->cmdidx != CMD_caddexpr
&& eap->cmdidx != CMD_laddexpr),
(linenr_T)0, (linenr_T)0, *eap->cmdlinep) > 0)
(linenr_T)0, (linenr_T)0, *eap->cmdlinep,
NULL) > 0)
{
#ifdef FEAT_AUTOCMD
if (au_name != NULL)

View File

@ -2132,6 +2132,9 @@ struct file_buffer
char_u *b_p_kp; /* 'keywordprg' */
#ifdef FEAT_LISP
int b_p_lisp; /* 'lisp' */
#endif
#ifdef FEAT_MBYTE
char_u *b_p_menc; /* 'makeencoding' */
#endif
char_u *b_p_mps; /* 'matchpairs' */
int b_p_ml; /* 'modeline' */

View File

@ -170,6 +170,7 @@ NEW_TESTS = test_arabic.res \
test_listlbr.res \
test_listlbr_utf8.res \
test_lua.res \
test_makeencoding.res \
test_man.res \
test_marks.res \
test_matchadd_conceal.res \

View File

@ -0,0 +1,67 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
# Test program for :make, :grep and :cgetfile.
from __future__ import print_function, unicode_literals
import locale
import io
import sys
def set_output_encoding(enc=None):
"""Set the encoding of stdout and stderr
arguments:
enc -- Encoding name.
If omitted, locale.getpreferredencoding() is used.
"""
if enc is None:
enc = locale.getpreferredencoding()
def get_text_writer(fo, **kwargs):
kw = dict(kwargs)
kw.setdefault('errors', 'backslashreplace') # use \uXXXX style
kw.setdefault('closefd', False)
if sys.version_info[0] < 3:
# Work around for Python 2.x
# New line conversion isn't needed here. Done in somewhere else.
writer = io.open(fo.fileno(), mode='w', newline='', **kw)
write = writer.write # save the original write() function
enc = locale.getpreferredencoding()
def convwrite(s):
if isinstance(s, bytes):
write(s.decode(enc)) # convert to unistr
else:
write(s)
try:
writer.flush() # needed on Windows
except IOError:
pass
writer.write = convwrite
else:
writer = io.open(fo.fileno(), mode='w', **kw)
return writer
sys.stdout = get_text_writer(sys.stdout, encoding=enc)
sys.stderr = get_text_writer(sys.stderr, encoding=enc)
def main():
enc = 'utf-8'
if len(sys.argv) > 1:
enc = sys.argv[1]
set_output_encoding(enc)
message_tbl = {
'utf-8': 'ÀÈÌÒÙ こんにちは 你好',
'latin1': 'ÀÈÌÒÙ',
'cp932': 'こんにちは',
'cp936': '你好',
}
print('Xfoobar.c(10) : %s (%s)' % (message_tbl[enc], enc))
if __name__ == "__main__":
main()

View File

@ -0,0 +1,106 @@
" Tests for 'makeencoding'.
if !has('multi_byte')
finish
endif
source shared.vim
let s:python = PythonProg()
if s:python == ''
" Can't run this test.
finish
endif
let s:script = 'test_makeencoding.py'
let s:message_tbl = {
\ 'utf-8': 'ÀÈÌÒÙ こんにちは 你好',
\ 'latin1': 'ÀÈÌÒÙ',
\ 'cp932': 'こんにちは',
\ 'cp936': '你好',
\}
" Tests for :cgetfile and :lgetfile.
func Test_getfile()
set errorfile=Xerror.txt
set errorformat=%f(%l)\ :\ %m
" :cgetfile
for enc in keys(s:message_tbl)
let &makeencoding = enc
exec "silent !" . s:python . " " . s:script . " " . enc . " > " . &errorfile
cgetfile
copen
call assert_equal("Xfoobar.c|10| " . s:message_tbl[enc] . " (" . enc . ")",
\ getline('.'))
cclose
endfor
" :lgetfile
for enc in keys(s:message_tbl)
let &makeencoding = enc
exec "silent !" . s:python . " " . s:script . " " . enc . " > " . &errorfile
lgetfile
lopen
call assert_equal("Xfoobar.c|10| " . s:message_tbl[enc] . " (" . enc . ")",
\ getline('.'))
lclose
endfor
call delete(&errorfile)
endfunc
" Tests for :grep and :lgrep.
func Test_grep()
let &grepprg = s:python
set grepformat=%f(%l)\ :\ %m
" :grep
for enc in keys(s:message_tbl)
let &makeencoding = enc
exec "silent grep! " . s:script . " " . enc
copen
call assert_equal("Xfoobar.c|10| " . s:message_tbl[enc] . " (" . enc . ")",
\ getline('.'))
cclose
endfor
" :lgrep
for enc in keys(s:message_tbl)
let &makeencoding = enc
exec "silent lgrep! " . s:script . " " . enc
lopen
call assert_equal("Xfoobar.c|10| " . s:message_tbl[enc] . " (" . enc . ")",
\ getline('.'))
lclose
endfor
endfunc
" Tests for :make and :lmake.
func Test_make()
let &makeprg = s:python
set errorformat=%f(%l)\ :\ %m
" :make
for enc in keys(s:message_tbl)
let &makeencoding = enc
exec "silent make! " . s:script . " " . enc
copen
call assert_equal("Xfoobar.c|10| " . s:message_tbl[enc] . " (" . enc . ")",
\ getline('.'))
cclose
endfor
" :lmake
for enc in keys(s:message_tbl)
let &makeencoding = enc
exec "silent lmake! " . s:script . " " . enc
lopen
call assert_equal("Xfoobar.c|10| " . s:message_tbl[enc] . " (" . enc . ")",
\ getline('.'))
lclose
endfor
endfunc

View File

@ -764,6 +764,8 @@ static char *(features[]) =
static int included_patches[] =
{ /* Add new patch number below this line */
/**/
420,
/**/
419,
/**/