aniani/vim
0
0
Fork 1
mirror of https://github.com/vim/vim.git synced 2025-09-27 04:14:06 -04:00
Code Activity

updated for version 7.3.427

Browse Source 
Problem:    readfile() can be slow with long lines.
Solution:   Use realloc() instead of alloc(). (John Little)
This commit is contained in:
Bram Moolenaar
2012-02-05 00:39:18 +01:00
parent 76b9b3696c
commit a489e1d9d6
2 changed files with 136 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

224
src/eval.c
View File

@@ -14325,22 +14325,19 @@ f_readfile(argvars, rettv)
typval_T *rettv; typval_T *rettv;
{ {
int binary = FALSE; int binary = FALSE;
int failed = FALSE;
char_u *fname; char_u *fname;
FILE *fd; FILE *fd;
listitem_T *li; char_u buf[(IOSIZE/256)*256]; /* rounded to avoid odd + 1 */
#define FREAD_SIZE 200 /* optimized for text lines */ int io_size = sizeof(buf);
char_u buf[FREAD_SIZE]; int readlen; /* size of last fread() */
int readlen; /* size of last fread() */ char_u *prev = NULL; /* previously read bytes, if any */
int buflen; /* nr of valid chars in buf[] */ long prevlen = 0; /* length of data in prev */
int filtd; /* how much in buf[] was NUL -> '\n' filtered */ long prevsize = 0; /* size of prev buffer */
int tolist; /* first byte in buf[] still to be put in list */ long maxline = MAXLNUM;
int chop; /* how many CR to chop off */ long cnt = 0;
char_u *prev = NULL; /* previously read bytes, if any */ char_u *p; /* position in buf */
int prevlen = 0; /* length of "prev" if not NULL */ char_u *start; /* start of current line */
char_u *s;
int len;
long maxline = MAXLNUM;
long cnt = 0;
if (argvars[1].v_type != VAR_UNKNOWN) if (argvars[1].v_type != VAR_UNKNOWN)
{ {
@@ -14362,49 +14359,61 @@ f_readfile(argvars, rettv)
return; return;
} }
filtd = 0;
while (cnt < maxline || maxline < 0) while (cnt < maxline || maxline < 0)
{ {
readlen = (int)fread(buf + filtd, 1, FREAD_SIZE - filtd, fd); readlen = (int)fread(buf, 1, io_size, fd);
buflen = filtd + readlen;
tolist = 0;
for ( ; filtd < buflen || readlen <= 0; ++filtd)
{
if (readlen <= 0 || buf[filtd] == '\n')
{
/* In binary mode add an empty list item when the last
* non-empty line ends in a '\n'. */
if (!binary && readlen == 0 && filtd == 0 && prev == NULL)
break;
/* Found end-of-line or end-of-file: add a text line to the /* This for loop processes what was read, but is also entered at end
* list. */ * of file so that either:
chop = 0; * - an incomplete line gets written
if (!binary) * - a "binary" file gets an empty line at the end if it ends in a
while (filtd - chop - 1 >= tolist * newline. */
&& buf[filtd - chop - 1] == '\r') for (p = buf, start = buf;
++chop; p < buf + readlen || (readlen <= 0 && (prevlen > 0 || binary));
len = filtd - tolist - chop; ++p)
if (prev == NULL) {
s = vim_strnsave(buf + tolist, len); if (*p == '\n' || readlen <= 0)
{
listitem_T *li;
char_u *s = NULL;
long_u len = p - start;
/* Finished a line. Remove CRs before NL. */
if (readlen > 0 && !binary)
{
while (len > 0 && start[len - 1] == '\r')
--len;
/* removal may cross back to the "prev" string */
if (len == 0)
while (prevlen > 0 && prev[prevlen - 1] == '\r')
--prevlen;
}
if (prevlen == 0)
s = vim_strnsave(start, len);
else else
{ {
s = alloc((unsigned)(prevlen + len + 1)); /* Change "prev" buffer to be the right size. This way
if (s != NULL) * the bytes are only copied once, and very long lines are
* allocated only once. */
if ((s = vim_realloc(prev, prevlen + len + 1)) != NULL)
{ {
mch_memmove(s, prev, prevlen); mch_memmove(s + prevlen, start, len);
vim_free(prev);
prev = NULL;
mch_memmove(s + prevlen, buf + tolist, len);
s[prevlen + len] = NUL; s[prevlen + len] = NUL;
prev = NULL; /* the list will own the string */
prevlen = prevsize = 0;
} }
} }
tolist = filtd + 1; if (s == NULL)
{
do_outofmem_msg((long_u) prevlen + len + 1);
failed = TRUE;
break;
}
li = listitem_alloc(); if ((li = listitem_alloc()) == NULL)
if (li == NULL)
{ {
vim_free(s); vim_free(s);
failed = TRUE;
break; break;
} }
li->li_tv.v_type = VAR_STRING; li->li_tv.v_type = VAR_STRING;
@@ -14412,74 +14421,109 @@ f_readfile(argvars, rettv)
li->li_tv.vval.v_string = s; li->li_tv.vval.v_string = s;
list_append(rettv->vval.v_list, li); list_append(rettv->vval.v_list, li);
if (++cnt >= maxline && maxline >= 0) start = p + 1; /* step over newline */
break; if ((++cnt >= maxline && maxline >= 0) || readlen <= 0)
if (readlen <= 0)
break; break;
} }
else if (buf[filtd] == NUL) else if (*p == NUL)
buf[filtd] = '\n'; *p = '\n';
#ifdef FEAT_MBYTE #ifdef FEAT_MBYTE
else if (buf[filtd] == 0xef /* Check for utf8 "bom"; U+FEFF is encoded as EF BB BF. Do this
&& enc_utf8 * when finding the BF and check the previous two bytes. */
&& filtd + 2 < buflen else if (*p == 0xbf && enc_utf8 && !binary)
&& !binary
&& buf[filtd + 1] == 0xbb
&& buf[filtd + 2] == 0xbf)
{ {
/* remove utf-8 byte order mark */ /* Find the two bytes before the 0xbf. If p is at buf, or buf
mch_memmove(buf + filtd, buf + filtd + 3, buflen - filtd - 3); * + 1, these may be in the "prev" string. */
--filtd; char_u back1 = p >= buf + 1 ? p[-1]
buflen -= 3; : prevlen >= 1 ? prev[prevlen - 1] : NUL;
} char_u back2 = p >= buf + 2 ? p[-2]
#endif : p == buf + 1 && prevlen >= 1 ? prev[prevlen - 1]
} : prevlen >= 2 ? prev[prevlen - 2] : NUL;
if (readlen <= 0)
break;
if (tolist == 0) if (back2 == 0xef && back1 == 0xbb)
{
if (buflen >= FREAD_SIZE / 2)
{
/* "buf" is full, need to move text to an allocated buffer */
if (prev == NULL)
{ {
prev = vim_strnsave(buf, buflen); char_u *dest = p - 2;
prevlen = buflen;
} /* Usually a BOM is at the beginning of a file, and so at
else * the beginning of a line; then we can just step over it.
{ */
s = alloc((unsigned)(prevlen + buflen)); if (start == dest)
if (s != NULL) start = p + 1;
else
{ {
mch_memmove(s, prev, prevlen); /* have to shuffle buf to close gap */
mch_memmove(s + prevlen, buf, buflen); int adjust_prevlen = 0;
vim_free(prev);
prev = s; if (dest < buf)
prevlen += buflen; {
adjust_prevlen = buf - dest; /* must be 1 or 2 */
dest = buf;
}
if (readlen > p - buf + 1)
mch_memmove(dest, p + 1, readlen - (p - buf) - 1);
readlen -= 3 - adjust_prevlen;
prevlen -= adjust_prevlen;
p = dest - 1;
} }
} }
filtd = 0;
} }
} #endif
else } /* for */
if (failed || (cnt >= maxline && maxline >= 0) || readlen <= 0)
break;
if (start < p)
{ {
mch_memmove(buf, buf + tolist, buflen - tolist); /* There's part of a line in buf, store it in "prev". */
filtd -= tolist; if (p - start + prevlen >= prevsize)
{
/* need bigger "prev" buffer */
char_u *newprev;
/* A common use case is ordinary text files and "prev" gets a
* fragment of a line, so the first allocation is made
* small, to avoid repeatedly 'allocing' large and
* 'reallocing' small. */
if (prevsize == 0)
prevsize = p - start;
else
{
long grow50pc = (prevsize * 3) / 2;
long growmin = (p - start) * 2 + prevlen;
prevsize = grow50pc > growmin ? grow50pc : growmin;
}
if ((newprev = vim_realloc(prev, prevsize)) == NULL)
{
do_outofmem_msg((long_u)prevsize);
failed = TRUE;
break;
}
prev = newprev;
}
/* Add the line part to end of "prev". */
mch_memmove(prev + prevlen, start, p - start);
prevlen += p - start;
} }
} } /* while */
/* /*
* For a negative line count use only the lines at the end of the file, * For a negative line count use only the lines at the end of the file,
* free the rest. * free the rest.
*/ */
if (maxline < 0) if (!failed && maxline < 0)
while (cnt > -maxline) while (cnt > -maxline)
{ {
listitem_remove(rettv->vval.v_list, rettv->vval.v_list->lv_first); listitem_remove(rettv->vval.v_list, rettv->vval.v_list->lv_first);
--cnt; --cnt;
} }
if (failed)
{
list_free(rettv->vval.v_list, TRUE);
/* readfile doc says an empty list is returned on error */
rettv->vval.v_list = list_alloc();
}
vim_free(prev); vim_free(prev);
fclose(fd); fclose(fd);
} }

2
src/version.c
View File

@@ -714,6 +714,8 @@ static char *(features[]) =
static int included_patches[] = static int included_patches[] =
{ /* Add new patch number below this line */ { /* Add new patch number below this line */
/**/
427,
/**/ /**/
426, 426,
/**/ /**/