fix tag search with !_TAG_FILE_ENCODING

fix-tag-search-with-encoding.diff

diff -r 15b934a16641 src/tag.c
--- a/src/tag.c	Wed Sep 14 19:04:40 2011 +0200
+++ b/src/tag.c	Wed Oct 05 08:49:02 2011 +0900
@@ -1360,13 +1360,9 @@
     char_u	*saved_pat = NULL;		/* copy of pat[] */
 #endif
 
-    /* Use two sets of variables for the pattern: "orgpat" holds the values
-     * for the original pattern and "convpat" converted from 'encoding' to
-     * encoding of the tags file.  "pats" point to either one of these. */
     pat_T	*pats;
     pat_T	orgpat;			/* holds unconverted pattern info */
 #ifdef FEAT_MBYTE
-    pat_T	convpat;		/* holds converted pattern info */
     vimconv_T	vimconv;
 #endif
 
@@ -1376,6 +1372,7 @@
     int		sort_error = FALSE;		/* tags file not sorted */
     int		linear;				/* do a linear search */
     int		sortic = FALSE;			/* tag file sorted in nocase */
+    int		tag_file_sorted = NUL;		/* !_TAG_FILE_SORTED value */
 #endif
     int		line_error = FALSE;		/* syntax error */
     int		has_re = (flags & TAG_REGEXP);	/* regexp used */
@@ -1701,6 +1698,30 @@
 	    }
 line_read_in:
 
+#ifdef FEAT_MBYTE
+	    if (vimconv.vc_type != CONV_NONE)
+	    {
+		char_u *conv_line;
+
+		len = STRLEN(lbuf);
+		conv_line = string_convert(&vimconv, lbuf, &len);
+		if (conv_line != NULL)
+		{
+		    /* Copy or swap lbuf and conv_line. */
+		    if (len + 1 > LSIZE)
+		    {
+			vim_free(lbuf);
+			lbuf = conv_line;
+		    }
+		    else
+		    {
+			vim_strncpy(lbuf, conv_line, LSIZE - 1);
+			vim_free(conv_line);
+		    }
+		}
+	    }
+#endif
+
 #ifdef FEAT_EMACS_TAGS
 	    /*
 	     * Emacs tags line with CTRL-L: New file name on next line.
@@ -1770,6 +1791,33 @@
 	     */
 	    if (state == TS_START)
 	    {
+		/* Read flags. */
+		if (STRNCMP(lbuf, "!_TAG_", 6) <= 0)
+		{
+#ifdef FEAT_TAG_BINS
+		    if (STRNCMP(lbuf, "!_TAG_FILE_SORTED\t", 18) == 0)
+		    {
+			tag_file_sorted = lbuf[18];
+			continue;
+		    }
+#endif
+#ifdef FEAT_MBYTE
+		    if (STRNCMP(lbuf, "!_TAG_FILE_ENCODING\t", 20) == 0)
+		    {
+			for (p = lbuf + 20; *p > ' ' && *p < 127; ++p)
+			    ;
+			*p = NUL;
+			/* Prepare for converting each line. */
+			convert_setup(&vimconv, lbuf + 20, p_enc);
+			continue;
+		    }
+#endif
+		    /* Ignore unknown flag. */
+		    continue;
+		}
+
+		/* Headers ends. */
+
 #ifdef FEAT_TAG_BINS
 		/*
 		 * When there is no tag head, or ignoring case, need to do a
@@ -1779,29 +1827,28 @@
 		 * When "!_TAG_FILE_SORTED" found: start binary search if
 		 * flag set.
 		 * For cscope, it's always linear.
+		 * When encoding conversion is enabled, use linear.
 		 */
+		if (linear)
+		    state = TS_LINEAR;
 # ifdef FEAT_CSCOPE
-		if (linear || use_cscope)
-# else
-		if (linear)
+		else if (use_cscope)
+		    state = TS_LINEAR;
 # endif
+# ifdef FEAT_MBYTE
+		else if (vimconv.vc_type != CONV_NONE)
 		    state = TS_LINEAR;
-		else if (STRNCMP(lbuf, "!_TAG_", 6) > 0)
+# endif
+		else if (tag_file_sorted == '1')
 		    state = TS_BINARY;
-		else if (STRNCMP(lbuf, "!_TAG_FILE_SORTED\t", 18) == 0)
+		else if (tag_file_sorted == '2')
 		{
-		    /* Check sorted flag */
-		    if (lbuf[18] == '1')
-			state = TS_BINARY;
-		    else if (lbuf[18] == '2')
-		    {
-			state = TS_BINARY;
-			sortic = TRUE;
-			pats->regmatch.rm_ic = (p_ic || !noic);
-		    }
-		    else
-			state = TS_LINEAR;
+		    state = TS_BINARY;
+		    sortic = TRUE;
+		    pats->regmatch.rm_ic = (p_ic || !noic);
 		}
+		else
+		    state = TS_LINEAR;
 
 		if (state == TS_BINARY && pats->regmatch.rm_ic && !sortic)
 		{
@@ -1841,36 +1888,12 @@
 		    continue;
 		}
 #endif
+
+                /* Start linear search in the start of the file. */
+                lseek(fileno(fp), (off_t)0L, SEEK_SET);
+                continue;
 	    }
 
-#ifdef FEAT_MBYTE
-	    if (lbuf[0] == '!' && pats == &orgpat
-			   && STRNCMP(lbuf, "!_TAG_FILE_ENCODING\t", 20) == 0)
-	    {
-		/* Convert the search pattern from 'encoding' to the
-		 * specified encoding. */
-		for (p = lbuf + 20; *p > ' ' && *p < 127; ++p)
-		    ;
-		*p = NUL;
-		convert_setup(&vimconv, p_enc, lbuf + 20);
-		if (vimconv.vc_type != CONV_NONE)
-		{
-		    convpat.pat = string_convert(&vimconv, pats->pat, NULL);
-		    if (convpat.pat != NULL)
-		    {
-			pats = &convpat;
-			pats->len = (int)STRLEN(pats->pat);
-			prepare_pats(pats, has_re);
-			pats->regmatch.rm_ic = orgpat.regmatch.rm_ic;
-		    }
-		}
-
-		/* Prepare for converting a match the other way around. */
-		convert_setup(&vimconv, lbuf + 20, p_enc);
-		continue;
-	    }
-#endif
-
 	    /*
 	     * Figure out where the different strings are in this line.
 	     * For "normal" tags: Do a quick check if the tag matches.
@@ -2187,35 +2210,6 @@
 		 */
 		if (ga_grow(&ga_match[mtt], 1) == OK)
 		{
-#ifdef FEAT_MBYTE
-		    char_u	*conv_line = NULL;
-		    char_u	*lbuf_line = lbuf;
-
-		    if (vimconv.vc_type != CONV_NONE)
-		    {
-			/* Convert the tag line from the encoding of the tags
-			 * file to 'encoding'.  Then parse the line again. */
-			conv_line = string_convert(&vimconv, lbuf, NULL);
-			if (conv_line != NULL)
-			{
-			    if (parse_tag_line(conv_line,
-#ifdef FEAT_EMACS_TAGS
-					is_etag,
-#endif
-					&tagp) == OK)
-				lbuf_line = conv_line;
-			    else
-				/* doesn't work, go back to unconverted line. */
-				(void)parse_tag_line(lbuf,
-#ifdef FEAT_EMACS_TAGS
-						     is_etag,
-#endif
-						     &tagp);
-			}
-		    }
-#else
-# define lbuf_line lbuf
-#endif
 		    if (help_only)
 		    {
 #ifdef FEAT_MULTI_LANG
@@ -2307,7 +2301,7 @@
 			 * without Emacs tags: <mtt><tag_fname><NUL><lbuf>
 			 */
 			len = (int)STRLEN(tag_fname)
-						 + (int)STRLEN(lbuf_line) + 3;
+						 + (int)STRLEN(lbuf) + 3;
 #ifdef FEAT_EMACS_TAGS
 			if (is_etag)
 			    len += (int)STRLEN(ebuf) + 1;
@@ -2337,7 +2331,7 @@
 			    else
 				*s++ = NUL;
 #endif
-			    STRCPY(s, lbuf_line);
+			    STRCPY(s, lbuf);
 			}
 		    }
 
@@ -2373,10 +2367,6 @@
 			else
 			    vim_free(mfp);
 		    }
-#ifdef FEAT_MBYTE
-		    /* Note: this makes the values in "tagp" invalid! */
-		    vim_free(conv_line);
-#endif
 		}
 		else    /* Out of memory! Just forget about the rest. */
 		{
@@ -2415,19 +2405,15 @@
 	}
 #endif
 #ifdef FEAT_MBYTE
-	if (pats == &convpat)
-	{
-	    /* Go back from converted pattern to original pattern. */
-	    vim_free(pats->pat);
-	    vim_free(pats->regmatch.regprog);
-	    orgpat.regmatch.rm_ic = pats->regmatch.rm_ic;
-	    pats = &orgpat;
-	}
 	if (vimconv.vc_type != CONV_NONE)
 	    convert_setup(&vimconv, NULL, NULL);
 #endif
 
 #ifdef FEAT_TAG_BINS
+	tag_file_sorted = NUL;
+#endif
+
+#ifdef FEAT_TAG_BINS
 	if (sort_error)
 	{
 	    EMSG2(_("E432: Tags file not sorted: %s"), tag_fname);
@@ -2461,7 +2447,8 @@
 #ifdef FEAT_TAG_BINS
       /* stop searching when already did a linear search, or when TAG_NOIC
        * used, and 'ignorecase' not set or already did case-ignore search */
-      if (stop_searching || linear || (!p_ic && noic) || pats->regmatch.rm_ic)
+      if (stop_searching || linear || (!p_ic && noic) || pats->regmatch.rm_ic
+	      || state == TS_LINEAR)
 	  break;
 # ifdef FEAT_CSCOPE
       if (use_cscope)