patch 9.0.1834: Some problems with xxd coloring

Problem:  Some problems with xxd coloring
Solution: Fix the following problems:

* Support colored output on Windows.
  SetConsoleMode() is required to enable ANSI color sequences.
* Support "NO_COLOR" environment variable.
  If "NO_COLOR" is defined and not empty, colored output should be
  disabled.
  See https://no-color.org/
* "-R" should only accept "always", "never" or "auto" as the parameter.
* Adjust help and documentation. "-R" cannot omit the parameter. Remove
  surrounding brackets.

Related #12131
closes: #12997
closes: #12991
closes: #12986

Signed-off-by: Christian Brabandt <cb@256bit.org>
Co-authored-by: K.Takata <kentkt@csc.jp>

runtime/doc/xxd.1

@@ -135,9 +135,9 @@ to read plain hexadecimal dumps without line number information and without a
 particular column layout. Additional whitespace and line breaks are allowed
 anywhere.
 .TP
-.IR \-R " "[WHEN]
+.IR \-R " " when
 In output the hex-value and the value are both colored with the same color depending on the hex-value. Mostly helping to differentiate printable and non-printable characters.
-.I WHEN
+.I \fIwhen\fP
 is
 .BR never ", " always ", or " auto .
 .TP

src/version.c

@@ -699,6 +699,8 @@ static char *(features[]) =
 
 static int included_patches[] =
 {   /* Add new patch number below this line */
+/**/
+    1834,
 /**/
     1833,
 /**/

src/xxd/xxd.c

@@ -87,10 +87,10 @@
 #endif
 #if defined(WIN32) || defined(CYGWIN)
 # include <io.h>	/* for setmode() */
-#else
+# include <windows.h>
-# ifdef UNIX
+#endif
+#ifdef UNIX
 # include <unistd.h>
-# endif
 #endif
 #include <stdlib.h>
 #include <string.h>	/* for strncmp() */
@@ -135,7 +135,7 @@ extern void perror __P((char *));
 # endif
 #endif
 
-char version[] = "xxd 2023-08-31 by Juergen Weigert et al.";
+char version[] = "xxd 2023-09-01 by Juergen Weigert et al.";
 #ifdef WIN32
 char osver[] = " (Win32)";
 #else
@@ -256,7 +256,7 @@ exit_with_usage(void)
 	  "", "");
 #endif
   fprintf(stderr, "    -u          use upper case hex letters.\n");
-  fprintf(stderr, "    -R [WHEN]   colorize the output; WHEN can be 'always', 'auto' or 'never'. Default: 'auto'.\n"),
+  fprintf(stderr, "    -R when     colorize the output; <when> can be 'always', 'auto' or 'never'. Default: 'auto'.\n"),
   fprintf(stderr, "    -v          show version: \"%s%s\".\n", version, osver);
   exit(1);
 }
@@ -547,6 +547,27 @@ begin_coloring_char (char *l, int *c, int e, int ebcdic)
   l[(*c)++] = 'm';
 }
 
+  static int
+enable_color(void)
+{
+#ifdef WIN32
+  DWORD   mode;
+  HANDLE  out;
+
+  if (!isatty(1))
+    return 0;
+
+  out = GetStdHandle(STD_OUTPUT_HANDLE);
+  GetConsoleMode(out, &mode);
+  mode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING;
+  return (int)SetConsoleMode(out, mode);
+#elif defined(UNIX)
+  return isatty(STDOUT_FILENO);
+#else
+  return 0;
+#endif
+}
+
   int
 main(int argc, char *argv[])
 {
@@ -564,10 +585,11 @@ main(int argc, char *argv[])
   char *varname = NULL;
   int addrlen = 9;
   int color = 0;
+  char *no_color;
 
-#ifdef UNIX
+  no_color = getenv("NO_COLOR");
-  color = isatty(STDOUT_FILENO);
+  if (no_color == NULL || no_color[0] == '\0')
-#endif
+    color = enable_color();
 
 #ifdef AMIGA
   /* This program doesn't work when started from the Workbench */
@@ -720,15 +742,27 @@ main(int argc, char *argv[])
         }
       else if (!STRNCMP(pp, "-R", 2))
         {
-	  if (!argv[2])
+	  char *pw = pp + 2;
-	    exit_with_usage();
+	  if (!pw[0])
-	  if (!STRNCMP(argv[2], "always", 2))
+	    {
-	    color = 1;
+	      pw = argv[2];
-	  else if (!STRNCMP(argv[2], "never", 1))
-	    color = 0;
 	      argv++;
 	      argc--;
 	    }
+	  if (!pw)
+	    exit_with_usage();
+	  if (!STRNCMP(pw, "always", 6))
+	    {
+	      (void)enable_color();
+	      color = 1;
+	    }
+	  else if (!STRNCMP(pw, "never", 5))
+	    color = 0;
+	  else if (!STRNCMP(pw, "auto", 4))
+	    ;	/* Do nothing. */
+	  else
+	    exit_with_usage();
+        }
       else if (!strcmp(pp, "--"))	/* end of options */
 	{
 	  argv++;