Terminal escape sequence handling in a non-TTY stream for Windows (#94)

To prevent screwing up terminal settings on windows when printing to the terminal, turn off the virtual termial before print out to console. The file call these funtions are from scp.c,  sftp.c and sshconnect(calls smprintf). The virtual termial are not enabled in scp and sftp. turn off it in vfmprintf is enough for now.

contrib/win32/win32compat/console.c

@@ -38,10 +38,6 @@
 
 #include "console.h"
 
-#ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING
-#define ENABLE_VIRTUAL_TERMINAL_PROCESSING  0x4
-#endif
-
 HANDLE	hOutputConsole = NULL;
 DWORD	dwSavedAttributes = 0;
 WORD	wStartingAttributes = 0;
@@ -1574,3 +1570,37 @@ ConMoveCursorTop(CONSOLE_SCREEN_BUFFER_INFO csbi)
 
 	ConSaveViewRect();
 }
+
+HANDLE
+get_console_handle(FILE *stream, DWORD * mode)
+{
+	int file_num = 0, ret = 0;
+	intptr_t lHandle = 0;
+	HANDLE hFile = NULL;
+	DWORD type = 0;
+
+	file_num = (_fileno)(stream);
+	if (file_num == -1) {
+		return -1;
+	}
+	lHandle = _get_osfhandle(file_num);
+	if (lHandle == -1 && errno == EBADF) {
+		return -1;
+	}
+	type = GetFileType((HANDLE)lHandle);
+	if (type == FILE_TYPE_CHAR && file_num >= 0 && file_num <= 2) {
+		if (file_num == 0)
+			hFile = GetStdHandle(STD_INPUT_HANDLE);
+		else if (file_num == 1)
+			hFile = GetStdHandle(STD_OUTPUT_HANDLE);
+		else if (file_num == 2)
+			hFile = GetStdHandle(STD_ERROR_HANDLE);
+
+		if ((hFile != NULL) &&
+			(hFile != INVALID_HANDLE_VALUE) &&
+			(GetFileType(hFile) == FILE_TYPE_CHAR) &&
+			GetConsoleMode(hFile, mode))
+			return hFile;
+	}
+	return INVALID_HANDLE_VALUE;
+}

contrib/win32/win32compat/console.h

@@ -79,6 +79,10 @@
 #define false FALSE
 #define bool BOOL
 
+#ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING
+#define ENABLE_VIRTUAL_TERMINAL_PROCESSING  0x4
+#endif
+
 typedef void *  SCREEN_HANDLE;
 
 int ConEnterRawMode(DWORD OutputHandle, BOOL fSmartInit);
@@ -138,4 +142,5 @@ void ConMoveVisibleWindow(int offset);
 int is_cursor_at_lastline_of_visible_window();
 void ConGetCursorPosition(int *x, int *y);
 void ConMoveCursorTop(CONSOLE_SCREEN_BUFFER_INFO csbi);
+HANDLE get_console_handle(FILE *, DWORD *);
 #endif

contrib/win32/win32compat/termio.c

@@ -146,7 +146,6 @@ WriteThread(_In_ LPVOID lpParameter)
 	struct w32_io* pio = (struct w32_io*)lpParameter;
 	char *respbuf = NULL;
 	size_t resplen = 0;	
-	DWORD dwSavedAttributes = ENABLE_PROCESSED_INPUT;
 	debug5("TermWrite thread, io:%p", pio);
 
 	pio->write_details.buf[write_status.to_transfer] = '\0';

contrib/win32/win32compat/win32-utf8.c

@@ -1,34 +1,51 @@
 /* 
- * Temporary Windows versions of functions implemented in utf8.c
+ * Windows versions of functions implemented in utf8.c
  */
 #include <stdio.h>
 #include <stdarg.h>
+#include <Windows.h>
+
+#include "console.h"
+
 
 int
-vfmprintf(FILE *f, const char *fmt, va_list list)
+vfmprintf(FILE *stream, const char *fmt, va_list ap)
 {
-	return vfprintf(f, fmt, list);
+	DWORD saved_mode = 0, new_mode = 0;
+	int ret;
+	HANDLE hFile;
+	hFile = get_console_handle(stream, &saved_mode);
+	if(hFile != INVALID_HANDLE_VALUE &&
+		((saved_mode & ENABLE_VIRTUAL_TERMINAL_PROCESSING) == ENABLE_VIRTUAL_TERMINAL_PROCESSING)) {
+			new_mode = saved_mode & (~ENABLE_VIRTUAL_TERMINAL_PROCESSING);
+			SetConsoleMode(hFile, new_mode);
+	}
+	
+	ret = vfprintf(stream, fmt, ap);
+	if (saved_mode != 0 && new_mode != saved_mode)
+		SetConsoleMode(hFile, saved_mode);
+	return ret;
 }
 
 int
 mprintf(const char *fmt, ...)
 {
 	int ret = 0;
-	va_list valist;
-	va_start(valist, fmt);
-	ret = vfmprintf(stdout, fmt, valist);
-	va_end(valist);
+	va_list ap;
+	va_start(ap, fmt);
+	ret = vfmprintf(stdout, fmt, ap);
+	va_end(ap);
 	return ret;
 }
 
 int
-fmprintf(FILE *f, const char *fmt, ...)
+fmprintf(FILE *stream, const char *fmt, ...)
 {
 	int ret = 0;
-	va_list valist;
-	va_start(valist, fmt);
-	ret = vfmprintf(f, fmt, valist);
-	va_end(valist);
+	va_list ap;
+	va_start(ap, fmt);
+	ret = vfmprintf(stream, fmt, ap);
+	va_end(ap);
 	return ret;
 }
 
@@ -49,6 +66,5 @@ snmprintf(char *buf, size_t len, int *written, const char *fmt, ...)
 void
 msetlocale(void)
 {
-	return;
 }