Merge branch 'anyoldname3-and-the-amazing-technicolour-terminal-emulator' into 'master'

Support coloured terminal output on Windows

See merge request OpenMW/openmw!4027

CMakeLists.txt

@@ -234,7 +234,7 @@ else()
 endif(APPLE)
 
 if (WIN32)
-    option(USE_DEBUG_CONSOLE "whether a debug console should be enabled for debug builds, if false debug output is redirected to Visual Studio output" ON)
+    option(USE_DEBUG_CONSOLE "Whether a console should be displayed if OpenMW isn't launched from the command line. Does not affect the Release configuration." ON)
 endif()
 
 if(MSVC)
@@ -710,9 +710,8 @@ if (WIN32)
         if (USE_DEBUG_CONSOLE AND BUILD_OPENMW)
             set_target_properties(openmw PROPERTIES LINK_FLAGS_DEBUG "/SUBSYSTEM:CONSOLE")
             set_target_properties(openmw PROPERTIES LINK_FLAGS_RELWITHDEBINFO "/SUBSYSTEM:CONSOLE")
-            set_target_properties(openmw PROPERTIES COMPILE_DEFINITIONS $<$<CONFIG:Debug>:_CONSOLE>)
         elseif (BUILD_OPENMW)
-            # Turn off debug console, debug output will be written to visual studio output instead
+            # Turn off implicit console, you won't see stdout unless launching OpenMW from a command line shell or look at openmw.log
             set_target_properties(openmw PROPERTIES LINK_FLAGS_DEBUG "/SUBSYSTEM:WINDOWS")
             set_target_properties(openmw PROPERTIES LINK_FLAGS_RELWITHDEBINFO "/SUBSYSTEM:WINDOWS")
         endif()

components/debug/debugging.cpp

@@ -61,6 +61,11 @@ namespace Debug
         bool outRedirected = isRedirected(STD_OUTPUT_HANDLE);
         bool errRedirected = isRedirected(STD_ERROR_HANDLE);
 
+        // Note: Do not spend three days reinvestigating this PowerShell bug thinking its our bug.
+        // https://gitlab.com/OpenMW/openmw/-/merge_requests/408#note_447467393
+        // The handles look valid, but GetFinalPathNameByHandleA can't tell what files they go to and writing to them
+        // doesn't work.
+
         if (AttachConsole(ATTACH_PARENT_PROCESS))
         {
             fflush(stdout);
@@ -73,16 +78,19 @@ namespace Debug
             {
                 _wfreopen(L"CON", L"r", stdin);
                 freopen("CON", "r", stdin);
+                std::cin.clear();
             }
             if (!outRedirected)
             {
                 _wfreopen(L"CON", L"w", stdout);
                 freopen("CON", "w", stdout);
+                std::cout.clear();
             }
             if (!errRedirected)
             {
                 _wfreopen(L"CON", L"w", stderr);
                 freopen("CON", "w", stderr);
+                std::cerr.clear();
             }
 
             return true;
@@ -256,9 +264,17 @@ namespace Debug
     private:
         static bool useColoredOutput()
         {
-            // Note: cmd.exe in Win10 should support ANSI colors, but in its own way.
 #if defined(_WIN32)
-            return 0;
+            if (getenv("NO_COLOR"))
+                return false;
+
+            DWORD mode;
+            if (GetConsoleMode(GetStdHandle(STD_ERROR_HANDLE), &mode) && mode & ENABLE_VIRTUAL_TERMINAL_PROCESSING)
+                return true;
+
+            // some console emulators may not use the Win32 API, so try the Unixy approach
+            char* term = getenv("TERM");
+            return term && GetFileType(GetStdHandle(STD_ERROR_HANDLE)) == FILE_TYPE_CHAR;
 #else
             char* term = getenv("TERM");
             bool useColor = term && !getenv("NO_COLOR") && isatty(fileno(stderr));