Skip to content

cleanup: refactor svccommio class #13843

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
benhillis wants to merge 2 commits into
base: master
Choose a base branch
from
Open

cleanup: refactor svccommio class #13843

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
cc603bd
cleanup: refactor svccommio class
Dec 6, 2025
9a845e8
add logging if setting code page fails
Dec 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
314 changes: 107 additions & 207 deletions src/windows/common/svccommio.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,265 +18,165 @@ Module Name:
#pragma hdrstop

namespace {
void ChangeConsoleMode(_In_ HANDLE File, _In_ DWORD ConsoleMode)
{
//
// Use the invalid parameter error code to detect the v1 console that does
// not support the provided mode. This can be improved in the future when
// a more elegant solution exists.
//
// N.B. Ignore failures setting the mode if the console has already
// disconnected.
//

if (!SetConsoleMode(File, ConsoleMode))
{
switch (GetLastError())
{
case ERROR_INVALID_PARAMETER:
THROW_HR(WSL_E_CONSOLE);

case ERROR_PIPE_NOT_CONNECTED:
break;

default:
THROW_LAST_ERROR();
}
}
bool IsConsoleHandle(_In_ HANDLE Handle)
{
DWORD Mode;
return GetFileType(Handle) == FILE_TYPE_CHAR && GetConsoleMode(Handle, &Mode);
}

void ConfigureStdHandles(_Inout_ LXSS_STD_HANDLES_INFO& StdHandlesInfo)
void TrySetConsoleMode(_In_ HANDLE Handle, _In_ DWORD Mode)
{
//
// Check stdin to see if it is a console or another device. If it is
// a console, configure it to raw processing mode and VT-100 support. If the
// force console I/O is requested, ignore stdin and get active console input
// handle instead.
//

UINT NewConsoleInputCP = 0;
DWORD NewConsoleInputMode = 0;
BOOLEAN IsConsoleInput = StdHandlesInfo.IsConsoleInput;
BOOLEAN IsConsoleOutput = StdHandlesInfo.IsConsoleOutput;
BOOLEAN IsConsoleError = StdHandlesInfo.IsConsoleError;
DWORD SavedInputMode = StdHandlesInfo.SavedInputMode;
DWORD SavedOutputMode = StdHandlesInfo.SavedOutputMode;
UINT SavedInputCP = StdHandlesInfo.SavedInputCP;
UINT SavedOutputCP = StdHandlesInfo.SavedOutputCP;
CONSOLE_SCREEN_BUFFER_INFO ScreenBufferInfo;
auto RestoreInputHandle = wil::scope_exit_log(WI_DIAGNOSTICS_INFO, [&] {
if (NewConsoleInputCP != 0)
{
SetConsoleCP(SavedInputCP);
}

if (NewConsoleInputMode != 0)
{
ChangeConsoleMode(StdHandlesInfo.InputHandle, SavedInputMode);
}
});

IsConsoleInput = FALSE;
if ((GetFileType(StdHandlesInfo.InputHandle) == FILE_TYPE_CHAR) && (GetConsoleMode(StdHandlesInfo.InputHandle, &SavedInputMode)))
if (!SetConsoleMode(Handle, Mode))
{
IsConsoleInput = TRUE;
NewConsoleInputMode = SavedInputMode;
WI_SetAllFlags(NewConsoleInputMode, (ENABLE_WINDOW_INPUT | ENABLE_VIRTUAL_TERMINAL_INPUT));
WI_ClearAllFlags(NewConsoleInputMode, (ENABLE_ECHO_INPUT | ENABLE_INSERT_MODE | ENABLE_LINE_INPUT | ENABLE_PROCESSED_INPUT));
ChangeConsoleMode(StdHandlesInfo.InputHandle, NewConsoleInputMode);

//
// Set the console input to the UTF-8 code page.
//

SavedInputCP = GetConsoleCP();
NewConsoleInputCP = CP_UTF8;
THROW_LAST_ERROR_IF(!::SetConsoleCP(NewConsoleInputCP));
}

bool RestoreMode = false;
bool RestoreCp = false;
auto RestoreOutput = wil::scope_exit([&] {
if (RestoreMode)
const auto Error = GetLastError();
if (Error != ERROR_INVALID_PARAMETER && Error != ERROR_PIPE_NOT_CONNECTED)
{
SetConsoleMode(StdHandlesInfo.ConsoleOutputHandle.get(), SavedOutputMode);
LOG_IF_WIN32_ERROR(Error);
}
}
}

if (RestoreCp)
{
SetConsoleOutputCP(SavedOutputCP);
}
});
} // namespace

//
// If there is a console output handle, save the output mode and codepage so
// it can be restored.
//
namespace wsl::windows::common {

if (StdHandlesInfo.ConsoleOutputHandle)
// ConsoleInput implementation
std::unique_ptr<ConsoleInput> ConsoleInput::Create(HANDLE Handle)
{
DWORD Mode;
if (GetFileType(Handle) == FILE_TYPE_CHAR && GetConsoleMode(Handle, &Mode))
{
THROW_LAST_ERROR_IF(!::GetConsoleMode(StdHandlesInfo.ConsoleOutputHandle.get(), &SavedOutputMode));

//
// Temporarily try both with and without the custom flag to disable newline
// auto return.
//

DWORD NewConsoleOutputMode = SavedOutputMode | ENABLE_PROCESSED_OUTPUT | ENABLE_VIRTUAL_TERMINAL_PROCESSING | DISABLE_NEWLINE_AUTO_RETURN;
if (SetConsoleMode(StdHandlesInfo.ConsoleOutputHandle.get(), NewConsoleOutputMode) == FALSE)
{
WI_ClearFlag(NewConsoleOutputMode, DISABLE_NEWLINE_AUTO_RETURN);
ChangeConsoleMode(StdHandlesInfo.ConsoleOutputHandle.get(), NewConsoleOutputMode);
}

RestoreMode = true;

//
// Set the console output to the UTF-8 code page.
//
return std::unique_ptr<ConsoleInput>(new ConsoleInput(Handle, Mode));
}

SavedOutputCP = GetConsoleOutputCP();
THROW_LAST_ERROR_IF(!::SetConsoleOutputCP(CP_UTF8));
return nullptr;
}

RestoreCp = true;
}
ConsoleInput::ConsoleInput(HANDLE Handle, DWORD SavedMode) : m_Handle(Handle), m_SavedMode(SavedMode)
{
// Save code page
m_SavedCodePage = GetConsoleCP();

//
// If the force console I/O is requested, ignore stdout and treat the
// console as the output handle.
//
// Configure for raw input with VT support
DWORD NewMode = m_SavedMode;
WI_SetAllFlags(NewMode, ENABLE_WINDOW_INPUT | ENABLE_VIRTUAL_TERMINAL_INPUT);
WI_ClearAllFlags(NewMode, ENABLE_ECHO_INPUT | ENABLE_INSERT_MODE | ENABLE_LINE_INPUT | ENABLE_PROCESSED_INPUT);
TrySetConsoleMode(Handle, NewMode);

IsConsoleOutput = FALSE;
if ((GetFileType(StdHandlesInfo.OutputHandle) == FILE_TYPE_CHAR) &&
(GetConsoleScreenBufferInfo(StdHandlesInfo.OutputHandle, &ScreenBufferInfo)))
{
IsConsoleOutput = TRUE;
}
// Set UTF-8 code page
SetConsoleCP(CP_UTF8);
}

IsConsoleError = FALSE;
if ((GetFileType(StdHandlesInfo.ErrorHandle) == FILE_TYPE_CHAR) &&
(GetConsoleScreenBufferInfo(StdHandlesInfo.ErrorHandle, &ScreenBufferInfo)))
ConsoleInput::~ConsoleInput()
{
if (m_Handle)
{
IsConsoleError = TRUE;
TrySetConsoleMode(m_Handle, m_SavedMode);
SetConsoleCP(m_SavedCodePage);
}

RestoreInputHandle.release();
RestoreOutput.release();
StdHandlesInfo.IsConsoleInput = IsConsoleInput;
StdHandlesInfo.IsConsoleOutput = IsConsoleOutput;
StdHandlesInfo.IsConsoleError = IsConsoleError;
StdHandlesInfo.SavedInputMode = SavedInputMode;
StdHandlesInfo.SavedOutputMode = SavedOutputMode;
StdHandlesInfo.SavedInputCP = SavedInputCP;
StdHandlesInfo.SavedOutputCP = SavedOutputCP;
}
} // namespace

wsl::windows::common::SvcCommIo::SvcCommIo()
// ConsoleOutput implementation
std::unique_ptr<ConsoleOutput> ConsoleOutput::Create()
{
_stdHandlesInfo.InputHandle = GetStdHandle(STD_INPUT_HANDLE);
_stdHandlesInfo.OutputHandle = GetStdHandle(STD_OUTPUT_HANDLE);
_stdHandlesInfo.ErrorHandle = GetStdHandle(STD_ERROR_HANDLE);
_stdHandlesInfo.ConsoleOutputHandle.reset(
wil::unique_hfile ConsoleHandle(
CreateFileW(L"CONOUT$", GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, OPEN_EXISTING, 0, nullptr));

ConfigureStdHandles(_stdHandlesInfo);
_stdHandles.StdIn.HandleType = LxssHandleInput;
_stdHandles.StdIn.Handle = HandleToUlong(_stdHandlesInfo.InputHandle);
_stdHandles.StdOut.HandleType = LxssHandleOutput;
_stdHandles.StdOut.Handle = HandleToUlong(_stdHandlesInfo.OutputHandle);
_stdHandles.StdErr.HandleType = LxssHandleOutput;
_stdHandles.StdErr.Handle = HandleToUlong(_stdHandlesInfo.ErrorHandle);

//
// N.B.: The console handle is not supposed to be closed, it is just copied
// from PEB.
//

if (_stdHandlesInfo.IsConsoleInput)
if (!ConsoleHandle)
{
_stdHandles.StdIn.Handle = LXSS_HANDLE_USE_CONSOLE;
_stdHandles.StdIn.HandleType = LxssHandleConsole;
return nullptr;
}

if (_stdHandlesInfo.IsConsoleOutput)
DWORD Mode;
if (GetConsoleMode(ConsoleHandle.get(), &Mode))
{
_stdHandles.StdOut.Handle = LXSS_HANDLE_USE_CONSOLE;
_stdHandles.StdOut.HandleType = LxssHandleConsole;
return std::unique_ptr<ConsoleOutput>(new ConsoleOutput(std::move(ConsoleHandle), Mode));
}

if (_stdHandlesInfo.IsConsoleError)
{
_stdHandles.StdErr.Handle = LXSS_HANDLE_USE_CONSOLE;
_stdHandles.StdErr.HandleType = LxssHandleConsole;
}
return nullptr;
}

wsl::windows::common::SvcCommIo::~SvcCommIo()
ConsoleOutput::ConsoleOutput(wil::unique_hfile ConsoleHandle, DWORD SavedMode) :
m_ConsoleHandle(std::move(ConsoleHandle)), m_SavedMode(SavedMode)
{
try
// Save code page
m_SavedCodePage = GetConsoleOutputCP();

// Configure for VT output with DISABLE_NEWLINE_AUTO_RETURN
DWORD NewMode = m_SavedMode;
WI_SetAllFlags(NewMode, ENABLE_PROCESSED_OUTPUT | ENABLE_VIRTUAL_TERMINAL_PROCESSING | DISABLE_NEWLINE_AUTO_RETURN);

// Try with DISABLE_NEWLINE_AUTO_RETURN first, fall back without it if not supported
if (!SetConsoleMode(m_ConsoleHandle.get(), NewMode))
{
RestoreConsoleMode();
WI_ClearFlag(NewMode, DISABLE_NEWLINE_AUTO_RETURN);
TrySetConsoleMode(m_ConsoleHandle.get(), NewMode);
}
CATCH_LOG()
}

PLXSS_STD_HANDLES
wsl::windows::common::SvcCommIo::GetStdHandles()
{
return &_stdHandles;
// Set UTF-8 code page
SetConsoleOutputCP(CP_UTF8);
}

COORD
wsl::windows::common::SvcCommIo::GetWindowSize() const
ConsoleOutput::~ConsoleOutput()
{
CONSOLE_SCREEN_BUFFER_INFOEX Info{};
Info.cbSize = sizeof(Info);
if (_stdHandlesInfo.IsConsoleOutput)
if (m_ConsoleHandle)
{
THROW_IF_WIN32_BOOL_FALSE(::GetConsoleScreenBufferInfoEx(_stdHandlesInfo.OutputHandle, &Info));
TrySetConsoleMode(m_ConsoleHandle.get(), m_SavedMode);
SetConsoleOutputCP(m_SavedCodePage);
}
else if (_stdHandlesInfo.IsConsoleError)
{
THROW_IF_WIN32_BOOL_FALSE(::GetConsoleScreenBufferInfoEx(_stdHandlesInfo.ErrorHandle, &Info));
}

return {
static_cast<short>(Info.srWindow.Right - Info.srWindow.Left + 1), static_cast<short>(Info.srWindow.Bottom - Info.srWindow.Top + 1)};
}

void wsl::windows::common::SvcCommIo::RestoreConsoleMode() const

/*++

Routine Description:
// SvcCommIo implementation
SvcCommIo::SvcCommIo()
{
const HANDLE InputHandle = GetStdHandle(STD_INPUT_HANDLE);
const HANDLE OutputHandle = GetStdHandle(STD_OUTPUT_HANDLE);
const HANDLE ErrorHandle = GetStdHandle(STD_ERROR_HANDLE);

Restores the saved input/output console mode.
// Configure input console
m_ConsoleInput = ConsoleInput::Create(InputHandle);

Arguments:
// Configure output console
m_ConsoleOutput = ConsoleOutput::Create();

None.
// Initialize the standard handles structure
const bool IsConsoleInput = m_ConsoleInput != nullptr;
m_StdHandles.StdIn.HandleType = IsConsoleInput ? LxssHandleConsole : LxssHandleInput;
m_StdHandles.StdIn.Handle = IsConsoleInput ? LXSS_HANDLE_USE_CONSOLE : HandleToUlong(InputHandle);

Return Value:
const bool IsConsoleOutput = IsConsoleHandle(OutputHandle);
m_StdHandles.StdOut.HandleType = IsConsoleOutput ? LxssHandleConsole : LxssHandleOutput;
m_StdHandles.StdOut.Handle = IsConsoleOutput ? LXSS_HANDLE_USE_CONSOLE : HandleToUlong(OutputHandle);

None.
const bool IsConsoleError = IsConsoleHandle(ErrorHandle);
m_StdHandles.StdErr.HandleType = IsConsoleError ? LxssHandleConsole : LxssHandleOutput;
m_StdHandles.StdErr.Handle = IsConsoleError ? LXSS_HANDLE_USE_CONSOLE : HandleToUlong(ErrorHandle);

--*/
// Cache a console handle for GetWindowSize
m_WindowSizeHandle = IsConsoleOutput ? OutputHandle : (IsConsoleError ? ErrorHandle : nullptr);
}

PLXSS_STD_HANDLES
SvcCommIo::GetStdHandles()
{
//
// Restore the console input and output modes.
//
return &m_StdHandles;
}

if (_stdHandlesInfo.ConsoleOutputHandle)
COORD
SvcCommIo::GetWindowSize() const
{
if (m_WindowSizeHandle)
{
ChangeConsoleMode(_stdHandlesInfo.ConsoleOutputHandle.get(), _stdHandlesInfo.SavedOutputMode);
SetConsoleOutputCP(_stdHandlesInfo.SavedOutputCP);
CONSOLE_SCREEN_BUFFER_INFOEX Info{};
Info.cbSize = sizeof(Info);
THROW_IF_WIN32_BOOL_FALSE(GetConsoleScreenBufferInfoEx(m_WindowSizeHandle, &Info));
return {
static_cast<short>(Info.srWindow.Right - Info.srWindow.Left + 1),
static_cast<short>(Info.srWindow.Bottom - Info.srWindow.Top + 1)};
}

if (_stdHandlesInfo.IsConsoleInput != FALSE)
{
ChangeConsoleMode(_stdHandlesInfo.InputHandle, _stdHandlesInfo.SavedInputMode);
SetConsoleCP(_stdHandlesInfo.SavedInputCP);
}
return {80, 24}; // Default size if no console
}

} // namespace wsl::windows::common
Loading