Fix stdio encoding issue: Enforce explicit UTF-8 for correct Unicode handling (#73)

* fix: Enforce UTF-8 encoding for stdio transport

This change replaces the default system encoding with an explicit UTF8Encoding (without BOM)
for both client and server transports. This ensures proper handling of Unicode characters,
including Chinese characters and emoji.

- Use UTF8Encoding explicitly for StreamReader and StreamWriter.
- Add tests for Chinese characters ("上下文伺服器") and emoji (🔍🚀👍) to confirm the fix.

Fixes #35.

* Remove relaxed JSON encoder

* Address PR feedback

---------

Co-authored-by: Stephen Toub <stoub@microsoft.com>
This commit is contained in:
Brandon Williams 2025-03-24 06:55:10 -07:00 • committed by GitHub
commit 36d5019de0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194

View file

@ -114,7 +114,7 @@ internal static partial class Log
internal static partial void TransportNotConnected(this ILogger logger, string endpointName);
[LoggerMessage(Level = LogLevel.Information, Message = "Transport sending message for {endpointName} with ID {messageId}, JSON {json}")]
internal static partial void TransportSendingMessage(this ILogger logger, string endpointName, string messageId, string json);
internal static partial void TransportSendingMessage(this ILogger logger, string endpointName, string messageId, string? json = null);
[LoggerMessage(Level = LogLevel.Information, Message = "Transport message sent for {endpointName} with ID {messageId}")]
internal static partial void TransportSentMessage(this ILogger logger, string endpointName, string messageId);
@ -347,4 +347,35 @@ internal static partial class Log
string endpointName,
string messageId,
string responseContent);
/// <summary>
/// Logs the byte representation of a message in UTF-8 encoding.
/// </summary>
/// <param name="logger">The logger to use.</param>
/// <param name="endpointName">The name of the endpoint.</param>
/// <param name="byteRepresentation">The byte representation as a hex string.</param>
[LoggerMessage(EventId = 39000, Level = LogLevel.Trace, Message = "Transport {EndpointName}: Message bytes (UTF-8): {ByteRepresentation}")]
private static partial void TransportMessageBytes(this ILogger logger, string endpointName, string byteRepresentation);
/// <summary>
/// Logs the byte representation of a message for diagnostic purposes.
/// This is useful for diagnosing encoding issues with non-ASCII characters.
/// </summary>
/// <param name="logger">The logger to use.</param>
/// <param name="endpointName">The name of the endpoint.</param>
/// <param name="message">The message to log bytes for.</param>
internal static void TransportMessageBytesUtf8(this ILogger logger, string endpointName, string message)
{
if (logger.IsEnabled(LogLevel.Trace))
{
var bytes = System.Text.Encoding.UTF8.GetBytes(message);
var byteRepresentation =
#if NET
Convert.ToHexString(bytes);
#else
BitConverter.ToString(bytes).Replace("-", " ");
#endif
logger.TransportMessageBytes(endpointName, byteRepresentation);
}
}
}