diff --git a/EncodingConverter.VisualStudio/Converter.cs b/EncodingConverter.VisualStudio/Converter.cs index f668f3a..ca8aabf 100644 --- a/EncodingConverter.VisualStudio/Converter.cs +++ b/EncodingConverter.VisualStudio/Converter.cs @@ -18,17 +18,7 @@ public static async Task ConvertEncodingAsync(IEnumerable files, Encod throw new FileNotFoundException("File Not Found!", file.FullName); } - string fullText; - - // Read file - using (StreamReader sr = new(file.FullName, detectTextEncoding(file.FullName, out _, 0))) { - fullText = await sr.ReadToEndAsync(); - } - - // Write file as new encoding - using (StreamWriter sw = new(file.FullName, false, newEncoding)) { - await sw.WriteAsync(fullText); - } + await ConverterCore.ConvertEncodingAsync(file, newEncoding); // Progress bar var pbTask = VS.StatusBar.ShowProgressAsync($"{i + 1}/{array.Length} file processed", i + 1, array.Length); @@ -40,116 +30,4 @@ public static async Task ConvertEncodingAsync(IEnumerable files, Encod await Task.WhenAll(pbTask, owTask); } } - - // From https://stackoverflow.com/questions/1025332/determine-a-strings-encoding-in-c-sharp - // Function to detect the encoding for UTF-7, UTF-8/16/32 (bom, no bom, little - // & big endian), and local default codepage, and potentially other codepages. - // 'taster' = number of bytes to check of the file (to save processing). Higher - // value is slower, but more reliable (especially UTF-8 with special characters - // later on may appear to be ASCII initially). If taster = 0, then taster - // becomes the length of the file (for maximum reliability). 'text' is simply - // the string with the discovered encoding applied to the file. - private static Encoding detectTextEncoding(string filename, out string text, int taster = 1000) { - var b = File.ReadAllBytes(filename); - - //////////////// First check the low hanging fruit by checking if a - //////////////// BOM/signature exists (sourced from http://www.unicode.org/faq/utf_bom.html#bom4) - if (b.Length >= 4 && b[0] == 0x00 && b[1] == 0x00 && b[2] == 0xFE && b[3] == 0xFF) { text = Encoding.GetEncoding("utf-32BE").GetString(b, 4, b.Length - 4); return Encoding.GetEncoding("utf-32BE"); } // UTF-32, big-endian - else if (b.Length >= 4 && b[0] == 0xFF && b[1] == 0xFE && b[2] == 0x00 && b[3] == 0x00) { text = Encoding.UTF32.GetString(b, 4, b.Length - 4); return Encoding.UTF32; } // UTF-32, little-endian - else if (b.Length >= 2 && b[0] == 0xFE && b[1] == 0xFF) { text = Encoding.BigEndianUnicode.GetString(b, 2, b.Length - 2); return Encoding.BigEndianUnicode; } // UTF-16, big-endian - else if (b.Length >= 2 && b[0] == 0xFF && b[1] == 0xFE) { text = Encoding.Unicode.GetString(b, 2, b.Length - 2); return Encoding.Unicode; } // UTF-16, little-endian - else if (b.Length >= 3 && b[0] == 0xEF && b[1] == 0xBB && b[2] == 0xBF) { text = Encoding.UTF8.GetString(b, 3, b.Length - 3); return Encoding.UTF8; } // UTF-8 - else if (b.Length >= 3 && b[0] == 0x2b && b[1] == 0x2f && b[2] == 0x76) { text = Encoding.UTF7.GetString(b, 3, b.Length - 3); return Encoding.UTF7; } // UTF-7 - - - //////////// If the code reaches here, no BOM/signature was found, so now - //////////// we need to 'taste' the file to see if can manually discover - //////////// the encoding. A high taster value is desired for UTF-8 - if (taster == 0 || taster > b.Length) { - taster = b.Length; // Taster size can't be bigger than the filesize obviously. - } - - - // Some text files are encoded in UTF8, but have no BOM/signature. Hence - // the below manually checks for a UTF8 pattern. This code is based off - // the top answer at: https://stackoverflow.com/questions/6555015/check-for-invalid-utf8 - // For our purposes, an unnecessarily strict (and terser/slower) - // implementation is shown at: https://stackoverflow.com/questions/1031645/how-to-detect-utf-8-in-plain-c - // For the below, false positives should be exceedingly rare (and would - // be either slightly malformed UTF-8 (which would suit our purposes - // anyway) or 8-bit extended ASCII/UTF-16/32 at a vanishingly long shot). - var i = 0; - var utf8 = false; - while (i < taster - 4) { - if (b[i] <= 0x7F) { i += 1; continue; } // If all characters are below 0x80, then it is valid UTF8, but UTF8 is not 'required' (and therefore the text is more desirable to be treated as the default codepage of the computer). Hence, there's no "utf8 = true;" code unlike the next three checks. - if (b[i] >= 0xC2 && b[i] < 0xE0 && b[i + 1] >= 0x80 && b[i + 1] < 0xC0) { i += 2; utf8 = true; continue; } - if (b[i] >= 0xE0 && b[i] < 0xF0 && b[i + 1] >= 0x80 && b[i + 1] < 0xC0 && b[i + 2] >= 0x80 && b[i + 2] < 0xC0) { i += 3; utf8 = true; continue; } - if (b[i] >= 0xF0 && b[i] < 0xF5 && b[i + 1] >= 0x80 && b[i + 1] < 0xC0 && b[i + 2] >= 0x80 && b[i + 2] < 0xC0 && b[i + 3] >= 0x80 && b[i + 3] < 0xC0) { i += 4; utf8 = true; continue; } - utf8 = false; break; - } - if (utf8) { - text = Encoding.UTF8.GetString(b); - return Encoding.UTF8; - } - - - // The next check is a heuristic attempt to detect UTF-16 without a BOM. - // We simply look for zeroes in odd or even byte places, and if a certain - // threshold is reached, the code is 'probably' UF-16. - var threshold = 0.1; // proportion of chars step 2 which must be zeroed to be diagnosed as utf-16. 0.1 = 10% - var count = 0; - for (var n = 0; n < taster; n += 2) { - if (b[n] == 0) { - count++; - } - } - - if (((double)count) / taster > threshold) { text = Encoding.BigEndianUnicode.GetString(b); return Encoding.BigEndianUnicode; } - count = 0; - for (var n = 1; n < taster; n += 2) { - if (b[n] == 0) { - count++; - } - } - - if (((double)count) / taster > threshold) { text = Encoding.Unicode.GetString(b); return Encoding.Unicode; } // (little-endian) - - - // Finally, a long shot - let's see if we can find "charset=xyz" or - // "encoding=xyz" to identify the encoding: - for (var n = 0; n < taster - 9; n++) { - if ( - ((b[n + 0] == 'c' || b[n + 0] == 'C') && (b[n + 1] == 'h' || b[n + 1] == 'H') && (b[n + 2] == 'a' || b[n + 2] == 'A') && (b[n + 3] == 'r' || b[n + 3] == 'R') && (b[n + 4] == 's' || b[n + 4] == 'S') && (b[n + 5] == 'e' || b[n + 5] == 'E') && (b[n + 6] == 't' || b[n + 6] == 'T') && (b[n + 7] == '=')) || - ((b[n + 0] == 'e' || b[n + 0] == 'E') && (b[n + 1] == 'n' || b[n + 1] == 'N') && (b[n + 2] == 'c' || b[n + 2] == 'C') && (b[n + 3] == 'o' || b[n + 3] == 'O') && (b[n + 4] == 'd' || b[n + 4] == 'D') && (b[n + 5] == 'i' || b[n + 5] == 'I') && (b[n + 6] == 'n' || b[n + 6] == 'N') && (b[n + 7] == 'g' || b[n + 7] == 'G') && (b[n + 8] == '=')) - ) { - if (b[n + 0] == 'c' || b[n + 0] == 'C') { - n += 8; - } else { - n += 9; - } - - if (b[n] == '"' || b[n] == '\'') { - n++; - } - - var oldn = n; - while (n < taster && (b[n] == '_' || b[n] == '-' || (b[n] >= '0' && b[n] <= '9') || (b[n] >= 'a' && b[n] <= 'z') || (b[n] >= 'A' && b[n] <= 'Z'))) { n++; } - var nb = new byte[n - oldn]; - Array.Copy(b, oldn, nb, 0, n - oldn); - try { - var internalEnc = Encoding.ASCII.GetString(nb); - text = Encoding.GetEncoding(internalEnc).GetString(b); - return Encoding.GetEncoding(internalEnc); - } catch { break; } // If C# doesn't recognize the name of the encoding, break. - } - } - - - // If all else fails, the encoding is probably (though certainly not - // definitely) the user's local codepage! One might present to the user a - // list of alternative encodings as shown here: https://stackoverflow.com/questions/8509339/what-is-the-most-common-encoding-of-each-language - // A full list can be found using Encoding.GetEncodings - text = Encoding.Default.GetString(b); - return Encoding.Default; - } } diff --git a/EncodingConverter.VisualStudio/EncodingConverter.VisualStudio.csproj b/EncodingConverter.VisualStudio/EncodingConverter.VisualStudio.csproj index 078c04a..4902c84 100644 --- a/EncodingConverter.VisualStudio/EncodingConverter.VisualStudio.csproj +++ b/EncodingConverter.VisualStudio/EncodingConverter.VisualStudio.csproj @@ -106,7 +106,12 @@ all - + + + {4787fd60-29a6-415e-bab9-0b909d0e9f8e} + EncodingConverter + + Designer diff --git a/EncodingConverter.VisualStudio/README.md b/EncodingConverter.VisualStudio/README.md new file mode 100644 index 0000000..dea7bc8 --- /dev/null +++ b/EncodingConverter.VisualStudio/README.md @@ -0,0 +1,6 @@ +# Encoding Converter Visual Studio Extension +A Visual Studio extension that allows you to easily convert the encoding of multiple files(System Encoding / UTF-8 with / without BOM). + +You can use this feature in Solution Explorer. + +![Image](./image.png) diff --git a/EncodingConverter.VisualStudio/image.png b/EncodingConverter.VisualStudio/image.png new file mode 100644 index 0000000..7eb35d0 Binary files /dev/null and b/EncodingConverter.VisualStudio/image.png differ diff --git a/EncodingConverter.sln b/EncodingConverter.sln index 9c2c18b..9ef5bd4 100644 --- a/EncodingConverter.sln +++ b/EncodingConverter.sln @@ -3,30 +3,24 @@ Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 VisualStudioVersion = 17.8.34525.116 MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EncodingConverter", "EncodingConverter\EncodingConverter.csproj", "{4787FD60-29A6-415E-BAB9-0B909D0E9F8E}" +EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EncodingConverter.VisualStudio", "EncodingConverter.VisualStudio\EncodingConverter.VisualStudio.csproj", "{9FFBDABA-E149-4EAF-8E0B-F2C5DFC1AFB5}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU - Debug|arm64 = Debug|arm64 - Debug|x86 = Debug|x86 Release|Any CPU = Release|Any CPU - Release|arm64 = Release|arm64 - Release|x86 = Release|x86 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution + {4787FD60-29A6-415E-BAB9-0B909D0E9F8E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4787FD60-29A6-415E-BAB9-0B909D0E9F8E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4787FD60-29A6-415E-BAB9-0B909D0E9F8E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4787FD60-29A6-415E-BAB9-0B909D0E9F8E}.Release|Any CPU.Build.0 = Release|Any CPU {9FFBDABA-E149-4EAF-8E0B-F2C5DFC1AFB5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {9FFBDABA-E149-4EAF-8E0B-F2C5DFC1AFB5}.Debug|Any CPU.Build.0 = Debug|Any CPU - {9FFBDABA-E149-4EAF-8E0B-F2C5DFC1AFB5}.Debug|arm64.ActiveCfg = Debug|arm64 - {9FFBDABA-E149-4EAF-8E0B-F2C5DFC1AFB5}.Debug|arm64.Build.0 = Debug|arm64 - {9FFBDABA-E149-4EAF-8E0B-F2C5DFC1AFB5}.Debug|x86.ActiveCfg = Debug|x86 - {9FFBDABA-E149-4EAF-8E0B-F2C5DFC1AFB5}.Debug|x86.Build.0 = Debug|x86 {9FFBDABA-E149-4EAF-8E0B-F2C5DFC1AFB5}.Release|Any CPU.ActiveCfg = Release|Any CPU {9FFBDABA-E149-4EAF-8E0B-F2C5DFC1AFB5}.Release|Any CPU.Build.0 = Release|Any CPU - {9FFBDABA-E149-4EAF-8E0B-F2C5DFC1AFB5}.Release|arm64.ActiveCfg = Release|arm64 - {9FFBDABA-E149-4EAF-8E0B-F2C5DFC1AFB5}.Release|arm64.Build.0 = Release|arm64 - {9FFBDABA-E149-4EAF-8E0B-F2C5DFC1AFB5}.Release|x86.ActiveCfg = Release|x86 - {9FFBDABA-E149-4EAF-8E0B-F2C5DFC1AFB5}.Release|x86.Build.0 = Release|x86 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/EncodingConverter/ConverterCore.cs b/EncodingConverter/ConverterCore.cs new file mode 100644 index 0000000..6fff28d --- /dev/null +++ b/EncodingConverter/ConverterCore.cs @@ -0,0 +1,133 @@ +using System.Text; + +namespace EncodingConverter; + +public static class ConverterCore { + public static async Task ConvertEncodingAsync(FileInfo file, Encoding newEncoding) { + string fullText; + + // Read file + using (StreamReader sr = new(file.FullName, detectTextEncoding(file.FullName, out _, 0))) { + fullText = await sr.ReadToEndAsync(); + } + + // Write file as new encoding +#pragma warning disable IDE0063 // 간단한 'using' 문 사용 + using (StreamWriter sw = new(file.FullName, false, newEncoding)) { + await sw.WriteAsync(fullText); + } +#pragma warning restore IDE0063 // 간단한 'using' 문 사용 + } + + // From https://stackoverflow.com/questions/1025332/determine-a-strings-encoding-in-c-sharp + // Function to detect the encoding for UTF-7, UTF-8/16/32 (bom, no bom, little + // & big endian), and local default codepage, and potentially other codepages. + // 'taster' = number of bytes to check of the file (to save processing). Higher + // value is slower, but more reliable (especially UTF-8 with special characters + // later on may appear to be ASCII initially). If taster = 0, then taster + // becomes the length of the file (for maximum reliability). 'text' is simply + // the string with the discovered encoding applied to the file. + private static Encoding detectTextEncoding(string filename, out string text, int taster = 1000) { + var b = File.ReadAllBytes(filename); + + //////////////// First check the low hanging fruit by checking if a + //////////////// BOM/signature exists (sourced from http://www.unicode.org/faq/utf_bom.html#bom4) + if (b.Length >= 4 && b[0] == 0x00 && b[1] == 0x00 && b[2] == 0xFE && b[3] == 0xFF) { text = Encoding.GetEncoding("utf-32BE").GetString(b, 4, b.Length - 4); return Encoding.GetEncoding("utf-32BE"); } // UTF-32, big-endian + else if (b.Length >= 4 && b[0] == 0xFF && b[1] == 0xFE && b[2] == 0x00 && b[3] == 0x00) { text = Encoding.UTF32.GetString(b, 4, b.Length - 4); return Encoding.UTF32; } // UTF-32, little-endian + else if (b.Length >= 2 && b[0] == 0xFE && b[1] == 0xFF) { text = Encoding.BigEndianUnicode.GetString(b, 2, b.Length - 2); return Encoding.BigEndianUnicode; } // UTF-16, big-endian + else if (b.Length >= 2 && b[0] == 0xFF && b[1] == 0xFE) { text = Encoding.Unicode.GetString(b, 2, b.Length - 2); return Encoding.Unicode; } // UTF-16, little-endian + else if (b.Length >= 3 && b[0] == 0xEF && b[1] == 0xBB && b[2] == 0xBF) { text = Encoding.UTF8.GetString(b, 3, b.Length - 3); return Encoding.UTF8; } // UTF-8 + else if (b.Length >= 3 && b[0] == 0x2b && b[1] == 0x2f && b[2] == 0x76) { text = Encoding.UTF7.GetString(b, 3, b.Length - 3); return Encoding.UTF7; } // UTF-7 + + + //////////// If the code reaches here, no BOM/signature was found, so now + //////////// we need to 'taste' the file to see if can manually discover + //////////// the encoding. A high taster value is desired for UTF-8 + if (taster == 0 || taster > b.Length) { + taster = b.Length; // Taster size can't be bigger than the filesize obviously. + } + + + // Some text files are encoded in UTF8, but have no BOM/signature. Hence + // the below manually checks for a UTF8 pattern. This code is based off + // the top answer at: https://stackoverflow.com/questions/6555015/check-for-invalid-utf8 + // For our purposes, an unnecessarily strict (and terser/slower) + // implementation is shown at: https://stackoverflow.com/questions/1031645/how-to-detect-utf-8-in-plain-c + // For the below, false positives should be exceedingly rare (and would + // be either slightly malformed UTF-8 (which would suit our purposes + // anyway) or 8-bit extended ASCII/UTF-16/32 at a vanishingly long shot). + var i = 0; + var utf8 = false; + while (i < taster - 4) { + if (b[i] <= 0x7F) { i += 1; continue; } // If all characters are below 0x80, then it is valid UTF8, but UTF8 is not 'required' (and therefore the text is more desirable to be treated as the default codepage of the computer). Hence, there's no "utf8 = true;" code unlike the next three checks. + if (b[i] >= 0xC2 && b[i] < 0xE0 && b[i + 1] >= 0x80 && b[i + 1] < 0xC0) { i += 2; utf8 = true; continue; } + if (b[i] >= 0xE0 && b[i] < 0xF0 && b[i + 1] >= 0x80 && b[i + 1] < 0xC0 && b[i + 2] >= 0x80 && b[i + 2] < 0xC0) { i += 3; utf8 = true; continue; } + if (b[i] >= 0xF0 && b[i] < 0xF5 && b[i + 1] >= 0x80 && b[i + 1] < 0xC0 && b[i + 2] >= 0x80 && b[i + 2] < 0xC0 && b[i + 3] >= 0x80 && b[i + 3] < 0xC0) { i += 4; utf8 = true; continue; } + utf8 = false; break; + } + if (utf8) { + text = Encoding.UTF8.GetString(b); + return Encoding.UTF8; + } + + + // The next check is a heuristic attempt to detect UTF-16 without a BOM. + // We simply look for zeroes in odd or even byte places, and if a certain + // threshold is reached, the code is 'probably' UF-16. + var threshold = 0.1; // proportion of chars step 2 which must be zeroed to be diagnosed as utf-16. 0.1 = 10% + var count = 0; + for (var n = 0; n < taster; n += 2) { + if (b[n] == 0) { + count++; + } + } + + if (((double)count) / taster > threshold) { text = Encoding.BigEndianUnicode.GetString(b); return Encoding.BigEndianUnicode; } + count = 0; + for (var n = 1; n < taster; n += 2) { + if (b[n] == 0) { + count++; + } + } + + if (((double)count) / taster > threshold) { text = Encoding.Unicode.GetString(b); return Encoding.Unicode; } // (little-endian) + + + // Finally, a long shot - let's see if we can find "charset=xyz" or + // "encoding=xyz" to identify the encoding: + for (var n = 0; n < taster - 9; n++) { + if ( + ((b[n + 0] == 'c' || b[n + 0] == 'C') && (b[n + 1] == 'h' || b[n + 1] == 'H') && (b[n + 2] == 'a' || b[n + 2] == 'A') && (b[n + 3] == 'r' || b[n + 3] == 'R') && (b[n + 4] == 's' || b[n + 4] == 'S') && (b[n + 5] == 'e' || b[n + 5] == 'E') && (b[n + 6] == 't' || b[n + 6] == 'T') && (b[n + 7] == '=')) || + ((b[n + 0] == 'e' || b[n + 0] == 'E') && (b[n + 1] == 'n' || b[n + 1] == 'N') && (b[n + 2] == 'c' || b[n + 2] == 'C') && (b[n + 3] == 'o' || b[n + 3] == 'O') && (b[n + 4] == 'd' || b[n + 4] == 'D') && (b[n + 5] == 'i' || b[n + 5] == 'I') && (b[n + 6] == 'n' || b[n + 6] == 'N') && (b[n + 7] == 'g' || b[n + 7] == 'G') && (b[n + 8] == '=')) + ) { + if (b[n + 0] == 'c' || b[n + 0] == 'C') { + n += 8; + } else { + n += 9; + } + + if (b[n] == '"' || b[n] == '\'') { + n++; + } + + var oldn = n; + while (n < taster && (b[n] == '_' || b[n] == '-' || (b[n] >= '0' && b[n] <= '9') || (b[n] >= 'a' && b[n] <= 'z') || (b[n] >= 'A' && b[n] <= 'Z'))) { n++; } + var nb = new byte[n - oldn]; + Array.Copy(b, oldn, nb, 0, n - oldn); + try { + var internalEnc = Encoding.ASCII.GetString(nb); + text = Encoding.GetEncoding(internalEnc).GetString(b); + return Encoding.GetEncoding(internalEnc); + } catch { break; } // If C# doesn't recognize the name of the encoding, break. + } + } + + + // If all else fails, the encoding is probably (though certainly not + // definitely) the user's local codepage! One might present to the user a + // list of alternative encodings as shown here: https://stackoverflow.com/questions/8509339/what-is-the-most-common-encoding-of-each-language + // A full list can be found using Encoding.GetEncodings + text = Encoding.Default.GetString(b); + return Encoding.Default; + } +} diff --git a/EncodingConverter/EncodingConverter.csproj b/EncodingConverter/EncodingConverter.csproj new file mode 100644 index 0000000..eec2689 --- /dev/null +++ b/EncodingConverter/EncodingConverter.csproj @@ -0,0 +1,6 @@ + + + netstandard2.0 + false + + \ No newline at end of file diff --git a/README.md b/README.md index 196f66d..cbb0b97 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ # Encoding Converter -An extension that allows you to easily convert the encoding of multiple files(System Encoding / UTF-8 with / without BOM). +Encoding Converter makes it easy to convert the encoding of a file. -![Image](./image.png) +Visual Studio Extension is avaliable. diff --git a/image.png b/image.png deleted file mode 100644 index f53d92e..0000000 Binary files a/image.png and /dev/null differ