从 UTF-8 字节数组创建字符串？

Question

考虑emoji ????。它是 U+1F619（十进制 128537）。我相信它的 UTF-8 字节数组是 240、159、152、151。

1. 给定 UTF-8 字节数组，我该如何显示它？我是否从字节数组创建std::string？是否有第三方库可以提供帮助？
2. 给定一个不同的表情符号，我怎样才能得到它的 UTF-8 字节数组？

目标平台：Windows。编译器：Visual C++ 2019。只是粘贴？？？？进入 Windows CMD 提示不起作用。我试过chcp 65001 和 Lucida 作为字体，但没有运气。

如有必要，我可以在 macOS 或 Linux 上执行此操作，但我更喜欢 Windows。

为了澄清......给定一个 400 字节的列表，假设 UTF-8，我如何显示相应的代码点？

Answer 1

C++ 对此有一个简单的解决方案。

#include <iostream>
#include <string>

int main(void) {
    std::string s = u8"?"; /* use std::u8string in c++20*/
    std::cout << s << std::endl;
    return 0;
}

这将允许您存储和打印任何 UTF-8 字符串。

请注意，Windows 命令提示符对这种东西很奇怪。最好使用 MSYS2 等替代方案。

Answer 2

这里是用于试验 unicode 的示例代码，用于转换 unicode 字符/字符串并在控制台中打印，假设您设置了正确的语言环境、控制台代码页并执行了足够的字符串转换，它适用于许多 unicode 字符（如果需要例如char32_t、char16_t 和char8_t 需要转换）。

除了要显示的字符不是那么容易，运行测试需要大量时间，这可以通过我修改下面的代码或通过了解所需的详细信息来改进，例如代码页（Windows 可能不支持），所以只要不觉得无聊就可以随意尝试;)

提示，最好添加代码写入文件，让它运行并在几个小时后检查文件中的结果。为此，您需要将 BOM 标记放入文件中，但不是在文件以 UTF 编码打开之前，您可以通过 wofstream::imbue() 对特定语言环境执行此操作，对于 BOM，它取决于字节序，它是 UTF-X LE 编码Windows 上的方案，其中 X 为 8、16 或 32，写入文件必须使用 wcout wchar_t 才能成功。

有关更多信息，请参阅代码注释，并尝试注释掉/取消注释部分代码以查看不同且更快的结果。

顺便说一句。这段代码的重点是尝试系统支持的所有可能的语言环境/代码页，直到您在控制台中看到您的笑脸或最终失败

#include <climits>
#include <locale>
#include <iostream>
#include <sstream>
#include <Windows.h>
#include <string_view>
#include <cassert>
#include <cwchar>
#include <limits>
#include <vector>
#include <string>

#pragma warning (push, 4)
#if !defined UNICODE && !defined _UNICODE
#error "Compile as unicode"
#endif

#define LINE __LINE__
// NOTE: change desired default code page here (unused)
#define CODE_PAGE CP_UTF8


// Error handling helper method
void StringCastError()
{
    std::wstring error = L"Unknown error";

    switch (GetLastError())
    {
    case ERROR_INSUFFICIENT_BUFFER:
        error = L"A supplied buffer size was not large enough, or it was incorrectly set to NULL";
        break;
    case ERROR_INVALID_FLAGS:
        error = L"The values supplied for flags were not valid";
        break;
    case ERROR_INVALID_PARAMETER:
        error = L"Any of the parameter values was invalid.";
        break;
    case ERROR_NO_UNICODE_TRANSLATION:
        error = L"Invalid Unicode was found in a string.";
        break;
    default:
        break;
    };

    std::wcerr << error << std::endl;
}

// Convert multybyte to wide string
static std::wstring StringCast(const std::string& param, int code_page)
{
    if (param.empty())
    {
        std::wcerr << L"ERROR: param string is empty" << std::endl;
        return std::wstring();
    }

    DWORD flags = MB_ERR_INVALID_CHARS;
    //flags |= MB_USEGLYPHCHARS;
    //flags |= MB_PRECOMPOSED;

    switch (code_page)
    {
    case 50220:
    case 50221:
    case 50222:
    case 50225:
    case 50227:
    case 50229:
    case 65000:
    case 42:
        flags = 0;
        break;
    case 54936:
    case CP_UTF8:
        flags = MB_ERR_INVALID_CHARS; // or 0
        break;
    default:
        if ((code_page >= 57002) && (code_page <= 57011))
            flags = 0;
        break;
    }

    const int source_char_size = static_cast<int>(param.size());
    int chars = MultiByteToWideChar(code_page, flags, param.c_str(), source_char_size, nullptr, 0);

    if (chars == 0)
    {
        StringCastError();
        return std::wstring();
    }

    std::wstring return_string(static_cast<const unsigned int>(chars), 0);
    chars = MultiByteToWideChar(code_page, flags, param.c_str(), source_char_size, &return_string[0], chars);

    if (chars == 0)
    {
        StringCastError();
        return std::wstring();
    }

    return return_string;
}

// Convert wide to multybyte string
std::string StringCast(const std::wstring& param, int code_page)
{
    if (param.empty())
    {
        std::wcerr << L"ERROR: param string is empty" << std::endl;
        return std::string();
    }

    DWORD flags = WC_ERR_INVALID_CHARS;
    //flags |= WC_COMPOSITECHECK;
    flags |= WC_NO_BEST_FIT_CHARS;

    switch (code_page)
    {
    case 50220:
    case 50221:
    case 50222:
    case 50225:
    case 50227:
    case 50229:
    case 65000:
    case 42:
        flags = 0;
        break;
    case 54936:
    case CP_UTF8:
        flags = WC_ERR_INVALID_CHARS; // or 0
        break;
    default:
        if ((code_page >= 57002) && (code_page <= 57011))
            flags = 0;
        break;
    }

    const int source_wchar_size = static_cast<int>(param.size());
    int chars = WideCharToMultiByte(code_page, flags, param.c_str(), source_wchar_size, nullptr, 0, nullptr, nullptr);

    if (chars == 0)
    {
        StringCastError();
        return std::string();
    }

    std::string return_string(static_cast<const unsigned int>(chars), 0);

    chars = WideCharToMultiByte(code_page, flags, param.c_str(), source_wchar_size, &return_string[0], chars, nullptr, nullptr);

    if (chars == 0)
    {
        StringCastError();
        return std::string();
    }

    return return_string;
}

// Console code page helper to adjust console
bool SetConsole(UINT code_page)
{
    if (IsValidCodePage(code_page) == 0)
    {
        std::wcerr << L"Code page is not valid: " << LINE << std::endl;
    }
    else if (SetConsoleCP(code_page) == 0)
    {
        std::wcerr << L"Failed to set console input code page line: " << LINE << std::endl;
    }
    else if (SetConsoleOutputCP(code_page) == 0)
    {
        std::wcerr << L"Failed to set console output code page: " << LINE << std::endl;
    }
    else
    {
        return true;
    }

    return false;
}

std::vector<std::string> locales;

// System locale enumerator to get all locales installed on system
BOOL LocaleEnumprocex(LPWSTR locale_name, [[maybe_unused]] DWORD locale_info, LPARAM code_page)
{
    locales.push_back(StringCast(locale_name, static_cast<int>(code_page)));
    return TRUE;    // continue drilling
}

// System code page enumerator to try out every possible supported/installed code page on system
BOOL CALLBACK EnumCodePagesProc(LPTSTR page_str)
{
    wchar_t* end;
    UINT code_page = std::wcstol(page_str, &end, 10);

    char char_buff[MB_LEN_MAX]{};
    char32_t target_char = U'?';

    std::mbstate_t state{};
    std::stringstream string_buff{};
    std::wstring wstr = L"";

    // convert UTF-32 to multibyte
    std::size_t ret = std::c32rtomb(char_buff, target_char, &state);

    if (ret == -1)
    {
        std::wcout << L"Conversion from char32_t failed: " << LINE << std::endl;
        return FALSE;
    }
    else
    {
        string_buff << std::string_view{ char_buff, ret };
        string_buff << '\0';

        if (string_buff.fail())
        {
            string_buff.clear();
            std::wcout << L"string_buff failed or bad line: " << LINE << std::endl;
            return FALSE;
        }

        // NOTE: CP_UTF8 gives good results, ex. CP_SYMBOL or code_page variable does not
        // To make stuff work, provide good code page
        wstr = StringCast(string_buff.str(), CP_UTF8 /* code_page */ /* CP_SYMBOL */);
    }

    // Try out every possible locale, this will take insane amount of time!
    // make sure to comment this range for out if you know the locale.
    for (auto loc : locales)
    {
        // locale used (comment out for testing)
        std::locale::global(std::locale(loc));

        if (SetConsole(code_page))
        {
            // HACK: put breakpoint here, and you'll see the string
            // is correctly encoded inside wstr (ex. mouse over wstr)
            // However it's not printed because console code page is likely wrong.
            assert(std::wcout.good() && string_buff.good());
            std::wcout << wstr << std::endl;

            // NOTE: commented out to avoid spamming the console, basically
            // hard to find correct code page if not impossible for CMD
            if (std::wcout.bad())
            {
                std::wcout.clear();
                //std::wcout << L"std::wcout Read/write error on i/o operation line:  " << LINE << std::endl;
            }
            else if (std::wcout.fail())
            {
                std::wcout.clear();
                //std::wcout << L"std::wcout Logical error on i/o operation line:  " << LINE << std::endl;
            }
        }
    }

    return TRUE;    // continue drilling
}

int main()
{
    // NOTE: can be also LOCALE_ALL, anything else than CP_UTF8 doesn't make sense here
    EnumSystemLocalesEx(LocaleEnumprocex, LOCALE_WINDOWS, static_cast<LPARAM>(CP_UTF8), 0);

    // NOTE: can also be CP_INSTALLED
    EnumSystemCodePagesW(EnumCodePagesProc, CP_SUPPORTED);

    // NOTE: following is just a test code to demonstrate these algorithms indeed work,
    // comment out 2 function above to test!
    std::mbstate_t state{};
    std::stringstream string_buff{};

    char char_buff[MB_LEN_MAX]{};

    // Test case for working char:
    std::locale::global(std::locale("ru_RU.utf8"));

    string_buff.clear();
    string_buff.str(std::string());

    // Russian (KOI8-R); Cyrillic (KOI8-R)
    if (SetConsole(20866))
    {
        char32_t char32_str[] = U"Познер обнародовал";

        for (char32_t c32 : char32_str)
        {
            std::size_t ret2 = std::c32rtomb(char_buff, c32, &state);

            if (ret2 == -1)
            {
                std::wcout << L"Conversion from char32_t failed line: " << LINE << std::endl;
            }
            else
            {
                string_buff << std::string_view{ char_buff, ret2 };
            }
        }

        string_buff << '\0';
        if (string_buff.fail())
        {
            string_buff.clear();
            std::wcout << L"string_buff failed or bad line:  " << LINE << std::endl;
        }

        std::wstring wstr = StringCast(string_buff.str(), CP_UTF8);
        std::wcout << wstr << std::endl;

        if (std::wcout.fail())
        {
            std::wcout.clear();
            std::wcout << L"std::wcout failed or bad line:  " << LINE << std::endl;
        }
    }
}

#pragma warning (pop)