From 20e89a48018af6983c28231d30d780fa812f2295 Mon Sep 17 00:00:00 2001 From: Gennaro Prota Date: Thu, 6 Aug 2026 15:43:44 +0200 Subject: [PATCH] Read wide strings back from XML archives as UTF-8 The XML output archive writes a `std::wstring` as UTF-8, through `mb_from_wchar`, but the input archive decoded it with `std::mbrtowc`, which follows the current locale. This means anything outside ASCII came back wrong, each byte having turned into one character. The text and binary archives were never affected, as they write the characters unchanged. The input archive now decodes with `wchar_from_mb`, the counterpart of the writer's `mb_from_wchar`. That iterator in turn had to stop taking a partial conversion for a failure: it fills its input buffer without regard to character boundaries, so a multibyte character can straddle the end, and the bytes left over are already carried to the next round. A conversion which really fails now throws instead of asserting. Fixes #298. --- .../boost/archive/impl/xml_iarchive_impl.ipp | 62 +++------- .../boost/archive/iterators/wchar_from_mb.hpp | 13 +- test/Jamfile.v2 | 1 + test/test_wstring_utf8.cpp | 115 ++++++++++++++++++ 4 files changed, 145 insertions(+), 46 deletions(-) create mode 100644 test/test_wstring_utf8.cpp diff --git a/include/boost/archive/impl/xml_iarchive_impl.ipp b/include/boost/archive/impl/xml_iarchive_impl.ipp index 2c7b04633..67de61da8 100644 --- a/include/boost/archive/impl/xml_iarchive_impl.ipp +++ b/include/boost/archive/impl/xml_iarchive_impl.ipp @@ -9,23 +9,19 @@ // See http://www.boost.org for updates, documentation, and revision history. #include +#include // copy #include // memcpy #include // NULL +#include // back_inserter #if defined(BOOST_NO_STDC_NAMESPACE) -namespace std{ +namespace std{ using ::memcpy; } // namespace std #endif #ifndef BOOST_NO_CWCHAR -#include // mbstate_t and mbrtowc -#if defined(BOOST_NO_STDC_NAMESPACE) -namespace std{ - using ::mbstate_t; - using ::mbrtowc; - } // namespace std -#endif +#include #endif // BOOST_NO_CWCHAR #include // RogueWave and Dinkumware @@ -67,23 +63,14 @@ xml_iarchive_impl::load(std::wstring &ws){ if(NULL != ws.data()) #endif ws.resize(0); - std::mbstate_t mbs = std::mbstate_t(); - const char * start = s.data(); - const char * end = start + s.size(); - while(start < end){ - wchar_t wc; - std::size_t count = std::mbrtowc(&wc, start, end - start, &mbs); - if(count == static_cast(-1)) - boost::serialization::throw_exception( - iterators::dataflow_exception( - iterators::dataflow_exception::invalid_conversion - ) - ); - if(count == static_cast(-2)) - continue; - start += count; - ws += wc; - } + // The text was written as utf8 by mb_from_wchar, so decode it with the + // facet which matches, rather than with whatever the current locale is. + typedef iterators::wchar_from_mb translator; + std::copy( + translator(s.data()), + translator(), + std::back_inserter(ws) + ); } #endif // BOOST_NO_STD_WSTRING @@ -100,24 +87,13 @@ xml_iarchive_impl::load(wchar_t * ws){ ) ); - std::mbstate_t mbs = std::mbstate_t(); - const char * start = s.data(); - const char * end = start + s.size(); - while(start < end){ - wchar_t wc; - std::size_t length = std::mbrtowc(&wc, start, end - start, &mbs); - if(static_cast(-1) == length) - boost::serialization::throw_exception( - iterators::dataflow_exception( - iterators::dataflow_exception::invalid_conversion - ) - ); - if(static_cast(-2) == length) - continue; - - start += length; - *ws++ = wc; - } + // see the comment in the std::wstring overload above + typedef iterators::wchar_from_mb translator; + ws = std::copy( + translator(s.data()), + translator(), + ws + ); *ws = L'\0'; } #endif // BOOST_NO_INTRINSIC_WCHAR_T diff --git a/include/boost/archive/iterators/wchar_from_mb.hpp b/include/boost/archive/iterators/wchar_from_mb.hpp index 294351b48..b2c1e6183 100644 --- a/include/boost/archive/iterators/wchar_from_mb.hpp +++ b/include/boost/archive/iterators/wchar_from_mb.hpp @@ -165,8 +165,7 @@ void wchar_from_mb::drain(){ const typename boost::iterators::iterator_value::type * input_new_start; typename iterator_value::type * next_available; - BOOST_ATTRIBUTE_UNUSED // redundant with ignore_unused below but clarifies intention - std::codecvt_base::result r = m_codecvt_facet.in( + const std::codecvt_base::result r = m_codecvt_facet.in( m_mbs, m_input.m_buffer.begin(), m_input.m_next_available, @@ -175,7 +174,15 @@ void wchar_from_mb::drain(){ m_output.m_buffer.end(), next_available ); - BOOST_ASSERT(std::codecvt_base::ok == r); + if(std::codecvt_base::error == r){ + boost::serialization::throw_exception( + dataflow_exception(dataflow_exception::invalid_conversion) + ); + } + // A partial result is normal here and not an error: the input buffer is + // filled without regard to character boundaries, so it can end in the + // middle of a multibyte character. Its remaining bytes are kept by the + // shift below and decoded once the rest of them have been read. m_output.m_next_available = next_available; m_output.m_next = m_output.m_buffer.begin(); diff --git a/test/Jamfile.v2 b/test/Jamfile.v2 index b79fb1576..fee62a270 100644 --- a/test/Jamfile.v2 +++ b/test/Jamfile.v2 @@ -118,6 +118,7 @@ test-suite "serialization" : [ test-bsl-run_files test_valarray ] [ test-bsl-run_files test_variant : A ] [ test-bsl-run_files test_version_value_type ] + [ test-bsl-run_files test_wstring_utf8 ] [ test-bsl-run_files test_vector : A ] [ test-bsl-run_files test_shared_ptr ] [ test-bsl-run_files test_shared_ptr_coexistence ] diff --git a/test/test_wstring_utf8.cpp b/test/test_wstring_utf8.cpp new file mode 100644 index 000000000..15418c51d --- /dev/null +++ b/test/test_wstring_utf8.cpp @@ -0,0 +1,115 @@ +/////////1/////////2/////////3/////////4/////////5/////////6/////////7/////////8 +// test_wstring_utf8.cpp + +// Copyright 2026 Gennaro Prota. +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +// See http://www.boost.org for updates, documentation, and revision history. + +// A std::wstring holding characters outside ASCII has to survive a round +// trip. The XML archives are the ones which have to encode it, since xml +// is text, and both ends have to agree on UTF-8. + +// Reported by schorsch1976 in +// https://github.com/boostorg/serialization/issues/298, with a reproducer +// showing that only the xml archive was affected while text and binary, +// which write the characters unchanged, were not. Thanks! + +// The characters are written as universal character names on purpose, so +// that the test does not depend on how the compiler reads this file. + +#include +#include +#include +#include + +#include +#if defined(BOOST_NO_STDC_NAMESPACE) +namespace std{ + using ::remove; +} +#endif + +#include +#include + +#include "test_tools.hpp" + +#ifndef BOOST_NO_STD_WSTRING + +struct data +{ + friend class boost::serialization::access; + template + void serialize(Archive & ar, const unsigned int /* file_version */){ + ar & BOOST_SERIALIZATION_NVP(value); + ar & BOOST_SERIALIZATION_NVP(tail); + } + std::wstring value; + int tail; +}; + +static void round_trip(const char * testfile, const std::wstring & w){ + { + data d; + d.value = w; + d.tail = 42; + test_ostream os(testfile, TEST_STREAM_FLAGS); + test_oarchive oa(os, TEST_ARCHIVE_FLAGS); + oa << BOOST_SERIALIZATION_NVP(d); + } + data d; + d.tail = 0; + test_istream is(testfile, TEST_STREAM_FLAGS); + test_iarchive ia(is, TEST_ARCHIVE_FLAGS); + ia >> BOOST_SERIALIZATION_NVP(d); + BOOST_CHECK(d.value == w); + // whatever follows the string has to be readable as well + BOOST_CHECK(42 == d.tail); +} + +int +test_main(int /* argc */, char * /* argv */ []) +{ + const char * testfile = boost::archive::tmpnam(NULL); + BOOST_REQUIRE(NULL != testfile); + + // two bytes and three bytes each once encoded as UTF-8 + const std::wstring two(L"\u00E9\u00DF"); + const std::wstring three(L"\u4F60\u597D"); + + round_trip(testfile, L""); + round_trip(testfile, L"plain ascii"); + round_trip(testfile, two); + round_trip(testfile, three); + round_trip(testfile, L"mixed " + two + L" and " + three + L" text"); + + // The decoder fills a 32 byte buffer without regard to character + // boundaries, so walk a multibyte character across that boundary. + for(std::size_t pad = 26; pad <= 38; ++pad){ + round_trip(testfile, std::wstring(pad, L'x') + three); + } + + // longer than the buffer, so that it is filled repeatedly + std::wstring big; + for(int i = 0; i < 40; ++i){ + big += three; + big += L"ascii"; + } + round_trip(testfile, big); + + std::remove(testfile); + return EXIT_SUCCESS; +} + +#else // BOOST_NO_STD_WSTRING + +int +test_main(int /* argc */, char * /* argv */ []) +{ + return EXIT_SUCCESS; +} + +#endif // BOOST_NO_STD_WSTRING