Read wide strings back from XML archives as UTF-8

include/boost/archive/impl/xml_iarchive_impl.ipp

@@ -9,23 +9,19 @@
 //  See http://www.boost.org for updates, documentation, and revision history.
 
 #include <boost/config.hpp>
+#include <algorithm> // copy
 #include <cstring> // memcpy
 #include <cstddef> // NULL
+#include <iterator> // back_inserter
 
 #if defined(BOOST_NO_STDC_NAMESPACE)
-namespace std{ 
+namespace std{
     using ::memcpy;
 } // namespace std
 #endif
 
 #ifndef BOOST_NO_CWCHAR
-#include <cwchar> // mbstate_t and mbrtowc
-#if defined(BOOST_NO_STDC_NAMESPACE)
-namespace std{ 
-    using ::mbstate_t;
-    using ::mbrtowc;
- } // namespace std
-#endif
+#include <boost/archive/iterators/wchar_from_mb.hpp>
 #endif // BOOST_NO_CWCHAR
 
 #include <boost/detail/workaround.hpp> // RogueWave and Dinkumware
@@ -67,23 +63,14 @@ xml_iarchive_impl<Archive>::load(std::wstring &ws){
     if(NULL != ws.data())
     #endif
     ws.resize(0);
-    std::mbstate_t mbs = std::mbstate_t();
-    const char * start = s.data();
-    const char * end = start + s.size();
-    while(start < end){
-        wchar_t wc;
-        std::size_t count = std::mbrtowc(&wc, start, end - start, &mbs);
-        if(count == static_cast<std::size_t>(-1))
-            boost::serialization::throw_exception(
-                iterators::dataflow_exception(
-                    iterators::dataflow_exception::invalid_conversion
-                )
-            );
-        if(count == static_cast<std::size_t>(-2))
-            continue;
-        start += count;
-        ws += wc;
-    }
+    // The text was written as utf8 by mb_from_wchar, so decode it with the
+    // facet which matches, rather than with whatever the current locale is.
+    typedef iterators::wchar_from_mb<const char *> translator;
+    std::copy(
+        translator(s.data()),
+        translator(),
+        std::back_inserter(ws)
+    );
 }
 #endif // BOOST_NO_STD_WSTRING
 
@@ -100,24 +87,13 @@ xml_iarchive_impl<Archive>::load(wchar_t * ws){
             )
         );
 
-    std::mbstate_t mbs = std::mbstate_t();
-    const char * start = s.data();
-    const char * end = start + s.size();
-    while(start < end){
-        wchar_t wc;
-        std::size_t length = std::mbrtowc(&wc, start, end - start, &mbs);
-        if(static_cast<std::size_t>(-1) == length)
-            boost::serialization::throw_exception(
-                iterators::dataflow_exception(
-                    iterators::dataflow_exception::invalid_conversion
-                )
-            );
-        if(static_cast<std::size_t>(-2) == length)
-            continue;
-
-        start += length;
-        *ws++ = wc;
-    }
+    // see the comment in the std::wstring overload above
+    typedef iterators::wchar_from_mb<const char *> translator;
+    ws = std::copy(
+        translator(s.data()),
+        translator(),
+        ws
+    );
     *ws = L'\0';
 }
 #endif // BOOST_NO_INTRINSIC_WCHAR_T

include/boost/archive/iterators/wchar_from_mb.hpp

@@ -165,8 +165,7 @@ void wchar_from_mb<Base>::drain(){
     const typename boost::iterators::iterator_value<Base>::type * input_new_start;
     typename iterator_value<this_t>::type * next_available;
 
-    BOOST_ATTRIBUTE_UNUSED // redundant with ignore_unused below but clarifies intention
-    std::codecvt_base::result r = m_codecvt_facet.in(
+    const std::codecvt_base::result r = m_codecvt_facet.in(
         m_mbs,
         m_input.m_buffer.begin(),
         m_input.m_next_available,
@@ -175,7 +174,15 @@ void wchar_from_mb<Base>::drain(){
         m_output.m_buffer.end(),
         next_available
     );
-    BOOST_ASSERT(std::codecvt_base::ok == r);
+    if(std::codecvt_base::error == r){
+        boost::serialization::throw_exception(
+            dataflow_exception(dataflow_exception::invalid_conversion)
+        );
+    }
+    // A partial result is normal here and not an error: the input buffer is
+    // filled without regard to character boundaries, so it can end in the
+    // middle of a multibyte character.  Its remaining bytes are kept by the
+    // shift below and decoded once the rest of them have been read.
     m_output.m_next_available = next_available;
     m_output.m_next = m_output.m_buffer.begin();
 

test/Jamfile.v2

@@ -118,6 +118,7 @@ test-suite "serialization" :
      [ test-bsl-run_files test_valarray ]
      [ test-bsl-run_files test_variant : A ]
      [ test-bsl-run_files test_version_value_type ]
+     [ test-bsl-run_files test_wstring_utf8 ]
      [ test-bsl-run_files test_vector : A ]
      [ test-bsl-run_files test_shared_ptr ]
      [ test-bsl-run_files test_shared_ptr_coexistence ]

test/test_wstring_utf8.cpp

@@ -0,0 +1,115 @@
+/////////1/////////2/////////3/////////4/////////5/////////6/////////7/////////8
+// test_wstring_utf8.cpp
+
+// Copyright 2026 Gennaro Prota.
+// Distributed under the Boost Software License, Version 1.0.
+// (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  See http://www.boost.org for updates, documentation, and revision history.
+
+// A std::wstring holding characters outside ASCII has to survive a round
+// trip.  The XML archives are the ones which have to encode it, since xml
+// is text, and both ends have to agree on UTF-8.
+
+// Reported by schorsch1976 in
+// https://github.com/boostorg/serialization/issues/298, with a reproducer
+// showing that only the xml archive was affected while text and binary,
+// which write the characters unchanged, were not.  Thanks!
+
+// The characters are written as universal character names on purpose, so
+// that the test does not depend on how the compiler reads this file.
+
+#include <cstddef>
+#include <cstdio>
+#include <fstream>
+#include <string>
+
+#include <boost/config.hpp>
+#if defined(BOOST_NO_STDC_NAMESPACE)
+namespace std{
+    using ::remove;
+}
+#endif
+
+#include <boost/serialization/nvp.hpp>
+#include <boost/serialization/string.hpp>
+
+#include "test_tools.hpp"
+
+#ifndef BOOST_NO_STD_WSTRING
+
+struct data
+{
+    friend class boost::serialization::access;
+    template<class Archive>
+    void serialize(Archive & ar, const unsigned int /* file_version */){
+        ar & BOOST_SERIALIZATION_NVP(value);
+        ar & BOOST_SERIALIZATION_NVP(tail);
+    }
+    std::wstring value;
+    int tail;
+};
+
+static void round_trip(const char * testfile, const std::wstring & w){
+    {
+        data d;
+        d.value = w;
+        d.tail = 42;
+        test_ostream os(testfile, TEST_STREAM_FLAGS);
+        test_oarchive oa(os, TEST_ARCHIVE_FLAGS);
+        oa << BOOST_SERIALIZATION_NVP(d);
+    }
+    data d;
+    d.tail = 0;
+    test_istream is(testfile, TEST_STREAM_FLAGS);
+    test_iarchive ia(is, TEST_ARCHIVE_FLAGS);
+    ia >> BOOST_SERIALIZATION_NVP(d);
+    BOOST_CHECK(d.value == w);
+    // whatever follows the string has to be readable as well
+    BOOST_CHECK(42 == d.tail);
+}
+
+int
+test_main(int /* argc */, char * /* argv */ [])
+{
+    const char * testfile = boost::archive::tmpnam(NULL);
+    BOOST_REQUIRE(NULL != testfile);
+
+    // two bytes and three bytes each once encoded as UTF-8
+    const std::wstring two(L"\u00E9\u00DF");
+    const std::wstring three(L"\u4F60\u597D");
+
+    round_trip(testfile, L"");
+    round_trip(testfile, L"plain ascii");
+    round_trip(testfile, two);
+    round_trip(testfile, three);
+    round_trip(testfile, L"mixed " + two + L" and " + three + L" text");
+
+    // The decoder fills a 32 byte buffer without regard to character
+    // boundaries, so walk a multibyte character across that boundary.
+    for(std::size_t pad = 26; pad <= 38; ++pad){
+        round_trip(testfile, std::wstring(pad, L'x') + three);
+    }
+
+    // longer than the buffer, so that it is filled repeatedly
+    std::wstring big;
+    for(int i = 0; i < 40; ++i){
+        big += three;
+        big += L"ascii";
+    }
+    round_trip(testfile, big);
+
+    std::remove(testfile);
+    return EXIT_SUCCESS;
+}
+
+#else // BOOST_NO_STD_WSTRING
+
+int
+test_main(int /* argc */, char * /* argv */ [])
+{
+    return EXIT_SUCCESS;
+}
+
+#endif // BOOST_NO_STD_WSTRING