fix(csv): do not read prose as a table

CHANGELOG.md

@@ -16,6 +16,10 @@ The release run heads these entries with the version and opens a fresh
 
 ## Unreleased
 
+- Prose is no longer read as a csv. A separator that every field follows with a
+  space, in fields long enough to be sentences, is punctuation; `a, b, c` with
+  short values is still a csv. One record is a line, not a table.
+
 ## v6.7.1 - 2026-08-16
 
 - A pdf whose fonts are not embedded sits where the file puts it: a recovered

src/odr/internal/csv/csv_util.cpp

@@ -43,10 +43,87 @@ std::optional<char> separator_directive(const std::string_view text) {
   return separator;
 }
 
-/// The field counts of every record @p dialect finds in @p text.
+/// Share of the fields after the first that must begin with a space before the
+/// separator is read as punctuation rather than a delimiter.
+constexpr double punctuation_spacing = 0.9;
+
+/// Mean field length, in characters, above which the fields read as prose
+/// rather than as values. Only ever consulted together with
+/// @ref punctuation_spacing.
+constexpr std::size_t prose_field_length = 12;
+
+/// The length of @p field in characters, without the spaces around it. Bytes
+/// would read four cjk characters as a sentence.
+std::size_t trimmed_length(const std::string_view field) {
+  constexpr std::string_view blanks = " \t";
+  const std::size_t first = field.find_first_not_of(blanks);
+  if (first == std::string_view::npos) {
+    return 0;
+  }
+  const std::string_view trimmed =
+      field.substr(first, field.find_last_not_of(blanks) - first + 1);
+  // every byte that is not a utf-8 continuation byte opens a character
+  return static_cast<std::size_t>(
+      std::ranges::count_if(trimmed, [](const char c) {
+        return (static_cast<unsigned char>(c) & 0xc0) != 0x80;
+      }));
+}
+
+/// What a record's fields look like — the evidence that its separator is
+/// punctuation rather than a delimiter.
+struct Fields final {
+  /// Fields after the first of their record - the ones a separator precedes.
+  std::size_t following{0};
+  /// How many of those begin with a space.
+  std::size_t space_led{0};
+  /// Every field's length in characters, spaces around it excluded.
+  std::size_t total_length{0};
+  std::size_t count{0};
+
+  Fields &operator+=(const Fields &other) noexcept {
+    following += other.following;
+    space_led += other.space_led;
+    total_length += other.total_length;
+    count += other.count;
+    return *this;
+  }
+
+  Fields &operator-=(const Fields &other) noexcept {
+    following -= other.following;
+    space_led -= other.space_led;
+    total_length -= other.total_length;
+    count -= other.count;
+    return *this;
+  }
+};
+
+Fields inspect(const std::vector<std::string> &fields) {
+  Fields result;
+  for (std::size_t i = 0; i < fields.size(); ++i) {
+    const std::string &field = fields[i];
+    result.total_length += trimmed_length(field);
+    ++result.count;
+
+    if (i == 0) {
+      continue;
+    }
+    ++result.following;
+    if (!field.empty() && field.front() == ' ') {
+      ++result.space_led;
+    }
+  }
+  return result;
+}
+
+/// What @p dialect finds in @p text: a field count per record, and what those
+/// fields look like.
 struct Scan final {
   std::vector<std::uint32_t> counts;
   bool unterminated{false};
+  Fields fields;
+  /// The last record's share of @ref fields, so a truncated one can be taken
+  /// back out.
+  Fields last_record;
 };
 
 Scan scan(const std::string_view text, const Dialect dialect) {
@@ -55,22 +132,32 @@ Scan scan(const std::string_view text, const Dialect dialect) {
   std::vector<std::string> fields;
   while (reader.read(fields)) {
     result.counts.push_back(static_cast<std::uint32_t>(fields.size()));
+    result.last_record = inspect(fields);
+    result.fields += result.last_record;
   }
   result.unterminated = reader.unterminated();
   return result;
 }
 
 /// How well @p dialect explains @p text: the field count most records carry,
-/// and the share of records carrying it.
+/// the share of records carrying it, and whether the separator is doing the
+/// work of a delimiter or of punctuation.
 struct Score final {
   std::uint32_t columns{0};
   double agreement{0.0};
+  std::size_t records{0};
+  /// Whether the separator reads as punctuation: every field after it opens
+  /// with a space, and the fields are long enough to be sentences. Prose that
+  /// puts a comma between clauses is a two column table without this, and a
+  /// table is the worse way to be wrong about text.
+  bool punctuation{false};
 };
 
 Score score(Scan scan, const bool complete) {
   // a sample cut mid-record says nothing about the record it cut
   if (!complete && !scan.counts.empty()) {
     scan.counts.pop_back();
+    scan.fields -= scan.last_record;
   }
   if (scan.counts.empty()) {
     return {};
@@ -83,8 +170,18 @@ Score score(Scan scan, const bool complete) {
   const auto modal = std::ranges::max_element(
       histogram, {}, [](const auto &entry) { return entry.second; });
 
-  return {modal->first, static_cast<double>(modal->second) /
-                            static_cast<double>(scan.counts.size())};
+  const bool spaced = scan.fields.following > 0 &&
+                      static_cast<double>(scan.fields.space_led) /
+                              static_cast<double>(scan.fields.following) >=
+                          punctuation_spacing;
+  const bool wordy =
+      scan.fields.count > 0 &&
+      scan.fields.total_length / scan.fields.count >= prose_field_length;
+
+  return {modal->first,
+          static_cast<double>(modal->second) /
+              static_cast<double>(scan.counts.size()),
+          scan.counts.size(), spaced && wordy};
 }
 
 } // namespace
@@ -190,7 +287,11 @@ csv::Probe csv::probe(const std::string_view text, const bool complete,
     }
 
     const Score scored = score(scanned, complete);
-    if (scored.columns < 2) {
+    if (scored.columns < 2 || scored.punctuation) {
+      continue;
+    }
+    // one record is a line, not a table, however many separators it holds
+    if (scored.records < 2) {
       continue;
     }
     if (scored.agreement > best.agreement ||

test/src/internal/csv/csv_file_test.cpp

@@ -119,6 +119,61 @@ TEST(CsvProbe, a_separator_that_only_sometimes_appears_loses) {
   EXPECT_EQ(result.columns, 2u);
 }
 
+/// Prose puts a comma between clauses and a space after it, which is a two
+/// column table by field count alone. The doubt breaks towards text: it is the
+/// readable way to be wrong about either one.
+TEST(CsvProbe, prose_is_not_a_table) {
+  EXPECT_FALSE(probe("Lorem ipsum dolor sit, consectetur adipiscing elit\n"
+                     "Lorem ipsum dolor sit, consectetur adipiscing elit\n"
+                     "Lorem ipsum dolor sit, consectetur adipiscing elit\n")
+                   .is_csv);
+
+  EXPECT_FALSE(probe("Sehr geehrte Damen und Herren, hiermit teile ich mit\n"
+                     "dass die Lieferung, wie besprochen, eingetroffen ist\n"
+                     "Mit freundlichen Gruessen, Ihre Buchhaltung heute\n")
+                   .is_csv);
+
+  // the same shape with a semicolon between the clauses
+  EXPECT_FALSE(probe("Lorem ipsum dolor sit; consectetur adipiscing elit\n"
+                     "Lorem ipsum dolor sit; consectetur adipiscing elit\n"
+                     "Lorem ipsum dolor sit; consectetur adipiscing elit\n")
+                   .is_csv);
+}
+
+/// The space after the separator is only evidence against alongside fields long
+/// enough to be sentences: a csv written with room to breathe is still a csv.
+TEST(CsvProbe, short_values_may_be_spaced_out) {
+  EXPECT_TRUE(
+      probe("name, age, city\nada, 36, london\nalan, 41, dublin\n").is_csv);
+  EXPECT_TRUE(probe("city, country\nlondon, england\nlyon, france\n").is_csv);
+}
+
+/// The prose length is a character count, so a value spelled in three byte
+/// characters is as short as the same value spelled in one byte ones.
+TEST(CsvProbe, short_values_stay_short_when_multibyte) {
+  EXPECT_TRUE(probe("東京都市, 日本国家\n"
+                    "大阪府市, 中国北京\n"
+                    "北京市区, 韓国首爾\n")
+                  .is_csv);
+}
+
+/// The record the sample cut through is evidence about where the bound fell,
+/// not about the file, and that holds for its fields as much as its width.
+TEST(CsvProbe, a_cut_record_does_not_argue_for_punctuation) {
+  const std::string text = "ab, cd\nef, gh\n"
+                           "a sentence long enough to look like prose,"
+                           " and a second one just as wordy as that";
+  EXPECT_FALSE(csv::probe(text, true).is_csv);
+  EXPECT_TRUE(csv::probe(text, false).is_csv);
+}
+
+/// However many separators it holds - a paragraph is one record, and one record
+/// is a line rather than a table.
+TEST(CsvProbe, a_single_record_is_not_a_table) {
+  EXPECT_FALSE(probe("a,b,c,d,e,f,g,h").is_csv);
+  EXPECT_FALSE(probe("a,b,c,d,e,f,g,h\n").is_csv);
+}
+
 TEST(CsvProbe, excel_declares_its_separator) {
   const csv::Probe result = probe("sep=;\na;b\n1;2\n");
   EXPECT_TRUE(result.is_csv);