diff --git a/CHANGELOG.md b/CHANGELOG.md index f224ec28..acf81d48 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,10 @@ The release run heads these entries with the version and opens a fresh ## Unreleased +- Prose is no longer read as a csv. A separator that every field follows with a + space, in fields long enough to be sentences, is punctuation; `a, b, c` with + short values is still a csv. One record is a line, not a table. + ## v6.7.1 - 2026-08-16 - A pdf whose fonts are not embedded sits where the file puts it: a recovered diff --git a/src/odr/internal/csv/csv_util.cpp b/src/odr/internal/csv/csv_util.cpp index 548db589..bfef6dc2 100644 --- a/src/odr/internal/csv/csv_util.cpp +++ b/src/odr/internal/csv/csv_util.cpp @@ -43,10 +43,87 @@ std::optional separator_directive(const std::string_view text) { return separator; } -/// The field counts of every record @p dialect finds in @p text. +/// Share of the fields after the first that must begin with a space before the +/// separator is read as punctuation rather than a delimiter. +constexpr double punctuation_spacing = 0.9; + +/// Mean field length, in characters, above which the fields read as prose +/// rather than as values. Only ever consulted together with +/// @ref punctuation_spacing. +constexpr std::size_t prose_field_length = 12; + +/// The length of @p field in characters, without the spaces around it. Bytes +/// would read four cjk characters as a sentence. +std::size_t trimmed_length(const std::string_view field) { + constexpr std::string_view blanks = " \t"; + const std::size_t first = field.find_first_not_of(blanks); + if (first == std::string_view::npos) { + return 0; + } + const std::string_view trimmed = + field.substr(first, field.find_last_not_of(blanks) - first + 1); + // every byte that is not a utf-8 continuation byte opens a character + return static_cast( + std::ranges::count_if(trimmed, [](const char c) { + return (static_cast(c) & 0xc0) != 0x80; + })); +} + +/// What a record's fields look like — the evidence that its separator is +/// punctuation rather than a delimiter. +struct Fields final { + /// Fields after the first of their record - the ones a separator precedes. + std::size_t following{0}; + /// How many of those begin with a space. + std::size_t space_led{0}; + /// Every field's length in characters, spaces around it excluded. + std::size_t total_length{0}; + std::size_t count{0}; + + Fields &operator+=(const Fields &other) noexcept { + following += other.following; + space_led += other.space_led; + total_length += other.total_length; + count += other.count; + return *this; + } + + Fields &operator-=(const Fields &other) noexcept { + following -= other.following; + space_led -= other.space_led; + total_length -= other.total_length; + count -= other.count; + return *this; + } +}; + +Fields inspect(const std::vector &fields) { + Fields result; + for (std::size_t i = 0; i < fields.size(); ++i) { + const std::string &field = fields[i]; + result.total_length += trimmed_length(field); + ++result.count; + + if (i == 0) { + continue; + } + ++result.following; + if (!field.empty() && field.front() == ' ') { + ++result.space_led; + } + } + return result; +} + +/// What @p dialect finds in @p text: a field count per record, and what those +/// fields look like. struct Scan final { std::vector counts; bool unterminated{false}; + Fields fields; + /// The last record's share of @ref fields, so a truncated one can be taken + /// back out. + Fields last_record; }; Scan scan(const std::string_view text, const Dialect dialect) { @@ -55,22 +132,32 @@ Scan scan(const std::string_view text, const Dialect dialect) { std::vector fields; while (reader.read(fields)) { result.counts.push_back(static_cast(fields.size())); + result.last_record = inspect(fields); + result.fields += result.last_record; } result.unterminated = reader.unterminated(); return result; } /// How well @p dialect explains @p text: the field count most records carry, -/// and the share of records carrying it. +/// the share of records carrying it, and whether the separator is doing the +/// work of a delimiter or of punctuation. struct Score final { std::uint32_t columns{0}; double agreement{0.0}; + std::size_t records{0}; + /// Whether the separator reads as punctuation: every field after it opens + /// with a space, and the fields are long enough to be sentences. Prose that + /// puts a comma between clauses is a two column table without this, and a + /// table is the worse way to be wrong about text. + bool punctuation{false}; }; Score score(Scan scan, const bool complete) { // a sample cut mid-record says nothing about the record it cut if (!complete && !scan.counts.empty()) { scan.counts.pop_back(); + scan.fields -= scan.last_record; } if (scan.counts.empty()) { return {}; @@ -83,8 +170,18 @@ Score score(Scan scan, const bool complete) { const auto modal = std::ranges::max_element( histogram, {}, [](const auto &entry) { return entry.second; }); - return {modal->first, static_cast(modal->second) / - static_cast(scan.counts.size())}; + const bool spaced = scan.fields.following > 0 && + static_cast(scan.fields.space_led) / + static_cast(scan.fields.following) >= + punctuation_spacing; + const bool wordy = + scan.fields.count > 0 && + scan.fields.total_length / scan.fields.count >= prose_field_length; + + return {modal->first, + static_cast(modal->second) / + static_cast(scan.counts.size()), + scan.counts.size(), spaced && wordy}; } } // namespace @@ -190,7 +287,11 @@ csv::Probe csv::probe(const std::string_view text, const bool complete, } const Score scored = score(scanned, complete); - if (scored.columns < 2) { + if (scored.columns < 2 || scored.punctuation) { + continue; + } + // one record is a line, not a table, however many separators it holds + if (scored.records < 2) { continue; } if (scored.agreement > best.agreement || diff --git a/test/src/internal/csv/csv_file_test.cpp b/test/src/internal/csv/csv_file_test.cpp index f55da88a..7438fafb 100644 --- a/test/src/internal/csv/csv_file_test.cpp +++ b/test/src/internal/csv/csv_file_test.cpp @@ -119,6 +119,61 @@ TEST(CsvProbe, a_separator_that_only_sometimes_appears_loses) { EXPECT_EQ(result.columns, 2u); } +/// Prose puts a comma between clauses and a space after it, which is a two +/// column table by field count alone. The doubt breaks towards text: it is the +/// readable way to be wrong about either one. +TEST(CsvProbe, prose_is_not_a_table) { + EXPECT_FALSE(probe("Lorem ipsum dolor sit, consectetur adipiscing elit\n" + "Lorem ipsum dolor sit, consectetur adipiscing elit\n" + "Lorem ipsum dolor sit, consectetur adipiscing elit\n") + .is_csv); + + EXPECT_FALSE(probe("Sehr geehrte Damen und Herren, hiermit teile ich mit\n" + "dass die Lieferung, wie besprochen, eingetroffen ist\n" + "Mit freundlichen Gruessen, Ihre Buchhaltung heute\n") + .is_csv); + + // the same shape with a semicolon between the clauses + EXPECT_FALSE(probe("Lorem ipsum dolor sit; consectetur adipiscing elit\n" + "Lorem ipsum dolor sit; consectetur adipiscing elit\n" + "Lorem ipsum dolor sit; consectetur adipiscing elit\n") + .is_csv); +} + +/// The space after the separator is only evidence against alongside fields long +/// enough to be sentences: a csv written with room to breathe is still a csv. +TEST(CsvProbe, short_values_may_be_spaced_out) { + EXPECT_TRUE( + probe("name, age, city\nada, 36, london\nalan, 41, dublin\n").is_csv); + EXPECT_TRUE(probe("city, country\nlondon, england\nlyon, france\n").is_csv); +} + +/// The prose length is a character count, so a value spelled in three byte +/// characters is as short as the same value spelled in one byte ones. +TEST(CsvProbe, short_values_stay_short_when_multibyte) { + EXPECT_TRUE(probe("東京都市, 日本国家\n" + "大阪府市, 中国北京\n" + "北京市区, 韓国首爾\n") + .is_csv); +} + +/// The record the sample cut through is evidence about where the bound fell, +/// not about the file, and that holds for its fields as much as its width. +TEST(CsvProbe, a_cut_record_does_not_argue_for_punctuation) { + const std::string text = "ab, cd\nef, gh\n" + "a sentence long enough to look like prose," + " and a second one just as wordy as that"; + EXPECT_FALSE(csv::probe(text, true).is_csv); + EXPECT_TRUE(csv::probe(text, false).is_csv); +} + +/// However many separators it holds - a paragraph is one record, and one record +/// is a line rather than a table. +TEST(CsvProbe, a_single_record_is_not_a_table) { + EXPECT_FALSE(probe("a,b,c,d,e,f,g,h").is_csv); + EXPECT_FALSE(probe("a,b,c,d,e,f,g,h\n").is_csv); +} + TEST(CsvProbe, excel_declares_its_separator) { const csv::Probe result = probe("sep=;\na;b\n1;2\n"); EXPECT_TRUE(result.is_csv);