diff --git a/CHANGELOG.md b/CHANGELOG.md index 98dda6c..ab0efee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,15 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.9.6] - 2026-08-12 + +### Changed + +- `cbq` uses the `bitnuc` implementation with `u8`-based encoding with better SIMD perf + - Does **not** change on-disk representation - block reading/writing is still done with u64 byte-boundaries and the format is unchanged. +- `cbq` blocks now use `bitnuc::ambiguous_bases` to find noncanonical bases instead of just `N` and adds them to the `npos` index. +- dependency both on `bitnuc-0.4` and `bitnuc-0.5` to keep 4-bit support for `bq` and `vbq` which is unsupported in `bitnuc-0.5.x`. + ## [0.9.5] - 2026-08-10 ### Fixed diff --git a/Cargo.toml b/Cargo.toml index b492d89..bb976a0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "binseq" -version = "0.9.5" +version = "0.9.6" edition = "2024" description = "A high efficiency binary format for sequencing data" license = "MIT" @@ -13,7 +13,8 @@ keywords = ["binary", "nucleotide", "sequencing", "genomics", "fastq"] [dependencies] anyhow = {version = "1.0.103", optional = true} auto_impl = "1.3.0" -bitnuc = "0.4.1" +bitnuc-deprec = { package ="bitnuc", version = "0.4.1" } +bitnuc = { version = "0.5.1" } bytemuck = { version = "1.25.1", features = ["derive", "extern_crate_alloc"] } byteorder = "1.5.0" itoa = "1.0.18" @@ -24,7 +25,7 @@ paraseq = { version = "0.4.14", optional = true } parking_lot = {version = "0.12.5", optional = true } rand = { version = "0.9.5", features = ["small_rng"] } sucds = "0.8.3" -thiserror = "2.0.18" +thiserror = "2.0.20" zstd = { version = "0.13.3", features = ["zstdmt"] } [dev-dependencies] diff --git a/examples/auto-write.rs b/examples/auto-write.rs index cffc16b..5a555a3 100644 --- a/examples/auto-write.rs +++ b/examples/auto-write.rs @@ -1,8 +1,8 @@ use std::{fs::File, io::BufWriter}; use anyhow::Result; +use binseq::BitSize; use binseq::{BinseqWriterBuilder, write::Format}; -use bitnuc::BitSize; use clap::Parser; type BoxedWriter = Box; diff --git a/examples/read.rs b/examples/read.rs index f060836..c362cb0 100644 --- a/examples/read.rs +++ b/examples/read.rs @@ -91,7 +91,7 @@ pub fn write_fastq_parts( sequence: &[u8], quality: &[u8], ) -> Result<(), std::io::Error> { - writer.write_all(b"@seq.")?; + writer.write_all(b"@")?; writer.write_all(index)?; writer.write_all(b"\n")?; writer.write_all(sequence)?; diff --git a/examples/write.rs b/examples/write.rs index c280998..f83a273 100644 --- a/examples/write.rs +++ b/examples/write.rs @@ -4,11 +4,11 @@ use std::{ }; use anyhow::{Result, bail}; +use binseq::BitSize; use binseq::{ SequencingRecordBuilder, write::{BinseqWriter, BinseqWriterBuilder, Format}, }; -use bitnuc::BitSize; use clap::Parser; use paraseq::{ Record, fastx, diff --git a/src/bq/header.rs b/src/bq/header.rs index eed8c2f..dd84593 100644 --- a/src/bq/header.rs +++ b/src/bq/header.rs @@ -4,7 +4,7 @@ //! The header contains metadata about the binary sequence data, including format version, //! sequence length, and other information necessary for proper interpretation of the data. -use bitnuc::BitSize; +use bitnuc_deprec::BitSize; use byteorder::{ByteOrder, LittleEndian}; use std::io::{Read, Write}; diff --git a/src/bq/reader.rs b/src/bq/reader.rs index fe230b9..679b210 100644 --- a/src/bq/reader.rs +++ b/src/bq/reader.rs @@ -13,7 +13,7 @@ use std::ops::Range; use std::path::Path; use std::sync::Arc; -use bitnuc::BitSize; +use bitnuc_deprec::BitSize; use bytemuck::cast_slice; use memmap2::Mmap; @@ -1013,7 +1013,7 @@ impl ParallelReader for MmapReader { mod tests { use super::*; use crate::BinseqRecord; - use bitnuc::BitSize; + use bitnuc_deprec::BitSize; const TEST_BQ_FILE: &str = "./data/subset.bq"; diff --git a/src/cbq/core/block.rs b/src/cbq/core/block.rs index b463f22..1045fd7 100644 --- a/src/cbq/core/block.rs +++ b/src/cbq/core/block.rs @@ -1,6 +1,5 @@ use std::io; -use bitnuc::BitSize; use bytemuck::{cast_slice, cast_slice_mut}; use sucds::Serializable; use sucds::mii_sequences::{EliasFano, EliasFanoBuilder}; @@ -9,7 +8,7 @@ use zstd::zstd_safe; use crate::cbq::core::utils::sized_compress; use crate::error::{CbqError, WriteError}; -use crate::{BinseqRecord, DEFAULT_QUALITY_SCORE, Result}; +use crate::{BinseqRecord, BitSize, DEFAULT_QUALITY_SCORE, Result}; use super::utils::{Span, calculate_offsets, extension_read, resize_uninit, slice_and_increment}; use super::{BlockHeader, BlockRange, FileHeader}; @@ -32,7 +31,10 @@ pub struct ColumnarBlock { pub(crate) npos: Vec, /// Reusable buffer for encoding sequences - ebuf: Vec, + /// + /// Byte-native 2-bit packing, zero-padded to whole 8-byte words to + /// remain bit-identical with the legacy u64-per-32-bases on-disk layout. + ebuf: Vec, /// An Elias-Fano encoding for the N-positions pub(crate) ef: Option, @@ -350,23 +352,23 @@ impl ColumnarBlock { Ok(()) } - /// Returns the expected length of the encoded sequence buffer + /// Returns the expected length of the encoded sequence buffer in bytes /// /// This is deterministically calculated based on the sequence length and the encoding scheme. + /// The on-disk format packs 32 bases per 8-byte word, so the buffer is padded to whole words. fn ebuf_len(&self) -> usize { - self.nuclen.div_ceil(32) + self.nuclen.div_ceil(32) * 8 } /// Encode the sequence into a compressed representation - fn encode_sequence(&mut self) -> Result<()> { - bitnuc::twobit::encode_with_invalid(&self.seq, &mut self.ebuf)?; - Ok(()) + fn encode_sequence(&mut self) { + bitnuc::encode_resize(&self.seq, &mut self.ebuf); + self.ebuf.resize(self.ebuf_len(), 0); // zero-pad the trailing partial word to match the legacy u64 layout } /// Find all positions of 'N' in the sequence fn fill_npos(&mut self) -> Result<()> { - self.npos - .extend(memchr::memchr_iter(b'N', &self.seq).map(|i| i as u64)); + bitnuc::ambiguous_bases(&self.seq, &mut self.npos); self.num_npos = self.npos.len(); // build Elias-Fano encoding for N positions @@ -412,7 +414,7 @@ impl ColumnarBlock { } // compress sequence - sized_compress(&mut self.z_seq, cast_slice(&self.ebuf), cctx)?; + sized_compress(&mut self.z_seq, &self.ebuf, cctx)?; // compress flags if !self.flags.is_empty() { @@ -467,9 +469,9 @@ impl ColumnarBlock { // decompress sequence { self.ebuf.resize(self.ebuf_len(), 0); - copy_decode(self.z_seq.as_slice(), cast_slice_mut(&mut self.ebuf))?; + copy_decode(self.z_seq.as_slice(), self.ebuf.as_mut_slice())?; - bitnuc::twobit::decode(&self.ebuf, self.nuclen, &mut self.seq)?; + bitnuc::decode_resize(&self.ebuf, self.nuclen, &mut self.seq)?; self.backfill_npos(); } @@ -519,7 +521,7 @@ impl ColumnarBlock { } // encode all sequences at once - self.encode_sequence()?; + self.encode_sequence(); // fill npos self.fill_npos()?; @@ -637,12 +639,12 @@ impl ColumnarBlock { let ebuf_len = self.ebuf_len(); resize_uninit(&mut self.ebuf, ebuf_len); dctx.decompress( - cast_slice_mut(&mut self.ebuf), + self.ebuf.as_mut_slice(), slice_and_increment(&mut byte_offset, header.len_z_seq, bytes), ) .map_err(|e| io::Error::other(zstd_safe::get_error_name(e)))?; - bitnuc::twobit::decode(&self.ebuf, self.nuclen, &mut self.seq)?; + bitnuc::decode_resize(&self.ebuf, self.nuclen, &mut self.seq)?; self.backfill_npos(); } @@ -1011,6 +1013,72 @@ mod tests { .unwrap() } + // ==================== On-disk compatibility with the legacy u64 packing ==================== + + /// The byte-native encoder (padded to whole 8-byte words) must stay + /// bit-identical with the legacy u64-per-32-bases packing so cbq files + /// remain interchangeable across binseq versions in both directions. + #[test] + fn test_ebuf_matches_legacy_word_stream() { + for len in [1usize, 5, 9, 31, 32, 33, 63, 64, 65, 127, 128, 129, 1000] { + let seq: Vec = (0..len).map(|i| b"ACGT"[(i * 7 + 3) % 4]).collect(); + + // new path, as performed by `encode_sequence` + let mut ebuf = Vec::new(); + bitnuc::encode_resize(&seq, &mut ebuf); + ebuf.resize(len.div_ceil(32) * 8, 0); + + // legacy path, as performed by the previous implementation + let mut legacy_words: Vec = Vec::new(); + bitnuc_deprec::twobit::encode_with_invalid(&seq, &mut legacy_words).unwrap(); + assert_eq!(ebuf, cast_slice::(&legacy_words), "len {len}"); + + // legacy words decode with the new decoder (old files, new reader) + let mut dbuf = Vec::new(); + bitnuc::decode_resize(cast_slice(&legacy_words), len, &mut dbuf).unwrap(); + assert_eq!(dbuf, seq, "len {len}"); + + // new bytes decode with the legacy decoder (new files, old reader) + let new_words: Vec = bytemuck::pod_collect_to_vec(&ebuf); + let mut old_dbuf = Vec::new(); + bitnuc_deprec::twobit::decode(&new_words, len, &mut old_dbuf).unwrap(); + assert_eq!(old_dbuf, seq, "len {len}"); + } + } + + /// N positions are restored from the Elias-Fano index regardless of how + /// the encoder packs them, so blocks written with either bitnuc decode + /// identically once `backfill_npos` runs. + #[test] + fn test_npos_backfill_covers_encoder_differences() { + let seq = b"ACGTNNACGTNACGTACGTACGTACGTACGTNNNNACGTACGTACGTACGTACGTACGTACGTN"; + + let header = unpaired_header(1 << 16); + let mut block = ColumnarBlock::new(header); + block + .push( + SequencingRecordBuilder::default() + .s_seq(seq) + .build() + .unwrap(), + ) + .unwrap(); + + let mut cctx = zstd_safe::CCtx::create(); + let mut buffer = Vec::new(); + let block_header = block.flush_to(&mut buffer, &mut cctx).unwrap().unwrap(); + + let mut cursor = + std::io::Cursor::new(buffer[std::mem::size_of::()..].to_vec()); + let mut reader_block = ColumnarBlock::new(header); + reader_block.read_from(&mut cursor, block_header).unwrap(); + reader_block.decompress_columns().unwrap(); + + let range = BlockRange::new(0, block_header.num_records); + let rec = reader_block.iter_records(range).next().unwrap(); + assert_eq!(rec.sseq(), seq); + } + // ==================== push()/validate_record() error paths ==================== #[test] diff --git a/src/error.rs b/src/error.rs index 7773e01..ef20cbb 100644 --- a/src/error.rs +++ b/src/error.rs @@ -44,9 +44,13 @@ pub enum Error { #[error("Error determining BINSEQ format: {0}")] FormatError(#[from] FormatError), - /// Errors from the bitnuc dependency for nucleotide encoding/decoding + /// Errors from the deprecated bitnuc dependency for nucleotide encoding/decoding (bq/vbq) #[error("Bitnuc error: {0}")] - BitnucError(#[from] bitnuc::Error), + BitnucError(#[from] bitnuc_deprec::Error), + + /// Errors from the bitnuc dependency for nucleotide encoding/decoding (cbq) + #[error("Bitnuc error: {0}")] + BitnucEncodingError(#[from] bitnuc::BitnucError), /// Conversion errors from anyhow errors #[cfg(feature = "anyhow")] diff --git a/src/lib.rs b/src/lib.rs index 00a71e9..7a3ccb6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -115,7 +115,7 @@ pub use record::{BinseqRecord, SequencingRecord, SequencingRecordBuilder}; pub use write::{BinseqWriter, BinseqWriterBuilder}; /// Re-export `bitnuc::BitSize` -pub use bitnuc::BitSize; +pub use bitnuc_deprec::BitSize; /// Default quality score for BINSEQ readers without quality scores pub(crate) const DEFAULT_QUALITY_SCORE: u8 = b'?'; diff --git a/src/record/binseq_record.rs b/src/record/binseq_record.rs index 246f201..7641799 100644 --- a/src/record/binseq_record.rs +++ b/src/record/binseq_record.rs @@ -1,5 +1,5 @@ use auto_impl::auto_impl; -use bitnuc::BitSize; +use bitnuc_deprec::BitSize; use crate::Result; diff --git a/src/vbq/header.rs b/src/vbq/header.rs index 6aa8ded..8d5fbf9 100644 --- a/src/vbq/header.rs +++ b/src/vbq/header.rs @@ -14,7 +14,7 @@ use std::io::{Read, Write}; -use bitnuc::BitSize; +use bitnuc_deprec::BitSize; use byteorder::{ByteOrder, LittleEndian}; use crate::error::{HeaderError, ReadError, Result}; diff --git a/src/vbq/reader.rs b/src/vbq/reader.rs index f9925ac..ef5b931 100644 --- a/src/vbq/reader.rs +++ b/src/vbq/reader.rs @@ -54,7 +54,7 @@ use std::ops::Range; use std::path::Path; use std::sync::Arc; -use bitnuc::BitSize; +use bitnuc_deprec::BitSize; use byteorder::{ByteOrder, LittleEndian}; use memmap2::Mmap; use zstd::zstd_safe; @@ -486,11 +486,11 @@ impl RecordBlock { match self.bitsize { BitSize::Two => { let num_bp = self.sequences.len() * 32; - bitnuc::twobit::decode(&self.sequences, num_bp, &mut self.dbuf) + bitnuc_deprec::twobit::decode(&self.sequences, num_bp, &mut self.dbuf) } BitSize::Four => { let num_bp = self.sequences.len() * 16; - bitnuc::fourbit::decode(&self.sequences, num_bp, &mut self.dbuf) + bitnuc_deprec::fourbit::decode(&self.sequences, num_bp, &mut self.dbuf) } }?; Ok(()) diff --git a/src/vbq/writer.rs b/src/vbq/writer.rs index 71d05f6..75e0d8b 100644 --- a/src/vbq/writer.rs +++ b/src/vbq/writer.rs @@ -64,7 +64,7 @@ use std::io::Write; -use bitnuc::BitSize; +use bitnuc_deprec::BitSize; use byteorder::{LittleEndian, WriteBytesExt}; use rand::SeedableRng; use rand::rngs::SmallRng; diff --git a/src/write.rs b/src/write.rs index 9d0ab17..30dd1c2 100644 --- a/src/write.rs +++ b/src/write.rs @@ -1500,7 +1500,7 @@ mod tests { #[test] fn test_configured_size_vbq_single_minimal() { - use bitnuc::BitSize; + use bitnuc_deprec::BitSize; let record = minimal_single_record(); // s_len (8) + x_len (8) + s_seq (32 nucs = 1 word = 8 bytes) let size = record.configured_size_vbq(false, false, false, false, BitSize::Two); @@ -1509,7 +1509,7 @@ mod tests { #[test] fn test_configured_size_vbq_single_with_flags() { - use bitnuc::BitSize; + use bitnuc_deprec::BitSize; let record = full_single_record(); // s_len (8) + x_len (8) + flag (8) + s_seq (8) let size = record.configured_size_vbq(false, true, false, false, BitSize::Two); @@ -1518,7 +1518,7 @@ mod tests { #[test] fn test_configured_size_vbq_single_with_all() { - use bitnuc::BitSize; + use bitnuc_deprec::BitSize; let record = full_single_record(); // s_len (8) + x_len (8) + flag (8) + s_seq (8) + s_qual (32) + s_header_len (8) + s_header (5) let size = record.configured_size_vbq(false, true, true, true, BitSize::Two); @@ -1527,7 +1527,7 @@ mod tests { #[test] fn test_configured_size_vbq_paired_minimal() { - use bitnuc::BitSize; + use bitnuc_deprec::BitSize; let record = full_paired_record(); // s_len (8) + x_len (8) + s_seq (8) + x_seq (8) let size = record.configured_size_vbq(true, false, false, false, BitSize::Two); @@ -1536,7 +1536,7 @@ mod tests { #[test] fn test_configured_size_vbq_paired_with_all() { - use bitnuc::BitSize; + use bitnuc_deprec::BitSize; let record = full_paired_record(); // s_len (8) + x_len (8) + flag (8) + s_seq (8) + x_seq (8) // + s_qual (32) + x_qual (32) @@ -1547,7 +1547,7 @@ mod tests { #[test] fn test_configured_size_vbq_paired_record_single_writer() { - use bitnuc::BitSize; + use bitnuc_deprec::BitSize; // A paired record being written to a single-end writer // should only count R1 data let record = full_paired_record(); @@ -1558,7 +1558,7 @@ mod tests { #[test] fn test_configured_size_vbq_four_bit_encoding() { - use bitnuc::BitSize; + use bitnuc_deprec::BitSize; let record = minimal_single_record(); // With 4-bit encoding: 2 nucleotides per byte, 16 per word // 32 nucleotides = 2 words = 16 bytes