rakata_formats/twoda/
reader.rs

1//! 2DA binary reader.
2
3use std::io::Read;
4
5use rakata_core::decode_text_strict;
6
7use super::{
8    binary, TwoDa, TwoDaBinaryError, TwoDaBinaryOptions, TwoDaRow, TWODA_MAGIC, TWODA_VERSION_V2B,
9};
10
11/// Reads a binary 2DA from a reader.
12///
13/// The stream is consumed from its current position.
14///
15/// # Errors
16///
17/// The same as [`read_twoda_with_options`] under the default options, which
18/// decode text as Windows-1252.
19#[cfg_attr(
20    feature = "tracing",
21    tracing::instrument(level = "debug", skip(reader))
22)]
23pub fn read_twoda<R: Read>(reader: &mut R) -> Result<TwoDa, TwoDaBinaryError> {
24    read_twoda_with_options(reader, TwoDaBinaryOptions::default())
25}
26
27/// Reads a binary 2DA from a reader with explicit text options.
28///
29/// The stream is consumed from its current position.
30///
31/// # Errors
32///
33/// [`TwoDaBinaryError::Io`] when the stream will not read to end, and whatever
34/// [`read_twoda_from_bytes_with_options`] reports for the bytes it collected.
35#[cfg_attr(
36    feature = "tracing",
37    tracing::instrument(level = "debug", skip(reader, options))
38)]
39pub fn read_twoda_with_options<R: Read>(
40    reader: &mut R,
41    options: TwoDaBinaryOptions,
42) -> Result<TwoDa, TwoDaBinaryError> {
43    let mut bytes = Vec::new();
44    reader.read_to_end(&mut bytes)?;
45    read_twoda_from_bytes_with_options(&bytes, options)
46}
47
48/// Reads a binary 2DA (`2DA V2.b`) from in-memory bytes.
49///
50/// # Errors
51///
52/// The same as [`read_twoda_from_bytes_with_options`] under the default
53/// options, which decode text as Windows-1252.
54#[cfg_attr(
55    feature = "tracing",
56    tracing::instrument(level = "debug", skip(bytes), fields(bytes_len = bytes.len()))
57)]
58pub fn read_twoda_from_bytes(bytes: &[u8]) -> Result<TwoDa, TwoDaBinaryError> {
59    read_twoda_from_bytes_with_options(bytes, TwoDaBinaryOptions::default())
60}
61
62/// Reads a binary 2DA (`2DA V2.b`) from in-memory bytes with explicit text options.
63///
64/// # Errors
65///
66/// [`TwoDaBinaryError::InvalidMagic`] when the signature is not `2DA `, and
67/// [`TwoDaBinaryError::InvalidVersion`] for a version other than `V2.b`.
68///
69/// [`TwoDaBinaryError::InvalidHeader`] covers every structural failure, and
70/// it is the arm most malformed files land on: a truncated header, a missing
71/// terminator for the column list or a row label, an offset table or cell
72/// region running past the end of `bytes`, a cell offset outside the cell
73/// region, a cell with no NUL terminator, and any cursor arithmetic that
74/// overflows. The message names which one.
75///
76/// [`TwoDaBinaryError::TextDecoding`] when a column name, row label or cell
77/// is not valid text in `options.text_encoding`, naming which one.
78#[cfg_attr(
79    feature = "tracing",
80    tracing::instrument(level = "debug", skip(bytes, options), fields(bytes_len = bytes.len()))
81)]
82pub fn read_twoda_from_bytes_with_options(
83    bytes: &[u8],
84    options: TwoDaBinaryOptions,
85) -> Result<TwoDa, TwoDaBinaryError> {
86    if bytes.len() < 9 {
87        return Err(TwoDaBinaryError::InvalidHeader(
88            "file smaller than 2DA header".into(),
89        ));
90    }
91
92    let magic = binary::read_fourcc(bytes, 0)?;
93    if magic != TWODA_MAGIC {
94        return Err(TwoDaBinaryError::InvalidMagic(magic));
95    }
96
97    let version = binary::read_fourcc(bytes, 4)?;
98    if version != TWODA_VERSION_V2B {
99        return Err(TwoDaBinaryError::InvalidVersion(version));
100    }
101
102    if bytes[8] != b'\n' {
103        return Err(TwoDaBinaryError::InvalidHeader(
104            "missing newline after 2DA version".into(),
105        ));
106    }
107
108    let (headers, mut cursor) = read_header_run(bytes, 9, options)?;
109
110    let row_count = usize::try_from(binary::read_u32(bytes, cursor)?)
111        .map_err(|_| TwoDaBinaryError::InvalidHeader("row count does not fit usize".into()))?;
112    cursor = cursor
113        .checked_add(4)
114        .ok_or_else(|| TwoDaBinaryError::InvalidHeader("row count cursor overflow".into()))?;
115
116    let mut rows = Vec::with_capacity(row_count);
117    for row_index in 0..row_count {
118        let label_end = find_delimiter(bytes, cursor).ok_or_else(|| {
119            TwoDaBinaryError::InvalidHeader(format!("missing delimiter for row label {row_index}"))
120        })?;
121        let label = decode_text_strict(&bytes[cursor..label_end], options.text_encoding).map_err(
122            |source| TwoDaBinaryError::TextDecoding {
123                context: format!("row label[{row_index}]"),
124                source,
125            },
126        )?;
127        rows.push(TwoDaRow {
128            label,
129            cells: vec![String::new(); headers.len()],
130        });
131        cursor = label_end
132            .checked_add(1)
133            .ok_or_else(|| TwoDaBinaryError::InvalidHeader("row label cursor overflow".into()))?;
134    }
135
136    let cell_count = row_count
137        .checked_mul(headers.len())
138        .ok_or_else(|| TwoDaBinaryError::InvalidHeader("cell count overflow".into()))?;
139    let offsets_bytes = cell_count
140        .checked_mul(2)
141        .ok_or_else(|| TwoDaBinaryError::InvalidHeader("offset table size overflow".into()))?;
142    if cursor
143        .checked_add(offsets_bytes + 2)
144        .is_none_or(|end| end > bytes.len())
145    {
146        return Err(TwoDaBinaryError::InvalidHeader(
147            "offset table exceeds file length".into(),
148        ));
149    }
150
151    let mut cell_offsets = Vec::with_capacity(cell_count);
152    for _ in 0..cell_count {
153        cell_offsets.push(binary::read_u16(bytes, cursor)?);
154        cursor = cursor
155            .checked_add(2)
156            .ok_or_else(|| TwoDaBinaryError::InvalidHeader("offset cursor overflow".into()))?;
157    }
158
159    let cell_data_size = usize::from(binary::read_u16(bytes, cursor)?);
160    cursor = cursor
161        .checked_add(2)
162        .ok_or_else(|| TwoDaBinaryError::InvalidHeader("cell data cursor overflow".into()))?;
163    let cell_data_end = cursor
164        .checked_add(cell_data_size)
165        .ok_or_else(|| TwoDaBinaryError::InvalidHeader("cell data end overflow".into()))?;
166    if cell_data_end > bytes.len() {
167        return Err(TwoDaBinaryError::InvalidHeader(
168            "cell string table exceeds file length".into(),
169        ));
170    }
171    let cell_data = &bytes[cursor..cell_data_end];
172
173    if !headers.is_empty() {
174        for (cell_index, offset) in cell_offsets.iter().enumerate() {
175            let offset = usize::from(*offset);
176            if offset >= cell_data_size {
177                return Err(TwoDaBinaryError::InvalidHeader(format!(
178                    "cell offset {offset} out of range for cell index {cell_index}"
179                )));
180            }
181
182            let value_end = find_byte(cell_data, offset, b'\0').ok_or_else(|| {
183                TwoDaBinaryError::InvalidHeader(format!(
184                    "missing NUL terminator for cell index {cell_index}"
185                ))
186            })?;
187            let row_index = cell_index / headers.len();
188            let column_index = cell_index % headers.len();
189            let value = decode_text_strict(&cell_data[offset..value_end], options.text_encoding)
190                .map_err(|source| TwoDaBinaryError::TextDecoding {
191                    context: format!("cell[{row_index}][{column_index}]"),
192                    source,
193                })?;
194            rows[row_index].cells[column_index] = value;
195        }
196    }
197
198    Ok(TwoDa { headers, rows })
199}
200
201/// Reads the column headers starting at `start`, and returns them with the
202/// offset just past the NUL that ends the run.
203///
204/// Each header is followed by a delimiter, a tab in most tables and a NUL in
205/// the zero-byte copies `global.rim` carries. The run ends at a NUL where a
206/// header would start: after the last header's tab in the tab form, and as the
207/// second of two NULs in the zero-byte form.
208///
209/// The first header's delimiter is the file's. A tab-form run whose last
210/// header lacks its tab ends at that header's NUL instead.
211fn read_header_run(
212    bytes: &[u8],
213    start: usize,
214    options: TwoDaBinaryOptions,
215) -> Result<(Vec<String>, usize), TwoDaBinaryError> {
216    let mut headers = Vec::new();
217    let mut cursor = start;
218    let mut delimiter = None;
219    loop {
220        match bytes.get(cursor) {
221            None => {
222                return Err(TwoDaBinaryError::InvalidHeader(
223                    "missing header terminator".into(),
224                ))
225            }
226            Some(b'\0') => return Ok((headers, cursor + 1)),
227            Some(_) => {}
228        }
229        let end = find_delimiter(bytes, cursor).ok_or_else(|| {
230            TwoDaBinaryError::InvalidHeader(format!(
231                "missing delimiter for header {}",
232                headers.len()
233            ))
234        })?;
235        let header =
236            decode_text_strict(&bytes[cursor..end], options.text_encoding).map_err(|source| {
237                TwoDaBinaryError::TextDecoding {
238                    context: format!("header[{}]", headers.len()),
239                    source,
240                }
241            })?;
242        headers.push(header);
243        let used = bytes[end];
244        if *delimiter.get_or_insert(used) == b'\t' && used == b'\0' {
245            return Ok((headers, end + 1));
246        }
247        cursor = end + 1;
248    }
249}
250
251fn find_byte(bytes: &[u8], start: usize, needle: u8) -> Option<usize> {
252    bytes
253        .get(start..)?
254        .iter()
255        .position(|byte| *byte == needle)
256        .map(|relative| start + relative)
257}
258
259/// Where the label starting at `start` ends: its tab, or its NUL in the
260/// zero-byte form. A label holds neither byte, so either one is its end.
261fn find_delimiter(bytes: &[u8], start: usize) -> Option<usize> {
262    bytes
263        .get(start..)?
264        .iter()
265        .position(|byte| matches!(*byte, b'\t' | b'\0'))
266        .map(|relative| start + relative)
267}
268
269#[cfg(test)]
270mod tests {
271    use super::*;
272    use crate::twoda::{
273        write_twoda_to_vec, write_twoda_to_vec_with_options, TwoDaBinaryOptions, TWODA_MAGIC,
274        TWODA_VERSION_V2B,
275    };
276    use rakata_core::TextEncoding;
277
278    #[test]
279    fn roundtrip_twoda_binary() {
280        let mut table = TwoDa::new(vec!["col3".into(), "col2".into(), "col1".into()]);
281        table
282            .push_row("10", vec!["ghi".into(), "def".into(), "abc".into()])
283            .expect("valid row");
284        table
285            .push_row("1", vec!["123".into(), "ghi".into(), "def".into()])
286            .expect("valid row");
287        table
288            .push_row("2", vec!["abc".into(), "".into(), "123".into()])
289            .expect("valid row");
290
291        let bytes = write_twoda_to_vec(&table).expect("write should succeed");
292        let parsed = read_twoda_from_bytes(&bytes).expect("read should succeed");
293
294        assert_eq!(parsed, table);
295        assert_eq!(parsed.cell(0, "col1"), Some("abc"));
296        assert_eq!(parsed.cell(2, "col2"), Some(""));
297    }
298
299    #[test]
300    fn writer_is_deterministic_for_synthetic_twoda() {
301        let mut table = TwoDa::new(vec!["col3".into(), "col2".into(), "col1".into()]);
302        table
303            .push_row("10", vec!["ghi".into(), "def".into(), "abc".into()])
304            .expect("valid row");
305        table
306            .push_row("1", vec!["123".into(), "ghi".into(), "def".into()])
307            .expect("valid row");
308        table
309            .push_row("2", vec!["abc".into(), "".into(), "123".into()])
310            .expect("valid row");
311
312        let first = write_twoda_to_vec(&table).expect("first write should succeed");
313        let second = write_twoda_to_vec(&table).expect("second write should succeed");
314        assert_eq!(first, second, "canonical 2DA writer output drifted");
315    }
316
317    fn sample_table() -> TwoDa {
318        let mut table = TwoDa::new(vec!["col3".into(), "col2".into(), "col1".into()]);
319        table
320            .push_row("10", vec!["ghi".into(), "def".into(), "abc".into()])
321            .expect("valid row");
322        table
323            .push_row("1", vec!["123".into(), "".into(), "def".into()])
324            .expect("valid row");
325        table
326    }
327
328    /// The zero-byte form of a tab-form table, made the way the page
329    /// describes the twinned copies: each delimiter tab, after a column header
330    /// and after a row label, becomes a NUL, and no other byte moves.
331    fn zero_byte_form(tabbed: &[u8]) -> Vec<u8> {
332        let mut bytes = tabbed.to_vec();
333        let headers_end = find_byte(&bytes, 9, b'\0').expect("header terminator");
334        for byte in &mut bytes[9..headers_end] {
335            if *byte == b'\t' {
336                *byte = b'\0';
337            }
338        }
339        let mut cursor = headers_end + 1;
340        let row_count = usize::try_from(binary::read_u32(&bytes, cursor).expect("row count"))
341            .expect("row count fits");
342        cursor += 4;
343        for _ in 0..row_count {
344            let tab = find_byte(&bytes, cursor, b'\t').expect("row label tab");
345            bytes[tab] = b'\0';
346            cursor = tab + 1;
347        }
348        bytes
349    }
350
351    #[test]
352    fn a_zero_byte_delimited_table_reads_like_its_tab_form() {
353        let table = sample_table();
354        let tabbed = write_twoda_to_vec(&table).expect("write");
355        let zeroed = zero_byte_form(&tabbed);
356
357        let differing = tabbed.iter().zip(&zeroed).filter(|(a, b)| a != b).count();
358        assert_eq!(
359            differing,
360            table.headers.len() + table.rows.len(),
361            "one byte per column and per row, as the twinned tables differ"
362        );
363        let parsed = read_twoda_from_bytes(&zeroed).expect("zero-byte form reads");
364        assert_eq!(parsed, table);
365        assert_eq!(
366            parsed,
367            read_twoda_from_bytes(&tabbed).expect("tab form reads")
368        );
369    }
370
371    #[test]
372    fn a_tab_form_run_without_its_last_tab_still_reads() {
373        // The last header's tab is missing, so its NUL closes the run.
374        let tabbed = write_twoda_to_vec(&sample_table()).expect("write");
375        let headers_end = find_byte(&tabbed, 9, b'\0').expect("header terminator");
376        assert_eq!(
377            tabbed[headers_end - 1],
378            b'\t',
379            "the writer ends the run with a tab"
380        );
381        let mut bytes = tabbed[..headers_end - 1].to_vec();
382        bytes.extend_from_slice(&tabbed[headers_end..]);
383
384        let parsed = read_twoda_from_bytes(&bytes).expect("reads");
385        assert_eq!(parsed, sample_table());
386    }
387
388    #[test]
389    fn the_writer_keeps_writing_tabs() {
390        let tabbed = write_twoda_to_vec(&sample_table()).expect("write");
391        assert!(tabbed[9..].starts_with(b"col3\tcol2\tcol1\t\0"));
392        let headers_end = find_byte(&tabbed, 9, b'\0').expect("header terminator");
393        assert_eq!(&tabbed[headers_end + 1 + 4..][..3], b"10\t");
394    }
395
396    #[test]
397    fn rejects_truncated_header() {
398        let bytes = vec![0_u8; 8];
399        let err = read_twoda_from_bytes(&bytes).expect_err("must fail");
400        assert!(matches!(err, TwoDaBinaryError::InvalidHeader(_)));
401    }
402
403    #[test]
404    fn rejects_invalid_magic() {
405        let mut bytes = vec![0_u8; 9];
406        bytes[0..4].copy_from_slice(b"BAD!");
407        bytes[4..8].copy_from_slice(&TWODA_VERSION_V2B);
408        bytes[8] = b'\n';
409
410        let err = read_twoda_from_bytes(&bytes).expect_err("must fail");
411        assert!(matches!(err, TwoDaBinaryError::InvalidMagic(_)));
412    }
413
414    #[test]
415    fn rejects_invalid_version() {
416        let mut bytes = vec![0_u8; 9];
417        bytes[0..4].copy_from_slice(&TWODA_MAGIC);
418        bytes[4..8].copy_from_slice(b"V2.0");
419        bytes[8] = b'\n';
420
421        let err = read_twoda_from_bytes(&bytes).expect_err("must fail");
422        assert!(matches!(err, TwoDaBinaryError::InvalidVersion(_)));
423    }
424
425    #[test]
426    fn writer_rejects_unencodable_text() {
427        let mut table = TwoDa::new(vec!["col".into()]);
428        table
429            .push_row("0", vec!["emoji \u{1f600}".into()])
430            .expect("valid table shape");
431
432        let err = write_twoda_to_vec(&table).expect_err("must fail");
433        match err {
434            TwoDaBinaryError::TextEncoding { context, source } => {
435                assert!(context.contains("cell[0][0]"));
436                assert_eq!(source.character, '\u{1f600}');
437            }
438            other => panic!("unexpected error variant: {other}"),
439        }
440    }
441
442    #[test]
443    fn writer_rejects_row_width_mismatch() {
444        let mut table = TwoDa::new(vec!["col1".into(), "col2".into()]);
445        table.rows.push(TwoDaRow {
446            label: "0".into(),
447            cells: vec!["only_one".into()],
448        });
449
450        let err = write_twoda_to_vec(&table).expect_err("must fail");
451        assert!(matches!(err, TwoDaBinaryError::InvalidTable(_)));
452    }
453
454    #[test]
455    fn writer_deduplicates_identical_cell_strings() {
456        let mut table = TwoDa::new(vec!["a".into(), "b".into()]);
457        table
458            .push_row("0", vec!["same".into(), "x".into()])
459            .expect("valid row");
460        table
461            .push_row("1", vec!["same".into(), "y".into()])
462            .expect("valid row");
463
464        let bytes = write_twoda_to_vec(&table).expect("write should succeed");
465        let offsets = extract_cell_offsets(&bytes).expect("must parse offsets");
466        assert_eq!(offsets.len(), 4);
467        assert_eq!(offsets[0], offsets[2]);
468    }
469
470    #[test]
471    fn configurable_encoding_supports_cp1250_data() {
472        let mut table = TwoDa::new(vec!["text".into()]);
473        table
474            .push_row("0", vec!["Za\u{017c}\u{00f3}\u{0142}\u{0107} g\u{0119}\u{015b}l\u{0105} ja\u{017a}\u{0144}".into()])
475            .expect("valid row");
476        let options = TwoDaBinaryOptions {
477            text_encoding: TextEncoding::Windows1250,
478        };
479
480        let bytes = write_twoda_to_vec_with_options(&table, options).expect("write should succeed");
481        let parsed =
482            read_twoda_from_bytes_with_options(&bytes, options).expect("read should succeed");
483        assert_eq!(
484            parsed.cell(0, "text"),
485            Some(
486                "Za\u{017c}\u{00f3}\u{0142}\u{0107} g\u{0119}\u{015b}l\u{0105} ja\u{017a}\u{0144}"
487            )
488        );
489    }
490
491    #[test]
492    fn default_encoding_rejects_cp1250_specific_characters() {
493        let mut table = TwoDa::new(vec!["text".into()]);
494        table
495            .push_row("0", vec!["Za\u{017c}\u{00f3}\u{0142}\u{0107} g\u{0119}\u{015b}l\u{0105} ja\u{017a}\u{0144}".into()])
496            .expect("valid row");
497
498        let err = write_twoda_to_vec(&table).expect_err("must fail");
499        assert!(matches!(err, TwoDaBinaryError::TextEncoding { .. }));
500    }
501
502    fn extract_cell_offsets(bytes: &[u8]) -> Result<Vec<u16>, TwoDaBinaryError> {
503        let mut cursor = 9;
504        let headers_end = find_byte(bytes, cursor, b'\0')
505            .ok_or_else(|| TwoDaBinaryError::InvalidHeader("missing header terminator".into()))?;
506        cursor = headers_end + 1;
507
508        let row_count = usize::try_from(binary::read_u32(bytes, cursor)?)
509            .map_err(|_| TwoDaBinaryError::InvalidHeader("row count conversion failed".into()))?;
510        cursor += 4;
511        for _ in 0..row_count {
512            let label_end = find_byte(bytes, cursor, b'\t')
513                .ok_or_else(|| TwoDaBinaryError::InvalidHeader("missing row label tab".into()))?;
514            cursor = label_end + 1;
515        }
516
517        // Test fixture uses exactly 2 headers.
518        let cell_count = row_count * 2;
519        let mut offsets = Vec::with_capacity(cell_count);
520        for _ in 0..cell_count {
521            offsets.push(binary::read_u16(bytes, cursor)?);
522            cursor += 2;
523        }
524        Ok(offsets)
525    }
526}
rakata_formats/twoda/reader.rs

rakata_formats/twoda/
reader.rs
