rakata_formats/twoda/
mod.rs

1//! 2DA `V2.b` binary reader and writer.
2//!
3//! 2DA is a compact table format used for gameplay metadata. The binary form
4//! stores column headers, row labels, a cell-offset matrix, and a shared string
5//! table.
6//!
7//! ## Shape of the binary form
8//!
9//! Six blocks and no offsets to any of them: signature, column headers, a row
10//! count, row labels, a cell offset matrix, and a shared string table. Parsing
11//! is strictly sequential, so a malformed block does not cause one bad lookup,
12//! it invalidates everything after it.
13//!
14//! Cell offsets are `u16`, which caps the string table at 64 KiB. Deduplication
15//! is what keeps real tables inside that, since a 2DA column is typically a few
16//! distinct values repeated down thousands of rows -- which is why this writer
17//! deduplicates rather than merely producing deterministic output. Writing each
18//! cell's string separately can overflow a table the game ships happily.
19//!
20//! Block contents and the text form's differences are in
21//! `docs/src/formats/text/2da.md`, with the engine's own load sequence.
22
23mod reader;
24mod writer;
25
26pub use reader::{
27    read_twoda, read_twoda_from_bytes, read_twoda_from_bytes_with_options, read_twoda_with_options,
28};
29pub use writer::{
30    write_twoda, write_twoda_to_vec, write_twoda_to_vec_with_options, write_twoda_with_options,
31};
32
33use std::io::Read;
34use thiserror::Error;
35
36#[cfg(feature = "serde")]
37use serde::{Deserialize, Serialize};
38
39use rakata_core::{DecodeTextError, EncodeTextError, TextEncoding};
40
41use crate::binary::{self, DecodeBinary, EncodeBinary};
42
43/// 2DA file type marker.
44const TWODA_MAGIC: [u8; 4] = *b"2DA ";
45/// Binary 2DA format version used by KotOR.
46const TWODA_VERSION_V2B: [u8; 4] = *b"V2.b";
47
48/// Encoding options for binary 2DA reader/writer operations.
49#[derive(Debug, Clone, Copy, PartialEq, Eq)]
50pub struct TwoDaBinaryOptions {
51    /// Text encoding used for headers, row labels, and cell values.
52    pub text_encoding: TextEncoding,
53}
54
55impl Default for TwoDaBinaryOptions {
56    fn default() -> Self {
57        Self {
58            text_encoding: TextEncoding::Windows1252,
59        }
60    }
61}
62
63/// In-memory representation of a binary 2DA table.
64///
65/// The structure keeps header order and row order stable so a write->read
66/// roundtrip remains deterministic.
67#[derive(Debug, Clone, PartialEq, Eq)]
68#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
69pub struct TwoDa {
70    /// Ordered column headers.
71    pub headers: Vec<String>,
72    /// Ordered table rows.
73    pub rows: Vec<TwoDaRow>,
74}
75
76impl TwoDa {
77    /// Creates an empty table with no headers and no rows.
78    pub fn empty() -> Self {
79        Self {
80            headers: Vec::new(),
81            rows: Vec::new(),
82        }
83    }
84
85    /// Creates an empty table with the provided headers.
86    pub fn new(headers: Vec<String>) -> Self {
87        Self {
88            headers,
89            rows: Vec::new(),
90        }
91    }
92
93    /// Appends a row, validating that cell count matches header count.
94    ///
95    /// # Errors
96    ///
97    /// [`TwoDaBinaryError::InvalidTable`] when `cells` is not as wide as the
98    /// header list, naming both widths. Nothing is appended in that case. The
99    /// cell contents are not checked here; a NUL or a tab inside one is
100    /// refused later, by the writer.
101    pub fn push_row(
102        &mut self,
103        label: impl Into<String>,
104        cells: Vec<String>,
105    ) -> Result<(), TwoDaBinaryError> {
106        if cells.len() != self.headers.len() {
107            return Err(TwoDaBinaryError::InvalidTable(format!(
108                "row width {} does not match header width {}",
109                cells.len(),
110                self.headers.len()
111            )));
112        }
113        self.rows.push(TwoDaRow {
114            label: label.into(),
115            cells,
116        });
117        Ok(())
118    }
119
120    /// Returns the number of rows.
121    pub fn row_count(&self) -> usize {
122        self.rows.len()
123    }
124
125    /// Returns the number of columns.
126    pub fn column_count(&self) -> usize {
127        self.headers.len()
128    }
129
130    /// Returns a cell by row index and column header name.
131    pub fn cell(&self, row_index: usize, column_name: &str) -> Option<&str> {
132        let column_index = self
133            .headers
134            .iter()
135            .position(|header| header == column_name)?;
136        self.rows
137            .get(row_index)
138            .and_then(|row| row.cells.get(column_index))
139            .map(String::as_str)
140    }
141}
142
143impl DecodeBinary for TwoDa {
144    type Error = TwoDaBinaryError;
145
146    fn decode_binary(bytes: &[u8]) -> Result<Self, Self::Error> {
147        read_twoda_from_bytes(bytes)
148    }
149}
150
151impl EncodeBinary for TwoDa {
152    type Error = TwoDaBinaryError;
153
154    fn encode_binary(&self) -> Result<Vec<u8>, Self::Error> {
155        write_twoda_to_vec(self)
156    }
157}
158
159/// One row in a 2DA table.
160#[derive(Debug, Clone, PartialEq, Eq)]
161#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
162pub struct TwoDaRow {
163    /// Row label written in the row-label table.
164    pub label: String,
165    /// Ordered cell values aligned with [`TwoDa::headers`].
166    pub cells: Vec<String>,
167}
168
169/// Errors produced while parsing or serializing binary 2DA data.
170#[derive(Debug, Error)]
171pub enum TwoDaBinaryError {
172    /// I/O read/write failure.
173    #[error(transparent)]
174    Io(#[from] std::io::Error),
175    /// Header magic does not match `2DA `.
176    #[error("invalid 2DA magic: {0:?}")]
177    InvalidMagic([u8; 4]),
178    /// Header version is unsupported.
179    #[error("invalid 2DA version: {0:?}")]
180    InvalidVersion([u8; 4]),
181    /// Header/body layout is invalid or truncated.
182    #[error("invalid 2DA header: {0}")]
183    InvalidHeader(String),
184    /// In-memory table content is structurally invalid for writing.
185    #[error("invalid 2DA table: {0}")]
186    InvalidTable(String),
187    /// Value cannot fit the target on-disk integer width.
188    #[error("value overflow while writing field `{0}`")]
189    ValueOverflow(&'static str),
190    /// Text cannot be represented losslessly as the configured target encoding.
191    #[error("2DA text encoding failed for {context}: {source}")]
192    TextEncoding {
193        /// Context of the value being encoded.
194        context: String,
195        /// Source encoding error with exact character location.
196        #[source]
197        source: EncodeTextError,
198    },
199    /// Text bytes cannot be decoded losslessly using the configured encoding.
200    #[error("2DA text decoding failed for {context}: {source}")]
201    TextDecoding {
202        /// Context of the value being decoded.
203        context: String,
204        /// Source decoding error with byte position details.
205        #[source]
206        source: DecodeTextError,
207    },
208}
209
210impl From<binary::BinaryLayoutError> for TwoDaBinaryError {
211    fn from(error: binary::BinaryLayoutError) -> Self {
212        Self::InvalidHeader(error.to_string())
213    }
214}
215
216fn validate_twoda(twoda: &TwoDa) -> Result<(), TwoDaBinaryError> {
217    for (index, header) in twoda.headers.iter().enumerate() {
218        if header.contains('\0') || header.contains('\t') {
219            return Err(TwoDaBinaryError::InvalidTable(format!(
220                "header[{index}] contains reserved delimiter"
221            )));
222        }
223    }
224    for (row_index, row) in twoda.rows.iter().enumerate() {
225        if row.cells.len() != twoda.headers.len() {
226            return Err(TwoDaBinaryError::InvalidTable(format!(
227                "row[{row_index}] width {} does not match header width {}",
228                row.cells.len(),
229                twoda.headers.len()
230            )));
231        }
232        if row.label.contains('\0') || row.label.contains('\t') {
233            return Err(TwoDaBinaryError::InvalidTable(format!(
234                "row label[{row_index}] contains reserved delimiter"
235            )));
236        }
237        for (column_index, value) in row.cells.iter().enumerate() {
238            if value.contains('\0') {
239                return Err(TwoDaBinaryError::InvalidTable(format!(
240                    "cell[{row_index}][{column_index}] contains NUL byte"
241                )));
242            }
243        }
244    }
245    Ok(())
246}
247
248//
249// Serde Support (CSV/JSON)
250//
251
252#[cfg(feature = "serde")]
253mod serde_impl {
254    use super::*;
255
256    pub mod csv_impl {
257        use super::*;
258        use csv::{ReaderBuilder, StringRecord, WriterBuilder};
259        use std::io::{Cursor, Write};
260
261        /// Reads a 2DA from CSV format.
262        ///
263        /// # Errors
264        ///
265        /// [`TwoDaBinaryError::InvalidTable`] for everything: a CSV the parser
266        /// rejects, a record it cannot read, and a row whose width does not
267        /// match the header row. The underlying `csv` error is carried as its
268        /// text rather than as a typed source, so a caller can show it and not
269        /// match on it.
270        pub fn read_twoda_from_csv<R: Read>(reader: &mut R) -> Result<TwoDa, TwoDaBinaryError> {
271            let mut csv_reader = ReaderBuilder::new()
272                .has_headers(true)
273                .flexible(true)
274                .trim(::csv::Trim::All)
275                .from_reader(reader);
276
277            let headers: Vec<String> = csv_reader
278                .headers()
279                .map_err(|e| TwoDaBinaryError::InvalidTable(e.to_string()))?
280                .iter()
281                .map(|s| s.to_string())
282                .collect();
283
284            // Interoperability Convention:
285            // The first column in 2DA CSVs is reserved for the row label.
286            // Some tools explicitly name it "label", while others leave it implicit.
287            // The first column is always stripped from the header list and treated as metadata.
288            let actual_headers = if headers.is_empty() {
289                Vec::new()
290            } else {
291                headers[1..].to_vec()
292            };
293
294            let mut twoda = TwoDa::new(actual_headers);
295
296            for result in csv_reader.records() {
297                let record = result.map_err(|e| TwoDaBinaryError::InvalidTable(e.to_string()))?;
298                if record.is_empty() {
299                    continue;
300                }
301
302                // First field is the row label
303                let label = record.get(0).unwrap_or_default().to_string();
304                // Remaining fields are cells
305                let cells: Vec<String> = record.iter().skip(1).map(|s| s.to_string()).collect();
306
307                // Ensure row width matches the header count.
308                if cells.len() != twoda.headers.len() {
309                    return Err(TwoDaBinaryError::InvalidTable(format!(
310                        "row width mismatch: expected {}, got {}",
311                        twoda.headers.len(),
312                        cells.len()
313                    )));
314                }
315
316                twoda.push_row(label, cells)?;
317            }
318
319            Ok(twoda)
320        }
321
322        /// Writes a 2DA to CSV format.
323        ///
324        /// # Errors
325        ///
326        /// [`TwoDaBinaryError::InvalidTable`] for a table the binary format
327        /// cannot express, checked in full before anything is written, and
328        /// for a write or flush the `csv` writer refuses. The same validation
329        /// as the binary writer, so a table that will not round-trip through
330        /// one will not round-trip through the other.
331        pub fn write_twoda_to_csv<W: Write>(
332            writer: &mut W,
333            twoda: &TwoDa,
334        ) -> Result<(), TwoDaBinaryError> {
335            validate_twoda(twoda)?;
336
337            let mut csv_writer = WriterBuilder::new().has_headers(true).from_writer(writer);
338
339            // Write header row: label column + actual column headers
340            let mut header_record = StringRecord::new();
341            header_record.push_field("label");
342            for header in &twoda.headers {
343                header_record.push_field(header);
344            }
345            csv_writer
346                .write_record(&header_record)
347                .map_err(|e| TwoDaBinaryError::InvalidTable(e.to_string()))?;
348
349            // Write data rows
350            for row in &twoda.rows {
351                let mut record = StringRecord::new();
352                record.push_field(&row.label);
353                for cell in &row.cells {
354                    record.push_field(cell);
355                }
356                csv_writer
357                    .write_record(&record)
358                    .map_err(|e| TwoDaBinaryError::InvalidTable(e.to_string()))?;
359            }
360
361            csv_writer
362                .flush()
363                .map_err(|e| TwoDaBinaryError::InvalidTable(e.to_string()))?;
364
365            Ok(())
366        }
367
368        /// Serializes a 2DA to a CSV byte vector.
369        ///
370        /// # Errors
371        ///
372        /// The same as [`write_twoda_to_csv`].
373        pub fn write_twoda_to_csv_vec(twoda: &TwoDa) -> Result<Vec<u8>, TwoDaBinaryError> {
374            let mut cursor = Cursor::new(Vec::new());
375            write_twoda_to_csv(&mut cursor, twoda)?;
376            Ok(cursor.into_inner())
377        }
378
379        /// Parses a 2DA from CSV bytes.
380        ///
381        /// # Errors
382        ///
383        /// The same as [`read_twoda_from_csv`].
384        pub fn read_twoda_from_csv_bytes(bytes: &[u8]) -> Result<TwoDa, TwoDaBinaryError> {
385            let mut cursor = Cursor::new(bytes);
386            read_twoda_from_csv(&mut cursor)
387        }
388    }
389
390    pub mod json_impl {
391        use super::*;
392        use serde_json::{from_slice, from_str, to_string_pretty, to_vec};
393
394        /// Serializes a 2DA to JSON.
395        ///
396        /// # Errors
397        ///
398        /// [`TwoDaBinaryError::InvalidTable`] carrying the `serde_json`
399        /// message. A `TwoDa` is strings all the way down, so nothing in it
400        /// resists serialization and this arm is not reachable in practice.
401        pub fn write_twoda_to_json(twoda: &TwoDa) -> Result<String, TwoDaBinaryError> {
402            to_string_pretty(twoda).map_err(|e| TwoDaBinaryError::InvalidTable(e.to_string()))
403        }
404
405        /// Serializes a 2DA to JSON bytes.
406        ///
407        /// # Errors
408        ///
409        /// The same as [`write_twoda_to_json`].
410        pub fn write_twoda_to_json_vec(twoda: &TwoDa) -> Result<Vec<u8>, TwoDaBinaryError> {
411            to_vec(twoda).map_err(|e| TwoDaBinaryError::InvalidTable(e.to_string()))
412        }
413
414        /// Deserializes a 2DA from JSON.
415        ///
416        /// # Errors
417        ///
418        /// [`TwoDaBinaryError::InvalidTable`] carrying the `serde_json`
419        /// message, for JSON that is malformed or does not have a `TwoDa`'s
420        /// shape. No 2DA-level validation runs here, so a table this accepts
421        /// can still be refused by the writer.
422        pub fn read_twoda_from_json(json: &str) -> Result<TwoDa, TwoDaBinaryError> {
423            from_str(json).map_err(|e| TwoDaBinaryError::InvalidTable(e.to_string()))
424        }
425
426        /// Deserializes a 2DA from JSON bytes.
427        ///
428        /// # Errors
429        ///
430        /// The same as [`read_twoda_from_json`].
431        pub fn read_twoda_from_json_bytes(bytes: &[u8]) -> Result<TwoDa, TwoDaBinaryError> {
432            from_slice(bytes).map_err(|e| TwoDaBinaryError::InvalidTable(e.to_string()))
433        }
434    }
435}
436
437#[cfg(feature = "serde")]
438pub use serde_impl::csv_impl::{
439    read_twoda_from_csv, read_twoda_from_csv_bytes, write_twoda_to_csv, write_twoda_to_csv_vec,
440};
441
442#[cfg(feature = "serde")]
443pub use serde_impl::json_impl::{
444    read_twoda_from_json, read_twoda_from_json_bytes, write_twoda_to_json, write_twoda_to_json_vec,
445};
446
447/// Auto-detects the format from file extension and reads accordingly.
448///
449/// Supported extensions (case-insensitive):
450/// - `.2da`, `.bif`, `.mod`, `.erf`, `.rim` (or unknown) -> binary 2DA
451/// - `.csv` -> CSV format
452/// - `.json` -> JSON format
453///
454/// # Errors
455///
456/// Whatever the format it dispatches to reports. Without the `serde` feature,
457/// `.csv` and `.json` are [`TwoDaBinaryError::InvalidTable`] saying the
458/// feature is needed, rather than falling through to the binary reader.
459///
460/// An unrecognised extension is read as binary rather than refused, so a name
461/// that names no format fails as a bad 2DA rather than as a bad name.
462#[cfg_attr(
463    feature = "tracing",
464    tracing::instrument(level = "debug", skip(reader))
465)]
466pub fn read_twoda_auto<R: Read>(
467    reader: &mut R,
468    name_or_extension: &str,
469) -> Result<TwoDa, TwoDaBinaryError> {
470    let ext = if let Some(idx) = name_or_extension.rfind('.') {
471        &name_or_extension[idx + 1..]
472    } else {
473        name_or_extension
474    };
475
476    match ext.to_lowercase().as_str() {
477        "csv" => {
478            #[cfg(feature = "serde")]
479            {
480                read_twoda_from_csv(reader)
481            }
482            #[cfg(not(feature = "serde"))]
483            Err(TwoDaBinaryError::InvalidTable(
484                "CSV format requires serde feature".into(),
485            ))
486        }
487        "json" => {
488            #[cfg(feature = "serde")]
489            {
490                // JSON requires reading to end to parse
491                let mut bytes = Vec::new();
492                reader.read_to_end(&mut bytes)?;
493                read_twoda_from_json_bytes(&bytes)
494            }
495            #[cfg(not(feature = "serde"))]
496            Err(TwoDaBinaryError::InvalidTable(
497                "JSON format requires serde feature".into(),
498            ))
499        }
500        _ => read_twoda(reader),
501    }
502}
rakata_formats/twoda/mod.rs

rakata_formats/twoda/
mod.rs
