rakata_core/
text.rs

1use encoding_rs::{
2    DecoderResult, Encoding, BIG5, EUC_KR, GBK, SHIFT_JIS, WINDOWS_1250, WINDOWS_1251,
3    WINDOWS_1252, WINDOWS_1253, WINDOWS_1254, WINDOWS_1255, WINDOWS_1256, WINDOWS_1257,
4    WINDOWS_1258, WINDOWS_874,
5};
6use thiserror::Error;
7
8use crate::LanguageId;
9
10/// Text encodings currently used by supported KotOR formats.
11#[derive(Debug, Clone, Copy, PartialEq, Eq)]
12#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
13pub enum TextEncoding {
14    /// Windows-1250 (Central/Eastern Europe).
15    Windows1250,
16    /// Windows-1251 (Cyrillic).
17    Windows1251,
18    /// Windows-1252 single-byte encoding used by TLK and many text payloads.
19    Windows1252,
20    /// Windows-1253 (Greek).
21    Windows1253,
22    /// Windows-1254 (Turkish).
23    Windows1254,
24    /// Windows-1255 (Hebrew).
25    Windows1255,
26    /// Windows-1256 (Arabic).
27    Windows1256,
28    /// Windows-1257 (Baltic).
29    Windows1257,
30    /// Windows-1258 (Vietnamese).
31    Windows1258,
32    /// Windows-874 (Thai).
33    Windows874,
34    /// Shift-JIS (Japanese, cp932-compatible family).
35    ShiftJis,
36    /// GBK (Simplified Chinese, cp936-compatible family).
37    Gbk,
38    /// Big5 (Traditional Chinese, cp950-compatible family).
39    Big5,
40    /// EUC-KR (Korean, cp949-compatible family).
41    EucKr,
42}
43
44/// Error returned when strict text encoding would lose information.
45#[derive(Debug, Clone, PartialEq, Eq, Error)]
46#[error("character {character:?} at char index {char_index} is not encodable as {encoding:?}")]
47pub struct EncodeTextError {
48    /// Character position (by `char` index) that could not be encoded.
49    pub char_index: usize,
50    /// Unicode character that is not representable in the target encoding.
51    pub character: char,
52    /// Target encoding that rejected the character.
53    pub encoding: TextEncoding,
54}
55
56/// Error returned when strict byte decoding encounters malformed input.
57#[derive(Debug, Clone, PartialEq, Eq, Error)]
58#[error("malformed byte sequence at byte index {byte_index} while decoding {encoding:?}")]
59pub struct DecodeTextError {
60    /// Byte offset where decoding first failed.
61    pub byte_index: usize,
62    /// Source encoding that rejected the byte sequence.
63    pub encoding: TextEncoding,
64}
65
66/// Error returned when a language ID does not map to a supported text encoding.
67#[derive(Debug, Clone, PartialEq, Eq, Error)]
68#[error("unsupported language id {} for text encoding", language_id.raw())]
69pub struct LanguageEncodingError {
70    /// Unsupported language ID.
71    pub language_id: LanguageId,
72}
73
74/// Decodes bytes into Unicode text.
75///
76/// For single-byte encodings such as Windows-1252, decoding is lossless across
77/// all byte values.
78pub fn decode_text(bytes: &[u8], encoding: TextEncoding) -> String {
79    let (decoded, _actual, _had_errors) = encoding_rs_codec(encoding).decode(bytes);
80    decoded.into_owned()
81}
82
83/// Decodes bytes into Unicode text using strict lossless behavior.
84///
85/// # Errors
86///
87/// [`DecodeTextError`] where the input holds a byte sequence `encoding` cannot
88/// represent, naming the first offending byte's index. Nothing is replaced;
89/// use [`decode_text`] for the lossy reading.
90pub fn decode_text_strict(bytes: &[u8], encoding: TextEncoding) -> Result<String, DecodeTextError> {
91    let codec = encoding_rs_codec(encoding);
92    let Some(decoded) = codec.decode_without_bom_handling_and_without_replacement(bytes) else {
93        let byte_index = first_malformed_byte_index(bytes, codec);
94        return Err(DecodeTextError {
95            byte_index,
96            encoding,
97        });
98    };
99    Ok(decoded.into_owned())
100}
101
102/// Encodes Unicode text into bytes using strict lossless behavior.
103///
104/// # Errors
105///
106/// [`EncodeTextError`] where a character is not representable in `encoding`,
107/// naming the first one and its index. Nothing is substituted.
108pub fn encode_text(input: &str, encoding: TextEncoding) -> Result<Vec<u8>, EncodeTextError> {
109    let codec = encoding_rs_codec(encoding);
110    let (encoded, _actual, had_errors) = codec.encode(input);
111    if !had_errors {
112        return Ok(encoded.into_owned());
113    }
114
115    let (char_index, character) = first_unencodable(input, codec);
116    Err(EncodeTextError {
117        char_index,
118        character,
119        encoding,
120    })
121}
122
123/// Resolves the text encoding for a KotOR language ID.
124///
125/// This mapping is shared by TLK and GFF localized-string paths so behavior
126/// stays centralized and avoids cross-crate drift.
127///
128/// # Errors
129///
130/// [`LanguageEncodingError`] for language IDs 70 through 72, the Armenian,
131/// Georgian and Tamil legacy codepages, which are the only ones deliberately
132/// left unmapped. Every other unrecognised ID falls back to Windows-1252
133/// rather than failing.
134pub fn text_encoding_for_language(
135    language_id: impl Into<LanguageId>,
136) -> Result<TextEncoding, LanguageEncodingError> {
137    let language_id = language_id.into();
138    let encoding = match language_id.raw() {
139        // Official KotOR releases.
140        0..=4 => TextEncoding::Windows1252,
141        5 => TextEncoding::Windows1250,
142        // Central/Eastern European.
143        32..=40 | 104 => TextEncoding::Windows1250,
144        // Cyrillic.
145        41..=46 => TextEncoding::Windows1251,
146        // Greek.
147        47 => TextEncoding::Windows1253,
148        // Turkish-family.
149        48..=50 => TextEncoding::Windows1254,
150        // Hebrew.
151        51 => TextEncoding::Windows1255,
152        // Arabic-family.
153        52..=54 => TextEncoding::Windows1256,
154        // Baltic.
155        55..=57 | 105 => TextEncoding::Windows1257,
156        // Vietnamese.
157        58 => TextEncoding::Windows1258,
158        // Thai.
159        59 => TextEncoding::Windows874,
160        // East Asian language families.
161        128 => TextEncoding::EucKr,
162        129 => TextEncoding::Big5,
163        130 => TextEncoding::Gbk,
164        131 => TextEncoding::ShiftJis,
165        // Optional enhancement track: add support for language IDs 70..=72
166        // (Armenian/Georgian/Tamil legacy codepages) only if downstream
167        // extended-localization use cases require it.
168        70..=72 => {
169            return Err(LanguageEncodingError { language_id });
170        }
171        // Defaults to Western European behavior used by most custom IDs.
172        _ => TextEncoding::Windows1252,
173    };
174    Ok(encoding)
175}
176
177fn encoding_rs_codec(encoding: TextEncoding) -> &'static Encoding {
178    match encoding {
179        TextEncoding::Windows1250 => WINDOWS_1250,
180        TextEncoding::Windows1251 => WINDOWS_1251,
181        TextEncoding::Windows1252 => WINDOWS_1252,
182        TextEncoding::Windows1253 => WINDOWS_1253,
183        TextEncoding::Windows1254 => WINDOWS_1254,
184        TextEncoding::Windows1255 => WINDOWS_1255,
185        TextEncoding::Windows1256 => WINDOWS_1256,
186        TextEncoding::Windows1257 => WINDOWS_1257,
187        TextEncoding::Windows1258 => WINDOWS_1258,
188        TextEncoding::Windows874 => WINDOWS_874,
189        TextEncoding::ShiftJis => SHIFT_JIS,
190        TextEncoding::Gbk => GBK,
191        TextEncoding::Big5 => BIG5,
192        TextEncoding::EucKr => EUC_KR,
193    }
194}
195
196fn first_unencodable(input: &str, codec: &'static Encoding) -> (usize, char) {
197    let mut buf = [0u8; 4];
198    for (char_index, ch) in input.chars().enumerate() {
199        let s = ch.encode_utf8(&mut buf);
200        let (_encoded, _actual, had_errors) = codec.encode(s);
201        if had_errors {
202            return (char_index, ch);
203        }
204    }
205    (0, '\u{FFFD}')
206}
207
208fn first_malformed_byte_index(bytes: &[u8], codec: &'static Encoding) -> usize {
209    let mut decoder = codec.new_decoder_without_bom_handling();
210    let mut output = String::new();
211    let mut input = bytes;
212    let mut consumed_total = 0usize;
213
214    loop {
215        let reserve = decoder
216            .max_utf8_buffer_length_without_replacement(input.len())
217            .unwrap_or(input.len().saturating_mul(4).saturating_add(16));
218        output.reserve(reserve.max(8));
219
220        let (result, read) = decoder.decode_to_string_without_replacement(input, &mut output, true);
221        consumed_total = consumed_total.saturating_add(read);
222        input = &input[read..];
223
224        match result {
225            DecoderResult::InputEmpty => return bytes.len(),
226            DecoderResult::OutputFull => continue,
227            DecoderResult::Malformed(_, _) => return consumed_total,
228        }
229    }
230}
231
232#[cfg(test)]
233mod tests {
234    use super::*;
235
236    #[test]
237    fn windows_1252_roundtrip_is_lossless_for_supported_text() {
238        let original = "“Hello” € test";
239        let encoded = encode_text(original, TextEncoding::Windows1252).expect("must encode");
240        let decoded = decode_text(&encoded, TextEncoding::Windows1252);
241        assert_eq!(decoded, original);
242    }
243
244    #[test]
245    fn strict_encoder_rejects_unencodable_characters() {
246        let err = encode_text("hello 😀", TextEncoding::Windows1252).expect_err("must fail");
247        assert_eq!(err.character, '😀');
248    }
249
250    #[test]
251    fn windows_1250_roundtrip_is_lossless_for_supported_text() {
252        let original = "Zażółć gęślą jaźń";
253        let encoded = encode_text(original, TextEncoding::Windows1250).expect("must encode");
254        let decoded = decode_text(&encoded, TextEncoding::Windows1250);
255        assert_eq!(decoded, original);
256    }
257
258    #[test]
259    fn strict_decoder_rejects_malformed_multibyte_sequences() {
260        let err = decode_text_strict(&[0x81], TextEncoding::ShiftJis).expect_err("must fail");
261        assert_eq!(err.encoding, TextEncoding::ShiftJis);
262    }
263
264    #[test]
265    fn strict_decoder_roundtrips_valid_multibyte_sequences() {
266        let encoded = encode_text("テスト", TextEncoding::ShiftJis).expect("must encode");
267        let decoded = decode_text_strict(&encoded, TextEncoding::ShiftJis).expect("must decode");
268        assert_eq!(decoded, "テスト");
269    }
270
271    #[test]
272    fn language_id_mapping_returns_expected_encodings() {
273        assert_eq!(
274            text_encoding_for_language(0).expect("english"),
275            TextEncoding::Windows1252
276        );
277        assert_eq!(
278            text_encoding_for_language(5).expect("polish"),
279            TextEncoding::Windows1250
280        );
281        assert_eq!(
282            text_encoding_for_language(59).expect("thai"),
283            TextEncoding::Windows874
284        );
285        assert_eq!(
286            text_encoding_for_language(131).expect("japanese"),
287            TextEncoding::ShiftJis
288        );
289    }
290
291    #[test]
292    fn language_id_mapping_rejects_unsupported_legacy_codepages() {
293        let err = text_encoding_for_language(70).expect_err("must fail");
294        assert_eq!(err.language_id, LanguageId::from_raw(70));
295    }
296}
rakata_core/text.rs

rakata_core/
text.rs
