rakata_formats/erf/
index.rs

1//! Lazy ERF-family index: tables in memory, entry bytes on demand.
2//!
3//! [`read_erf`](super::read_erf) buffers a whole archive and decodes every
4//! resource up front, which is the wrong shape for a game install: mounting
5//! a module or a save means paying for megabytes of resource data to answer
6//! lookups that touch a handful of entries. [`ErfIndex`] parses the header,
7//! the localized strings, and both entry tables once, then reads an entry's
8//! bytes only when asked for them.
9//!
10//! Because the backing store is a [`SectionReader`](crate::section_reader::SectionReader), an index can be built
11//! over a window *inside* another archive. That is what makes a save game
12//! readable without unpacking it: per-module archives live at offsets inside
13//! `SAVEGAME.sav`, and [`ErfIndex::entry_section`] hands back the window to
14//! index one in place.
15
16use std::collections::HashMap;
17use std::fs::File;
18use std::io::{Read, Seek};
19use std::path::Path;
20use std::sync::{Arc, Mutex};
21
22use rakata_core::{ResRef, ResourceTypeCode, StrRef};
23
24use super::layout::{self, ErfHeader};
25use super::{ErfBinaryError, ErfFileType, ErfLocalizedString, ErfReadOptions, FILE_HEADER_SIZE};
26use crate::archive::EntryDefect;
27use crate::section_reader::SectionReader;
28
29/// One indexed resource: its identity and where its bytes live.
30///
31/// Offsets are relative to the start of the archive, which for a nested
32/// archive is the start of its window rather than the start of the file.
33#[derive(Debug, Clone, PartialEq, Eq)]
34pub struct ErfIndexEntry {
35    /// Resource name.
36    pub resref: ResRef,
37    /// Resource type code.
38    pub resource_type: ResourceTypeCode,
39    /// Byte offset of the resource data within the archive, as declared by
40    /// the entry table.
41    pub offset: u64,
42    /// Byte length of the resource data, as declared by the entry table.
43    pub size: u64,
44    /// Set when the entry cannot be read, leaving the rest of the archive
45    /// usable. Reading a defective entry returns this as an error.
46    pub defect: Option<EntryDefect>,
47}
48
49impl ErfIndexEntry {
50    /// Returns whether this entry's bytes can be read.
51    pub fn is_readable(&self) -> bool {
52        self.defect.is_none()
53    }
54}
55
56/// A lazily-read ERF-family archive.
57///
58/// Holds the archive's tables and metadata; resource bytes are read from the
59/// backing source on each request. Cheap to construct relative to
60/// [`read_erf`](super::read_erf) on any archive whose data dwarfs its tables.
61#[derive(Debug)]
62pub struct ErfIndex<R = File> {
63    section: SectionReader<R>,
64    file_type: ErfFileType,
65    build_year: u32,
66    build_day: u32,
67    description_strref: StrRef,
68    localized_strings: Vec<ErfLocalizedString>,
69    entries: Vec<ErfIndexEntry>,
70    lookup: HashMap<(ResRef, ResourceTypeCode), usize>,
71}
72
73impl ErfIndex<File> {
74    /// Opens an archive from disk and indexes it.
75    ///
76    /// Errors when the file cannot be opened or its tables do not parse.
77    pub fn open(path: impl AsRef<Path>) -> Result<Self, ErfBinaryError> {
78        Self::open_with_options(path, ErfReadOptions::default())
79    }
80
81    /// Opens an archive from disk with explicit read options.
82    pub fn open_with_options(
83        path: impl AsRef<Path>,
84        options: ErfReadOptions,
85    ) -> Result<Self, ErfBinaryError> {
86        let file = File::open(path)?;
87        let len = file.metadata()?.len();
88        Self::new_with_options(
89            SectionReader::whole(Arc::new(Mutex::new(file)), len),
90            options,
91        )
92    }
93}
94
95impl<R: Read + Seek> ErfIndex<R> {
96    /// Indexes the archive occupying `section`.
97    pub fn new(section: SectionReader<R>) -> Result<Self, ErfBinaryError> {
98        Self::new_with_options(section, ErfReadOptions::default())
99    }
100
101    /// Indexes the archive occupying `section` with explicit read options.
102    ///
103    /// Reads the header and both entry tables immediately. A table that will
104    /// not parse fails here, because nothing about the archive is knowable
105    /// without it. An entry whose declared byte range escapes the archive
106    /// does not: it is marked with a defect and the rest stays readable.
107    pub fn new_with_options(
108        section: SectionReader<R>,
109        options: ErfReadOptions,
110    ) -> Result<Self, ErfBinaryError> {
111        let header_bytes = read_region(&section, 0, u64_from(FILE_HEADER_SIZE)?, "ERF header")?;
112        let header = ErfHeader::parse(&header_bytes, options)?;
113
114        let localized_strings = if header.language_count > 0 {
115            let block = read_region(
116                &section,
117                u64_from(header.localized_strings_offset)?,
118                u64_from(header.localized_string_size)?,
119                "localized string block",
120            )?;
121            layout::parse_localized_strings(&block, header.language_count)?
122        } else {
123            Vec::new()
124        };
125
126        let keys_table = read_region(
127            &section,
128            u64_from(header.keys_offset)?,
129            u64_from(header.keys_table_size()?)?,
130            "keys table",
131        )?;
132        let resources_table = read_region(
133            &section,
134            u64_from(header.resources_offset)?,
135            u64_from(header.resources_table_size()?)?,
136            "resources table",
137        )?;
138
139        let mut entries = Vec::with_capacity(header.entry_count);
140        let mut lookup = HashMap::with_capacity(header.entry_count);
141        for key_index in 0..header.entry_count {
142            let key = layout::parse_key_entry(&keys_table, key_index)?;
143            if key.resource_id >= header.entry_count {
144                return Err(ErfBinaryError::InvalidData(format!(
145                    "keys[{key_index}] references missing resource id {}",
146                    key.resource_id
147                )));
148            }
149            let (data_offset, data_size) =
150                layout::parse_resource_entry(&resources_table, key.resource_id)?;
151            let offset = u64_from(data_offset)?;
152            let size = u64_from(data_size)?;
153            // An entry pointing outside the file poisons only itself: the
154            // rest of the archive stays readable, and a caller wanting
155            // strictness inspects the defects.
156            let defect = EntryDefect::check_bounds(offset, size, section.len());
157
158            // First entry wins on a duplicate name, matching the eager
159            // reader's first-match lookup.
160            lookup
161                .entry((key.resref, key.resource_type))
162                .or_insert(entries.len());
163            entries.push(ErfIndexEntry {
164                resref: key.resref,
165                resource_type: key.resource_type,
166                offset,
167                size,
168                defect,
169            });
170        }
171
172        Ok(Self {
173            section,
174            file_type: header.file_type,
175            build_year: header.build_year,
176            build_day: header.build_day,
177            description_strref: header.description_strref,
178            localized_strings,
179            entries,
180            lookup,
181        })
182    }
183
184    /// Reads the bytes of the resource named `resref` with `resource_type`.
185    ///
186    /// Returns `Ok(None)` when the archive holds no such resource, and `Err`
187    /// only when a matching entry exists but its bytes cannot be read.
188    pub fn resolve(
189        &self,
190        resref: &ResRef,
191        resource_type: ResourceTypeCode,
192    ) -> Result<Option<Vec<u8>>, ErfBinaryError> {
193        let Some(index) = self.lookup.get(&(*resref, resource_type)) else {
194            return Ok(None);
195        };
196        self.read_entry(*index).map(Some)
197    }
198
199    /// Reads the bytes of the entry at `index` in table order.
200    ///
201    /// Errors when `index` is out of range, the entry carries a defect, or
202    /// the read itself fails.
203    pub fn read_entry(&self, index: usize) -> Result<Vec<u8>, ErfBinaryError> {
204        let entry = self.entries.get(index).ok_or_else(|| {
205            ErfBinaryError::InvalidData(format!("resource index {index} is out of range"))
206        })?;
207        if let Some(defect) = &entry.defect {
208            return Err(ErfBinaryError::InvalidHeader(format!(
209                "resource data[{index}] ({}) is unreadable: {defect}",
210                entry.resref
211            )));
212        }
213        self.entry_section(index)
214            .ok_or_else(|| {
215                ErfBinaryError::InvalidHeader(format!("resource data[{index}] exceeds file bounds"))
216            })
217            .and_then(|section| {
218                section.read_all().map_err(|source| {
219                    ErfBinaryError::InvalidData(format!(
220                        "failed reading {} bytes for {}: {source}",
221                        entry.size, entry.resref
222                    ))
223                })
224            })
225    }
226
227    /// Returns a window over the entry's bytes without reading them.
228    ///
229    /// This is how a nested archive is opened in place: pass the returned
230    /// window to [`ErfIndex::new`] to index an archive stored as an entry of
231    /// this one. Returns `None` when `index` is out of range or the entry
232    /// carries a defect.
233    pub fn entry_section(&self, index: usize) -> Option<SectionReader<R>> {
234        let entry = self.entries.get(index)?;
235        if entry.defect.is_some() {
236            return None;
237        }
238        self.section.section(entry.offset, entry.size)
239    }
240
241    /// Returns a window over the named resource's bytes without reading them.
242    pub fn resource_section(
243        &self,
244        resref: &ResRef,
245        resource_type: ResourceTypeCode,
246    ) -> Option<SectionReader<R>> {
247        let index = *self.lookup.get(&(*resref, resource_type))?;
248        self.entry_section(index)
249    }
250
251    /// Iterates every resource, pairing its entry with freshly-read bytes.
252    ///
253    /// Each item is read on demand, so a caller that stops early pays only
254    /// for what it consumed.
255    pub fn iter_resources(
256        &self,
257    ) -> impl Iterator<Item = Result<(&ErfIndexEntry, Vec<u8>), ErfBinaryError>> + '_ {
258        self.entries
259            .iter()
260            .enumerate()
261            .map(move |(index, entry)| self.read_entry(index).map(|bytes| (entry, bytes)))
262    }
263}
264
265impl<R> ErfIndex<R> {
266    /// Returns the indexed entries in table order.
267    pub fn entries(&self) -> &[ErfIndexEntry] {
268        &self.entries
269    }
270
271    /// Returns the number of indexed resources.
272    pub fn len(&self) -> usize {
273        self.entries.len()
274    }
275
276    /// Returns whether the archive holds no resources.
277    pub fn is_empty(&self) -> bool {
278        self.entries.is_empty()
279    }
280
281    /// Returns the archive's container signature.
282    pub fn file_type(&self) -> ErfFileType {
283        self.file_type
284    }
285
286    /// Returns the archive's build year field.
287    pub fn build_year(&self) -> u32 {
288        self.build_year
289    }
290
291    /// Returns the archive's build day field.
292    pub fn build_day(&self) -> u32 {
293        self.build_day
294    }
295
296    /// Returns the archive's description string reference.
297    pub fn description_strref(&self) -> StrRef {
298        self.description_strref
299    }
300
301    /// Returns the archive's localized description strings.
302    ///
303    /// These are parsed up front: they are kilobyte-scale at most and are
304    /// useful metadata for tooling that lists archives.
305    pub fn localized_strings(&self) -> &[ErfLocalizedString] {
306        &self.localized_strings
307    }
308
309    /// Returns whether the archive holds the named resource.
310    ///
311    /// True even when the entry is defective: the archive claims the resource
312    /// exists, it just cannot be read.
313    pub fn contains(&self, resref: &ResRef, resource_type: ResourceTypeCode) -> bool {
314        self.lookup.contains_key(&(*resref, resource_type))
315    }
316
317    /// Iterates the entries that cannot be read, with their table positions.
318    ///
319    /// Whatever mounts this archive is expected to surface these rather than
320    /// let a partially-readable archive pass as intact.
321    pub fn defects(&self) -> impl Iterator<Item = (usize, &ErfIndexEntry, &EntryDefect)> + '_ {
322        self.entries
323            .iter()
324            .enumerate()
325            .filter_map(|(index, entry)| entry.defect.as_ref().map(|defect| (index, entry, defect)))
326    }
327
328    /// Returns whether any entry in the archive is unreadable.
329    pub fn has_defects(&self) -> bool {
330        self.entries.iter().any(|entry| entry.defect.is_some())
331    }
332}
333
334/// Reads a named region out of the backing section.
335///
336/// Maps an out-of-bounds region onto the same "exceeds file bounds" wording
337/// the eager reader produces for the equivalent failure.
338fn read_region<R: Read + Seek>(
339    section: &SectionReader<R>,
340    offset: u64,
341    len: u64,
342    name: &str,
343) -> Result<Vec<u8>, ErfBinaryError> {
344    section
345        .section(offset, len)
346        .ok_or_else(|| ErfBinaryError::InvalidHeader(format!("{name} exceeds file bounds")))?
347        .read_all()
348        .map_err(ErfBinaryError::Io)
349}
350
351/// Widens a table-derived `usize` to `u64` for offset arithmetic.
352fn u64_from(value: usize) -> Result<u64, ErfBinaryError> {
353    u64::try_from(value)
354        .map_err(|_| ErfBinaryError::InvalidHeader("offset exceeds addressable range".into()))
355}
356
357#[cfg(test)]
358mod tests {
359    use super::*;
360    use crate::erf::{read_erf_from_bytes, write_erf_to_vec, Erf, ErfResource};
361    use std::io::Cursor;
362
363    const TEST_ERF: &[u8] = include_bytes!(concat!(
364        env!("CARGO_MANIFEST_DIR"),
365        "/../../fixtures/test.erf"
366    ));
367
368    fn index_over(bytes: &[u8]) -> ErfIndex<Cursor<Vec<u8>>> {
369        let len = u64::try_from(bytes.len()).expect("fixture fits in u64");
370        let source = Arc::new(Mutex::new(Cursor::new(bytes.to_vec())));
371        ErfIndex::new(SectionReader::whole(source, len)).expect("fixture indexes")
372    }
373
374    fn archive_with(resources: Vec<ErfResource>) -> Vec<u8> {
375        let mut erf = Erf::new(ErfFileType::Erf);
376        erf.resources = resources;
377        write_erf_to_vec(&erf).expect("archive writes")
378    }
379
380    #[test]
381    fn indexes_the_same_resources_the_eager_reader_finds() {
382        let eager = read_erf_from_bytes(TEST_ERF).expect("fixture parses eagerly");
383        let index = index_over(TEST_ERF);
384
385        assert_eq!(index.len(), eager.resources.len());
386        assert_eq!(index.file_type(), eager.file_type);
387
388        for resource in &eager.resources {
389            let bytes = index
390                .resolve(&resource.resref, resource.resource_type)
391                .expect("resolve succeeds")
392                .expect("resource is present");
393            assert_eq!(bytes, resource.data, "bytes differ for {}", resource.resref);
394        }
395    }
396
397    #[test]
398    fn a_missing_resource_is_a_clean_miss_not_an_error() {
399        let index = index_over(TEST_ERF);
400        let absent = ResRef::new("nope").expect("valid resref");
401
402        let found = index
403            .resolve(&absent, ResourceTypeCode::from_raw_id(0xFFFF))
404            .expect("a miss is not an error");
405
406        assert!(found.is_none());
407        assert!(!index.contains(&absent, ResourceTypeCode::from_raw_id(0xFFFF)));
408    }
409
410    #[test]
411    fn iterating_yields_every_resource_with_its_bytes() {
412        let eager = read_erf_from_bytes(TEST_ERF).expect("fixture parses eagerly");
413        let index = index_over(TEST_ERF);
414
415        let collected = index
416            .iter_resources()
417            .collect::<Result<Vec<_>, _>>()
418            .expect("iteration succeeds");
419
420        assert_eq!(collected.len(), eager.resources.len());
421        for ((entry, bytes), expected) in collected.iter().zip(&eager.resources) {
422            assert_eq!(entry.resref, expected.resref);
423            assert_eq!(bytes, &expected.data);
424        }
425    }
426
427    #[test]
428    fn a_nested_archive_is_indexed_in_place() {
429        // The save-game shape: an archive stored as a resource inside another.
430        let inner_bytes = archive_with(vec![ErfResource {
431            resref: ResRef::new("buried").expect("valid resref"),
432            resource_type: ResourceTypeCode::from_raw_id(2037),
433            data: b"treasure".to_vec(),
434        }]);
435        let outer_bytes = archive_with(vec![ErfResource {
436            resref: ResRef::new("nested").expect("valid resref"),
437            resource_type: ResourceTypeCode::from_raw_id(2057),
438            data: inner_bytes.clone(),
439        }]);
440
441        let outer = index_over(&outer_bytes);
442        let window = outer
443            .resource_section(
444                &ResRef::new("nested").expect("valid resref"),
445                ResourceTypeCode::from_raw_id(2057),
446            )
447            .expect("nested window exists");
448        let inner = ErfIndex::new(window).expect("nested archive indexes");
449
450        let bytes = inner
451            .resolve(
452                &ResRef::new("buried").expect("valid resref"),
453                ResourceTypeCode::from_raw_id(2037),
454            )
455            .expect("nested resolve succeeds")
456            .expect("nested resource is present");
457
458        assert_eq!(bytes, b"treasure");
459        // The nested index never copied the inner archive out of the outer one.
460        assert_eq!(inner.len(), 1);
461    }
462
463    #[test]
464    fn a_truncated_archive_either_fails_to_index_or_reports_defects() {
465        // Which of the two depends on where the cut lands: losing the tables
466        // makes the archive unknowable, while losing only payload leaves the
467        // tables describing entries that now point past the end.
468        let truncated = &TEST_ERF[..TEST_ERF.len() / 2];
469        let len = u64::try_from(truncated.len()).expect("fits in u64");
470        let source = Arc::new(Mutex::new(Cursor::new(truncated.to_vec())));
471
472        match ErfIndex::new(SectionReader::whole(source, len)) {
473            Err(_) => {}
474            Ok(index) => assert!(
475                index.has_defects(),
476                "a truncated archive must not index clean"
477            ),
478        }
479    }
480
481    #[test]
482    fn one_out_of_bounds_entry_does_not_cost_the_whole_archive() {
483        // Point the first resource past the end of the file, leaving the rest
484        // of the tables intact: the salvage case a damaged save presents.
485        let mut bytes = TEST_ERF.to_vec();
486        let header = crate::erf::layout::ErfHeader::parse(&bytes, ErfReadOptions::default())
487            .expect("fixture header parses");
488        let resources_offset = header.resources_offset;
489        bytes[resources_offset..resources_offset + 4].copy_from_slice(&u32::MAX.to_le_bytes());
490
491        let index = index_over(&bytes);
492        let eager_count = read_erf_from_bytes(TEST_ERF)
493            .expect("fixture parses eagerly")
494            .resources
495            .len();
496
497        // Every entry is still indexed, and exactly one is unreadable.
498        assert_eq!(index.len(), eager_count);
499        assert_eq!(index.defects().count(), 1);
500        assert!(index.has_defects());
501
502        let (position, entry, _) = index.defects().next().expect("one defect");
503        assert!(!entry.is_readable());
504        assert!(index.read_entry(position).is_err());
505        assert!(index.entry_section(position).is_none());
506
507        // The healthy neighbours still serve their bytes.
508        for other in 0..index.len() {
509            if other != position {
510                assert!(
511                    index.read_entry(other).is_ok(),
512                    "entry {other} should still read"
513                );
514            }
515        }
516    }
517
518    #[test]
519    fn entry_metadata_is_available_without_reading_bytes() {
520        let index = index_over(TEST_ERF);
521
522        assert!(!index.is_empty());
523        for entry in index.entries() {
524            assert!(entry.offset + entry.size <= u64::try_from(TEST_ERF.len()).expect("fits"));
525        }
526    }
527}
rakata_formats/erf/index.rs

rakata_formats/erf/
index.rs
