rakata_formats/erf/
index.rs

1//! Lazy ERF-family index: tables in memory, entry bytes on demand.
2//!
3//! [`read_erf`](super::read_erf) buffers a whole archive and decodes every
4//! resource up front, which is the wrong shape for a game install: mounting
5//! a module or a save means paying for megabytes of resource data to answer
6//! lookups that touch a handful of entries. [`ErfIndex`] parses the header,
7//! the localized strings, and both entry tables once, then reads an entry's
8//! bytes only when asked for them.
9//!
10//! Because the backing store is a [`SectionReader`](crate::section_reader::SectionReader), an index can be built
11//! over a window *inside* another archive. That is what makes a save game
12//! readable without unpacking it: per-module archives live at offsets inside
13//! `SAVEGAME.sav`, and [`ErfIndex::entry_section`] hands back the window to
14//! index one in place.
15
16use std::collections::HashMap;
17use std::fs::File;
18use std::io::{Read, Seek};
19use std::path::Path;
20use std::sync::{Arc, Mutex};
21
22use rakata_core::{ResRef, ResourceTypeCode, StrRef, MAX_RESREF_LEN};
23
24use super::layout::{self, ErfHeader};
25use super::{ErfBinaryError, ErfFileType, ErfLocalizedString, ErfReadOptions, FILE_HEADER_SIZE};
26use crate::archive::EntryDefect;
27use crate::section_reader::SectionReader;
28
29/// One indexed resource: its identity and where its bytes live.
30///
31/// Offsets are relative to the start of the archive, which for a nested
32/// archive is the start of its window rather than the start of the file.
33#[derive(Debug, Clone, PartialEq, Eq)]
34pub struct ErfIndexEntry {
35    /// Resource name.
36    pub resref: ResRef,
37    /// The key exactly as the file spelled it.
38    ///
39    /// The lazy counterpart to
40    /// [`ErfResource::name_as_read`](super::ErfResource::name_as_read), and
41    /// always present here because an index is only ever read from a file.
42    /// Carried so a rewrite assembled through this index puts back the casing
43    /// the archive had rather than the folded form a [`ResRef`] keeps.
44    pub name_as_read: [u8; MAX_RESREF_LEN],
45    /// Resource type code.
46    pub resource_type: ResourceTypeCode,
47    /// Byte offset of the resource data within the archive, as declared by
48    /// the entry table.
49    pub offset: u64,
50    /// Byte length of the resource data, as declared by the entry table.
51    pub size: u64,
52    /// Set when the entry cannot be read, leaving the rest of the archive
53    /// usable. Reading a defective entry returns this as an error.
54    pub defect: Option<EntryDefect>,
55}
56
57impl ErfIndexEntry {
58    /// Returns whether this entry's bytes can be read.
59    pub fn is_readable(&self) -> bool {
60        self.defect.is_none()
61    }
62}
63
64/// A lazily-read ERF-family archive.
65///
66/// Holds the archive's tables and metadata; resource bytes are read from the
67/// backing source on each request. Cheap to construct relative to
68/// [`read_erf`](super::read_erf) on any archive whose data dwarfs its tables.
69#[derive(Debug)]
70pub struct ErfIndex<R = File> {
71    section: SectionReader<R>,
72    /// The parsed header, kept whole rather than copied field by field: its
73    /// table offsets are what anything rewriting the archive needs, and
74    /// re-deriving them would mean re-parsing with read options the index
75    /// does not keep.
76    header: ErfHeader,
77    localized_strings: Vec<ErfLocalizedString>,
78    entries: Vec<ErfIndexEntry>,
79    lookup: HashMap<(ResRef, ResourceTypeCode), usize>,
80}
81
82impl ErfIndex<File> {
83    /// Opens an archive from disk and indexes it.
84    ///
85    /// # Errors
86    ///
87    /// The same as [`Self::open_with_options`] under the default options.
88    pub fn open(path: impl AsRef<Path>) -> Result<Self, ErfBinaryError> {
89        Self::open_with_options(path, ErfReadOptions::default())
90    }
91
92    /// Opens an archive from disk with explicit read options.
93    ///
94    /// # Errors
95    ///
96    /// [`ErfBinaryError::Io`] when `path` will not open or its length cannot
97    /// be read, and whatever [`Self::new_with_options`] reports for the file
98    /// it opened.
99    pub fn open_with_options(
100        path: impl AsRef<Path>,
101        options: ErfReadOptions,
102    ) -> Result<Self, ErfBinaryError> {
103        let file = File::open(path)?;
104        let len = file.metadata()?.len();
105        Self::new_with_options(
106            SectionReader::whole(Arc::new(Mutex::new(file)), len),
107            options,
108        )
109    }
110}
111
112impl<R: Read + Seek> ErfIndex<R> {
113    /// Indexes the archive occupying `section`.
114    ///
115    /// # Errors
116    ///
117    /// The same as [`Self::new_with_options`] under the default options.
118    pub fn new(section: SectionReader<R>) -> Result<Self, ErfBinaryError> {
119        Self::new_with_options(section, ErfReadOptions::default())
120    }
121
122    /// Indexes the archive occupying `section` with explicit read options.
123    ///
124    /// Reads the header and both entry tables immediately. A table that will
125    /// not parse fails here, because nothing about the archive is knowable
126    /// without it. An entry whose declared byte range escapes the archive
127    /// does not: it is marked with a defect and the rest stays readable.
128    ///
129    /// # Errors
130    ///
131    /// [`ErfBinaryError::InvalidHeader`] when the header or either table
132    /// escapes the section's bounds, [`ErfBinaryError::InvalidMagic`] and
133    /// [`ErfBinaryError::InvalidVersion`] for a signature or version
134    /// `options.input` does not accept, and [`ErfBinaryError::InvalidData`]
135    /// when a key names a resource id past the entry count.
136    ///
137    /// [`ErfBinaryError::InvalidResRef`] for an entry name that is not a valid
138    /// resref, [`ErfBinaryError::TextDecoding`] for a localized string that is
139    /// not valid text in the archive's encoding, and [`ErfBinaryError::Io`]
140    /// when the backing source fails.
141    ///
142    /// An entry whose declared data range escapes the archive is not one of
143    /// these. It carries a defect instead, and only reading that entry fails.
144    pub fn new_with_options(
145        section: SectionReader<R>,
146        options: ErfReadOptions,
147    ) -> Result<Self, ErfBinaryError> {
148        let header_bytes = read_region(&section, 0, u64_from(FILE_HEADER_SIZE)?, "ERF header")?;
149        let header = ErfHeader::parse(&header_bytes, options)?;
150
151        let localized_strings = if header.language_count > 0 {
152            let block = read_region(
153                &section,
154                u64_from(header.localized_strings_offset)?,
155                u64_from(header.localized_string_size)?,
156                "localized string block",
157            )?;
158            layout::parse_localized_strings(&block, header.language_count)?
159        } else {
160            Vec::new()
161        };
162
163        let keys_table = read_region(
164            &section,
165            u64_from(header.keys_offset)?,
166            u64_from(header.keys_table_size()?)?,
167            "keys table",
168        )?;
169        let resources_table = read_region(
170            &section,
171            u64_from(header.resources_offset)?,
172            u64_from(header.resources_table_size()?)?,
173            "resources table",
174        )?;
175
176        let mut entries = Vec::with_capacity(header.entry_count);
177        let mut lookup = HashMap::with_capacity(header.entry_count);
178        for key_index in 0..header.entry_count {
179            let key = layout::parse_key_entry(&keys_table, key_index)?;
180            if key.resource_id >= header.entry_count {
181                return Err(ErfBinaryError::InvalidData(format!(
182                    "keys[{key_index}] references missing resource id {}",
183                    key.resource_id
184                )));
185            }
186            let (data_offset, data_size) =
187                layout::parse_resource_entry(&resources_table, key.resource_id)?;
188            let offset = u64_from(data_offset)?;
189            let size = u64_from(data_size)?;
190            // An entry pointing outside the file poisons only itself: the
191            // rest of the archive stays readable, and a caller wanting
192            // strictness inspects the defects.
193            let defect = EntryDefect::check_bounds(offset, size, section.len());
194
195            // First entry wins on a duplicate name, matching the eager
196            // reader's first-match lookup.
197            lookup
198                .entry((key.resref, key.resource_type))
199                .or_insert(entries.len());
200            entries.push(ErfIndexEntry {
201                resref: key.resref,
202                name_as_read: key.raw_resref,
203                resource_type: key.resource_type,
204                offset,
205                size,
206                defect,
207            });
208        }
209
210        Ok(Self {
211            section,
212            header,
213            localized_strings,
214            entries,
215            lookup,
216        })
217    }
218
219    /// Reads the bytes of the resource named `resref` with `resource_type`.
220    ///
221    /// Returns `Ok(None)` when the archive holds no such resource.
222    ///
223    /// # Errors
224    ///
225    /// Only once the lookup has matched, so an error here means a damaged
226    /// archive rather than a miss: the terms [`Self::read_entry`] gives.
227    pub fn resolve(
228        &self,
229        resref: &ResRef,
230        resource_type: ResourceTypeCode,
231    ) -> Result<Option<Vec<u8>>, ErfBinaryError> {
232        let Some(index) = self.lookup.get(&(*resref, resource_type)) else {
233            return Ok(None);
234        };
235        self.read_entry(*index).map(Some)
236    }
237
238    /// Reads the bytes of the entry at `index` in table order.
239    ///
240    /// # Errors
241    ///
242    /// [`ErfBinaryError::InvalidData`] when `index` is past the end of the
243    /// table or the backing read comes up short,
244    /// [`ErfBinaryError::InvalidHeader`] when the entry carries the defect
245    /// [`Self::new_with_options`] leaves on one whose range escapes the
246    /// archive, and [`ErfBinaryError::Io`] when the source fails.
247    pub fn read_entry(&self, index: usize) -> Result<Vec<u8>, ErfBinaryError> {
248        let entry = self.entries.get(index).ok_or_else(|| {
249            ErfBinaryError::InvalidData(format!("resource index {index} is out of range"))
250        })?;
251        if let Some(defect) = &entry.defect {
252            return Err(ErfBinaryError::InvalidHeader(format!(
253                "resource data[{index}] ({}) is unreadable: {defect}",
254                entry.resref
255            )));
256        }
257        self.entry_section(index)
258            .ok_or_else(|| {
259                ErfBinaryError::InvalidHeader(format!("resource data[{index}] exceeds file bounds"))
260            })
261            .and_then(|section| {
262                section.read_all().map_err(|source| {
263                    ErfBinaryError::InvalidData(format!(
264                        "failed reading {} bytes for {}: {source}",
265                        entry.size, entry.resref
266                    ))
267                })
268            })
269    }
270
271    /// Returns a window over the entry's bytes without reading them.
272    ///
273    /// This is how a nested archive is opened in place: pass the returned
274    /// window to [`ErfIndex::new`] to index an archive stored as an entry of
275    /// this one. Returns `None` when `index` is out of range or the entry
276    /// carries a defect.
277    pub fn entry_section(&self, index: usize) -> Option<SectionReader<R>> {
278        let entry = self.entries.get(index)?;
279        if entry.defect.is_some() {
280            return None;
281        }
282        self.section.section(entry.offset, entry.size)
283    }
284
285    /// Returns a window over the named resource's bytes without reading them.
286    pub fn resource_section(
287        &self,
288        resref: &ResRef,
289        resource_type: ResourceTypeCode,
290    ) -> Option<SectionReader<R>> {
291        let index = *self.lookup.get(&(*resref, resource_type))?;
292        self.entry_section(index)
293    }
294
295    /// Iterates every resource, pairing its entry with freshly-read bytes.
296    ///
297    /// Each item is read on demand, so a caller that stops early pays only
298    /// for what it consumed.
299    pub fn iter_resources(
300        &self,
301    ) -> impl Iterator<Item = Result<(&ErfIndexEntry, Vec<u8>), ErfBinaryError>> + '_ {
302        self.entries
303            .iter()
304            .enumerate()
305            .map(move |(index, entry)| self.read_entry(index).map(|bytes| (entry, bytes)))
306    }
307}
308
309impl<R> ErfIndex<R> {
310    /// Returns the indexed entries in table order.
311    pub fn entries(&self) -> &[ErfIndexEntry] {
312        &self.entries
313    }
314
315    /// Returns the number of indexed resources.
316    pub fn len(&self) -> usize {
317        self.entries.len()
318    }
319
320    /// Returns whether the archive holds no resources.
321    pub fn is_empty(&self) -> bool {
322        self.entries.is_empty()
323    }
324
325    /// Returns the archive's container signature.
326    pub fn file_type(&self) -> ErfFileType {
327        self.header.file_type
328    }
329
330    /// Returns the archive's build year field.
331    pub fn build_year(&self) -> u32 {
332        self.header.build_year
333    }
334
335    /// Returns the archive's build day field.
336    pub fn build_day(&self) -> u32 {
337        self.header.build_day
338    }
339
340    /// Returns the archive's description string reference.
341    pub fn description_strref(&self) -> StrRef {
342        self.header.description_strref
343    }
344
345    /// Returns the parsed header, table offsets included.
346    pub(super) fn header(&self) -> &ErfHeader {
347        &self.header
348    }
349
350    /// Returns the window the archive occupies in its backing source.
351    pub(super) fn section(&self) -> &SectionReader<R> {
352        &self.section
353    }
354
355    /// Returns the archive's localized description strings.
356    ///
357    /// These are parsed up front: they are kilobyte-scale at most and are
358    /// useful metadata for tooling that lists archives.
359    pub fn localized_strings(&self) -> &[ErfLocalizedString] {
360        &self.localized_strings
361    }
362
363    /// Returns the table position of the named resource, if it is there.
364    ///
365    /// First entry wins on a duplicate name, matching
366    /// [`resolve`](Self::resolve). The position is what
367    /// [`rewrite_erf`](super::rewrite_erf) takes to name an entry, since a
368    /// name is ambiguous in an archive that carries the same one twice.
369    pub fn position(&self, resref: &ResRef, resource_type: ResourceTypeCode) -> Option<usize> {
370        self.lookup.get(&(*resref, resource_type)).copied()
371    }
372
373    /// Returns whether the archive holds the named resource.
374    ///
375    /// True even when the entry is defective: the archive claims the resource
376    /// exists, it just cannot be read.
377    pub fn contains(&self, resref: &ResRef, resource_type: ResourceTypeCode) -> bool {
378        self.lookup.contains_key(&(*resref, resource_type))
379    }
380
381    /// Iterates the entries that cannot be read, with their table positions.
382    ///
383    /// Whatever mounts this archive is expected to surface these rather than
384    /// let a partially-readable archive pass as intact.
385    pub fn defects(&self) -> impl Iterator<Item = (usize, &ErfIndexEntry, &EntryDefect)> + '_ {
386        self.entries
387            .iter()
388            .enumerate()
389            .filter_map(|(index, entry)| entry.defect.as_ref().map(|defect| (index, entry, defect)))
390    }
391
392    /// Returns whether any entry in the archive is unreadable.
393    pub fn has_defects(&self) -> bool {
394        self.entries.iter().any(|entry| entry.defect.is_some())
395    }
396}
397
398/// Reads a named region out of the backing section.
399///
400/// Maps an out-of-bounds region onto the same "exceeds file bounds" wording
401/// the eager reader produces for the equivalent failure.
402fn read_region<R: Read + Seek>(
403    section: &SectionReader<R>,
404    offset: u64,
405    len: u64,
406    name: &str,
407) -> Result<Vec<u8>, ErfBinaryError> {
408    section
409        .section(offset, len)
410        .ok_or_else(|| ErfBinaryError::InvalidHeader(format!("{name} exceeds file bounds")))?
411        .read_all()
412        .map_err(ErfBinaryError::Io)
413}
414
415/// Widens a table-derived `usize` to `u64` for offset arithmetic.
416fn u64_from(value: usize) -> Result<u64, ErfBinaryError> {
417    u64::try_from(value)
418        .map_err(|_| ErfBinaryError::InvalidHeader("offset exceeds addressable range".into()))
419}
420
421#[cfg(test)]
422mod tests {
423    use super::*;
424    use crate::erf::{read_erf_from_bytes, write_erf_to_vec, Erf, ErfResource};
425    use std::io::Cursor;
426
427    const TEST_ERF: &[u8] = include_bytes!(concat!(
428        env!("CARGO_MANIFEST_DIR"),
429        "/../../fixtures/test.erf"
430    ));
431
432    fn index_over(bytes: &[u8]) -> ErfIndex<Cursor<Vec<u8>>> {
433        let len = u64::try_from(bytes.len()).expect("fixture fits in u64");
434        let source = Arc::new(Mutex::new(Cursor::new(bytes.to_vec())));
435        ErfIndex::new(SectionReader::whole(source, len)).expect("fixture indexes")
436    }
437
438    fn archive_with(resources: Vec<ErfResource>) -> Vec<u8> {
439        let mut erf = Erf::new(ErfFileType::Erf);
440        erf.resources = resources;
441        write_erf_to_vec(&erf).expect("archive writes")
442    }
443
444    #[test]
445    fn indexes_the_same_resources_the_eager_reader_finds() {
446        let eager = read_erf_from_bytes(TEST_ERF).expect("fixture parses eagerly");
447        let index = index_over(TEST_ERF);
448
449        assert_eq!(index.len(), eager.resources.len());
450        assert_eq!(index.file_type(), eager.file_type);
451
452        for resource in &eager.resources {
453            let bytes = index
454                .resolve(&resource.resref, resource.resource_type)
455                .expect("resolve succeeds")
456                .expect("resource is present");
457            assert_eq!(bytes, resource.data, "bytes differ for {}", resource.resref);
458        }
459    }
460
461    #[test]
462    fn a_missing_resource_is_a_clean_miss_not_an_error() {
463        let index = index_over(TEST_ERF);
464        let absent = ResRef::new("nope").expect("valid resref");
465
466        let found = index
467            .resolve(&absent, ResourceTypeCode::from_raw_id(0xFFFF))
468            .expect("a miss is not an error");
469
470        assert!(found.is_none());
471        assert!(!index.contains(&absent, ResourceTypeCode::from_raw_id(0xFFFF)));
472    }
473
474    #[test]
475    fn iterating_yields_every_resource_with_its_bytes() {
476        let eager = read_erf_from_bytes(TEST_ERF).expect("fixture parses eagerly");
477        let index = index_over(TEST_ERF);
478
479        let collected = index
480            .iter_resources()
481            .collect::<Result<Vec<_>, _>>()
482            .expect("iteration succeeds");
483
484        assert_eq!(collected.len(), eager.resources.len());
485        for ((entry, bytes), expected) in collected.iter().zip(&eager.resources) {
486            assert_eq!(entry.resref, expected.resref);
487            assert_eq!(bytes, &expected.data);
488        }
489    }
490
491    #[test]
492    fn a_nested_archive_is_indexed_in_place() {
493        // The save-game shape: an archive stored as a resource inside another.
494        let inner_bytes = archive_with(vec![ErfResource {
495            name_as_read: None,
496            resref: ResRef::new("buried").expect("valid resref"),
497            resource_type: ResourceTypeCode::from_raw_id(2037),
498            data: b"treasure".to_vec(),
499        }]);
500        let outer_bytes = archive_with(vec![ErfResource {
501            name_as_read: None,
502            resref: ResRef::new("nested").expect("valid resref"),
503            resource_type: ResourceTypeCode::from_raw_id(2057),
504            data: inner_bytes.clone(),
505        }]);
506
507        let outer = index_over(&outer_bytes);
508        let window = outer
509            .resource_section(
510                &ResRef::new("nested").expect("valid resref"),
511                ResourceTypeCode::from_raw_id(2057),
512            )
513            .expect("nested window exists");
514        let inner = ErfIndex::new(window).expect("nested archive indexes");
515
516        let bytes = inner
517            .resolve(
518                &ResRef::new("buried").expect("valid resref"),
519                ResourceTypeCode::from_raw_id(2037),
520            )
521            .expect("nested resolve succeeds")
522            .expect("nested resource is present");
523
524        assert_eq!(bytes, b"treasure");
525        // The nested index never copied the inner archive out of the outer one.
526        assert_eq!(inner.len(), 1);
527    }
528
529    #[test]
530    fn a_truncated_archive_either_fails_to_index_or_reports_defects() {
531        // Which of the two depends on where the cut lands: losing the tables
532        // makes the archive unknowable, while losing only payload leaves the
533        // tables describing entries that now point past the end.
534        let truncated = &TEST_ERF[..TEST_ERF.len() / 2];
535        let len = u64::try_from(truncated.len()).expect("fits in u64");
536        let source = Arc::new(Mutex::new(Cursor::new(truncated.to_vec())));
537
538        match ErfIndex::new(SectionReader::whole(source, len)) {
539            Err(_) => {}
540            Ok(index) => assert!(
541                index.has_defects(),
542                "a truncated archive must not index clean"
543            ),
544        }
545    }
546
547    #[test]
548    fn one_out_of_bounds_entry_does_not_cost_the_whole_archive() {
549        // Point the first resource past the end of the file, leaving the rest
550        // of the tables intact: the salvage case a damaged save presents.
551        let mut bytes = TEST_ERF.to_vec();
552        let header = crate::erf::layout::ErfHeader::parse(&bytes, ErfReadOptions::default())
553            .expect("fixture header parses");
554        let resources_offset = header.resources_offset;
555        bytes[resources_offset..resources_offset + 4].copy_from_slice(&u32::MAX.to_le_bytes());
556
557        let index = index_over(&bytes);
558        let eager_count = read_erf_from_bytes(TEST_ERF)
559            .expect("fixture parses eagerly")
560            .resources
561            .len();
562
563        // Every entry is still indexed, and exactly one is unreadable.
564        assert_eq!(index.len(), eager_count);
565        assert_eq!(index.defects().count(), 1);
566        assert!(index.has_defects());
567
568        let (position, entry, _) = index.defects().next().expect("one defect");
569        assert!(!entry.is_readable());
570        assert!(index.read_entry(position).is_err());
571        assert!(index.entry_section(position).is_none());
572
573        // The healthy neighbours still serve their bytes.
574        for other in 0..index.len() {
575            if other != position {
576                assert!(
577                    index.read_entry(other).is_ok(),
578                    "entry {other} should still read"
579                );
580            }
581        }
582    }
583
584    #[test]
585    fn entry_metadata_is_available_without_reading_bytes() {
586        let index = index_over(TEST_ERF);
587
588        assert!(!index.is_empty());
589        for entry in index.entries() {
590            assert!(entry.offset + entry.size <= u64::try_from(TEST_ERF.len()).expect("fits"));
591        }
592    }
593}
rakata_formats/erf/index.rs

rakata_formats/erf/
index.rs
