maxminddb/
reader.rs

1//! MaxMind DB reader implementation.
2
3use std::collections::HashSet;
4use std::fs;
5use std::net::IpAddr;
6use std::path::Path;
7
8use ipnetwork::IpNetwork;
9use serde::Deserialize;
10
11#[cfg(feature = "mmap")]
12pub use memmap2::Mmap;
13#[cfg(feature = "mmap")]
14use memmap2::MmapOptions;
15#[cfg(feature = "mmap")]
16use std::fs::File;
17
18use crate::decoder;
19use crate::error::MaxMindDbError;
20use crate::metadata::Metadata;
21use crate::result::{LookupResult, LookupSource, NetworkKind};
22use crate::within::{IpInt, Within, WithinNode, WithinOptions};
23
24/// Size of the data section separator (16 zero bytes).
25const DATA_SECTION_SEPARATOR_SIZE: usize = 16;
26const METADATA_START_MARKER: &[u8] = b"\xab\xcd\xefMaxMind.com";
27
28/// A reader for the MaxMind DB format. The lifetime `'data` is tied to the
29/// lifetime of the underlying buffer holding the contents of the database file.
30///
31/// The `Reader` supports both file-based and memory-mapped access to MaxMind
32/// DB files, including GeoIP2 and GeoLite2 databases.
33///
34/// # Features
35///
36/// - **`mmap`**: Enable memory-mapped file access for better performance
37/// - **`unsafe-str-decode`**: Skip UTF-8 validation when deserializing trusted
38///   database strings into Rust `str` or `String` values. Cross-runtime format
39///   adapters should prefer [`crate::deserialize_any_with_raw_strings()`].
40pub struct Reader<S: AsRef<[u8]>> {
41    pub(crate) buf: S,
42    /// Database metadata.
43    metadata: Metadata,
44    record_size: u16,
45    /// Cached `Metadata::node_count` for `Reader` search-tree traversal.
46    /// Use this instead of `metadata.node_count` for traversal invariants.
47    node_count: usize,
48    /// Cached bytes per node derived from `Metadata::record_size` for `Reader`.
49    /// Use this instead of `metadata.record_size` in lookup hot paths.
50    node_byte_size: usize,
51    pub(crate) ipv4_start: usize,
52    /// Bit depth at which ipv4_start was found (0-96). Used to calculate
53    /// correct prefix lengths for IPv4 lookups in IPv6 databases.
54    pub(crate) ipv4_start_bit_depth: usize,
55    pub(crate) pointer_base: usize,
56    pub(crate) data_section_len: usize,
57    pub(crate) metadata_start: usize,
58}
59
60impl<S: AsRef<[u8]>> std::fmt::Debug for Reader<S> {
61    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
62        f.debug_struct("Reader")
63            .field("buf_len", &self.buf.as_ref().len())
64            .field("metadata", &self.metadata)
65            .field("ipv4_start", &self.ipv4_start)
66            .field("ipv4_start_bit_depth", &self.ipv4_start_bit_depth)
67            .field("pointer_base", &self.pointer_base)
68            .field("data_section_len", &self.data_section_len)
69            .field("metadata_start", &self.metadata_start)
70            .finish_non_exhaustive()
71    }
72}
73
74#[cfg(feature = "mmap")]
75impl Reader<Mmap> {
76    /// Open a MaxMind DB database file by memory mapping it.
77    ///
78    /// # Safety
79    ///
80    /// The caller must ensure that the database file is not modified or
81    /// truncated while the `Reader` exists. Modifying or truncating the
82    /// file while it is memory-mapped will result in undefined behavior.
83    ///
84    /// # Example
85    ///
86    /// ```
87    /// # #[cfg(feature = "mmap")]
88    /// # {
89    /// // SAFETY: The database file will not be modified while the reader exists.
90    /// let reader = unsafe {
91    ///     maxminddb::Reader::open_mmap("test-data/test-data/GeoIP2-City-Test.mmdb")
92    /// }.unwrap();
93    /// # }
94    /// ```
95    pub unsafe fn open_mmap<P: AsRef<Path>>(database: P) -> Result<Reader<Mmap>, MaxMindDbError> {
96        let file_read = File::open(database)?;
97        let mmap = MmapOptions::new()
98            .map(&file_read)
99            .map_err(MaxMindDbError::Mmap)?;
100        Reader::from_source(mmap)
101    }
102}
103
104impl Reader<Vec<u8>> {
105    /// Open a MaxMind DB database file by loading it into memory.
106    ///
107    /// # Example
108    ///
109    /// ```
110    /// let reader = maxminddb::Reader::open_readfile(
111    ///     "test-data/test-data/GeoIP2-City-Test.mmdb").unwrap();
112    /// ```
113    pub fn open_readfile<P: AsRef<Path>>(database: P) -> Result<Reader<Vec<u8>>, MaxMindDbError> {
114        let buf: Vec<u8> = fs::read(&database)?; // IO error converted via #[from]
115        Reader::from_source(buf)
116    }
117}
118
119impl<'de, S: AsRef<[u8]>> Reader<S> {
120    /// Open a MaxMind DB database from anything that implements AsRef<[u8]>
121    ///
122    /// # Example
123    ///
124    /// ```
125    /// use std::fs;
126    /// let buf = fs::read("test-data/test-data/GeoIP2-City-Test.mmdb").unwrap();
127    /// let reader = maxminddb::Reader::from_source(buf).unwrap();
128    /// ```
129    pub fn from_source(buf: S) -> Result<Reader<S>, MaxMindDbError> {
130        let metadata_start = find_metadata_start(buf.as_ref())?;
131        // find_metadata_start returns the offset after the marker; the marker
132        // bytes are not part of the data section and must stay out of limits.
133        let data_section_end = metadata_marker_start(metadata_start)?;
134        let metadata_bytes = &buf.as_ref()[metadata_start..];
135        let mut type_decoder = decoder::Decoder::new(metadata_bytes, 0);
136        let metadata = Metadata::deserialize(&mut type_decoder)
137            .map_err(|error| error.with_invalid_database_offset_base(metadata_start))?;
138        validate_metadata_for_reader(&metadata)?;
139
140        let search_tree_size =
141            search_tree_size_bytes(metadata.node_count as usize, metadata.record_size as usize)?;
142        let record_size = metadata.record_size;
143        let node_count = metadata.node_count as usize;
144        let node_byte_size = record_size as usize / 4;
145        let pointer_base = search_tree_size
146            .checked_add(DATA_SECTION_SEPARATOR_SIZE)
147            .ok_or_else(|| {
148                MaxMindDbError::invalid_database(
149                    "the MaxMind DB file's search tree extends beyond the file",
150                )
151            })?;
152        validate_search_tree_layout(pointer_base, data_section_end)?;
153        let data_section_len = data_section_end - pointer_base;
154
155        let mut reader = Reader {
156            buf,
157            record_size,
158            node_count,
159            node_byte_size,
160            pointer_base,
161            data_section_len,
162            metadata_start,
163            metadata,
164            ipv4_start: 0,
165            ipv4_start_bit_depth: 0,
166        };
167        let (ipv4_start, ipv4_start_bit_depth) = reader.find_ipv4_start();
168        reader.ipv4_start = ipv4_start;
169        reader.ipv4_start_bit_depth = ipv4_start_bit_depth;
170
171        Ok(reader)
172    }
173
174    /// Returns database metadata.
175    ///
176    /// Metadata is validated when the reader is created and exposed by
177    /// reference so it cannot be mutated independently of cached reader state.
178    #[inline]
179    pub fn metadata(&self) -> &Metadata {
180        &self.metadata
181    }
182
183    /// Lookup an IP address in the database.
184    ///
185    /// Returns a [`LookupResult`] that can be used to:
186    /// - Check if data exists with [`has_data()`](LookupResult::has_data)
187    /// - Get the network containing the IP with [`network()`](LookupResult::network)
188    /// - Decode the full record with [`decode()`](LookupResult::decode)
189    /// - Decode a specific path with [`decode_path()`](LookupResult::decode_path)
190    ///
191    /// # Examples
192    ///
193    /// Basic city lookup:
194    /// ```
195    /// # use maxminddb::geoip2;
196    /// # use std::net::IpAddr;
197    /// # fn main() -> Result<(), maxminddb::MaxMindDbError> {
198    /// let reader = maxminddb::Reader::open_readfile(
199    ///     "test-data/test-data/GeoIP2-City-Test.mmdb")?;
200    ///
201    /// let ip: IpAddr = "89.160.20.128".parse().unwrap();
202    /// let result = reader.lookup(ip)?;
203    ///
204    /// if let Some(city) = result.decode::<geoip2::City>()? {
205    ///     // Access nested structs directly - no Option unwrapping needed
206    ///     if let Some(name) = city.city.names.english {
207    ///         println!("City: {}", name);
208    ///     }
209    /// } else {
210    ///     println!("No data found for IP {}", ip);
211    /// }
212    /// # Ok(())
213    /// # }
214    /// ```
215    ///
216    /// Selective field access:
217    /// ```
218    /// # use maxminddb::{path, Reader};
219    /// # use std::net::IpAddr;
220    /// # fn main() -> Result<(), maxminddb::MaxMindDbError> {
221    /// let reader = Reader::open_readfile(
222    ///     "test-data/test-data/GeoIP2-City-Test.mmdb")?;
223    /// let ip: IpAddr = "89.160.20.128".parse().unwrap();
224    ///
225    /// let result = reader.lookup(ip)?;
226    /// let country_code: Option<String> = result.decode_path(&path!["country", "iso_code"])?;
227    ///
228    /// println!("Country: {:?}", country_code);
229    /// # Ok(())
230    /// # }
231    /// ```
232    #[inline]
233    pub fn lookup(&'de self, address: IpAddr) -> Result<LookupResult<'de, S>, MaxMindDbError> {
234        match address {
235            IpAddr::V4(v4) => {
236                let (pointer, prefix_len) = self.find_address_in_tree_v4(v4.into());
237
238                // For IPv4 addresses in IPv6 databases, adjust prefix_len to reflect
239                // the actual bit depth in the tree. The ipv4_start_bit_depth tells us
240                // how deep in the IPv6 tree we were when we found the IPv4 subtree.
241                let prefix_len = if self.metadata.ip_version == 6 {
242                    self.ipv4_start_bit_depth + prefix_len
243                } else {
244                    prefix_len
245                };
246
247                self.lookup_result(pointer, prefix_len as u8, address)
248            }
249            IpAddr::V6(v6) => {
250                if self.metadata.ip_version == 4 {
251                    return Err(MaxMindDbError::invalid_input(
252                        "cannot look up IPv6 address in IPv4-only database",
253                    ));
254                }
255
256                let (pointer, prefix_len) = self.find_address_in_tree_v6(v6.into());
257                self.lookup_result(pointer, prefix_len as u8, address)
258            }
259        }
260    }
261
262    /// Iterate over all networks in the database.
263    ///
264    /// This is a convenience method equivalent to calling [`within()`](Self::within)
265    /// with `0.0.0.0/0` for IPv4-only databases or `::/0` for IPv6 databases.
266    ///
267    /// # Arguments
268    ///
269    /// * `options` - Controls which networks are yielded. Use [`Default::default()`]
270    ///   for standard behavior.
271    ///
272    /// # Examples
273    ///
274    /// Iterate over all networks with default options:
275    /// ```
276    /// use maxminddb::{geoip2, Reader};
277    ///
278    /// let reader = Reader::open_readfile(
279    ///     "test-data/test-data/GeoIP2-City-Test.mmdb").unwrap();
280    ///
281    /// let mut count = 0;
282    /// for result in reader.networks(Default::default()).unwrap() {
283    ///     let lookup = result.unwrap();
284    ///     count += 1;
285    ///     if count >= 10 { break; }
286    /// }
287    /// ```
288    pub fn networks(&'de self, options: WithinOptions) -> Result<Within<'de, S>, MaxMindDbError> {
289        let cidr = if self.metadata.ip_version == 6 {
290            IpNetwork::V6("::/0".parse().unwrap())
291        } else {
292            IpNetwork::V4("0.0.0.0/0".parse().unwrap())
293        };
294        self.within(cidr, options)
295    }
296
297    /// Iterate over IP networks within a CIDR range.
298    ///
299    /// Returns an iterator that yields [`LookupResult`] for each network in the
300    /// database that falls within the specified CIDR range.
301    ///
302    /// # Arguments
303    ///
304    /// * `cidr` - The CIDR range to iterate over.
305    /// * `options` - Controls which networks are yielded. Use [`Default::default()`]
306    ///   for standard behavior (skip aliases, skip networks without data, include
307    ///   empty values).
308    ///
309    /// # Examples
310    ///
311    /// Iterate over all IPv4 networks:
312    /// ```
313    /// use ipnetwork::IpNetwork;
314    /// use maxminddb::{geoip2, Reader};
315    ///
316    /// let reader = Reader::open_readfile(
317    ///     "test-data/test-data/GeoIP2-City-Test.mmdb").unwrap();
318    ///
319    /// let ipv4_all = IpNetwork::V4("0.0.0.0/0".parse().unwrap());
320    /// let mut count = 0;
321    /// for result in reader.within(ipv4_all, Default::default()).unwrap() {
322    ///     let lookup = result.unwrap();
323    ///     let network = lookup.network().unwrap();
324    ///     let city: geoip2::City = lookup.decode().unwrap().unwrap();
325    ///     let city_name = city.city.names.english;
326    ///     println!("Network: {}, City: {:?}", network, city_name);
327    ///     count += 1;
328    ///     if count >= 10 { break; } // Limit output for example
329    /// }
330    /// ```
331    ///
332    /// Search within a specific subnet:
333    /// ```
334    /// use ipnetwork::IpNetwork;
335    /// use maxminddb::{geoip2, Reader};
336    ///
337    /// let reader = Reader::open_readfile(
338    ///     "test-data/test-data/GeoIP2-City-Test.mmdb").unwrap();
339    ///
340    /// let subnet = IpNetwork::V4("192.168.0.0/16".parse().unwrap());
341    /// for result in reader.within(subnet, Default::default()).unwrap() {
342    ///     match result {
343    ///         Ok(lookup) => {
344    ///             let network = lookup.network().unwrap();
345    ///             println!("Found: {}", network);
346    ///         }
347    ///         Err(e) => eprintln!("Error: {}", e),
348    ///     }
349    /// }
350    /// ```
351    ///
352    /// Include networks without data:
353    /// ```
354    /// use ipnetwork::IpNetwork;
355    /// use maxminddb::{Reader, WithinOptions};
356    ///
357    /// let reader = Reader::open_readfile(
358    ///     "test-data/test-data/MaxMind-DB-test-mixed-24.mmdb").unwrap();
359    ///
360    /// let opts = WithinOptions::default().include_networks_without_data();
361    /// for result in reader.within("1.0.0.0/8".parse().unwrap(), opts).unwrap() {
362    ///     let lookup = result.unwrap();
363    ///     if !lookup.has_data() {
364    ///         println!("Network {} has no data", lookup.network().unwrap());
365    ///     }
366    /// }
367    /// ```
368    pub fn within(
369        &'de self,
370        cidr: IpNetwork,
371        options: WithinOptions,
372    ) -> Result<Within<'de, S>, MaxMindDbError> {
373        if self.metadata.ip_version == 4 && matches!(cidr, IpNetwork::V6(_)) {
374            return Err(MaxMindDbError::invalid_input(
375                "cannot iterate IPv6 network in IPv4-only database",
376            ));
377        }
378        let ip_address = cidr.network();
379        let prefix_len = cidr.prefix() as usize;
380        let ip_int = IpInt::new(ip_address);
381        let bit_count = ip_int.bit_count();
382
383        let mut node = self.start_node(bit_count);
384        let node_count = self.node_count;
385        let has_ipv4_subtree = self.has_ipv4_subtree();
386
387        let mut stack: Vec<WithinNode> = Vec::with_capacity(bit_count - prefix_len);
388
389        // `bit_count == 32` means the caller requested an IPv4 CIDR. In an
390        // IPv6 database with no IPv4 subtree, `start_node(32)` can already be a
391        // terminal IPv6 record reached by walking the all-zero prefix. Do not
392        // read that terminal value as a tree node; yield the containing IPv6
393        // network instead, matching lookup behavior.
394        if bit_count == 32
395            && self.metadata.ip_version == 6
396            && !has_ipv4_subtree
397            && node >= node_count
398        {
399            stack.push(WithinNode {
400                node,
401                ip_int: IpInt::V6(0),
402                prefix_len: self.ipv4_start_bit_depth,
403            });
404
405            return Ok(Within {
406                reader: self,
407                node_count,
408                has_ipv4_subtree,
409                stack,
410                options,
411            });
412        }
413
414        // Traverse down the tree to the level that matches the cidr mark
415        let mut depth = 0_usize;
416        for i in 0..prefix_len {
417            // `read_node` is only valid for internal search-tree nodes.
418            if node >= node_count {
419                // We've hit a data node or dead end before we exhausted our prefix.
420                // This means the requested CIDR is contained in a single record.
421                break;
422            }
423
424            let bit = ip_int.get_bit(i);
425            node = self.read_node(node, bit as usize);
426            depth = i + 1; // We've now traversed i+1 bits (bits 0 through i)
427
428            if node >= node_count {
429                // We've hit a data node or dead end before we exhausted our prefix.
430                // This means the requested CIDR is contained in a single record.
431                break;
432            }
433        }
434
435        // Always push the node - it could be:
436        // - A data node (> node_count): will be yielded as a single record
437        // - The empty node (== node_count): will be skipped unless include_networks_without_data
438        // - An internal node (< node_count): will be traversed to find all contained records
439        stack.push(WithinNode {
440            node,
441            ip_int,
442            prefix_len: depth,
443        });
444
445        let within = Within {
446            reader: self,
447            node_count,
448            has_ipv4_subtree,
449            stack,
450            options,
451        };
452
453        Ok(within)
454    }
455
456    // Pointer 0 means "not found" because normalize_lookup_result collapses both
457    // the placeholder empty node (`node == node_count`) and an unfinished internal
458    // terminal (`node < node_count`, i.e. bits exhausted while still on a tree
459    // node) into 0, so neither path reaches resolve_data_pointer with a non-data
460    // value.
461    #[inline(always)]
462    fn lookup_result(
463        &'de self,
464        pointer: usize,
465        prefix_len: u8,
466        address: IpAddr,
467    ) -> Result<LookupResult<'de, S>, MaxMindDbError> {
468        let network_kind = match address {
469            IpAddr::V4(_) if self.metadata.ip_version == 6 && self.has_ipv4_subtree() => {
470                NetworkKind::V4InV6Subtree
471            }
472            IpAddr::V4(_) if self.metadata.ip_version == 6 => NetworkKind::V6,
473            IpAddr::V4(_) => NetworkKind::V4,
474            IpAddr::V6(_) => NetworkKind::V6,
475        };
476        if pointer == 0 {
477            Ok(LookupResult::new_not_found(
478                self,
479                prefix_len,
480                address,
481                LookupSource::Lookup,
482                network_kind,
483            ))
484        } else {
485            let data_offset = self.resolve_data_pointer(pointer)?;
486            Ok(LookupResult::new_found(
487                self,
488                data_offset,
489                prefix_len,
490                address,
491                LookupSource::Lookup,
492                network_kind,
493            ))
494        }
495    }
496
497    #[inline(always)]
498    fn find_address_in_tree_v4(&self, ip: u32) -> (usize, usize) {
499        let buf = self.buf.as_ref();
500        let node_count = self.node_count;
501
502        match self.record_size {
503            24 => find_address_in_tree_v4::<RecordSize24>(buf, self.ipv4_start, node_count, ip),
504            28 => find_address_in_tree_v4::<RecordSize28>(buf, self.ipv4_start, node_count, ip),
505            32 => find_address_in_tree_v4::<RecordSize32>(buf, self.ipv4_start, node_count, ip),
506            _ => unreachable!("record_size is validated in Reader::from_source"),
507        }
508    }
509
510    #[inline(always)]
511    fn find_address_in_tree_v6(&self, ip: u128) -> (usize, usize) {
512        let buf = self.buf.as_ref();
513        let node_count = self.node_count;
514
515        match self.record_size {
516            24 => find_address_in_tree_v6::<RecordSize24>(buf, node_count, ip),
517            28 => find_address_in_tree_v6::<RecordSize28>(buf, node_count, ip),
518            32 => find_address_in_tree_v6::<RecordSize32>(buf, node_count, ip),
519            _ => unreachable!("record_size is validated in Reader::from_source"),
520        }
521    }
522
523    #[inline]
524    fn start_node(&self, length: usize) -> usize {
525        if length == 128 {
526            0
527        } else {
528            self.ipv4_start
529        }
530    }
531
532    #[inline]
533    pub(crate) fn has_ipv4_subtree(&self) -> bool {
534        self.metadata.ip_version == 6 && self.ipv4_start < self.node_count
535    }
536
537    /// Find the IPv4 start node and the bit depth at which it was found.
538    /// Returns (node, depth) where depth is how far into the tree we traversed.
539    fn find_ipv4_start(&self) -> (usize, usize) {
540        if self.metadata.ip_version != 6 {
541            return (0, 0);
542        }
543
544        // We are looking up an IPv4 address in an IPv6 tree. Skip over the
545        // first 96 nodes.
546        let mut node: usize = 0;
547        for i in 0_u8..96 {
548            if node >= self.node_count {
549                return (node, i as usize);
550            }
551            node = self.read_node(node, 0);
552        }
553        (node, 96)
554    }
555
556    #[inline(always)]
557    pub(crate) fn read_node(&self, node_number: usize, index: usize) -> usize {
558        let buf = self.buf.as_ref();
559
560        match self.record_size {
561            24 => RecordSize24::read_node(buf, node_number, index),
562            28 => RecordSize28::read_node(buf, node_number, index),
563            32 => RecordSize32::read_node(buf, node_number, index),
564            _ => unreachable!("record_size is validated in Reader::from_source"),
565        }
566    }
567
568    /// Resolves a pointer from the search tree to an offset in the data section.
569    #[inline]
570    pub(crate) fn resolve_data_pointer(&self, pointer: usize) -> Result<usize, MaxMindDbError> {
571        let resolved = pointer
572            .checked_sub(self.node_count)
573            .and_then(|p| p.checked_sub(DATA_SECTION_SEPARATOR_SIZE))
574            .ok_or_else(|| {
575                MaxMindDbError::invalid_database(
576                    "the MaxMind DB file's data pointer resolves to an invalid location",
577                )
578            })?;
579        // Reject offsets at or beyond the marker-excluding data section length.
580        if resolved >= self.data_section_len {
581            return Err(MaxMindDbError::invalid_database(
582                "the MaxMind DB file's data pointer resolves to an invalid location",
583            ));
584        }
585
586        Ok(resolved)
587    }
588
589    /// Performs comprehensive validation of the MaxMind DB file.
590    ///
591    /// This method validates:
592    /// - Metadata section: format versions, required fields, and value constraints
593    /// - Search tree: traverses all networks to verify tree structure integrity
594    /// - Data section separator: validates the 16-byte separator between tree and data
595    /// - Data section: verifies all data records referenced by the search tree
596    ///
597    /// The verifier is stricter than the MaxMind DB specification and may return
598    /// errors on some databases that are still readable by normal operations.
599    /// This method is useful for:
600    /// - Validating database files after download or generation
601    /// - Debugging database corruption issues
602    /// - Ensuring database integrity in critical applications
603    ///
604    /// Note: Verification traverses the entire database and retains visited data
605    /// offsets for the duration of the call. It may be slow and use memory
606    /// proportional to the number of distinct referenced values on large files.
607    /// Verification validates each shared target once. For each metadata or
608    /// data section, it permits up to eight times the section's byte length in
609    /// work units: one per visited value and one per string byte validated.
610    /// Exceeding this allowance returns [`MaxMindDbError::ResourceLimit`],
611    /// bounding repeated scans of overlapping payloads. Later deserialization
612    /// independently applies the per-operation container and payload limits
613    /// documented on [`crate::LookupResult::decode()`]. The method is
614    /// thread-safe and can be called on an active Reader.
615    ///
616    /// # Example
617    ///
618    /// ```
619    /// use maxminddb::Reader;
620    ///
621    /// let reader = Reader::open_readfile("test-data/test-data/GeoIP2-City-Test.mmdb").unwrap();
622    /// reader.verify().expect("Database should be valid");
623    /// ```
624    pub fn verify(&self) -> Result<(), MaxMindDbError> {
625        let metadata_start = find_metadata_start(self.buf.as_ref())?;
626        let data_section_end = metadata_marker_start(metadata_start)?;
627        self.verify_metadata(metadata_start, data_section_end)?;
628        self.verify_database(data_section_end)
629    }
630
631    fn verify_metadata(
632        &self,
633        metadata_start: usize,
634        data_section_end: usize,
635    ) -> Result<(), MaxMindDbError> {
636        // Metadata deserialization intentionally ignores unknown fields. Verify
637        // the original value graph as well so an invalid or cyclic pointer in
638        // an unknown field cannot hide behind the cached typed representation.
639        let metadata_bytes = &self.buf.as_ref()[metadata_start..];
640        let mut decoder = decoder::Decoder::new(metadata_bytes, 0);
641        decoder
642            .skip_value_for_verification(&mut decoder::VerificationState::new(metadata_bytes.len()))
643            .map_err(|error| error.with_invalid_database_offset_base(metadata_start))?;
644
645        let m = &self.metadata;
646
647        validate_metadata_for_reader(m)?;
648        if m.database_type.is_empty() {
649            return Err(MaxMindDbError::invalid_database(
650                "database_type - Expected: non-empty string Actual: \"\"",
651            ));
652        }
653        if m.description.is_empty() {
654            return Err(MaxMindDbError::invalid_database(
655                "description - Expected: non-empty map Actual: {}",
656            ));
657        }
658        validate_search_tree_layout(self.pointer_base, data_section_end)?;
659        Ok(())
660    }
661
662    fn verify_database(&self, data_section_end: usize) -> Result<(), MaxMindDbError> {
663        let offsets = self.verify_search_tree()?;
664        self.verify_data_section_separator()?;
665        self.verify_data_section(offsets, data_section_end)
666    }
667
668    fn verify_search_tree(&self) -> Result<HashSet<usize>, MaxMindDbError> {
669        let mut offsets = HashSet::new();
670        let opts = WithinOptions::default().include_networks_without_data();
671
672        // Maximum number of networks we can expect in a valid database.
673        // A database with N nodes can have at most 2N data entries (each leaf node
674        // can have data). We add some margin for safety.
675        let max_iterations = self.node_count.saturating_mul(3);
676        let mut iteration_count = 0usize;
677
678        for result in self.networks(opts)? {
679            let lookup = result?;
680            if let Some(offset) = lookup.offset() {
681                offsets.insert(offset);
682            }
683
684            iteration_count += 1;
685            if iteration_count > max_iterations {
686                return Err(MaxMindDbError::invalid_database(format!(
687                    "search tree appears to have a cycle or invalid structure (exceeded {max_iterations} iterations)"
688                )));
689            }
690        }
691        Ok(offsets)
692    }
693
694    fn verify_data_section_separator(&self) -> Result<(), MaxMindDbError> {
695        let separator_start = self.node_count * self.node_byte_size;
696        let separator_end = separator_start + DATA_SECTION_SEPARATOR_SIZE;
697
698        if separator_end > self.buf.as_ref().len() {
699            return Err(MaxMindDbError::invalid_database_at(
700                "data section separator extends past end of file",
701                separator_start,
702            ));
703        }
704
705        let separator = &self.buf.as_ref()[separator_start..separator_end];
706
707        for &b in separator {
708            if b != 0 {
709                return Err(MaxMindDbError::invalid_database_at(
710                    format!("unexpected byte in data separator: {separator:?}"),
711                    separator_start,
712                ));
713            }
714        }
715        Ok(())
716    }
717
718    fn verify_data_section(
719        &self,
720        offsets: HashSet<usize>,
721        data_section_end: usize,
722    ) -> Result<(), MaxMindDbError> {
723        let data_section = &self.buf.as_ref()[self.pointer_base..data_section_end];
724        let mut verification_state = decoder::VerificationState::new(data_section.len());
725
726        // Verify each offset from the search tree points to valid, decodable data
727        for &offset in &offsets {
728            if offset >= data_section.len() {
729                return Err(MaxMindDbError::invalid_database_at(
730                    format!(
731                        "search tree pointer is beyond data section (len: {})",
732                        data_section.len()
733                    ),
734                    offset,
735                )
736                .with_invalid_database_offset_base(self.pointer_base));
737            }
738
739            let mut dec = decoder::Decoder::new(data_section, offset);
740
741            // Try to skip/decode the value to verify it's valid
742            dec.skip_value_for_verification(&mut verification_state)
743                .map_err(|error| error.with_invalid_database_offset_base(self.pointer_base))?;
744        }
745
746        Ok(())
747    }
748}
749
750fn validate_record_size(record_size: u16) -> Result<(), MaxMindDbError> {
751    if matches!(record_size, 24 | 28 | 32) {
752        Ok(())
753    } else {
754        Err(MaxMindDbError::invalid_database(format!(
755            "record_size - Expected: 24, 28, or 32 Actual: {}",
756            record_size
757        )))
758    }
759}
760
761pub(crate) fn validate_metadata_for_reader(metadata: &Metadata) -> Result<(), MaxMindDbError> {
762    if metadata.binary_format_major_version != 2 {
763        return Err(MaxMindDbError::invalid_database(format!(
764            "binary_format_major_version - Expected: 2 Actual: {}",
765            metadata.binary_format_major_version
766        )));
767    }
768    // Minor format versions are intended to be forward-compatible.
769    if metadata.ip_version != 4 && metadata.ip_version != 6 {
770        return Err(MaxMindDbError::invalid_database(format!(
771            "ip_version - Expected: 4 or 6 Actual: {}",
772            metadata.ip_version
773        )));
774    }
775    if metadata.node_count == 0 {
776        return Err(MaxMindDbError::invalid_database(
777            "node_count - Expected: positive integer Actual: 0",
778        ));
779    }
780    metadata.build_time()?;
781    validate_record_size(metadata.record_size)
782}
783
784fn search_tree_size_bytes(node_count: usize, record_size: usize) -> Result<usize, MaxMindDbError> {
785    node_count
786        .checked_mul(record_size)
787        .map(|size| size / 4)
788        .ok_or_else(|| {
789            MaxMindDbError::invalid_database(
790                "search tree size calculation overflowed or is impossibly large",
791            )
792        })
793}
794
795fn validate_search_tree_layout(
796    pointer_base: usize,
797    data_section_end: usize,
798) -> Result<(), MaxMindDbError> {
799    if pointer_base > data_section_end {
800        return Err(MaxMindDbError::invalid_database(
801            "the MaxMind DB file's search tree extends beyond the metadata section",
802        ));
803    }
804    Ok(())
805}
806
807trait SearchTreeRecord {
808    fn read_node(buf: &[u8], node_number: usize, index: usize) -> usize;
809}
810
811struct RecordSize24;
812
813impl SearchTreeRecord for RecordSize24 {
814    #[inline(always)]
815    fn read_node(buf: &[u8], node_number: usize, index: usize) -> usize {
816        // Both four-byte windows stay inside the six-byte node. The left
817        // child occupies the high three bytes; the right occupies the low three.
818        let offset = node_number * 6 + index * 2;
819        let bytes: [u8; 4] = buf[offset..offset + 4].try_into().unwrap();
820        let word = u32::from_be_bytes(bytes);
821        ((word >> ((1 - index) * 8)) & 0x00FF_FFFF) as usize
822    }
823}
824
825struct RecordSize28;
826
827impl SearchTreeRecord for RecordSize28 {
828    #[inline(always)]
829    fn read_node(buf: &[u8], node_number: usize, index: usize) -> usize {
830        // The left window ends at the shared nibble byte; the right starts
831        // there. Each child and its shared nibble fit in a single word.
832        let offset = node_number * 7 + index * 3;
833        let bytes: [u8; 4] = buf[offset..offset + 4].try_into().unwrap();
834        let word = u32::from_be_bytes(bytes);
835        if index == 0 {
836            // The first three bytes hold bits 23..0, and the shared byte's
837            // high nibble holds bits 27..24. Move that nibble above the bytes.
838            ((word >> 8) | ((word << 20) & 0x0F00_0000)) as usize
839        } else {
840            (word & 0x0FFF_FFFF) as usize
841        }
842    }
843}
844
845struct RecordSize32;
846
847impl SearchTreeRecord for RecordSize32 {
848    #[inline(always)]
849    fn read_node(buf: &[u8], node_number: usize, index: usize) -> usize {
850        let offset = node_number * 8 + index * 4;
851        let bytes: [u8; 4] = buf[offset..offset + 4].try_into().unwrap();
852        u32::from_be_bytes(bytes) as usize
853    }
854}
855
856#[inline(always)]
857fn find_address_in_tree_v4<R: SearchTreeRecord>(
858    buf: &[u8],
859    start_node: usize,
860    node_count: usize,
861    ip: u32,
862) -> (usize, usize) {
863    let mut node = start_node;
864    let mut prefix_len = 32;
865
866    for i in 0..32 {
867        if node >= node_count {
868            prefix_len = i;
869            break;
870        }
871        let bit = ((ip >> (31 - i)) & 1) as usize;
872        node = R::read_node(buf, node, bit);
873    }
874
875    normalize_lookup_result(node, node_count, prefix_len)
876}
877
878#[inline(always)]
879fn find_address_in_tree_v6<R: SearchTreeRecord>(
880    buf: &[u8],
881    node_count: usize,
882    ip: u128,
883) -> (usize, usize) {
884    let mut node = 0;
885    let mut prefix_len = 128;
886
887    for i in 0..128 {
888        if node >= node_count {
889            prefix_len = i;
890            break;
891        }
892        let bit = ((ip >> (127 - i)) & 1) as usize;
893        node = R::read_node(buf, node, bit);
894    }
895
896    normalize_lookup_result(node, node_count, prefix_len)
897}
898
899// Map both "not found" outcomes onto pointer 0:
900//   - `node == node_count`: the placeholder empty terminal in the search tree.
901//   - `node < node_count`: bits exhausted while still on an internal node
902//     (a partially-specified address that did not reach a record).
903// Anything strictly greater than `node_count` is a data-section pointer that
904// the caller must resolve via `resolve_data_pointer`.
905#[inline(always)]
906fn normalize_lookup_result(node: usize, node_count: usize, prefix_len: usize) -> (usize, usize) {
907    if node <= node_count {
908        (0, prefix_len)
909    } else {
910        (node, prefix_len)
911    }
912}
913
914fn find_metadata_start(buf: &[u8]) -> Result<usize, MaxMindDbError> {
915    memchr::memmem::rfind(buf, METADATA_START_MARKER)
916        .map(|x| x + METADATA_START_MARKER.len())
917        .ok_or_else(|| {
918            MaxMindDbError::invalid_database("could not find MaxMind DB metadata in file")
919        })
920}
921
922fn metadata_marker_start(metadata_start: usize) -> Result<usize, MaxMindDbError> {
923    metadata_start
924        .checked_sub(METADATA_START_MARKER.len())
925        .ok_or_else(|| MaxMindDbError::invalid_database("invalid metadata marker location"))
926}
927
928#[cfg(test)]
929mod tests {
930    use super::{RecordSize24, RecordSize28, RecordSize32, SearchTreeRecord};
931
932    #[test]
933    fn packed_28_bit_node_matches_spec_layout() {
934        // The MMDB specification places each child's most-significant nibble
935        // in the shared byte: [left low 24][left high 4 | right high 4][right low 24].
936        // https://maxmind.github.io/MaxMind-DB/#28-bits-medium-database-one-node-is-7-bytes
937        let node = [0x23, 0x45, 0x67, 0x18, 0x9A, 0xBC, 0xDE];
938        assert_eq!(RecordSize28::read_node(&node, 0, 0), 0x0123_4567);
939        assert_eq!(RecordSize28::read_node(&node, 0, 1), 0x089A_BCDE);
940    }
941
942    #[test]
943    fn packed_nodes_round_trip_both_children() {
944        type ReadNode = fn(&[u8], usize, usize) -> usize;
945        let readers: [(usize, ReadNode); 3] = [
946            (24, RecordSize24::read_node),
947            (28, RecordSize28::read_node),
948            (32, RecordSize32::read_node),
949        ];
950        for (bits, read_node) in readers {
951            let node_size = bits / 4;
952            let mask = u32::MAX >> (32 - bits);
953            let mut state = 0x4D59_5DF4_D0F3_3173_u64;
954            let mut buf = [0u8; 24];
955            for sample in 0..65_536 {
956                state = state
957                    .wrapping_mul(6_364_136_223_846_793_005)
958                    .wrapping_add(1_442_695_040_888_963_407);
959                let (left, right) = match sample {
960                    0 => (0, mask),
961                    1 => (mask, 0),
962                    _ => (state as u32 & mask, (state >> 32) as u32 & mask),
963                };
964                // Exercise every shared-nibble combination for 28-bit records.
965                let (left, right) = if bits == 28 && sample >= 2 {
966                    (
967                        (left & 0x00FF_FFFF) | ((sample as u32 & 15) << 24),
968                        (right & 0x00FF_FFFF) | (((sample as u32 >> 4) & 15) << 24),
969                    )
970                } else {
971                    (left, right)
972                };
973                let node = sample % 3;
974                let start = node * node_size;
975                let end = start + node_size;
976                let encoded = &mut buf[start..end];
977                match bits {
978                    24 => {
979                        encoded[..3].copy_from_slice(&left.to_be_bytes()[1..]);
980                        encoded[3..].copy_from_slice(&right.to_be_bytes()[1..]);
981                    }
982                    28 => {
983                        encoded[..3].copy_from_slice(&left.to_be_bytes()[1..]);
984                        encoded[3] = ((left >> 20) as u8 & 0xF0) | (right >> 24) as u8;
985                        encoded[4..].copy_from_slice(&right.to_be_bytes()[1..]);
986                    }
987                    32 => {
988                        encoded[..4].copy_from_slice(&left.to_be_bytes());
989                        encoded[4..].copy_from_slice(&right.to_be_bytes());
990                    }
991                    _ => unreachable!(),
992                }
993                // Include unaligned nodes and end the buffer at this node:
994                // neither child may require a byte from the following node.
995                assert_eq!(read_node(&buf[..end], node, 0), left as usize);
996                assert_eq!(read_node(&buf[..end], node, 1), right as usize);
997            }
998        }
999    }
1000}
maxminddb/reader.rs

maxminddb/
reader.rs
