Save/load of struct works or fails depending on member vector length

Question

I have defined the following structs with custom load/save methods using serde and bincode:

use std::{
    fs::File,
    io::{Read, Seek, SeekFrom, Write},
};

use serde::{Deserialize, Serialize};
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct Histogram {
    pub bars: Vec<f64>,
}

#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct Histograms {
    pub num_bars: usize,
    pub num_histograms: usize,
    pub vec: Vec<Histogram>,
}

fn main() {
    println!("Hello, world!");
}

impl Histogram {
    pub fn new(num_bars: usize) -> Histogram {
        Histogram {
            bars: vec![0.0; num_bars],
        }
    }
}

impl Histograms {
    pub fn new(num_histograms: usize, num_bars: usize) -> Histograms {
        let histograms = Histograms {
            vec: vec![Histogram::new(num_bars); num_histograms],
            num_histograms,
            num_bars,
        };
        histograms
    }

    pub fn save(&self, filename: &str) {
        let mut file = File::create(format!("{}{}", "path", filename)).unwrap();
        let bin = bincode::serialize(&self).unwrap();
        Write::write_all(&mut file, &bin).unwrap();
    }

    pub fn load(filename: &str) -> Histograms {
        let mut file = File::open(format!("{}{}", "path", filename)).unwrap();
        let mut vec: Vec<u8> = vec![0; file.seek(SeekFrom::End(0)).unwrap() as usize];
        file.seek(SeekFrom::Start(0)).unwrap();
        file.read(&mut vec).unwrap();
        let result: Histograms = bincode::deserialize(&vec).unwrap();
        result
    }
}

The strange thing now is, that the following test shows me that save/load works properly if the length of the vec(member of histograms) is small, but it fails(i don't get any error, just the resulting histograms instance is wrong) with large values like for instance 10000000. Precisely i get a value of 5263431 from where on it is not correct anymore.

mod tests {
    use core::panic;

    use super::*;
    #[test]
    fn save_load_test() {
        let histogramms = Histograms::new(10000000, 50);
        histogramms.save("debug_test");
        let histogramms1 = Histograms::load("debug_test");
        assert_eq!(histogramms.vec.len(), histogramms1.vec.len());
        let mut failed = false;
        for i in 0..histogramms.vec.len() {
            if histogramms.vec[i].bars.len() != histogramms1.vec[i].bars.len() {
                failed = true;
                println!("first i that failed: {}", i);
                println!(
                    "histogramms.vec.bars.len: {} histogramms1.vec.bars.len: {}",
                    histogramms.vec[i].bars.len(),
                    histogramms1.vec[i].bars.len()
                );
                break;
            }
        }
        if failed {
            panic!()
        }
    }
}

Any ideas what is going wrong?

Answer 1

The Read::read() function pulls some bytes from the source, but it is in no way guaranteed to read all of them. The only guarantee you get is that you will get some bytes if there are any left (at least if your buffer doesn't have length zero).

The easiest way to fix this is to use the std::fs::read() function instead:

pub fn load(filename: &str) -> Histograms {
    let vec = std::fs::read(format!("{}{}", "path", filename)).unwrap();
    bincode::deserialize(&vec).unwrap()
}

An even better fix is to directly deserialize from the file, without reading the whole file into memory first:

pub fn save(&self, filename: &str) {
    let mut w = BufWriter::new(File::create(format!("{}{}", "path", filename)).unwrap());
    bincode::serialize_into(&mut w, &self).unwrap();
    w.flush().unwrap();
}

pub fn load(filename: &str) -> Histograms {
    let file = File::open(format!("{}{}", "path", filename)).unwrap();
    bincode::deserialize_from(BufReader::new(file)).unwrap()
}

(Of course you should add proper error handling for real-world code.)