1

I have a piece of serde code which does what I want, but I don't like how it does it. I'm looking for help with figuring out on how to improve it.

Playground

use std::any::Any;

trait Model: std::fmt::Debug + Any {
    fn as_any(&self) -> &dyn Any;
}

impl Model for Generator {
    fn as_any(&self) -> &dyn Any {
        self
    }
}
impl Model for Connector {
    fn as_any(&self) -> &dyn Any {
        self
    }
}

#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct Generator {
    id: String,
    #[serde(rename = "sourceID")]
    source_id: String,
}

#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct Connector {
    id: String,
    #[serde(rename = "sourceID")]
    source_id: String,
}

#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
#[serde(tag = "type")]
enum AllModels {
    Generator(Generator),
    Connector(Connector),
}

fn main() {
    let data = r#"
- sourceID: "generator-01"
  id: "connector-01"
  type: "Generator"
- sourceID: "geneiator-01"
  type: "Connector"
  id: "connector-01"
"#;
    let p: Vec<Box<dyn Model>> = serde_yaml::from_str::<Vec<AllModels>>(&data)
        .unwrap()
        .into_iter()
        .collect();
    println!("{:?}", p);
    let l = serde_yaml::to_string(&p.into_iter().collect::<Vec<AllModels>>());
    println!("{:?}", l);
}

impl std::iter::FromIterator<AllModels> for Vec<Box<dyn Model>> {
    fn from_iter<I: IntoIterator<Item = AllModels>>(iter: I) -> Self {
        let mut v: Vec<Box<dyn Model>> = Vec::new();
        for i in iter {
            match i {
                AllModels::Generator(d) => {
                    v.push(Box::new(d));
                }
                AllModels::Connector(d) => {
                    v.push(Box::new(d));
                }
            }
        }
        v
    }
}

impl std::iter::FromIterator<std::boxed::Box<dyn Model>> for std::vec::Vec<AllModels> {
    fn from_iter<I: IntoIterator<Item = Box<dyn Model>>>(iter: I) -> Self {
        let mut v: Vec<AllModels> = Vec::new();
        for i in iter {
            if let Some(model) = i.as_any().downcast_ref::<Generator>() {
                v.push(AllModels::Generator(model.clone()));
            } else if let Some(model) = i.as_any().downcast_ref::<Connector>() {
                v.push(AllModels::Connector(model.clone()));
            }
        }
        v
    }
}

What I'm trying to achieve is to de/serialize yaml into one of multiple structs, dynamically choosing to which struct should it deserialize to based on value of type field in yaml it parses. e.g.

- id: Foo
  source: Bar
  type: Connector

should be parsed into struct Connector

I figured I could use enum representation to deal with that, however, it produces undesired side effect - by default following yaml:

- id: Foo
  source: Bar
  type: Connector
- id: Foo
  source: Bar
  type: Generator

will be parsed as:

[Connector(Connector{...}), Generator(Generator{...})]

so my structs are wrapped in enum variants. In order to "unwrap it" I figured I could implement FromIterator<AllModels> for Vec<Box<dyn Model>> , thanks to which and type conversion/coercion(not sure which one is the right word) the output changes to:

[Connector{...}, Generator{...}]

so far so good.

Two issues I'm having with this solution, are:

  1. code repetition - for each new struct (Connector,Generator,...) I have to update enum AllModels and match arm inside FromIterator implementation - the latter is what bothers me the most. I could do it with macro probably, but I haven't learned how to write them, and before I do so, I'd like to explore other possible solutions
  2. extra iteration - in order to convert from Vec<enum variant> to Vec<struct> I need to do the following: let p: Vec<Box<dyn Model>> = serde_yaml::from_str::<Vec<AllModels>>(&data).unwrap().into_iter().collect();. I'd prefer if the conversion would happen without an extra iteration

I have considered a few of options, but I'm not able to figure how to implement them...

A. serde container attribute from/into docs

#[serde(from = "FromType")] - the way I think it would work is by force-converting my enum variant straight into desired struct, with no extra iteration and no code repetition. However, I fail to implement it - Playground.

When I'm trying to add from attribute

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "type")]
enum AllModels {
  Gene(Generator),
  Connector(Connector),
}

the compiler will yell at me

 ❯ cargo r
   Compiling serdeissue v0.1.0 (/sandbox/serdeissue)
error[E0277]: the trait bound `Box<dyn Model>: From<AllModels>` is not satisfied
  --> src/main.rs:21:24
   |
21 | #[derive(Debug, Clone, Serialize, Deserialize)]
   |                        ^^^^^^^^^ the trait `From<AllModels>` is not implemented for `Box<dyn Model>`
   |
   = help: the following implementations were found:
             <Box<(dyn StdError + 'a)> as From<E>>
             <Box<(dyn StdError + 'static)> as From<&str>>
             <Box<(dyn StdError + 'static)> as From<Cow<'a, str>>>
             <Box<(dyn StdError + 'static)> as From<std::string::String>>
           and 22 others
   = note: required because of the requirements on the impl of `Into<Box<dyn Model>>` for `AllModels`
   = note: required by `into`
   = note: this error originates in a derive macro (in Nightly builds, run with -Z macro-backtrace for more info)

error[E0277]: the trait bound `dyn Model: Serialize` is not satisfied
   --> src/main.rs:21:24
    |
21  | #[derive(Debug, Clone, Serialize, Deserialize)]
    |                        ^^^^^^^^^ the trait `Serialize` is not implemented for `dyn Model`
    |
   ::: /home/marcin/.cargo/registry/src/github.com-1ecc6299db9ec823/serde-1.0.123/src/ser/mod.rs:247:18
    |
247 |     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
    |                  - required by this bound in `serialize`
    |
    = note: required because of the requirements on the impl of `Serialize` for `Box<dyn Model>`
    = note: this error originates in a derive macro (in Nightly builds, run with -Z macro-backtrace for more info)

error[E0277]: the trait bound `dyn Model: Deserialize<'_>` is not satisfied
   --> src/main.rs:21:35
    |
21  | #[derive(Debug, Clone, Serialize, Deserialize)]
    |                                   ^^^^^^^^^^^ the trait `Deserialize<'_>` is not implemented for `dyn Model`
    |
   ::: /home/marcin/.cargo/registry/src/github.com-1ecc6299db9ec823/serde-1.0.123/src/de/mod.rs:539:12
    |
539 |         D: Deserializer<'de>;
    |            ----------------- required by this bound in `_::_serde::Deserialize::deserialize`
    |
    = note: required because of the requirements on the impl of `Deserialize<'_>` for `Box<dyn Model>`
    = note: this error originates in a derive macro (in Nightly builds, run with -Z macro-backtrace for more info)

error[E0277]: the trait bound `AllModels: From<Box<dyn Model>>` is not satisfied
   --> src/main.rs:21:35
    |
21  | #[derive(Debug, Clone, Serialize, Deserialize)]
    |                                   ^^^^^^^^^^^ the trait `From<Box<dyn Model>>` is not implemented for `AllModels`
    |
   ::: /home/marcin/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/convert/mod.rs:372:1
    |
372 | pub trait From<T>: Sized {
    | ------------------------ required by this bound in `From`
    |
    = note: this error originates in a derive macro (in Nightly builds, run with -Z macro-backtrace for more info)

My angle of attack is following: use error msg to copy-pasteroni-dummy-implementoni missing trait bounds:

impl From<AllModels> for Box<dyn Model> {
    fn from(am: AllModels) -> Self {
        Box::new(Generator{id:String::from("arst"),source_id:String::from("arst")})
    }
}
impl Serialize for dyn Model {
    fn serialize(&self) -> Self {
        Box::new(Generator{id:String::from("arst"),source_id:String::from("arst")})
    }
}

impl Deserialize<'_> for dyn Model {}
impl From<Box<dyn Model>> for AllModels {
    fn from(dm: Box<dyn Model>) -> Self {
        AllModels::Gene(Generator{id:String::from("arst"),source_id:String::from("arst")})
    }
}

but then this happens:

 ❯ cargo r
   Compiling serdeissue v0.1.0 (/sandbox/serdeissue)
error[E0277]: the size for values of type `(dyn Model + 'static)` cannot be known at compilation time
   --> src/main.rs:75:6
    |
75  | impl Deserialize<'_> for dyn Model {}
    |      ^^^^^^^^^^^^^^^ doesn't have a size known at compile-time
    |
   ::: /home/marcin/.cargo/registry/src/github.com-1ecc6299db9ec823/serde-1.0.123/src/de/mod.rs:530:29
    |
530 | pub trait Deserialize<'de>: Sized {
    |                             ----- required by this bound in `Deserialize`
    |
    = help: the trait `Sized` is not implemented for `(dyn Model + 'static)`

and that's a game over for me

B. erased-serde

this seems to be the right tool for the job, but again, I run into problems when implementing it (no wonder - I have no idea what I'm doing:)

use erased_serde::{Deserializer,  Serializer, serialize_trait_object};
use serde::{Serialize, Deserialize};


#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Generator {
  id: String,
  source: String,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Connector {
  id: String,
  source: String,
}

trait Model: std::fmt::Debug + erased_serde::Serialize {}
erased_serde::serialize_trait_object!(Model);
impl Model for Generator {}
impl Model for Connector {}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "type", into = "Box<dyn Model>", from = "Box<dyn Model>")]
enum AllModels {
  Generator(Generator),
  Connector(Connector),
}

fn main() {
  let data = r#"
- source: "generator-01"
  id: "g-01"
  type: "Generator"
- source: "connector-01"
  type: "Connector"
  id: "c-01"
"#;
  let p: Vec<Box<dyn Model>> = serde_yaml::from_str(&data).unwrap();
  println!("{:?}", p);
}


impl From<AllModels> for Box<dyn Model> {
    fn from(am: AllModels) -> Self {
        Box::new(Generator{id:String::from("arst"),source_id:String::from("arst")})
    }
}
// impl Serialize for dyn Model {
//     fn serialize(&self) -> Self {
//         Box::new(Generator{id:String::from("arst"),source_id:String::from("arst")})
//     }
// }
impl Deserialize<'_> for dyn Model {}
impl From<Box<dyn Model>> for AllModels {
    fn from(dm: Box<dyn Model>) -> Self {
        AllModels::Generator(Generator{id:String::from("arst"),source_id:String::from("arst")})
    }
}

// impl std::convert::From<AllModels> for Box<dyn Model> {
//     fn from(m: AllModels) -> Self {
//        Box::new(Generator {source_id: String::from("i"), id: String::from("r")})
//     }
//     }

I get this error when compiling:

 ❯ cargo r
   Compiling serdeissue v0.1.0 (/sandbox/serdeissue)
warning: unused imports: `Deserializer`, `Serializer`, `serialize_trait_object`
 --> src/main.rs:1:20
  |
1 | use erased_serde::{Deserializer,  Serializer, serialize_trait_object};
  |                    ^^^^^^^^^^^^   ^^^^^^^^^^  ^^^^^^^^^^^^^^^^^^^^^^
  |
  = note: `#[warn(unused_imports)]` on by default

error[E0277]: the size for values of type `(dyn Model + 'static)` cannot be known at compilation time
   --> src/main.rs:76:6
    |
76  | impl Deserialize<'_> for dyn Model {}
    |      ^^^^^^^^^^^^^^^ doesn't have a size known at compile-time
    |
   ::: /home/marcin/.cargo/registry/src/github.com-1ecc6299db9ec823/serde-1.0.123/src/de/mod.rs:530:29
    |
530 | pub trait Deserialize<'de>: Sized {
    |                             ----- required by this bound in `Deserialize`
    |
    = help: the trait `Sized` is not implemented for `(dyn Model + 'static)`

which is something I thought erased_serde could help with, and maybe it does, but I have no clue how to implement it.

Sadly I can't use typetag crate since it doesn't support wasm compilation target which I need. I am not considering using #[serde(serialize_with = "path")] for each enum variant, since it makes my issue #1 much worse than it currently is.

I'm also aware of this question How to implement `serde::Serialize` for a boxed trait object? however the code provided by @dtolnay doesn't compile and I don't know how to fix it

❯ cargo r
   Compiling serdeissue v0.1.0 (/sandbox/serdeissue)
error[E0603]: module `export` is private
   --> src/main.rs:168:10
    |
168 | #[derive(Serialize)]
    |          ^^^^^^^^^ private module
    |
note: the module `export` is defined here
   --> /home/marcin/.cargo/registry/src/github.com-1ecc6299db9ec823/serde-1.0.123/src/lib.rs:275:5
    |
275 | use self::__private as export;
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^

error[E0603]: module `export` is private
   --> src/main.rs:173:10
    |
173 | #[derive(Serialize)]
    |          ^^^^^^^^^ private module
    |
note: the module `export` is defined here
   --> /home/marcin/.cargo/registry/src/github.com-1ecc6299db9ec823/serde-1.0.123/src/lib.rs:275:5
    |
275 | use self::__private as export;
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^

error[E0603]: module `export` is private
   --> src/main.rs:179:10
    |
179 | #[derive(Serialize)]
    |          ^^^^^^^^^ private module
    |
note: the module `export` is defined here
   --> /home/marcin/.cargo/registry/src/github.com-1ecc6299db9ec823/serde-1.0.123/src/lib.rs:275:5
    |
275 | use self::__private as export;
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^

error[E0603]: module `export` is private
   --> src/main.rs:184:10
    |
184 | #[derive(Serialize)]
    |          ^^^^^^^^^ private module
    |
note: the module `export` is defined here
   --> /home/marcin/.cargo/registry/src/github.com-1ecc6299db9ec823/serde-1.0.123/src/lib.rs:275:5
    |
275 | use self::__private as export;
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^

warning: trait objects without an explicit `dyn` are deprecated
   --> src/main.rs:176:22
    |
176 |     widgets: Vec<Box<WidgetTrait>>,
    |                      ^^^^^^^^^^^ help: use `dyn`: `dyn WidgetTrait`
    |
    = note: `#[warn(bare_trait_objects)]` on by default

It looks like the feature I'm looking for is waiting to be implemented here: https://github.com/serde-rs/serde/issues/1402

There is also this issue https://github.com/serde-rs/serde/issues/1350 which suggests manual Deserializer implementation Playground. The playground code indicates this could help with my issue #2 "extra iteration", however there's still some repetition involved in code implementation, therefore I'm still looking for a better answer.

Edit: I'm also considering Enum or trait object, can't figure what's the right approach for evaluating whether I need one or the other.

Marcin
  • 179
  • 2
  • 14

1 Answers1

4

I found a solution which satisfies me:

  • slick
  • no code repetition
  • no dynamic dispatch, no extra iterations
  • uses enum and serde(tag="...")
    • type field can be out of order in json/yaml (doesn't have to be first)
    • type field is included when serializing back to json/yaml

The trick was to use enum_dispatch.

use serde::{Serialize, Deserialize};
use enum_dispatch::enum_dispatch;


#[derive(Debug, Clone, Serialize, Deserialize)]
struct Generator {
    source_id: String,
    id: String,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
struct Gate {
    source_id: String,
    id: String,
}

#[enum_dispatch]
trait Model {fn time_advance(self, a:i32,b:i32) -> i32;} 
impl Model for Generator { fn time_advance(self,a:i32,b:i32) -> i32 {a+b} }
impl Model for Gate { fn time_advance(self,a:i32,b:i32) -> i32 {a+b} }

#[enum_dispatch(Model)]
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "type")]
enum ModelTypes {
    Generator(Generator),
    Gate(Gate),
}

fn main() {
    let s = r#"
- source_id: "generator-01"
  id: "connector-01"
  type: "Generator"
- source_id: "geneiator-01"
  type: "Gate"
  id: "connector-01"
"#;
    let data: Vec<ModelTypes> = serde_yaml::from_str(s).unwrap();
    println!("{:?}", serde_yaml::to_string(&data));
    for d in data {
        println!("{:?}", d.time_advance(4, 2));
    }
}

[package]
name = "enum_unpack"
version = "0.1.0"
authors = ["---"]
edition = "2018"

[dependencies]
serde = { version = "1.0.124", features = ["derive"] }
serde_yaml = "0.8.1"
enum_dispatch = "0.3.5"

output:

> cargo run
   Compiling enum_unpack v0.1.0 (.../enum_unpack)
    Finished dev [unoptimized + debuginfo] target(s) in 0.66s
     Running `target/debug/enum_unpack`
Ok("---\n- type: Generator\n  source_id: generator-01\n  id: connector-01\n- type: Gate\n  source_id: geneiator-01\n  id: connector-01\n")
6
6
Marcin
  • 179
  • 2
  • 14