Swift JSONDecode decoding arrays fails if single element decoding fails

Question

While using Swift4 and Codable protocols I got the following problem - it looks like there is no way to allow JSONDecoder to skip elements in an array. For example, I have the following JSON:

[
    {
        "name": "Banana",
        "points": 200,
        "description": "A banana grown in Ecuador."
    },
    {
        "name": "Orange"
    }
]

And a Codable struct:

struct GroceryProduct: Codable {
    var name: String
    var points: Int
    var description: String?
}

When decoding this json

let decoder = JSONDecoder()
let products = try decoder.decode([GroceryProduct].self, from: json)

Resulting products is empty. Which is to be expected, due to the fact that the second object in JSON has no "points" key, while points is not optional in GroceryProduct struct.

Question is how can I allow JSONDecoder to "skip" invalid object?

Answer 1

One option is to use a wrapper type that attempts to decode a given value; storing nil if unsuccessful:

struct FailableDecodable<Base : Decodable> : Decodable {

    let base: Base?

    init(from decoder: Decoder) throws {
        let container = try decoder.singleValueContainer()
        self.base = try? container.decode(Base.self)
    }
}

We can then decode an array of these, with your GroceryProduct filling in the Base placeholder:

import Foundation

let json = """
[
    {
        "name": "Banana",
        "points": 200,
        "description": "A banana grown in Ecuador."
    },
    {
        "name": "Orange"
    }
]
""".data(using: .utf8)!


struct GroceryProduct : Codable {
    var name: String
    var points: Int
    var description: String?
}

let products = try JSONDecoder()
    .decode([FailableDecodable<GroceryProduct>].self, from: json)
    .compactMap { $0.base } // .flatMap in Swift 4.0

print(products)

// [
//    GroceryProduct(
//      name: "Banana", points: 200,
//      description: Optional("A banana grown in Ecuador.")
//    )
// ]

We're then using .compactMap { $0.base } to filter out nil elements (those that threw an error on decoding).

This will create an intermediate array of [FailableDecodable<GroceryProduct>], which shouldn't be an issue; however if you wish to avoid it, you could always create another wrapper type that decodes and unwraps each element from an unkeyed container:

struct FailableCodableArray<Element : Codable> : Codable {

    var elements: [Element]

    init(from decoder: Decoder) throws {

        var container = try decoder.unkeyedContainer()

        var elements = [Element]()
        if let count = container.count {
            elements.reserveCapacity(count)
        }

        while !container.isAtEnd {
            if let element = try container
                .decode(FailableDecodable<Element>.self).base {

                elements.append(element)
            }
        }

        self.elements = elements
    }

    func encode(to encoder: Encoder) throws {
        var container = encoder.singleValueContainer()
        try container.encode(elements)
    }
}

You would then decode as:

let products = try JSONDecoder()
    .decode(FailableCodableArray<GroceryProduct>.self, from: json)
    .elements

print(products)

// [
//    GroceryProduct(
//      name: "Banana", points: 200,
//      description: Optional("A banana grown in Ecuador.")
//    )
// ]

Answer 2

I would create a new type Throwable, which can wrap any type conforming to Decodable:

enum Throwable<T: Decodable>: Decodable {
    case success(T)
    case failure(Error)

    init(from decoder: Decoder) throws {
        do {
            let decoded = try T(from: decoder)
            self = .success(decoded)
        } catch let error {
            self = .failure(error)
        }
    }
}

For decoding an array of GroceryProduct (or any other Collection):

let decoder = JSONDecoder()
let throwables = try decoder.decode([Throwable<GroceryProduct>].self, from: json)
let products = throwables.compactMap { $0.value }

where value is a computed property introduced in an extension on Throwable:

extension Throwable {
    var value: T? {
        switch self {
        case .failure(_):
            return nil
        case .success(let value):
            return value
        }
    }
}

I would opt for using a enum wrapper type (over a Struct) because it may be useful to keep track of the errors that are thrown as well as their indices.

Swift 5

For Swift 5 Consider using the Result enum e.g.

struct Throwable<T: Decodable>: Decodable {
    let result: Result<T, Error>

    init(from decoder: Decoder) throws {
        result = Result(catching: { try T(from: decoder) })
    }
}

To unwrap the decoded value use the get() method on the result property:

let products = throwables.compactMap { try? $0.result.get() }

Answer 3

The problem is that when iterating over a container, the container.currentIndex isn’t incremented so you can try to decode again with a different type.

Because the currentIndex is read only, a solution is to increment it yourself successfully decoding a dummy. I took @Hamish solution, and wrote a wrapper with a custom init.

This problem is a current Swift bug: https://bugs.swift.org/browse/SR-5953

The solution posted here is a workaround in one of the comments. I like this option because I’m parsing a bunch of models the same way on a network client, and I wanted the solution to be local to one of the objects. That is, I still want the others to be discarded.

I explain better in my github https://github.com/phynet/Lossy-array-decode-swift4

import Foundation

    let json = """
    [
        {
            "name": "Banana",
            "points": 200,
            "description": "A banana grown in Ecuador."
        },
        {
            "name": "Orange"
        }
    ]
    """.data(using: .utf8)!

    private struct DummyCodable: Codable {}

    struct Groceries: Codable 
    {
        var groceries: [GroceryProduct]

        init(from decoder: Decoder) throws {
            var groceries = [GroceryProduct]()
            var container = try decoder.unkeyedContainer()
            while !container.isAtEnd {
                if let route = try? container.decode(GroceryProduct.self) {
                    groceries.append(route)
                } else {
                    _ = try? container.decode(DummyCodable.self) // <-- TRICK
                }
            }
            self.groceries = groceries
        }
    }

    struct GroceryProduct: Codable {
        var name: String
        var points: Int
        var description: String?
    }

    let products = try JSONDecoder().decode(Groceries.self, from: json)

    print(products)

Answer 4

There are two options:

1. Declare all members of the struct as optional whose keys can be missing

   struct GroceryProduct: Codable {
       var name: String
       var points : Int?
       var description: String?
   }
   

2. Write a custom initializer to assign default values in the nil case.

   struct GroceryProduct: Codable {
       var name: String
       var points : Int
       var description: String
   
       init(from decoder: Decoder) throws {
           let values = try decoder.container(keyedBy: CodingKeys.self)
           name = try values.decode(String.self, forKey: .name)
           points = try values.decodeIfPresent(Int.self, forKey: .points) ?? 0
           description = try values.decodeIfPresent(String.self, forKey: .description) ?? ""
       }
   }
   

Answer 5

A solution made possible by Swift 5.1, using the property wrapper:

@propertyWrapper
struct IgnoreFailure<Value: Decodable>: Decodable {
    var wrappedValue: [Value] = []

    private struct _None: Decodable {}

    init(from decoder: Decoder) throws {
        var container = try decoder.unkeyedContainer()
        while !container.isAtEnd {
            if let decoded = try? container.decode(Value.self) {
                wrappedValue.append(decoded)
            }
            else {
                // item is silently ignored.
                try? container.decode(_None.self)
            }
        }
    }
}

And then the usage:

let json = """
{
    "products": [
        {
            "name": "Banana",
            "points": 200,
            "description": "A banana grown in Ecuador."
        },
        {
            "name": "Orange"
        }
    ]
}
""".data(using: .utf8)!

struct GroceryProduct: Decodable {
    var name: String
    var points: Int
    var description: String?
}

struct ProductResponse: Decodable {
    @IgnoreFailure
    var products: [GroceryProduct]
}


let response = try! JSONDecoder().decode(ProductResponse.self, from: json)
print(response.products) // Only contains banana.

Note: The property wrapper things will only works if the response can be wrapped in a struct (i.e: not a top level array). In that case, you can still wrap it manually (with a typealias for better readability):

typealias ArrayIgnoringFailure<Value: Decodable> = IgnoreFailure<Value>

let response = try! JSONDecoder().decode(ArrayIgnoringFailure<GroceryProduct>.self, from: json)
print(response.wrappedValue) // Only contains banana.

Answer 6

Ive put @sophy-swicz solution, with some modifications, into an easy to use extension

fileprivate struct DummyCodable: Codable {}

extension UnkeyedDecodingContainer {

    public mutating func decodeArray<T>(_ type: T.Type) throws -> [T] where T : Decodable {

        var array = [T]()
        while !self.isAtEnd {
            do {
                let item = try self.decode(T.self)
                array.append(item)
            } catch let error {
                print("error: \(error)")

                // hack to increment currentIndex
                _ = try self.decode(DummyCodable.self)
            }
        }
        return array
    }
}
extension KeyedDecodingContainerProtocol {
    public func decodeArray<T>(_ type: T.Type, forKey key: Self.Key) throws -> [T] where T : Decodable {
        var unkeyedContainer = try self.nestedUnkeyedContainer(forKey: key)
        return try unkeyedContainer.decodeArray(type)
    }
}

Just call it like this

init(from decoder: Decoder) throws {

    let container = try decoder.container(keyedBy: CodingKeys.self)

    self.items = try container.decodeArray(ItemType.self, forKey: . items)
}

For the example above:

let json = """
[
    {
        "name": "Banana",
        "points": 200,
        "description": "A banana grown in Ecuador."
    },
    {
        "name": "Orange"
    }
]
""".data(using: .utf8)!

struct Groceries: Codable 
{
    var groceries: [GroceryProduct]

    init(from decoder: Decoder) throws {
        var container = try decoder.unkeyedContainer()
        groceries = try container.decodeArray(GroceryProduct.self)
    }
}

struct GroceryProduct: Codable {
    var name: String
    var points: Int
    var description: String?
}

let products = try JSONDecoder().decode(Groceries.self, from: json)
print(products)

Answer 7

Instead, You can also do like this:

struct GroceryProduct: Decodable {
    var name: String
    var points: Int
    var description: String?
}'

and then in while getting it:

'let groceryList = try JSONDecoder().decode(Array<GroceryProduct>.self, from: responseData)'

Answer 8

Swift 5

Inspired with previous answers I decode inside Result enum extension.

What do you think about it?

extension Result: Decodable where Success: Decodable, Failure == DecodingError {

    public init(from decoder: Decoder) throws {

        let container: SingleValueDecodingContainer = try decoder.singleValueContainer()

        do {

            self = .success(try container.decode(Success.self))

        } catch {

            if let decodingError = error as? DecodingError {
                self = .failure(decodingError)
            } else {
                self = .failure(DecodingError.dataCorrupted(.init(codingPath: [], debugDescription: error.localizedDescription)))
            }
        }
    }
    
}

Usage

let listResult = try? JSONDecoder().decode([Result<SomeObject, DecodingError>].self, from: ##YOUR DATA##)

let list: [SomeObject] = listResult.compactMap {try? $0.get()}

Answer 9

Unfortunately Swift 4 API doesn't have failable initializer for init(from: Decoder).

Only one solution that I see is implementing custom decoding, giving default value for optional fields and possible filter with needed data:

struct GroceryProduct: Codable {
    let name: String
    let points: Int?
    let description: String

    private enum CodingKeys: String, CodingKey {
        case name, points, description
    }

    init(from decoder: Decoder) throws {
        let container = try decoder.container(keyedBy: CodingKeys.self)
        name = try container.decode(String.self, forKey: .name)
        points = try? container.decode(Int.self, forKey: .points)
        description = (try? container.decode(String.self, forKey: .description)) ?? "No description"
    }
}

// for test
let dict = [["name": "Banana", "points": 100], ["name": "Nut", "description": "Woof"]]
if let data = try? JSONSerialization.data(withJSONObject: dict, options: []) {
    let decoder = JSONDecoder()
    let result = try? decoder.decode([GroceryProduct].self, from: data)
    print("rawResult: \(result)")

    let clearedResult = result?.filter { $0.points != nil }
    print("clearedResult: \(clearedResult)")
}

Answer 10

I improved on @Hamish's for the case, that you want this behaviour for all arrays:

private struct OptionalContainer<Base: Codable>: Codable {
    let base: Base?
    init(from decoder: Decoder) throws {
        let container = try decoder.singleValueContainer()
        base = try? container.decode(Base.self)
    }
}

private struct OptionalArray<Base: Codable>: Codable {
    let result: [Base]
    init(from decoder: Decoder) throws {
        let container = try decoder.singleValueContainer()
        let tmp = try container.decode([OptionalContainer<Base>].self)
        result = tmp.compactMap { $0.base }
    }
}

extension Array where Element: Codable {
    init(from decoder: Decoder) throws {
        let optionalArray = try OptionalArray<Element>(from: decoder)
        self = optionalArray.result
    }
}

Answer 11

You made the description optional, you should also make the points field optional if there is a chance it could be nil, such as this:

struct GroceryProduct: Codable {
    var name: String
    var points: Int?
    var description: String?
}

Just make sure you safe-unwrap it however you see fit for it's use. I'm guessing nil points == 0 in the actual use case so an example could be:

let products = try JSONDecoder().decode([GroceryProduct].self, from: json)
for product in products {
    let name = product.name
    let points = product.points ?? 0
    let description = product.description ?? ""
    ProductView(name, points, description)
}

or in-line:

let products = try JSONDecoder().decode([GroceryProduct].self, from: json)
for product in products {
    ProductView(product.name, product.points ?? 0, product.description ?? "")
}

Answer 12

@Hamish's answer is great. However, you can reduce FailableCodableArray to:

struct FailableCodableArray<Element : Codable> : Codable {

    var elements: [Element]

    init(from decoder: Decoder) throws {
        let container = try decoder.singleValueContainer()
        let elements = try container.decode([FailableDecodable<Element>].self)
        self.elements = elements.compactMap { $0.wrapped }
    }

    func encode(to encoder: Encoder) throws {
        var container = encoder.singleValueContainer()
        try container.encode(elements)
    }
}

Answer 13

I had a similar issue recently, but slightly different.

struct Person: Codable {
    var name: String
    var age: Int
    var description: String?
    var friendnamesArray:[String]?
}

In this case, if one of the element in friendnamesArray is nil, the whole object is nil while decoding.

And the right way to handle this edge case is to declare the string array[String] as array of optional strings[String?] as below,

struct Person: Codable {
    var name: String
    var age: Int
    var description: String?
    var friendnamesArray:[String?]?
}

Answer 14

I come up with this KeyedDecodingContainer.safelyDecodeArray that provides a simple interface:

extension KeyedDecodingContainer {

/// The sole purpose of this `EmptyDecodable` is allowing decoder to skip an element that cannot be decoded.
private struct EmptyDecodable: Decodable {}

/// Return successfully decoded elements even if some of the element fails to decode.
func safelyDecodeArray<T: Decodable>(of type: T.Type, forKey key: KeyedDecodingContainer.Key) -> [T] {
    guard var container = try? nestedUnkeyedContainer(forKey: key) else {
        return []
    }
    var elements = [T]()
    elements.reserveCapacity(container.count ?? 0)
    while !container.isAtEnd {
        /*
         Note:
         When decoding an element fails, the decoder does not move on the next element upon failure, so that we can retry the same element again
         by other means. However, this behavior potentially keeps `while !container.isAtEnd` looping forever, and Apple does not offer a `.skipFailable`
         decoder option yet. As a result, `catch` needs to manually skip the failed element by decoding it into an `EmptyDecodable` that always succeed.
         See the Swift ticket https://bugs.swift.org/browse/SR-5953.
         */
        do {
            elements.append(try container.decode(T.self))
        } catch {
            if let decodingError = error as? DecodingError {
                Logger.error("\(#function): skipping one element: \(decodingError)")
            } else {
                Logger.error("\(#function): skipping one element: \(error)")
            }
            _ = try? container.decode(EmptyDecodable.self) // skip the current element by decoding it into an empty `Decodable`
        }
    }
    return elements
}
}

The potentially infinite loop while !container.isAtEnd is a concern, and it's addressed by using EmptyDecodable.

Answer 15

A much simpler attempt: Why don't you declare points as optional or make the array contain optional elements

let products = [GroceryProduct?]

Answer 16

Features:

• Simple use. One line in Decodable instance: let array: CompactDecodableArray<Int>
• Is decoded with standard mapping mechanism: JSONDecoder().decode(Model.self, from: data)
• skips incorrect elements (returns array with only successful mapped elements)

Details

• Xcode 12.1 (12A7403)
• Swift 5.3

Solution

class CompactDecodableArray<Element>: Decodable where Element: Decodable {
    private(set) var elements = [Element]()
    required init(from decoder: Decoder) throws {
        guard var unkeyedContainer = try? decoder.unkeyedContainer() else { return }
        while !unkeyedContainer.isAtEnd {
            if let value = try? unkeyedContainer.decode(Element.self) {
                elements.append(value)
            } else {
                unkeyedContainer.skip()
            }
        }
    }
}

// https://forums.swift.org/t/pitch-unkeyeddecodingcontainer-movenext-to-skip-items-in-deserialization/22151/17

struct Empty: Decodable { }

extension UnkeyedDecodingContainer {
    mutating func skip() { _ = try? decode(Empty.self) }
}

Usage

struct Model2: Decodable {
    let num: Int
    let str: String
}

struct Model: Decodable {
    let num: Int
    let str: String
    let array1: CompactDecodableArray<Int>
    let array2: CompactDecodableArray<Int>?
    let array4: CompactDecodableArray<Model2>
}

let dictionary: [String : Any] = ["num": 1, "str": "blablabla",
                                  "array1": [1,2,3],
                                  "array3": [1,nil,3],
                                  "array4": [["num": 1, "str": "a"], ["num": 2]]
]

let data = try! JSONSerialization.data(withJSONObject: dictionary)
let object = try JSONDecoder().decode(Model.self, from: data)
print("1. \(object.array1.elements)")
print("2. \(object.array2?.elements)")
print("3. \(object.array4.elements)")

Console

1. [1, 2, 3]
2. nil
3. [__lldb_expr_25.Model2(num: 1, str: "a")]