How to properly parallelize 2 functions and catch errors?

Question

I'm practising golang and I have no idea how to catch errors.

What I expect:

FetchTickerData runs
It calls 2 different functions at the same time: fetchPriceTicket and fetchWhatToMine
If one of functions returns an error then FetchTickerData returns that error
If all is ok it process data from both sources and return it

I can't figure how to catch errors. I wrote this code but I don't think it's correct solution and it doesn't work. What's better way to do that?

package main

import "net/http"
import (
    "github.com/tidwall/gjson"
    "time"
    "io/ioutil"
    "fmt"
)

var client = &http.Client{Timeout: 10 * time.Second}

type Ticker struct {
}

func FetchTickerData() (error, *gjson.Result, *gjson.Result) {
    whatToMine := make(chan *gjson.Result)
    currency := make(chan *gjson.Result)
    err := make(chan error)
    counter := 0 // This variable indicates if both data was fetched

    go func() {
        innerError, json := fetchWhatToMine()

        fmt.Print(innerError)
        if innerError != nil {
            err <- innerError
            // Stop handler immediately
            whatToMine <- nil
            currency <- nil
            return
        }

        whatToMine <- json
        counter = counter + 1

        if counter == 2 {
            fmt.Print("err pushed")
            err <- nil
        }
    }()

    go func() {
        innerError, json := fetchPriceTicket()
        fmt.Print(innerError)

        if innerError != nil {
            err <- innerError
            whatToMine <- nil
            currency <- nil
            return
        }

        currency <- json
        counter = counter + 1

        if counter == 2 {
            fmt.Print("err pushed")
            err <- nil
        }
    }()

    return <-err, <-whatToMine, <-currency
}

func fetchPriceTicket() (error, *gjson.Result) {
    resp, err := client.Get("https://api.coinmarketcap.com/v1/ticker/")

    if err != nil {
        return err, nil
    }
    defer resp.Body.Close()

    body, _ := ioutil.ReadAll(resp.Body)
    json := gjson.GetBytes(body, "");
    return nil, &json;
}

func fetchWhatToMine() (error, *gjson.Result) {
    resp, err := client.Get("https://whattomine.com/coins.json")

    if err != nil {
        return err, nil
    }
    defer resp.Body.Close()

    body, _ := ioutil.ReadAll(resp.Body)
    json := gjson.GetBytes(body, "");
    return nil, &json;
}

UPD: if I replace return <-err, <-whatToMine, <-currency with return nil, <-whatToMine, <-currency it returns data that I expect but doesn't return error if there is.

UPD: There's a second version of code:

package main

import "net/http"
import (
    "github.com/tidwall/gjson"
    "time"
    "io/ioutil"
    "context"
    "fmt"
)

var client = &http.Client{Timeout: 10 * time.Second}

type Ticker struct {
}

func main() {
    ticker, coins, err := FetchTickerData()

    fmt.Print("Everything is null! ", ticker, coins, err)
    if err != nil {
        fmt.Print(err)
        return
    }
    fmt.Print("Bitcoin price in usd: ", ticker.Array()[0].Get("price_usd"))
}

func FetchTickerData() (*gjson.Result, *gjson.Result, error) {
    ctx, cancel := context.WithCancel(context.Background())
    defer cancel()

    var result1, result2 *gjson.Result
    var err1, err2 error

    go func() {
        result1, err1 = fetchJson(ctx, "https://api.coinmarketcap.com/v1/ticker/")
        if err1 != nil {
            cancel() // Abort the context, so the other function can abort early
        }
    }()

    go func() {
        result2, err2 = fetchJson(ctx, "https://whattomine.com/coins.json")
        if err2 != nil {
            cancel() // Abort the context, so the other function can abort early
        }
    }()

    if err1 == context.Canceled || err1 == nil {
        return result1, result2, err2
    }
    return result1, result2, err1
}

func fetchJson(ctx context.Context, url string) (*gjson.Result, error) {
    req, err := http.NewRequest(http.MethodGet, url, nil)
    if err != nil {
        return nil, err
    }

    req = req.WithContext(ctx)
    resp, err := client.Do(req)
    if err != nil {
        return nil, err
    }
    defer resp.Body.Close()

    body, err := ioutil.ReadAll(resp.Body)
    if err != nil {
        return nil, err
    }
    fmt.Print("I don't know why this body isn't printed ", string(body))
    json := gjson.ParseBytes(body)
    return &json, nil
}

For some reasons http requests are not working here and there is no error. Ideas?

Everything is null! <nil> <nil> <nil>panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x0 pc=0x11f7843]

goroutine 1 [running]:
main.main()
    /Users/andrey/go/src/tickerUpdater/fetchTicker.go:25 +0x183

Make sure you always return an error as the *last* value from a function. — Jonathan Hall, Feb 05 '18 at 14:36
@Flimzy does it matter which argument (last or first or second) is error? — blits, Feb 05 '18 at 14:44
Yes. Go convention is that error is always the last one. And standard Go tools will complain otherwise. — Jonathan Hall, Feb 05 '18 at 14:45
You should also use 'go fmt'--it will provide for a more uniform coding convention, and remove unnecessary semicolons, etc. — Jonathan Hall, Feb 05 '18 at 14:53
Possible duplicate of [Close multiple goroutine if an error occurs in one in go](https://stackoverflow.com/questions/45500836/close-multiple-goroutine-if-an-error-occurs-in-one-in-go/45502591#45502591). — icza, Feb 05 '18 at 16:26
You have a significant race condition with regards to the `counter` variable. It is incremented and inspected by both goroutines, which can trample on each other if they occur at the same time. Use a mutex, or better yet, use a channel (or a pair of them) or a waitgroup to indicate when the two goroutines are finished. — Kaedys, Feb 05 '18 at 19:18

Jonathan Hall · Answer 1 · 2018-02-05T15:36:58.277

This is the perfect use-case for the context package. I have removed some of your boilerplate, and your second function; you'll want to add that back in for your actual code.

func FetchTickerData() (*gjson.Result, *gjson.Result, error) {
    ctx, cancel := context.WithCancel(context.Background())
    defer cancel()

    var result1, result2 *gjson.Result
    var err1, err2 error
    var wg sync.WaitGroup

    wg.Add(1)
    go func() {
        defer wg.Done()
        result1, err1 := fetchPriceTicket(ctx)
        if err1 != nil {
            cancel() // Abort the context, so the other function can abort early
        }()
    }

    wg.Add(1)
    go func() {
        defer wg.Done()
        result2, err2 := fetchWhatToMine(ctx)
        if err2 != nil {
            cancel() // Abort the context, so the other function can abort early
        }
    }()

    wg.Wait()

    // if err1 == context.Canceled, that means the second goroutine had
    // an error and aborted the first goroutine, so return err2.
    // If err1 == nil, err2 may still be set, so return it in this case
    // as well.
    if err1 == context.Canceled || err1 == nil {
        return result1, result2, err2
    }
    return result1, result2, err1
}

func fetchPriceTicket(ctx context.Context) (*gjson.Result, error) {
    req, err := http.NewRequest(http.MethodGet, "https://api.coinmarketcap.com/v1/ticker/", nil)
    if err != nil {
        return nil, err
    }

    req = req.WithContext(ctx)
    resp, err := client.Do(req)
    if err != nil {
        return nil, err
    }
    defer resp.Body.Close()

    body, err := ioutil.ReadAll(resp.Body)
    if err != nil {
        return nil, err
    }
    json := gjson.GetBytes(body, "")
    return &json, nil
}

score 4 · Accepted Answer · answered Feb 05 '18 at 15:39

Since you aren't using channels any more, the main go routine finishes before the execution of the other two goroutines starts, hence exiting the program. you should use waitgroups to block the main goroutine until the other two finish their work.

package main

import "net/http"
import (
    "context"
    "fmt"
    "io/ioutil"
    "sync"
    "time"

    "github.com/tidwall/gjson"
)

var client = &http.Client{Timeout: 10 * time.Second}

type Ticker struct {
}

func main() {
    ticker, coins, err := FetchTickerData()

    fmt.Print("Everything is null! ", ticker, coins, err)
    if err != nil {
        fmt.Print(err)
        return
    }
    fmt.Print("Bitcoin price in usd: ", ticker.Array()[0].Get("price_usd"))
}

func FetchTickerData() (*gjson.Result, *gjson.Result, error) {
    var wg sync.WaitGroup
    ctx, cancel := context.WithCancel(context.Background())
    defer cancel()

    var result1, result2 *gjson.Result
    var err1, err2 error

    wg.Add(2)
    go func() {
        defer wg.Done()
        result1, err1 = fetchJson(ctx, "https://api.coinmarketcap.com/v1/ticker/")
        if err1 != nil {
            cancel() // Abort the context, so the other function can abort early
        }
    }()

    go func() {
        defer wg.Done()
        result2, err2 = fetchJson(ctx, "https://whattomine.com/coins.json")
        if err2 != nil {
            cancel() // Abort the context, so the other function can abort early
        }
    }()

    wg.Wait()

    if err1 == context.Canceled || err1 == nil {
        return result1, result2, err2
    }
    return result1, result2, err1
}

func fetchJson(ctx context.Context, url string) (*gjson.Result, error) {
    req, err := http.NewRequest(http.MethodGet, url, nil)
    if err != nil {
        return nil, err
    }

    req = req.WithContext(ctx)
    resp, err := client.Do(req)
    if err != nil {
        return nil, err
    }
    defer resp.Body.Close()

    body, err := ioutil.ReadAll(resp.Body)
    if err != nil {
        return nil, err
    }
    fmt.Print("I don't know why this body isn't printed ", string(body))
    json := gjson.ParseBytes(body)
    return &json, nil
}

How to properly parallelize 2 functions and catch errors?

2 Answers2