2

I'm trying to run a pretty straightforward parser of coinmarketcap. I only have two tables: currency and rate, so I add the new currencies on the fly and insert a new rate after that.

package main

import (
    "log"
    "os"
    "../helpers"
    "fmt"
    "database/sql"
    _ "github.com/lib/pq"
    "encoding/json"
    "strconv"
)

func chk(err error) {
    if err != nil {
        log.Panic(err)
    }
}

type Rate struct {
    Name   string `json:"name"`
    Symbol string `json:"symbol"`
    Price  string `json:"price_usd"`
}

func getCreateCurrency(db *sql.DB, rate Rate) int {
    var id int

    res, err := db.Query("SELECT id FROM currency WHERE symbol = $1", rate.Symbol)
    chk(err)
    defer res.Close()

    if res.Next() {
        err = res.Scan(&id)
        chk(err)
        return id
    }

    err = db.QueryRow(`
        INSERT INTO currency
        (name, symbol)
        VALUES ($1, $2)
        RETURNING id
    `, rate.Name, rate.Symbol).Scan(&id)
    chk(err)

    return id
}

func parseRates(db *sql.DB) {
    for {
        ratesBody, err := helpers.GetHttpBody("https://api.coinmarketcap.com/v1/ticker/?limit=0")
        chk(err)

        rates := make([]Rate, 0)
        json.Unmarshal([]byte(ratesBody), &rates)

        for _, rate := range rates {
            currencyId := getCreateCurrency(db, rate)
            price, err := strconv.ParseFloat(rate.Price, 64)
            if err != nil {
                continue
            }

            db.QueryRow(`INSERT INTO rate (currency_id, rate) VALUES ($1, $2)`, currencyId, price)

            fmt.Println(rate.Symbol, rate.Price)
        }

        break
    }
}

func main() {
    cfg, err := helpers.GetConfig(os.Args[1])
    chk(err)

    db, err := sql.Open("postgres", cfg.DbConnection)
    chk(err)
    defer db.Close()

    parseRates(db)
}

All of a sudden a got an error exactly after 100 loop iterations:

panic: pq: sorry, too many clients already

The problem somewhere at the getCreateCurrency function. I get the hung connection somehow, because if I turn it off all works just fine.

Of course I have max_connections = 100 at postgres.conf, but I thought I only use one connection, because I event don't use async functionality here.

I use macos, pg 9.6, go 1.9.2.

James May
  • 1,371
  • 3
  • 20
  • 37
  • 4
    The `parseRates`'s call to `db.QueryRow`... you're not handling the result. Either use `db.Exec` or call `Scan` on the returned row. – mkopriva Jan 02 '18 at 10:48
  • 1
    For more details see this answer: https://stackoverflow.com/questions/47706147/how-to-reuse-a-single-postgres-db-connection-for-row-inserts-in-go/47711577#47711577 – mkopriva Jan 02 '18 at 10:49
  • Thank you a lot !!! – James May Jan 02 '18 at 11:04

0 Answers0