I'm trying to run a pretty straightforward parser of coinmarketcap. I only have two tables: currency and rate, so I add the new currencies on the fly and insert a new rate after that.
package main
import (
"log"
"os"
"../helpers"
"fmt"
"database/sql"
_ "github.com/lib/pq"
"encoding/json"
"strconv"
)
func chk(err error) {
if err != nil {
log.Panic(err)
}
}
type Rate struct {
Name string `json:"name"`
Symbol string `json:"symbol"`
Price string `json:"price_usd"`
}
func getCreateCurrency(db *sql.DB, rate Rate) int {
var id int
res, err := db.Query("SELECT id FROM currency WHERE symbol = $1", rate.Symbol)
chk(err)
defer res.Close()
if res.Next() {
err = res.Scan(&id)
chk(err)
return id
}
err = db.QueryRow(`
INSERT INTO currency
(name, symbol)
VALUES ($1, $2)
RETURNING id
`, rate.Name, rate.Symbol).Scan(&id)
chk(err)
return id
}
func parseRates(db *sql.DB) {
for {
ratesBody, err := helpers.GetHttpBody("https://api.coinmarketcap.com/v1/ticker/?limit=0")
chk(err)
rates := make([]Rate, 0)
json.Unmarshal([]byte(ratesBody), &rates)
for _, rate := range rates {
currencyId := getCreateCurrency(db, rate)
price, err := strconv.ParseFloat(rate.Price, 64)
if err != nil {
continue
}
db.QueryRow(`INSERT INTO rate (currency_id, rate) VALUES ($1, $2)`, currencyId, price)
fmt.Println(rate.Symbol, rate.Price)
}
break
}
}
func main() {
cfg, err := helpers.GetConfig(os.Args[1])
chk(err)
db, err := sql.Open("postgres", cfg.DbConnection)
chk(err)
defer db.Close()
parseRates(db)
}
All of a sudden a got an error exactly after 100 loop iterations:
panic: pq: sorry, too many clients already
The problem somewhere at the getCreateCurrency function. I get the hung connection somehow, because if I turn it off all works just fine.
Of course I have max_connections = 100 at postgres.conf, but I thought I only use one connection, because I event don't use async functionality here.
I use macos, pg 9.6, go 1.9.2.