My goal: is to set a rate limit of 600 requests per minute, which is reset at the next minute. My intend was to do this via the http.client setting a RoundTrip with a limit.wait(). So that I can set different limits for different http.clients() and have the limiting handled via roundtrip rather than adding complexity to my code elsewhere.
The issue is that the rate limit is not honoured, I still exceed the number of requests allowed and setting a timeout produces a fatal panic net/http: request canceled (Client.Timeout exceeded while awaiting headers)
I have created a barebones main.go that replicates the issue. Note that the 64000 loop is a realistic scenario for me.
Update: setting ratelimiter: rate.NewLimiter(10, 10), still exceeds the 600 rate limit somehow and produces errors Context deadline exceeded with the set timeout.
package main
import (
"fmt"
"io/ioutil"
"net/http"
"sync"
"time"
"golang.org/x/time/rate"
)
var client http.Client
// ThrottledTransport Rate Limited HTTP Client
type ThrottledTransport struct {
roundTripperWrap http.RoundTripper
ratelimiter *rate.Limiter
}
func (c *ThrottledTransport) RoundTrip(r *http.Request) (*http.Response, error) {
err := c.ratelimiter.Wait(r.Context()) // This is a blocking call. Honors the rate limit
if err != nil {
return nil, err
}
return c.roundTripperWrap.RoundTrip(r)
}
// NewRateLimitedTransport wraps transportWrap with a rate limitter
func NewRateLimitedTransport(transportWrap http.RoundTripper) http.RoundTripper {
return &ThrottledTransport{
roundTripperWrap: transportWrap,
//ratelimiter: rate.NewLimiter(rate.Every(limitPeriod), requestCount),
ratelimiter: rate.NewLimiter(10, 10),
}
}
func main() {
concurrency := 20
var ch = make(chan int, concurrency)
var wg sync.WaitGroup
wg.Add(concurrency)
for i := 0; i < concurrency; i++ {
go func() {
for {
a, ok := <-ch
if !ok { // if there is nothing to do and the channel has been closed then end the goroutine
wg.Done()
return
}
resp, err := client.Get("https://api.guildwars2.com/v2/items/12452")
if err != nil {
fmt.Println(err)
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
fmt.Println(err)
}
fmt.Println(a, ":", string(body[4:29]))
}
}()
}
client = http.Client{}
client.Timeout = time.Second * 10
// Rate limits 600 requests per 60 seconds via RoundTripper
transport := NewRateLimitedTransport(http.DefaultTransport)
client.Transport = transport
for i := 0; i < 64000; i++ {
ch <- i // add i to the queue
}
wg.Wait()
fmt.Println("done")
}
