1

I have simple REST API written in go. I was test calling it many times using different processes with python request library. But I ran into requests.exceptions.ConnectionError (more detailed message below). This was surprising to me because I expected the server to be able to handle parallel requests. Out of curiosity I implemented the python version in go, but I was not able to reproduce the error!

Here is the code for my REST API:

package main

import (
    "net/http"
    "strconv"
    "fmt"
)

func make_result(w http.ResponseWriter, r *http.Request) {

    fmt.Println(r)

    err := r.ParseForm()
    if err != nil {
        panic(err)
    }

    number_string := r.Form["Number"][0]
    number, err := strconv.Atoi(number_string)
    if err != nil {
        panic(err)
    }

    fmt.Fprint(w, fmt.Sprint(number * 5))

}

I am calling it with python, timer.py:

import requests
from time import time
import random as rn
import sys


def time_get_requests(n_iterations):
    start = time()
    total = 0
    for i in xrange(n_iterations):
        if i % 1000 == 0:
            print "processed: {} requests, time taken: {}".format(i, time() - start)
        n = rn.randint(0, 1000)

        temp = requests.get("http://localhost:8000/get_result", params={"Number": n})

        total += int(temp.content)
    print "Total: {}".format(total)
    print "Total time: {}".format(time() - start)

if __name__ == "__main__":
    time_get_requests(int(sys.argv[1]))

I am calling using go, timer.go:

package main

import(
    "time"
    "math/rand"
    "strconv"
    "log"
    "os"

    "github.com/parnurzeal/gorequest"

)

func main() {
    rand.Seed( time.Now().UTC().UnixNano())
    n_iterations, err := strconv.Atoi(os.Args[1])
    if err != nil {
        panic(err)
    }
    total := 0
    start := time.Now()
    request := gorequest.New()
    for i := 0; i < n_iterations; i++ {
        if i % 1000 == 0 {
            log.Printf("%d: Time taken: %v\n", i, time.Now().Sub(start))
        }
        rand_int := rand.Intn(1000)
        rand_int_str := strconv.Itoa(rand_int)
        _, body, errs := request.
            Get("http://localhost:8000/get_result").
            Param("Number", rand_int_str).
            End()
        if len(errs) > 0 {
            log.Fatalf("\033[0;31m%v\033[0m\n", errs)
        }
        int_body, err := strconv.Atoi(body)
        if err != nil {
            panic(err)
        }
        total += int_body
    }
    log.Printf("Total: %d\n", total)
    total_time_taken := time.Now().Sub(start)
    log.Printf("Total time: %v\n", total_time_taken)
}

I am calling it like this in my linux shell:

for i in {0..7}; do python timer.py 20000 & done

and

for i in {0..7}; do go run timer.go 20000 & done

(When I run this on one processor both lines run fine)

The error message that I am getting:

    resp = self.send(prep, **send_kwargs)
  File "/usr/lib/python2.7/dist-packages/requests/sessions.py", line 558, in sen
d
    raise ConnectionError(e)
requests.exceptions.ConnectionError: HTTPConnectionPool(host='localhost', port=8
000): Max retries exceeded with url: /get_result?Number=850 (Caused by <class 's
ocket.error'>: [Errno 99] Cannot assign requested address)
r = adapter.send(request, **kwargs)
  File "/usr/lib/python2.7/dist-packages/requests/adapters.py", line 378, in send

My main question: Why is my python part (timer.py) leads to this error? What should I do to fix it? (I don't know whether my problem is with my REST API or how I am using requests).

My secondary question: Why does timer.go work properly?

I am using: go 1.5, python 2.7.6, requests 2.2.1, ubuntu 14.04.

Akavall
  • 82,592
  • 51
  • 207
  • 251
  • Did you try with [`wrk`](https://github.com/wg/wrk)? – kostix Nov 05 '15 at 11:39
  • To be more clear, this might indicate the exhaustion of the TCP ephemeral ports for your Python client. Hence I'd check with a *proper* tool first which is able to proprly manage its budget of sockets to create a as-constant-as-possible workload onto the service under test. – kostix Nov 05 '15 at 11:42
  • @kostix, Thanks for the info! I could not figure out how to use `wrk` for an api that takes parameters and asked about it here http://stackoverflow.com/questions/33548940/is-there-a-way-to-pass-parameters-to-get-request-using-wrk, if this is interesting to you. For an api that does not take parameters, I could not reproduce the error using `requests`. – Akavall Nov 05 '15 at 15:51

0 Answers0