Memory consumption with large array with Echo or Gin framework

Question

I have a memory problem when I try to send a large array with Echo (and Gin too). After the request, memory is not free.

package main

import (
    "net/http"
    "strconv"

    "github.com/labstack/echo"
)

type User struct {
    Username  string
    Password  string
    Lastname  string
    Firstname string
}

func main() {
    e := echo.New()
    e.GET("/", func(c echo.Context) error {
        var user User
        users := make([]User, 0)

        for i := 0; i < 100000; i++ {
            user = User{
                Username:  "ffgfgfghhfghfhgfgfhgfghfghfhgfhgfh" + strconv.Itoa(i),
                Password:  "gjgjghjgjhgjhghjfrserhkhjhklljjkbhjvftxersgdghjjkhkljkbhftd",
                Lastname:  "njuftydfhgjkjlkjlkjlkhjkhu",
                Firstname: "jkggkjkl,,lm,kljkvgf"}

            users = append(users, user)
        }

        defer func() {
            users = nil
        }()
        return c.JSON(http.StatusOK, users)
    })
    e.Logger.Fatal(e.Start(":1323"))
}

To test, I run request in parallel and I have these results :

• 1 request : 300Mo
• 5 requests : 1.5Go
• 10 requests : 3.1Go
• more : my PC freeze :)

How can I reduce memory consumption?

EDIT

It works well if I don't have to process the data.
If, for example, I get 100,000 lines from the database. And then I need to process them to return a JSON with several levels. In this case, I am obliged to create an array or map.
But the problem is that memory is never released. And it increases with each request and it is even worse in the case of parallel requests.

Here an example:

import (
    "strconv"
    "time"
)

type sqlDataType struct {
    ApplicationID        int
    ApplicationName      string
    ApplicationCreatedAt time.Time
    ApplicationUpdatedAt time.Time
    ModuleID             int
    ModuleName           string
    ModuleCreatedAt      time.Time
    ModuleUpdatedAt      time.Time
    ActionID             int
    ActionName           string
    ActionCreatedAt      time.Time
    ActionUpdatedAt      time.Time
}

type DataApplicationType struct {
    Name      string
    CreatedAt time.Time
    UpdatedAt time.Time
    Modules   map[int]dataModuleType
}

type dataModuleType struct {
    Name      string
    CreatedAt time.Time
    UpdatedAt time.Time
    Actions   map[int]dataActionType
}

type dataActionType struct {
    Name      string
    CreatedAt time.Time
    UpdatedAt time.Time
}

// InitData inits data for test
func InitData() map[int]DataApplicationType {
    data := make(map[int]DataApplicationType)

    const nbApplications = 10
    const nbModules = 1000
    const nbActions = 100000

    sqlData := make([]sqlDataType, 0)
    for i := 0; i < nbActions; i++ {
        line := sqlDataType{
            ApplicationID:        (i % nbApplications) + 1,
            ApplicationName:      "Application " + strconv.Itoa((i%nbApplications)+1),
            ApplicationCreatedAt: time.Now(),
            ApplicationUpdatedAt: time.Now(),
            ModuleID:             (i % nbModules) + 1,
            ModuleName:           "Module " + strconv.Itoa((i%nbModules)+1),
            ModuleCreatedAt:      time.Now(),
            ModuleUpdatedAt:      time.Now(),
            ActionID:             i + 1,
            ActionName:           "Action " + strconv.Itoa(i+1),
            ActionCreatedAt:      time.Now(),
            ActionUpdatedAt:      time.Now(),
        }

        sqlData = append(sqlData, line)
    }

    nbData := len(sqlData)
    for i := 0; i < nbData; i++ {
        if _, ok := data[sqlData[i].ApplicationID]; !ok {
            dac := new(dataActionType)
            dac.Name = sqlData[i].ActionName
            dac.CreatedAt = sqlData[i].ActionCreatedAt
            dac.UpdatedAt = sqlData[i].ActionUpdatedAt

            dmo := new(dataModuleType)
            dmo.Name = sqlData[i].ModuleName
            dmo.CreatedAt = sqlData[i].ModuleCreatedAt
            dmo.UpdatedAt = sqlData[i].ModuleUpdatedAt
            dmo.Actions = make(map[int]dataActionType)
            dmo.Actions[sqlData[i].ActionID] = *dac

            dap := new(DataApplicationType)
            dap.Name = sqlData[i].ApplicationName
            dap.CreatedAt = sqlData[i].ApplicationCreatedAt
            dap.UpdatedAt = sqlData[i].ApplicationUpdatedAt
            dap.Modules = make(map[int]dataModuleType)
            dap.Modules[sqlData[i].ModuleID] = *dmo

            data[sqlData[i].ApplicationID] = *dap
        }

        if _, ok := data[sqlData[i].ApplicationID].Modules[sqlData[i].ModuleID]; !ok {
            dac := new(dataActionType)
            dac.Name = sqlData[i].ActionName
            dac.CreatedAt = sqlData[i].ActionCreatedAt
            dac.UpdatedAt = sqlData[i].ActionUpdatedAt

            dmo := new(dataModuleType)
            dmo.Name = sqlData[i].ModuleName
            dmo.CreatedAt = sqlData[i].ModuleCreatedAt
            dmo.UpdatedAt = sqlData[i].ModuleUpdatedAt
            dmo.Actions = make(map[int]dataActionType)
            dmo.Actions[sqlData[i].ActionID] = *dac

            data[sqlData[i].ApplicationID].Modules[sqlData[i].ModuleID] = *dmo
        }

        if _, ok := data[sqlData[i].ApplicationID].Modules[sqlData[i].ModuleID].Actions[sqlData[i].ActionID]; !ok {
            dac := new(dataActionType)
            dac.Name = sqlData[i].ActionName
            dac.CreatedAt = sqlData[i].ActionCreatedAt
            dac.UpdatedAt = sqlData[i].ActionUpdatedAt

            data[sqlData[i].ApplicationID].Modules[sqlData[i].ModuleID].Actions[sqlData[i].ActionID] = *dac
        }
    }

    return data
}

In the main.go:

func main() {
    // Lancement de Cobra
    // commands.Execute()
    go issues.InitData()
    go issues.InitData()
    go issues.InitData()
    go issues.InitData()
    go issues.InitData()

    time.Sleep(60 * time.Second)
}

This script needs around 500 Mo of memory and does not release it whereas the map hasn't even been transformed into JSON.

How can I reduce memory consumption and/or I can have a stable memory consumption state with many calls?

Thanks for you help

Answer 1

Allocated memory isn't immediately returned back to the OS, see Cannot free memory once occupied by bytes.Buffer; and Freeing unused memory?

Your answer gives little improvement, because you are still building the (Go) array (or rather slice) in memory, and once it's done, only then you proceed to marshal it into the response. You also create a new encoder for each item, you marshal a single item and you just throw it away. You may use json.Encoder to marshal multiple items. You also flush the response after each item, that's also terribly inefficient. That defeats the purpose of all internal buffering...

Instead you may marshal them as soon as an item (User) is ready, so you don't have to keep all in memory. And don't flush after every user, it's enough to do it once in the end, which isn't necessary as once you return from the handler, the server will flush all buffered data.

Do it something like this:

e.GET("/", func(c echo.Context) error {
    c.Response().WriteHeader(http.StatusOK)

    enc := json.NewEncoder(c.Response())
    for i := 0; i < 100000; i++ {
        user := User{
            Username:  "ffgfgfghhfghfhgfgfhgfghfghfhgfhgfh" + strconv.Itoa(i),
            Password:  "gjgjghjgjhgjhghjfrserhkhjhklljjkbhjvftxersgdghjjkhkljkbhfd",
            Lastname:  "njuftydfhgjkjlkjlkjlkhjkhu",
            Firstname: "jkggkjkl,,lm,kljkvgf",
        }
        if err := enc.Encode(user); err != nil {
            return err
        }
    }

    return nil
})

One thing to note here: the above code does not send a JSON array to the output, it sends a series of JSON objects. If this is not suitable to you and you do need to send a single JSON array, simply "frame" the data and insert a comma between items:

e.GET("/", func(c echo.Context) error {
    resp := c.Response()
    resp.WriteHeader(http.StatusOK)

    if _, err := io.WriteString(resp, "["); err != nil {
        return err
    }
    enc := json.NewEncoder(resp)
    for i := 0; i < 100000; i++ {
        if i > 0 {
            if _, err := io.WriteString(resp, ","); err != nil {
                return err
            }
        }
        user := User{
            Username:  "ffgfgfghhfghfhgfgfhgfghfghfhgfhgfh" + strconv.Itoa(i),
            Password:  "gjgjghjgjhgjhghjfrserhkhjhklljjkbhjvftxersgdghjjkhkljkbhft",
            Lastname:  "njuftydfhgjkjlkjlkjlkhjkhu",
            Firstname: "jkggkjkl,,lm,kljkvgf",
        }
        if err := enc.Encode(user); err != nil {
            return err
        }
    }
    if _, err := io.WriteString(resp, "]"); err != nil {
        return err
    }

    return nil
})