1

i have problem in Allocation failed - JavaScript heap out of memory
how i can solve this problem and what is way for solve this problem. Knowing edit files size 50GB how i can solve this and why i get this problem. then i use the a lot function and not solve it with me and i split my file to 600 files same problem

i use this

const path = require('path');
const LineByLineReader = require('line-by-line');

async function processDir(pathToDir) {
    const dir = await fs.promises.opendir(pathToDir);
    const processFilePromises = [];
    for await (const entry of dir) {
        processFilePromises.push(processFile(path.resolve(pathToDir, entry.name)));
    }
    return Promise.all(processFilePromises);
}

async function processFile(file) {
    return new Promise(resolve => {
        const lr = new LineByLineReader(file);
        lr.on('line', (line) => {
           // process the line here
        });

        lr.on('end', () => {
            resolve();
        });
    })
}

processDir('./path-to-your-dir').catch(console.error);

the error is:


{
  "header": {
    "reportVersion": 1,
    "event": "Allocation failed - JavaScript heap out of memory",
    "trigger": "FatalError",
    "filename": "report.20200829.092658.11508.0.001.json",
    "dumpEventTime": "2020-08-29T09:26:58Z",
    "dumpEventTimeStamp": "1598718418779",
    "processId": 11508,
    "cwd": "C:\\Users\\LENOVO\\proxy",
    "commandLine": [
      "C:\\Program Files\\nodejs\\node.exe",
      "app"
    ],
    "nodejsVersion": "v12.13.0",
    "wordSize": 64,
    "arch": "x64",
    "platform": "win32",
    "componentVersions": {
      "node": "12.13.0",
      "v8": "7.7.299.13-node.12",
      "uv": "1.32.0",
      "zlib": "1.2.11",
      "brotli": "1.0.7",
      "ares": "1.15.0",
      "modules": "72",
      "nghttp2": "1.39.2",
      "napi": "5",
      "llhttp": "1.1.4",
      "http_parser": "2.8.0",
      "openssl": "1.1.1d",
      "cldr": "35.1",
      "icu": "64.2",
      "tz": "2019a",
      "unicode": "12.1"
    },
    "release": {
      "name": "node",
      "lts": "Erbium",
      "headersUrl": "https://nodejs.org/download/release/v12.13.0/node-v12.13.0-headers.tar.gz",
      "sourceUrl": "https://nodejs.org/download/release/v12.13.0/node-v12.13.0.tar.gz",
      "libUrl": "https://nodejs.org/download/release/v12.13.0/win-x64/node.lib"
    },
    "osName": "Windows_NT",
    "osRelease": "10.0.19041",
    "osVersion": "Windows 10 Pro",
    "osMachine": "x86_64",
    "cpus": [
      {
        "model": "Intel(R) Core(TM) i7-5500U CPU @ 2.40GHz",
        "speed": 2394,
        "user": 277453,
        "nice": 0,
        "sys": 153328,
        "idle": 273687,
        "irq": 4875
      },
      {
        "model": "Intel(R) Core(TM) i7-5500U CPU @ 2.40GHz",
        "speed": 2394,
        "user": 299093,
        "nice": 0,
        "sys": 112828,
        "idle": 292359,
        "irq": 1781
      },
      {
        "model": "Intel(R) Core(TM) i7-5500U CPU @ 2.40GHz",
        "speed": 2394,
        "user": 331765,
        "nice": 0,
        "sys": 109937,
        "idle": 262578,
        "irq": 750
      },
      {
        "model": "Intel(R) Core(TM) i7-5500U CPU @ 2.40GHz",
        "speed": 2394,
        "user": 330187,
        "nice": 0,
        "sys": 103500,
        "idle": 270593,
        "irq": 421
      }
    ],
    "networkInterfaces": [
      {
        "name": "Wi-Fi",
        "internal": false,
        "mac": "34:e6:ad:77:4e:58",
        "address": "fe80::1cc9:77f3:40b4:5f2d",
        "netmask": "ffff:ffff:ffff:ffff::",
        "family": "IPv6",
        "scopeid": 14
      },
      {
        "name": "Wi-Fi",
        "internal": false,
        "mac": "34:e6:ad:77:4e:58",
        "address": "192.168.100.10",
        "netmask": "255.255.255.0",
        "family": "IPv4"
      },
      {
        "name": "Loopback Pseudo-Interface 1",
        "internal": true,
        "mac": "00:00:00:00:00:00",
        "address": "::1",
        "netmask": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff",
        "family": "IPv6",
        "scopeid": 0
      },
      {
        "name": "Loopback Pseudo-Interface 1",
        "internal": true,
        "mac": "00:00:00:00:00:00",
        "address": "127.0.0.1",
        "netmask": "255.0.0.0",
        "family": "IPv4"
      }
    ],
    "host": "DESKTOP-RPO6L6P"
  },
  "javascriptStack": {
    "message": "No stack.",
    "stack": [
      "Unavailable."
    ]
  },
  "nativeStack": [
    {
      "pc": "0x00007ff794d31389",
      "symbol": "std::basic_ostream<char,std::char_traits<char> >::operator<<+10873"
    },
    {
      "pc": "0x00007ff794d357cc",
      "symbol": "std::basic_ostream<char,std::char_traits<char> >::operator<<+28348"
    },
    {
      "pc": "0x00007ff794d34778",
      "symbol": "std::basic_ostream<char,std::char_traits<char> >::operator<<+24168"
    },
    {
      "pc": "0x00007ff794e2332b",
      "symbol": "v8::base::CPU::has_sse+37723"
    },
    {
      "pc": "0x00007ff795626f4e",
      "symbol": "v8::Isolate::ReportExternalAllocationLimitReached+94"
    },
    {
      "pc": "0x00007ff79560ef91",
      "symbol": "v8::SharedArrayBuffer::Externalize+833"
    },
    {
      "pc": "0x00007ff7954dc85c",
      "symbol": "v8::internal::Heap::EphemeronKeyWriteBarrierFromCode+1436"
    },
    {
      "pc": "0x00007ff7954e7c00",
      "symbol": "v8::internal::Heap::ProtectUnprotectedMemoryChunks+1312"
    },
    {
      "pc": "0x00007ff7954e4734",
      "symbol": "v8::internal::Heap::PageFlagsAreConsistent+3204"
    },
    {
      "pc": "0x00007ff7954d9fc3",
      "symbol": "v8::internal::Heap::CollectGarbage+1283"
    },
    {
      "pc": "0x00007ff7954d8794",
      "symbol": "v8::internal::Heap::AddRetainedMap+2356"
    },
    {
      "pc": "0x00007ff7954f9a65",
      "symbol": "v8::internal::Factory::NewFillerObject+53"
    },
    {
      "pc": "0x00007ff795266239",
      "symbol": "v8::internal::interpreter::JumpTableTargetOffsets::iterator::operator=+4057"
    },
    {
      "pc": "0x00007ff795a52c6d",
      "symbol": "v8::internal::SetupIsolateDelegate::SetupHeap+567949"
    },
    {
      "pc": "0x000000a36e6bc578",
      "symbol": ""
    }
  ],
  "javascriptHeap": {
    "totalMemory": 2151895040,
    "totalCommittedMemory": 2151895040,
    "usedMemory": 1999598008,
    "availableMemory": 48001464,
    "memoryLimit": 2197815296,
    "heapSpaces": {
      "read_only_space": {
        "memorySize": 262144,
        "committedMemory": 262144,
        "capacity": 261872,
        "used": 32296,
        "available": 229576
      },
      "new_space": {
        "memorySize": 2097152,
        "committedMemory": 2097152,
        "capacity": 1047488,
        "used": 266832,
        "available": 780656
      },
      "old_space": {
        "memorySize": 2146021376,
        "committedMemory": 2146021376,
        "capacity": 1996596824,
        "used": 1996573336,
        "available": 23488
      },
      "code_space": {
        "memorySize": 950272,
        "committedMemory": 950272,
        "capacity": 679936,
        "used": 679936,
        "available": 0
      },
      "map_space": {
        "memorySize": 1052672,
        "committedMemory": 1052672,
        "capacity": 593280,
        "used": 593280,
        "available": 0
      },
      "large_object_space": {
        "memorySize": 1462272,
        "committedMemory": 1462272,
        "capacity": 1448776,
        "used": 1448776,
        "available": 0
      },
      "code_large_object_space": {
        "memorySize": 49152,
        "committedMemory": 49152,
        "capacity": 3552,
        "used": 3552,
        "available": 0
      },
      "new_large_object_space": {
        "memorySize": 0,
        "committedMemory": 0,
        "capacity": 1047488,
        "used": 0,
        "available": 1047488
      }
    }
  },
  "resourceUsage": {
    "userCpuSeconds": 248.64,
    "kernelCpuSeconds": 75.953,
    "cpuConsumptionPercent": 135.247,
    "maxRss": 2484764672,
    "pageFaults": {
      "IORequired": 778624,
      "IONotRequired": 0
    },
    "fsActivity": {
      "reads": 700,
      "writes": 165
    }
  },
  "libuv": [
  ],
  "environmentVariables": {
    "": "::=::\\",
    "ALLUSERSPROFILE": "C:\\ProgramData",
    "APPDATA": "C:\\Users\\LENOVO\\AppData\\Roaming",
    "CommonProgramFiles": "C:\\Program Files\\Common Files",
    "CommonProgramFiles(x86)": "C:\\Program Files (x86)\\Common Files",
    "CommonProgramW6432": "C:\\Program Files\\Common Files",
    "COMPUTERNAME": "DESKTOP-RPO6L6P",
    "ComSpec": "C:\\Windows\\system32\\cmd.exe",
    "DriverData": "C:\\Windows\\System32\\Drivers\\DriverData",
    "HOMEDRIVE": "C:",
    "HOMEPATH": "\\Users\\LENOVO",
    "LOCALAPPDATA": "C:\\Users\\LENOVO\\AppData\\Local",
    "LOGONSERVER": "\\\\DESKTOP-RPO6L6P",
    "NUMBER_OF_PROCESSORS": "4",
    "OneDrive": "C:\\Users\\LENOVO\\OneDrive",
    "OS": "Windows_NT",
    "Path": "C:\\Program Files (x86)\\Common Files\\Oracle\\Java\\javapath;C:\\Windows\\system32;C:\\Windows;C:\\Windows\\System32\\Wbem;C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\;C:\\Windows\\System32\\OpenSSH\\;C:\\Program Files\\nodejs\\;C:\\Program Files\\Git\\cmd;C:\\MinGW\\bin;C:\\Users\\LENOVO\\AppData\\Local\\Programs\\Python\\Python37\\Scripts\\;C:\\Users\\LENOVO\\AppData\\Local\\Programs\\Python\\Python37\\;C:\\Users\\LENOVO\\AppData\\Local\\Microsoft\\WindowsApps;C:\\Users\\LENOVO\\AppData\\Roaming\\npm;C:\\Users\\LENOVO\\AppData\\Local\\Programs\\Microsoft VS Code\\bin;C:\\Users\\LENOVO\\AppData\\Local\\GitHubDesktop\\bin",
    "PATHEXT": ".COM;.EXE;.BAT;.CMD;.VBS;.VBE;.JS;.JSE;.WSF;.WSH;.MSC;.CPL",
    "PROCESSOR_ARCHITECTURE": "AMD64",
    "PROCESSOR_IDENTIFIER": "Intel64 Family 6 Model 61 Stepping 4, GenuineIntel",
    "PROCESSOR_LEVEL": "6",
    "PROCESSOR_REVISION": "3d04",
    "ProgramData": "C:\\ProgramData",
    "ProgramFiles": "C:\\Program Files",
    "ProgramFiles(x86)": "C:\\Program Files (x86)",
    "ProgramW6432": "C:\\Program Files",
    "PSModulePath": "C:\\Users\\LENOVO\\Documents\\WindowsPowerShell\\Modules;C:\\Program Files\\WindowsPowerShell\\Modules;C:\\Windows\\system32\\WindowsPowerShell\\v1.0\\Modules",
    "PUBLIC": "C:\\Users\\Public",
    "SESSIONNAME": "Console",
    "SystemDrive": "C:",
    "SystemRoot": "C:\\Windows",
    "TEMP": "C:\\Users\\LENOVO\\AppData\\Local\\Temp",
    "TMP": "C:\\Users\\LENOVO\\AppData\\Local\\Temp",
    "USERDOMAIN": "DESKTOP-RPO6L6P",
    "USERDOMAIN_ROAMINGPROFILE": "DESKTOP-RPO6L6P",
    "USERNAME": "LENOVO",
    "USERPROFILE": "C:\\Users\\LENOVO",
    "windir": "C:\\Windows"
  },
  "sharedObjects": [
    "C:\\Program Files\\nodejs\\node.exe",
    "C:\\Windows\\SYSTEM32\\ntdll.dll",
    "C:\\Windows\\System32\\KERNEL32.DLL",
    "C:\\Windows\\System32\\KERNELBASE.dll",
    "C:\\Windows\\System32\\WS2_32.dll",
    "C:\\Windows\\System32\\RPCRT4.dll",
    "C:\\Windows\\SYSTEM32\\dbghelp.dll",
    "C:\\Windows\\System32\\ucrtbase.dll",
    "C:\\Windows\\System32\\ADVAPI32.dll",
    "C:\\Windows\\System32\\msvcrt.dll",
    "C:\\Windows\\System32\\sechost.dll",
    "C:\\Windows\\System32\\USER32.dll",
    "C:\\Windows\\System32\\win32u.dll",
    "C:\\Windows\\System32\\GDI32.dll",
    "C:\\Windows\\System32\\gdi32full.dll",
    "C:\\Windows\\System32\\msvcp_win.dll",
    "C:\\Windows\\System32\\PSAPI.DLL",
    "C:\\Windows\\System32\\CRYPT32.dll",
    "C:\\Windows\\SYSTEM32\\IPHLPAPI.DLL",
    "C:\\Windows\\SYSTEM32\\USERENV.dll",
    "C:\\Windows\\System32\\bcrypt.dll",
    "C:\\Windows\\SYSTEM32\\WINMM.dll",
    "C:\\Windows\\System32\\IMM32.DLL",
    "C:\\Windows\\SYSTEM32\\powrprof.dll",
    "C:\\Windows\\SYSTEM32\\UMPDC.dll",
    "C:\\Windows\\system32\\uxtheme.dll",
    "C:\\Windows\\System32\\combase.dll",
    "C:\\Windows\\system32\\mswsock.dll",
    "C:\\Windows\\SYSTEM32\\kernel.appcore.dll",
    "C:\\Windows\\System32\\bcryptprimitives.dll",
    "C:\\Windows\\SYSTEM32\\CRYPTBASE.DLL",
    "C:\\Windows\\System32\\NSI.dll",
    "C:\\Windows\\SYSTEM32\\dhcpcsvc6.DLL",
    "C:\\Windows\\SYSTEM32\\dhcpcsvc.DLL",
    "C:\\Windows\\SYSTEM32\\DNSAPI.dll",
    "C:\\Windows\\system32\\napinsp.dll",
    "C:\\Windows\\system32\\pnrpnsp.dll",
    "C:\\Windows\\system32\\wshbth.dll",
    "C:\\Windows\\system32\\NLAapi.dll",
    "C:\\Windows\\System32\\winrnr.dll",
    "C:\\Windows\\System32\\rasadhlp.dll",
    "C:\\Windows\\System32\\fwpuclnt.dll"
  ]
}
TR Alruwaily
  • 21
  • 1
  • Well, you're processing all the files in the directory in parallel. That means that every single file is open at once. The simplest way to reduce memory usage would be to process one at a time. You could also process N at a time where N is a number between 1 and 10. – jfriend00 Aug 29 '20 at 22:02
  • can you edit my code and send how it work ? – TR Alruwaily Aug 30 '20 at 01:08

1 Answers1

0

Well, you're processing all the files in the directory in parallel. That means that every single file is open at once. The simplest way to reduce memory usage would be to process one at a time.

You could do them sequentially like this:

const path = require('path');
const LineByLineReader = require('line-by-line');

async function processDir(pathToDir) {
    const dir = await fs.promises.opendir(pathToDir);
    const results = [];
    for await (const entry of dir) {
        let result = await processFile(path.join(pathToDir, entry.name));
        results.push(result);
    }
    return results;
}

async function processFile(file) {
    return new Promise(resolve => {
        const lr = new LineByLineReader(file);
        lr.on('line', (line) => {
           // process the line here
        });

        lr.on('end', () => {
            resolve();
        });
    })
}

processDir('./path-to-your-dir').catch(console.error);

You could also process N at a time where N is a number between 1 and 10. For that, I would probably use a previously created function like mapConcurrent() here.

jfriend00
  • 683,504
  • 96
  • 985
  • 979
  • please can you create this together ? because i do all function not work with me and i trust you , you can help me and to do that – TR Alruwaily Aug 30 '20 at 01:30
  • @TRAlruwaily - I don't know what that last comment means. Please explain. – jfriend00 Aug 30 '20 at 03:38
  • how i can create `mapConcurrent()` and add my code ? can you do that ? – TR Alruwaily Aug 30 '20 at 17:57
  • @TRAlruwaily - I'm sorry, but we don't just write code for you. We answer questions when you get stuck trying something yourself. Get the list of files into an array and it should be very easy to use `mapConcurrent()` with that array. If you get stuck, then start a new question and show the code you have and describe where you got stuck. – jfriend00 Aug 30 '20 at 18:12
  • @TRAlruwaily - Also, if you [search](https://stackoverflow.com/search?q=mapconcurrent) for `mapConcurrent()` here, there are 19 answers that use it that should also give you some info. – jfriend00 Aug 30 '20 at 18:48