Node.js로 대용량 txt 파일 읽기

데이터셋을 읽어보려 했다.

시도 1

const fs = require('fs');

fs.readFile('2019-Nov.csv', 'utf8', function (err, data) {
  console.log(data);
  console.log(err);
});

RangeError [ERR_FS_FILE_TOO_LARGE]: File size (9006762395) is greater than 2 GiB
    at new NodeError (node:internal/errors:387:5)
    at FSReqCallback.readFileAfterStat [as oncomplete] (node:fs:344:11) {
  code: 'ERR_FS_FILE_TOO_LARGE'
}

시도 2

한 줄씩 읽으면 된다

const fs = require('fs');
const readline = require('readline');
const stream = fs.createReadStream('2019-Nov.csv');
const rl = readline.createInterface({ input: stream });
let data = [];
let t0, t1;
rl.on('line', (row) => {
  t0 = performance.now().toFixed(2);
  // data.push(row);
  console.log(row);
});

rl.on('close', () => {
  console.log(data);
  t1 = performance.now().toFixed(2);
});

console.log(` 열리는데 총 소요 시간:${t1 - t0}`);

<--- Last few GCs --->

[1911:0x120008000]   658154 ms: Mark-sweep 3824.8 (4141.6) -> 3813.2 (4143.3) MB, 955.4 / 0.0 ms  (average mu = 0.257, current mu = 0.207) task scavenge might not succeed
[1911:0x120008000]   659357 ms: Mark-sweep 3826.4 (4143.3) -> 3814.8 (4144.8) MB, 956.0 / 0.0 ms  (average mu = 0.233, current mu = 0.206) task scavenge might not succeed


<--- JS stacktrace --->

FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory
 1: 0x1043a58d8 node::Abort() [/usr/local/bin/node]
 2: 0x1043a5a60 node::ModifyCodeGenerationFromStrings(v8::Local<v8::Context>, v8::Local<v8::Value>, bool) [/usr/local/bin/node]
 3: 0x1044e736c v8::Utils::ReportOOMFailure(v8::internal::Isolate*, char const*, bool) [/usr/local/bin/node]
 4: 0x1044e7300 v8::internal::V8::FatalProcessOutOfMemory(v8::internal::Isolate*, char const*, bool) [/usr/local/bin/node]
 5: 0x10466a81c v8::internal::Heap::GarbageCollectionReasonToString(v8::internal::GarbageCollectionReason) [/usr/local/bin/node]
 6: 0x10466933c v8::internal::Heap::CollectGarbage(v8::internal::AllocationSpace, v8::internal::GarbageCollectionReason, v8::GCCallbackFlags) [/usr/local/bin/node]
 7: 0x1046e330c v8::internal::ScavengeJob::Task::RunInternal() [/usr/local/bin/node]
 8: 0x104401850 node::PerIsolatePlatformData::RunForegroundTask(std::__1::unique_ptr<v8::Task, std::__1::default_delete<v8::Task> >) [/usr/local/bin/node]
 9: 0x1044004e8 node::PerIsolatePlatformData::FlushForegroundTasksInternal() [/usr/local/bin/node]
10: 0x104c0835c uv__async_io [/usr/local/bin/node]
11: 0x104c1a0a8 uv__io_poll [/usr/local/bin/node]
12: 0x104c087ec uv_run [/usr/local/bin/node]
13: 0x1042f66d4 node::SpinEventLoop(node::Environment*) [/usr/local/bin/node]
14: 0x1043dd328 node::NodeMainInstance::Run(int*, node::Environment*) [/usr/local/bin/node]
15: 0x1043dd008 node::NodeMainInstance::Run() [/usr/local/bin/node]
16: 0x104379a40 node::Start(int, char**) [/usr/local/bin/node]
17: 0x108fc108c