memory - Golang， fatal error : out of memory on 1 TB RAM machine

Question

我的代码包含一个巨大的 uint8 slice ，其中包含近 5.9 亿个元素。我将此数组保留在代码中以使其尽可能快地运行。代码的最终大小为 1.3 GB。

当我尝试编译它时，它引发了 fatal error :内存不足。与以下

# command-line-arguments
fatal error: out of memory

runtime stack:
runtime.throw(0x8fb3f2, 0xd)
    /usr/local/go/src/runtime/panic.go:566 +0x95
runtime.(*mcache).refill(0x7f5c2afa3ba8, 0x1440000000a, 0x7f57dc46d5e0)
    /usr/local/go/src/runtime/mcache.go:123 +0x160
runtime.(*mcache).nextFree.func1()
    /usr/local/go/src/runtime/malloc.go:505 +0x33
runtime.systemstack(0xc420044000)
    /usr/local/go/src/runtime/asm_amd64.s:298 +0x79
runtime.mstart()
    /usr/local/go/src/runtime/proc.go:1079

goroutine 1 [running]:
runtime.systemstack_switch()
    /usr/local/go/src/runtime/asm_amd64.s:252 fp=0x131f300c970 sp=0x131f300c968
runtime.(*mcache).nextFree(0x7f5c2afa3ba8, 0xa, 0x0, 0x131f300ca80, 0x1)
    /usr/local/go/src/runtime/malloc.go:506 +0xb2 fp=0x131f300c9c8 sp=0x131f300c970
runtime.mallocgc(0x90, 0x8ed060, 0x1, 0x115f3f9c)
    /usr/local/go/src/runtime/malloc.go:658 +0x809 fp=0x131f300ca68 sp=0x131f300c9c8
runtime.newobject(0x8ed060, 0xe00b46a450)
    /usr/local/go/src/runtime/malloc.go:785 +0x38 fp=0x131f300ca98 sp=0x131f300ca68
cmd/compile/internal/gc.Nod(0xd6c186c643, 0xfca1028090, 0xd6c186c480, 0x1)
    /usr/local/go/src/cmd/compile/internal/gc/subr.go:335 +0x31 fp=0x131f300cac0 sp=0x131f300ca98
cmd/compile/internal/gc.arraylit(0x0, 0x1, 0xc420663290, 0xfca1028090, 0x131f300d6c0)
    /usr/local/go/src/cmd/compile/internal/gc/sinit.go:698 +0x18f fp=0x131f300cb48 sp=0x131f300cac0
cmd/compile/internal/gc.slicelit(0x0, 0xc420663290, 0xfca1018000, 0x131f300d6c0)
    /usr/local/go/src/cmd/compile/internal/gc/sinit.go:765 +0xb36 fp=0x131f300cc58 sp=0x131f300cb48
cmd/compile/internal/gc.anylit(0x0, 0xc420663290, 0xfca1018000, 0x131f300d6c0)
    /usr/local/go/src/cmd/compile/internal/gc/sinit.go:1106 +0xac3 fp=0x131f300cd30 sp=0x131f300cc58
cmd/compile/internal/gc.oaslit(0xfca1017ef0, 0x131f300d6c0, 0xfca1018000)
    /usr/local/go/src/cmd/compile/internal/gc/sinit.go:1192 +0x104 fp=0x131f300cd60 sp=0x131f300cd30
cmd/compile/internal/gc.walkexpr(0xfca1017ef0, 0x131f300d6c0, 0x0)
    /usr/local/go/src/cmd/compile/internal/gc/walk.go:725 +0xe9c fp=0x131f300d5b8 sp=0x131f300cd60
cmd/compile/internal/gc.walkstmt(0xfca1017ef0, 0xfca1018090)
    /usr/local/go/src/cmd/compile/internal/gc/walk.go:192 +0xf45 fp=0x131f300d7f0 sp=0x131f300d5b8
cmd/compile/internal/gc.walkstmtlist(0x131f3184000, 0x21, 0x40)
    /usr/local/go/src/cmd/compile/internal/gc/walk.go:80 +0x4d fp=0x131f300d820 sp=0x131f300d7f0
cmd/compile/internal/gc.walk(0xc42065ccf0)
    /usr/local/go/src/cmd/compile/internal/gc/walk.go:65 +0x1e4 fp=0x131f300d8f0 sp=0x131f300d820
cmd/compile/internal/gc.compile(0xc42065ccf0)
    /usr/local/go/src/cmd/compile/internal/gc/pgen.go:391 +0x1d4 fp=0x131f300dba0 sp=0x131f300d8f0
cmd/compile/internal/gc.funccompile(0xc42065ccf0)
    /usr/local/go/src/cmd/compile/internal/gc/dcl.go:1287 +0x186 fp=0x131f300dc18 sp=0x131f300dba0
cmd/compile/internal/gc.Main()
    /usr/local/go/src/cmd/compile/internal/gc/main.go:467 +0x19f8 fp=0x131f300de60 sp=0x131f300dc18
cmd/compile/internal/amd64.Main()
    /usr/local/go/src/cmd/compile/internal/amd64/galign.go:93 +0x2fa fp=0x131f300de98 sp=0x131f300de60
main.main()
    /usr/local/go/src/cmd/compile/main.go:33 +0x2a3 fp=0x131f300df28 sp=0x131f300de98
runtime.main()
    /usr/local/go/src/runtime/proc.go:183 +0x1f4 fp=0x131f300df80 sp=0x131f300df28
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:2086 +0x1 fp=0x131f300df88 sp=0x131f300df80

我认为机器的 RAM 足以完成此类任务，但无法通过此问题。我用另一个包含 2100 万个元素的片段尝试了该代码，它运行良好。这是一项非常艰巨的计算任务，并且由于它需要非常快地运行，所以我不能使用外部文件来获取数据。

已经优化了变量类型。

有什么想法吗？

Answer 1

我以字节的形式从外部二进制文件加载数据。 @BJ Black 提供的以下代码有帮助。

    buf, err := ioutil.ReadFile("somefile.bin")
    if err != nil { panic(err.Error()) }
    // Coerce type
    uintbuf := []uint8(buf)