postgresql - 如何将行从 STDIN 导入到 Postgresql？

Question

在 Python 中，我有以下内容可以在不使用文件的情况下将行批量加载到 Postgresql:

import csv
import subprocess

mylist, keys = [{'name': 'fred'}, {'name': 'mary'}], ['name']
p = subprocess.Popen(['psql', 'mydb', '-U', 'openupitsme', '-h', 'my.ip.address', '--no-password', '-c',
    '\COPY tester(%s) FROM STDIN (FORMAT CSV)' % ', '.join(keys),
    '--set=ON_ERROR_STOP=false'
    ], stdin=subprocess.PIPE
)
for d in mylist:
    dict_writer = csv.DictWriter(p.stdin, keys, quoting=csv.QUOTE_MINIMAL)
    dict_writer.writerow(d)
p.stdin.close()

我正尝试在 Go 中完成相同的任务。我目前正在将行写入一个文件，然后导入它们，然后删除该文件。我想像在 Python 中一样从 STDIN 导入行。我有:

package main

import (
    "database/sql"
    "log"
    "os"
    "os/exec"

    _ "github.com/lib/pq"
)

var (
    err error
    db  *sql.DB
)

func main() {
    var err error
    fh := "/path/to/my/file.txt"
    f, err := os.Create(fh)
    if err != nil {
        panic(err)
    }
    defer f.Close()
    defer os.Remove(fh)
    rows := []string{"fred", "mary"}
    for _, n := range rows {
        _, err = f.WriteString(n + "\n")
        if err != nil {
            panic(err)
        }
    }
    // dump to postgresql
    c := exec.Command("psql", "mydb", "-U", "openupitsme", "-h", "my.ip.address", "--no-password",
        "-c", `\COPY tester(customer) FROM `+fh)
    if out, err := c.CombinedOutput(); err != nil {
        log.Println(string(out), err)
    }
}

编辑: 更进一步，但这不是插入记录:

    keys := []string{"link", "domain"}
    records := [][]string{
        {"first_name", "last_name"},
        {"Rob", "Pike"},
        {"Ken", "Thompson"},
        {"Robert", "Griesemer"},
    }

    cmd := exec.Command("psql")
    stdin, err := cmd.StdinPipe()
    if err != nil {
        log.Println(err)
    }
    stdout, err := cmd.StdoutPipe()
    if err != nil {
        log.Println(err)
    }
    if err := cmd.Start(); err != nil {
        log.Println(err)
    }
    go func() {
        _, err = io.WriteString(stdin, "search -U meyo -h 1.2.3.4 -p 1111 --no-password -c ")
        if err != nil {
            log.Println(err)
        }
        _, err := io.WriteString(stdin, fmt.Sprintf("COPY links(%s) FROM STDIN (FORMAT CSV)", strings.Join(keys, ",")))
        if err != nil {
            log.Println(err)
        }
        w := csv.NewWriter(stdin)
        if err := w.WriteAll(records); err != nil {
            log.Fatalln("error writing record to csv:", err)
        }
        w.Flush()
        if err := w.Error(); err != nil {
            log.Fatal(err)
        }
        if err != nil {
            log.Println(err)
        }
        stdin.Close()
    }()

    done := make(chan bool)
    go func() {
        _, err := io.Copy(os.Stdout, stdout)
        if err != nil {
            log.Fatal(err)
        }
        stdout.Close()
        done <- true
    }()
    <-done

    if err := cmd.Wait(); err != nil {
        log.Println(err, cmd.Args, stdout)
    }

没有插入任何记录，我得到一个无用的错误:

exit status 2

Answer 1

github.com/lib/pq 包文档实际上有an example如何做你想做的事。这是整个程序的改编文本:

package main

import (
    "database/sql"
    "log"

    "github.com/lib/pq"
)

func main() {
    records := [][]string{
        {"Rob", "Pike"},
        {"Ken", "Thompson"},
        {"Robert", "Griesemer"},
    }

    db, err := sql.Open("postgres", "dbname=postgres user=postgres password=postgres")
    if err != nil {
        log.Fatalf("open: %v", err)
    }
    if err = db.Ping(); err != nil {
        log.Fatalf("open ping: %v", err)
    }
    defer db.Close()

    txn, err := db.Begin()
    if err != nil {
        log.Fatalf("begin: %v", err)
    }

    stmt, err := txn.Prepare(pq.CopyIn("test", "first_name", "last_name"))
    if err != nil {
        log.Fatalf("prepare: %v", err)
    }

    for _, r := range records {
        _, err = stmt.Exec(r[0], r[1])
        if err != nil {
            log.Fatalf("exec: %v", err)
        }
    }

    _, err = stmt.Exec()
    if err != nil {
        log.Fatalf("exec: %v", err)
    }

    err = stmt.Close()
    if err != nil {
        log.Fatalf("stmt close: %v", err)
    }

    err = txn.Commit()
    if err != nil {
        log.Fatalf("commit: %v", err)
    }
}

在我的机器上，这会在大约 2 秒内导入 1 000 000 条记录。