go - 解析 Freebase RDF 时 golang 内存不足

Question

我正在使用 Golang 中的 XML 包解析压缩和流式传输的 Freebase RDF 的三元组。但是，我收到内存不足错误。

我必须进行垃圾收集吗？我怎样才能做到这一点？将该三元组写入 XML 文件后如何清除内存？

这是我的代码:http://play.golang.org/p/dWvbtcs7wy

主要包

import(
    "bufio"
    "flag"
    "fmt"
    "io"
    "net/url"
    "os"
    "regexp"
    "strings"
 )

 var inputFile = flag.String("infile", "freebase-rdf", "Input file path")
 var filter, _ = regexp.Compile("^file:.*|^talk:.*|^special:.*|^wikipedia:.*|^wiktionary:.*|^user:.*|^user_talk:.*")
type Redirect struct {
     Title string `xml:"title,attr"`
}

type Page struct {
    Title    string `xml:"title"`
    Abstract string `xml:""`
}

func CanonicaliseTitle(title string) string{
     can := strings.ToLower(title)
     can = strings.Replace(can, " ", "_", -1)
     can = url.QueryEscape(can)
     return can
 }

 func convertFreebaseId(uri string) string{
     if strings.HasPrefix(uri, "<") && strings.HasSuffix(uri, ">") {
        var id = uri[1 : len(uri)-1]
        id = strings.Replace(id, "http://rdf.freebase.com/ns", "", -1)
        id = strings.Replace(id, ".", "/", -1)
        return id
     }
     return uri
 }

 func parseTriple(line string) (string, string, string){
     var parts = strings.Split(line, "\t")
     subject := convertFreebaseId(parts[0])
     predicate := convertFreebaseId(parts[1])
     object := convertFreebaseId(parts[2])
     return subject, predicate, object
 }

 var (
     validRegexp = regexp.MustCompile("^[A-Za-z0-9][A-Za-z0-9_-]*$")
     englishRegexp = regexp.MustCompile("@en")
 )

 func validTitle(content []string) bool{
     for _, v := range content{
         if !englishRegexp.MatchString(v) && len(v) > 1 && strings.Index(v, "[]") != -1{
         }  
     }
     return true
  }

 func validText(content []string) bool{
     for _, v := range content{
         if !validRegexp.MatchString(v) && len(v) > 1 && strings.Index(v, "[]") != -1{
             return false
         }
     }
     return true
 }

 func processTopic(id string, properties map[string][]string, file io.Writer){
     if validTitle(properties["/type/object/name"]) && validText(properties["/common/document/text"]){
         fmt.Fprintf(file, "<card>\n")
         fmt.Fprintf(file, "<title>\"%s\"</title>\n", properties["/type/object/name"])
         fmt.Fprintf(file, "<image>\"%s/%s\"</image>\n", "https://usercontent.googleapis.com/freebase/v1/image", id)
         fmt.Fprintf(file, "<text>\"%s\"</text>\n", properties["/common/document/text"])
         fmt.Fprintf(file, "<facts>\n")
         for k, v := range properties{
             for _, value := range v{
                  fmt.Fprintf(file, "<fact property=\"%s\">%s</fact>\n", k, value)
             }
         }
         fmt.Fprintf(file, "</facts>\n")
         fmt.Fprintf(file, "</card>\n")
     }
 }

 func main(){
     var current_mid = ""
     current_topic := make(map[string][]string)
     f, err := os.Open(*inputFile)
     if err != nil {
         fmt.Println(err)
         return
     }
     r := bufio.NewReader(f)
     xmlFile, _ := os.Create("freebase.xml")
     line, err := r.ReadString('\n')
     for err == nil{
         subject, predicate, object := parseTriple(line)
         if subject == current_mid{
             current_topic[predicate] = append(current_topic[predicate], object)
         }else if len(current_mid) > 0{
              processTopic(current_mid, current_topic, xmlFile)
              current_topic = make(map[string][]string)
         }
         current_mid = subject
         line, err = r.ReadString('\n')
     }
     processTopic(current_mid, current_topic, xmlFile)
     if err != io.EOF {
         fmt.Println(err)
         return
     }
 }

Answer 1

我不确定这是你的问题，虽然阅读你的代码似乎你没有泄漏任何东西 - 但你可以使用 SetGCPercent() http://golang.org/pkg/runtime/debug/#SetGCPercent 稍微调整 GC 行为

根据 TFM，当新分配的数据与上一次收集后剩余的实时数据的比率达到此百分比时，将触发一次收集。。默认率为 100%，这意味着对于进行大量小分配并持有大量 RAM 的程序，开销可能会很大。我有一次 HTTP 缓存占用了缓存大小的 200% 以上。尝试将百分比调整到 10% 左右，看看是否有帮助。