http - 多线程Go for HTTP get

Question

我在一个 API 中使用 Go 来获取我所在城市的公交车频率，但是当我尝试使 HTTP Get 访问许多 URL 时，我有点卡在了线程上。 如果没有并发，程序需要超过 16 分钟才能完成 1500 个 url 调用以获取 HTTP 状态代码，我试图使用并发，但在阅读了很多帖子后我不明白 goroutines 是如何工作的......

想法是制作一个函数并更改请求的数量，如下所示:

 go getBusPostStatus(600, 800)

但我完全坚持...

代码如下:

package main

import (
"fmt"
"net/http"
"strconv"
"time"
)
var i int = 0
var convStr string
var message = make(chan string)

/*func main(){
    for i = 0; i < 1500; i++ {
        z = strconv.Itoa(i)
        url := "http://www.urbanosdezaragoza.es/frm_esquemaparadatime.php?poste=" + z
        resp, err := http.Get(url)
        if err != nil {
            fmt.Println("Houston, we've got problems")
        }else{
            if resp.StatusCode == 200{
                fmt.Println("OK: El poste "+z+" existe")
            }else{
                fmt.Println("WARN: El poste "+z+" NO existe")
            }
        }
    }
}*/

//Return 2 houndred posts
func returnTH(c chan string){
 for i = 0; i < 200; i++ {  
    convStr = strconv.Itoa(i)
    url := "http://www.urbanosdezaragoza.es/frm_esquemaparadatime.php?poste=" + convStr
    resp, err := http.Get(url)
    if err != nil {
        fmt.Println("Houston, we've got problems")
    }else{
        if resp.StatusCode == 200{
            //fmt.Println("OK: El poste "+z+" existe")
            c <- "OK: The bus post "+convStr+" exists"
        }else{
            //fmt.Println("WARN: El poste "+z+" NO existe")
            c <- "WARN: The bus post "+convStr+" does not exist"
        }   
    }
  }
}
func returnFH(z chan string){
  for i = 201; i < 400; i++ {   
   convStr = strconv.Itoa(i)
   url := "http://www.urbanosdezaragoza.es/frm_esquemaparadatime.php?poste=" + convStr
    resp, err := http.Get(url)
    if err != nil {
        fmt.Println("Houston, we've got problems")
    }else{
        if resp.StatusCode == 200{
            //fmt.Println("OK: El poste "+z+" existe")
            z <- "OK: The bus post "+convStr+" exists"
        }else{
            //fmt.Println("WARN: El poste "+z+" NO existe")
            z <- "WARN: The bus post "+convStr+" does not exist"
        }   
    } 
  }
}

func threadPrint(c, z chan string){
   for {
       threadOne := <- c
       threadTwo := <- z
       fmt.Println(threadOne)
       fmt.Println(threadTwo)
   }
}
func main(){
    start := time.Now()
    var c chan string = make(chan string)
    var z chan string = make(chan string)
    //for i = 0; i < 1500; i++{
    go returnTH(c)
    go returnFH(z)
    go threadPrint(c,z)
    /*go getBusPostStatus(400, 600)
    go getBusPostStatus(600, 800)
    go getBusPostStatus(800, 1000)
    go getBusPostStatus(1000, 1200)
    go getBusPostStatus(1200, 1400)
    go getBusPostStatus(1400, 1500)*/
    //}
    timeExec:= time.Since(start)
    fmt.Println("Time to exec code = ", timeExec)

    /*var input string
    fmt.Scanln(&input)
    fmt.Println("done")*/
}

非常感谢!

Answer 1

以下是一个简化的示例代码，它使用 goroutine 和 channel 并发请求 100 次并打印结果。希望这段代码有所帮助。

package main

import (
    "fmt"
    "math/rand"
    "time"
)

func main() {
    rep := 100
    results := make(chan string)

    // Use goroutine to send multiple time-consuming jobs to the channel.
    for i := 0; i < rep; i++ {
        go func(num int) {
            results <- mockHTTPRequest(num)
        }(i)
    }

    // Receive results from the channel and use them.
    for i := 0; i < rep; i++ {
        fmt.Println(<-results)
    }
}

func mockHTTPRequest(num int) string {
    timeDelay := rand.Intn(5000)
    time.Sleep(time.Duration(timeDelay) * time.Millisecond)
    if timeDelay%2 == 0 {
        return fmt.Sprintf("OK: The bus post %v exists", num)
    }
    return fmt.Sprintf("WARN: The bus post %v does not exist", num)
}

您可以在 https://play.golang.org/p/RR34roRIl4 上运行此代码.