如何使用 Golang 构建多线程爬虫框架_Golang 并发抓取与任务分配讲解

func fetch(url string, ch chan string) {
    resp, err := http.Get(url)
    if err != nil {
        ch <- fmt.Sprintf("Error fetching %s: %v", url, err)
        return
    }
    defer resp.Body.Close()
    ch <- fmt.Sprintf("Fetched %s with status %s", url, resp.Status)
}

ch := make(chan string)
for _, url := range urls {
    go fetch(url, ch)
}
for range urls {
    fmt.Println(<-ch)
}

type Task struct {
    URL string
    Fn  func(*http.Response)
}
<p>tasks := make(chan Task, 100)</p><p>for i := 0; i < 10; i++ { // 10 个 worker
go func() {
for task := range tasks {
resp, err := http.Get(task.URL)
if err == nil {
task.Fn(resp)
resp.Body.Close()
}
}
}()
}</p><p>// 提交任务
tasks <- Task{
URL: "<a href="https://www.php.cn/link/b05edd78c294dcf6d960190bf5bde635">https://www.php.cn/link/b05edd78c294dcf6d960190bf5bde635</a>",
Fn: func(resp *http.Response) {
// 处理响应
},
}
close(tasks)</p>

limiter := time.Tick(time.Millisecond * 200) // 每 200ms 允许一次请求
for _, url := range urls {
    <-limiter
    go fetch(url, ch)
}