Version1
UrlsStatue 使用普通map，不能保证并发安全，所以要加一个互斥锁
package mainimport ("fmt""sync"
)type Fetcher interface {// Fetch 返回 URL 所指向页面的 body 内容，// 并将该页面上找到的所有 URL 放到一个切片中。Fetch(url string) (body string, urls []string, err error)
}var url_statue = UrlsStatue{vis: make(map[string]bool)}// Crawl 用 fetcher 从某个 URL 开始递归的爬取页面，直到达到最大深度。
func Crawl(url string, depth int, fetcher Fetcher, wg *sync.WaitGroup) {// TODO: 并行地爬取 URL。// TODO: 不重复爬取页面。// 下面并没有实现上面两种情况：defer wg.Done()if depth <= 0 {return}body, urls, err := fetcher.Fetch(url)if err != nil {fmt.Println(err)return}fmt.Printf("found: %s %q\n", url, body)for _, u := range urls {flag := falseurl_statue.mu.Lock() // 加锁查看该网站是否遍历过if url_statue.vis[u] == false {flag = trueurl_statue.vis[u] = true}url_statue.mu.Unlock()if flag == true {wg.Add(1)go func(u string) {//fmt.Println(url)Crawl(u, depth-1, fetcher, wg)}(u)} // 没遍历过则递归遍历}return
}func main() {var wg sync.WaitGroupwg.Add(1)start := "https://golang.org/"url_statue.vis[start] = trueCrawl(start, 4, fetcher, &wg)wg.Wait()
}// fakeFetcher 是待填充结果的 Fetcher。
type fakeFetcher map[string]*fakeResulttype fakeResult struct {body stringurls []string
}// 创建记录每个url是否遍历过的结构体，里面加锁是为了让并发的线程互斥地使用map
type UrlsStatue struct {mu sync.Mutexvis map[string]bool
}func (f fakeFetcher) Fetch(url string) (string, []string, error) {if res, ok := f[url]; ok {return res.body, res.urls, nil}return "", nil, fmt.Errorf("not found: %s", url)
}// fetcher 是填充后的 fakeFetcher。
var fetcher = fakeFetcher{"https://golang.org/": &fakeResult{"The Go Programming Language",[]string{"https://golang.org/pkg/","https://golang.org/cmd/",},},"https://golang.org/pkg/": &fakeResult{"Packages",[]string{"https://golang.org/","https://golang.org/cmd/","https://golang.org/pkg/fmt/","https://golang.org/pkg/os/",},},"https://golang.org/pkg/fmt/": &fakeResult{"Package fmt",[]string{"https://golang.org/","https://golang.org/pkg/",},},"https://golang.org/pkg/os/": &fakeResult{"Package os",[]string{"https://golang.org/","https://golang.org/pkg/",},},
}Version2
使用sync.map，该结构体本身是并发安全的。
package mainimport ("fmt""sync"
)type Fetcher interface {// Fetch 返回 URL 所指向页面的 body 内容，// 并将该页面上找到的所有 URL 放到一个切片中。Fetch(url string) (body string, urls []string, err error)
}var url_statue = UrlsStatue{}// Crawl 用 fetcher 从某个 URL 开始递归的爬取页面，直到达到最大深度。
func Crawl(url string, depth int, fetcher Fetcher, wg *sync.WaitGroup) {// TODO: 并行地爬取 URL。// TODO: 不重复爬取页面。// 下面并没有实现上面两种情况：defer wg.Done()if depth <= 0 {return}body, urls, err := fetcher.Fetch(url)if err != nil {fmt.Println(err)return}fmt.Printf("found: %s %q\n", url, body)for _, u := range urls {flag := false//url_statue.mu.Lock() // 加锁查看该网站是否遍历过if _, ok := url_statue.vis.Load(u); !ok {flag = trueurl_statue.vis.Store(u, true)}//url_statue.mu.Unlock()if flag == true { // 没遍历过则递归遍历wg.Add(1)go func(u string) {//fmt.Println(url)Crawl(u, depth-1, fetcher, wg)}(u)}}return
}func main() {var wg sync.WaitGroupwg.Add(1)start := "https://golang.org/"url_statue.vis.Store(start, true)Crawl(start, 4, fetcher, &wg)wg.Wait()
}// fakeFetcher 是待填充结果的 Fetcher。
type fakeFetcher map[string]*fakeResulttype fakeResult struct {body stringurls []string
}type UrlsStatue struct {mu sync.Mutex//vis map[string]boolvis sync.Map
}func (f fakeFetcher) Fetch(url string) (string, []string, error) {if res, ok := f[url]; ok {return res.body, res.urls, nil}return "", nil, fmt.Errorf("not found: %s", url)
}// fetcher 是填充后的 fakeFetcher。
var fetcher = fakeFetcher{"https://golang.org/": &fakeResult{"The Go Programming Language",[]string{"https://golang.org/pkg/","https://golang.org/cmd/",},},"https://golang.org/pkg/": &fakeResult{"Packages",[]string{"https://golang.org/","https://golang.org/cmd/","https://golang.org/pkg/fmt/","https://golang.org/pkg/os/",},},"https://golang.org/pkg/fmt/": &fakeResult{"Package fmt",[]string{"https://golang.org/","https://golang.org/pkg/",},},"https://golang.org/pkg/os/": &fakeResult{"Package os",[]string{"https://golang.org/","https://golang.org/pkg/",},},
}
sync.Map对外暴露的方法
// 根据一个key读取值，返回值会返回对应的值和该值是否存在
func (m *Map) Load(key any) (value any, ok bool)// 存储一个键值对
func (m *Map) Store(key, value any)// 删除一个键值对
func (m *Map) Delete(key any)// 如果该key已存在，就返回原有的值，否则将新的值存入并返回，当成功读取到值时，loaded为true，否则为false
func (m *Map) LoadOrStore(key, value any) (actual any, loaded bool)// 删除一个键值对，并返回其原有的值，loaded的值取决于key是否存在
func (m *Map) LoadAndDelete(key any) (value any, loaded bool)// 遍历Map，当f()返回false时，就会停止遍历
func (m *Map) Range(f func(key, value any) bool) 
