A Tour of Go 70 Exercise: Web Crawler

演習： ウェブクローラ( web crawler )を並列化する。

　なにはともあれ、あ、つぅぁー、おぶ、ごぉー完走ｗ＝

package main

import (
	"errors"
	"fmt"
	//dbgfmt "fmt"
	"sync"
)

type Fetcher interface {
    // Fetch returns the body of URL and
    // a slice of URLs found on that page.
    Fetch(url string) (body string, urls []string, err error)
}


var fetched = struct {
	m map[string]error
	sync.Mutex
}{m : make(map[string]error)}

var loading = errors.New("url load in progress")

func crawlImpl(url string, depth int, fetcher Fetcher) {
	if depth <= 0 {
		return
	}


	fetched.Lock()
	if _, ok := fetched.m[url]; ok {
		fetched.Unlock()
		fmt.Println("<- Done with %v, already fetched.", url)
		return
	}
	fetched.m[url] = loading
	fetched.Unlock()


	/*
	s := make([]string, 4-depth)
	dbgfmt.Println(s,body,url,depth)
	dbgfmt.Println("@",body,urls,err)
	*/
	
	body, urls, err := fetcher.Fetch(url)

	fetched.Lock()
	fetched.m[url] = err
	fetched.Unlock()

	if err != nil {
		fmt.Println("<- Error on %v: %v",url,err)
		return
	}
	fmt.Printf("Found: %s %q\n", url, body)
	done := make(chan bool)
	for i, u := range urls {
		fmt.Println("-> Crowling child %v/%v of %v : %v.",i, len(urls), url, u)
		go func(url string){
			crawlImpl(u, depth-1, fetcher)
			done <- true
		}(u)
	}
	for i := range urls {
                fmt.Println("<- [%v] %v/%v Waiting for child %v.", url, i, len(urls))
                <-done
        }
        fmt.Println("<- Done with %v", url)
}

// Crawl uses fetcher to recursively crawl
// pages starting with url, to a maximum of depth.
func Crawl(url string, depth int, fetcher Fetcher) {
    // TODO: Fetch URLs in parallel.
    // TODO: Don't fetch the same URL twice.
    // This implementation doesn't do either:

	crawlImpl(url,depth,fetcher)

	for url,err := range fetched.m {
		if err != nil {
			fmt.Println("%v failed: %v", url, err)
		} else {
			fmt.Println("%v was fetched", url)
		}
	}
	return
}

func main() {
    Crawl("http://golang.org/", 4, fetcher)
}

// fakeFetcher is Fetcher that returns canned results.
type fakeFetcher map[string]*fakeResult

type fakeResult struct {
    body string
    urls []string
}

func (f *fakeFetcher) Fetch(url string) (string, []string, error) {
    if res, ok := (*f)[url]; ok {
        return res.body, res.urls, nil
    }
    return "", nil, fmt.Errorf("not found: %s", url)
}

// fetcher is a populated fakeFetcher.
var fetcher = &fakeFetcher{
    "http://golang.org/": &fakeResult{
        "The Go Programming Language",
        []string{
            "http://golang.org/pkg/",
            "http://golang.org/cmd/",
        },
    },
    "http://golang.org/pkg/": &fakeResult{
        "Packages",
        []string{
            "http://golang.org/",
            "http://golang.org/cmd/",
            "http://golang.org/pkg/fmt/",
            "http://golang.org/pkg/os/",
        },
    },
    "http://golang.org/pkg/fmt/": &fakeResult{
        "Package fmt",
        []string{
            "http://golang.org/",
            "http://golang.org/pkg/",
        },
    },
    "http://golang.org/pkg/os/": &fakeResult{
        "Package os",
        []string{
            "http://golang.org/",
            "http://golang.org/pkg/",
        },
    },
}