蜘蛛抓取

• 类似jQuery github.com/PuerkitoBio/goquery

  res, err := http.Get("http://www.yoby123.cn")
    if err != nil {
        log.Fatal(err)
    }
    defer res.Body.Close()
    if res.StatusCode != 200 {
        log.Fatalf("status code error: %d %s", res.StatusCode, res.Status)
    }
    doc, _ := goquery.NewDocumentFromReader(res.Body)
  
    body, _ := io.ReadAll(res.Body)
    str:=string(body)
    doc,_:=goquery.NewDocumentFromReader(strings.NewReader(str))
  
    var sc []string//且片,选择支持类 #等
    doc.Find("span.cfont1").Each(func(i int,s *goquery.Selection){
        sc=append(sc,s.Text())
    })
    str:=doc.Find("title").Text() 内容
    fmt.Println(sc)

import “github.com/gocolly/colly”
c := colly.NewCollector()
c.OnHTML(“a[href]”, func(e *colly.HTMLElement) {
link := e.Attr(“href”)
// Print link
fmt.Printf(“Link found: %q -> %s\n”, e.Text, link)
// Visit link found on page
// Only those links are visited which are in AllowedDomains
c.Visit(e.Request.AbsoluteURL(link))
})

c.OnRequest(func(r *colly.Request) {
fmt.Println(“Visiting”, r.URL)
})
c.Visit(“http://go-colly.org/")