Your First Scraper

Fetching a page

net/http is all you need for the request itself:

package main

import (
  "fmt"
  "io"
  "net/http"
  "time"
)

func main() {
  req, err := http.NewRequest("GET", "https://example.com", nil)
  if err != nil {
    panic(err)
  }
  // Identify your crawler honestly, with a way to contact you.
  req.Header.Set("User-Agent", "nasigolang-example-bot/1.0 (+https://example.com/bot)")

  client := &http.Client{Timeout: 10 * time.Second}
  resp, err := client.Do(req)
  if err != nil {
    panic(err)
  }
  defer resp.Body.Close()

  body, err := io.ReadAll(resp.Body)
  if err != nil {
    panic(err)
  }
  fmt.Println(string(body)[:200]) // first 200 chars
}

Always set a Timeout on your http.Client — a scraper with no timeout will eventually hang forever on a slow or unresponsive server.

Parsing HTML with goquery

The standard library’s html package parses HTML into a tree, but goquery wraps it with a jQuery-like API that’s much nicer for extraction:

go get github.com/PuerkitoBio/goquery

package main

import (
  "fmt"
  "log"
  "net/http"

  "github.com/PuerkitoBio/goquery"
)

type Article struct {
  Title string
  Link  string
}

func main() {
  req, _ := http.NewRequest("GET", "https://example.com/blog", nil)
  req.Header.Set("User-Agent", "nasigolang-example-bot/1.0 (+https://example.com/bot)")

  resp, err := http.DefaultClient.Do(req)
  if err != nil {
    log.Fatal(err)
  }
  defer resp.Body.Close()

  doc, err := goquery.NewDocumentFromReader(resp.Body)
  if err != nil {
    log.Fatal(err)
  }

  var articles []Article
  doc.Find("article h2 a").Each(func(i int, s *goquery.Selection) {
    href, _ := s.Attr("href")
    articles = append(articles, Article{
      Title: s.Text(),
      Link:  href,
    })
  })

  for _, a := range articles {
    fmt.Println(a.Title, "->", a.Link)
  }
}

doc.Find(selector) takes ordinary CSS selectors, so anything you’d write in document.querySelectorAll in the browser works here too.

Checking the response before parsing

Real scrapers check status codes and content type before assuming they got HTML:

if resp.StatusCode != http.StatusOK {
  log.Printf("unexpected status %d for %s", resp.StatusCode, url)
  return
}
if ct := resp.Header.Get("Content-Type"); !strings.Contains(ct, "text/html") {
  log.Printf("unexpected content type %q for %s", ct, url)
  return
}