How to Build a Web Scraper in Go

Web
2026-04-17T12:05:42+00:00
Build a Go web scraper using net/http to fetch pages and golang.org/x/net/html to parse and extract data.

Use the net/http standard library to fetch pages and encoding/json or golang.org/x/net/html to parse the response.

package main

import (
	"fmt"
	"net/http"
	"golang.org/x/net/html"
)

func main() {
	resp, err := http.Get("https://example.com")
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()

	doc, err := html.Parse(resp.Body)
	if err != nil {
		panic(err)
	}

	var traverse func(*html.Node)
	traverse = func(n *html.Node) {
		if n.Type == html.ElementNode && n.Data == "a" {
			for _, a := range n.Attr {
				if a.Key == "href" {
					fmt.Println(a.Val)
				}
			}
		}
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			traverse(c)
		}
	}
	traverse(doc)
}