http.Client默认的User-Agent是“Go-http-client/1.1”，在做页面抓取的时候容易被服务端拒绝，需要在请求前重新设置一个模拟的User-Agent。具体代码如下：

package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"time"
)

func main() {
	client := &http.Client{
		Timeout: 2 * time.Second,
	}

	req, _ := http.NewRequest("GET", "http://127.0.0.1:8080", nil)
	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36")

	resp, err := client.Do(req)
	if err != nil {
		fmt.Printf("%s\n", err.Error())
		return
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Printf("%s\n", err.Error())
		return
	}
	fmt.Printf("%s\n", string(body))
}

服务端验证代码：

package main

import (
        "fmt"
        "io"
        "log"
        "net/http"
)

func main() {
        // Hello world, the web server

        helloHandler := func(w http.ResponseWriter, r *http.Request) {
                fmt.Printf("%v\n", r.UserAgent())
                io.WriteString(w, "Hello, world!\n")
        }

        http.HandleFunc("/", helloHandler)
        log.Fatal(http.ListenAndServe(":8080", nil))
}

请求后，服务端输出：

Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36