2019-05-09

go爬虫的代理设置

package main

import (
    "crypto/tls"
    "fmt"
    "io/ioutil"
    "net/http"
    "net/url"
    "os"
    "time"
)

func main() {

    // 设置的ip代理
    ip_proxy := "http://117.60.2.184:8888"
    proxy := func(_ *http.Request) (*url.URL, error) {
        return url.Parse(ip_proxy)
    }

    tr := &http.Transport{
        // 忽略证书验证(接收任何证书)
        TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
        // 代理设置
        Proxy: proxy,
    }

    client := &http.Client{
        Transport: tr,
        Timeout:   time.Second * 5,
    }

    resp, err := client.Get("https://www.douban.com/")
    if err != nil {
        fmt.Println(err)
        return
    }

    defer resp.Body.Close()

    body, _ := ioutil.ReadAll(resp.Body)
    //fmt.Printf("%s\n", body)
    fmt.Println(len(body))

    // 将响应信息写入HTML文件中
    f, err1 := os.Create("filename")
    if err1 != nil {
        fmt.Println("create file error:", err1)
        return
    }

    f.WriteString(string(body))
    f.Close()

    fmt.Println(resp.StatusCode)

}