From cc27f1a35ba45beb6f71b0fdba6d26c51651c914 Mon Sep 17 00:00:00 2001 From: guonaihong Date: Sat, 10 Dec 2022 22:53:22 +0800 Subject: [PATCH] Issue343 (#345) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 完成issue * 新加测试 --- README.md | 16 +++- dataflow/dataflow.go | 13 ++- dataflow/dataflow_auto_decode_body_test.go | 91 +++++++++++++++++++ dataflow/dataflow_middleware_test.go | 6 +- dataflow/req.go | 6 +- debug/debug_trace.go | 4 +- go.mod | 11 +-- go.sum | 5 +- interface/README.md | 8 ++ middler/do.go | 7 ++ middler/request_use_interface.go | 19 ++++ middler/response_use_interface.go | 22 +++++ .../rsp/autodecodebody/autodecodebody.go | 48 ++++++++++ .../rsp/autodecodebody/autodecodebody_test.go | 76 ++++++++++++++++ setting/setting.go | 1 - version.go | 2 +- 16 files changed, 307 insertions(+), 28 deletions(-) create mode 100644 dataflow/dataflow_auto_decode_body_test.go create mode 100644 interface/README.md create mode 100644 middler/do.go create mode 100644 middler/request_use_interface.go create mode 100644 middler/response_use_interface.go create mode 100644 middleware/rsp/autodecodebody/autodecodebody.go create mode 100644 middleware/rsp/autodecodebody/autodecodebody_test.go diff --git a/README.md b/README.md index 7cfff79..0f27b52 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,7 @@ gout 是go写的http 客户端，为提高工作效率而开发 - [callback](#callback) - [get *http.Response](#get-response) - [multiple binding functions](#multiple-binding-functions) + - [Auto decode body](#auto-decode-body) - [Set request timeout](#Set-request-timeout) - [proxy](#proxy) - [socks5](#socks5) @@ -569,7 +570,7 @@ import ( "fmt" "github.com/gin-gonic/gin" "github.com/guonaihong/gout" - middler "github.com/guonaihong/gout/interface" + "github.com/guonaihong/gout/middler" "io/ioutil" "log" "net/http" @@ -1124,7 +1125,7 @@ func main() { ``` ### multiple binding functions -支持绑定多个对象 +支持绑定多个对象, BindXXX函数可以多次调用。例子里面是BindJSON和BindBody ```go var responseStruct struct { Name string `json:"name"` @@ -1147,6 +1148,17 @@ func main() { log.Println(responseStr) } +``` +### Auto decode body +响应头里面指明压缩格式，使用AutoDecodeBody接口可以自动解压。 +```go +//Content-Encoding: gzip +//Content-Encoding: deflate +//Content-Encoding: br +//gzip由标准库原生支持，不需要使用AutoDecodeBody接口，后两种由gout支持. +func main() { + gout.GET(url).AutoDecodeBody().BindBody(&s).Do() +} ``` ## Set request timeout setimeout是request级别的超时方案。相比http.Client级别，更灵活。 diff --git a/dataflow/dataflow.go b/dataflow/dataflow.go index 21a21e4..e30559e 100644 --- a/dataflow/dataflow.go +++ b/dataflow/dataflow.go @@ -12,7 +12,8 @@ import ( "github.com/guonaihong/gout/debug" "github.com/guonaihong/gout/decode" "github.com/guonaihong/gout/encode" - api "github.com/guonaihong/gout/interface" + "github.com/guonaihong/gout/middler" + "github.com/guonaihong/gout/middleware/rsp/autodecodebody" "github.com/guonaihong/gout/setting" "golang.org/x/net/proxy" ) @@ -378,7 +379,7 @@ func (df *DataFlow) SetBasicAuth(username, password string) *DataFlow { } // Request middleware -func (df *DataFlow) RequestUse(reqModify ...api.RequestMiddler) *DataFlow { +func (df *DataFlow) RequestUse(reqModify ...middler.RequestMiddler) *DataFlow { if len(reqModify) > 0 { df.reqModify = append(df.reqModify, reqModify...) } @@ -386,7 +387,7 @@ func (df *DataFlow) RequestUse(reqModify ...api.RequestMiddler) *DataFlow { } // Response middleware -func (df *DataFlow) ResponseUse(responseModify ...api.ResponseMiddler) *DataFlow { +func (df *DataFlow) ResponseUse(responseModify ...middler.ResponseMiddler) *DataFlow { if len(responseModify) > 0 { df.responseModify = append(df.responseModify, responseModify...) } @@ -420,6 +421,12 @@ func (df *DataFlow) NoAutoContentType() *DataFlow { return df } +// https://github.com/guonaihong/gout/issues/343 +// content-encoding会指定response body的压缩方法，支持常用的压缩，gzip, deflate, br等 +func (df *DataFlow) AutoDecodeBody() *DataFlow { + return df.ResponseUse(middler.WithResponseMiddlerFunc(autodecodebody.AutoDecodeBody)) +} + func (df *DataFlow) IsDebug() bool { return df.Setting.Debug } diff --git a/dataflow/dataflow_auto_decode_body_test.go b/dataflow/dataflow_auto_decode_body_test.go new file mode 100644 index 0000000..1be7001 --- /dev/null +++ b/dataflow/dataflow_auto_decode_body_test.go @@ -0,0 +1,91 @@ +package dataflow + +import ( + "bytes" + "compress/gzip" + "compress/zlib" + "log" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/andybalholm/brotli" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" +) + +var test_autoDecodeBody_data = "test auto decode boyd function" + +// 测试服务 +func create_AutoDecodeBody() *httptest.Server { + r := gin.New() + r.GET("/gzip", func(c *gin.Context) { + + var buf bytes.Buffer + + zw := gzip.NewWriter(&buf) + // Setting the Header fields is optional. + zw.Name = "a-new-hope.txt" + zw.Comment = "an epic space opera by George Lucas" + zw.ModTime = time.Date(1977, time.May, 25, 0, 0, 0, 0, time.UTC) + _, err := zw.Write([]byte(test_autoDecodeBody_data)) + if err != nil { + log.Fatal(err) + } + + if err := zw.Close(); err != nil { + log.Fatal(err) + } + c.Header("Content-Encoding", "gzip") + c.String(200, buf.String()) + }) + + r.GET("/br", func(c *gin.Context) { + + c.Header("Content-Encoding", "br") + var buf bytes.Buffer + w := brotli.NewWriter(&buf) + w.Write([]byte(test_autoDecodeBody_data)) + w.Flush() + w.Close() + c.String(200, buf.String()) + }) + + r.GET("/deflate", func(c *gin.Context) { + + var buf bytes.Buffer + w := zlib.NewWriter(&buf) + w.Write([]byte(test_autoDecodeBody_data)) + w.Close() + c.Header("Content-Encoding", "deflate") + c.String(200, buf.String()) + }) + + r.GET("/compress", func(c *gin.Context) { + c.Header("Content-Encoding", "compress") + }) + return httptest.NewServer(http.HandlerFunc(r.ServeHTTP)) +} + +func Test_AutoDecodeBody(t *testing.T) { + ts := create_AutoDecodeBody() + var err error + for _, path := range []string{"/gzip", "/br", "/deflate"} { + s := "" + if path == "/gzip" { + err = New().GET(ts.URL + path).Debug(true).BindBody(&s).Do() + } else { + err = New().GET(ts.URL + path).AutoDecodeBody().Debug(true).BindBody(&s).Do() + + } + assert.NoError(t, err) + assert.Equal(t, s, test_autoDecodeBody_data) + } +} + +func Test_AutoDecodeBody_Fail(t *testing.T) { + ts := create_AutoDecodeBody() + err := New().GET(ts.URL + "/compress").AutoDecodeBody().Do() + assert.Error(t, err) +} diff --git a/dataflow/dataflow_middleware_test.go b/dataflow/dataflow_middleware_test.go index eb61627..6f61319 100644 --- a/dataflow/dataflow_middleware_test.go +++ b/dataflow/dataflow_middleware_test.go @@ -12,7 +12,7 @@ import ( core "github.com/guonaihong/gout/core" - api "github.com/guonaihong/gout/interface" + "github.com/guonaihong/gout/middler" "github.com/stretchr/testify/assert" ) @@ -24,7 +24,7 @@ func (d *demoRequestMiddler) ModifyRequest(req *http.Request) error { return nil } -func demoRequest() api.RequestMiddler { +func demoRequest() middler.RequestMiddler { return &demoRequestMiddler{} } @@ -65,7 +65,7 @@ func (d *demoResponseMiddler) ModifyResponse(response *http.Response) error { return nil } } -func demoResponse() api.ResponseMiddler { +func demoResponse() middler.ResponseMiddler { return &demoResponseMiddler{} } diff --git a/dataflow/req.go b/dataflow/req.go index 85f3c4a..44c7e5c 100644 --- a/dataflow/req.go +++ b/dataflow/req.go @@ -15,7 +15,7 @@ import ( "github.com/guonaihong/gout/debug" "github.com/guonaihong/gout/decode" "github.com/guonaihong/gout/encode" - api "github.com/guonaihong/gout/interface" + "github.com/guonaihong/gout/middler" "github.com/guonaihong/gout/setting" ) @@ -54,9 +54,9 @@ type Req struct { c context.Context Err error - reqModify []api.RequestMiddler + reqModify []middler.RequestMiddler - responseModify []api.ResponseMiddler + responseModify []middler.ResponseMiddler req *http.Request diff --git a/debug/debug_trace.go b/debug/debug_trace.go index 25d8be8..b27a110 100644 --- a/debug/debug_trace.go +++ b/debug/debug_trace.go @@ -11,7 +11,7 @@ import ( "time" "github.com/guonaihong/gout/color" - api "github.com/guonaihong/gout/interface" + "github.com/guonaihong/gout/middler" ) type TraceInfo struct { @@ -25,7 +25,7 @@ type TraceInfo struct { w io.Writer } -func (t *TraceInfo) StartTrace(opt *Options, needTrace bool, req *http.Request, do api.Do) (*http.Response, error) { +func (t *TraceInfo) StartTrace(opt *Options, needTrace bool, req *http.Request, do middler.Do) (*http.Response, error) { w := opt.Write var dnsStart, connStart, reqStart, tlsStart, waitResponseStart, respStart time.Time var dnsDuration, connDuration, reqDuration, tlsDuration, waitResponeDuration time.Duration diff --git a/go.mod b/go.mod index f1b3af9..5ec8fa4 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/guonaihong/gout go 1.18 require ( + github.com/andybalholm/brotli v1.0.4 github.com/gin-gonic/gin v1.7.0 github.com/go-playground/locales v0.13.0 github.com/go-playground/universal-translator v0.17.0 @@ -19,23 +20,13 @@ require ( require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/gin-contrib/sse v0.1.0 // indirect - github.com/go-playground/assert/v2 v2.0.1 // indirect github.com/golang/protobuf v1.5.0 // indirect - github.com/google/go-cmp v0.5.5 // indirect - github.com/google/gofuzz v1.0.0 // indirect github.com/json-iterator/go v1.1.9 // indirect - github.com/kisielk/godepgraph v0.0.0-20220719222756-573dc89cecc8 // indirect github.com/leodido/go-urn v1.2.0 // indirect github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/stretchr/objx v0.1.0 // indirect - github.com/ugorji/go v1.1.7 // indirect github.com/ugorji/go/codec v1.1.7 // indirect golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 // indirect golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect - golang.org/x/text v0.3.2 // indirect - golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e // indirect - golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 // indirect - gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 // indirect ) diff --git a/go.sum b/go.sum index c72e968..7931c1c 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= +github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -23,8 +25,6 @@ github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/kisielk/godepgraph v0.0.0-20220719222756-573dc89cecc8 h1:4q/BeJpElGXg5jX05HFJEyZxkM/cjQGQvr1FzCBnvRA= -github.com/kisielk/godepgraph v0.0.0-20220719222756-573dc89cecc8/go.mod h1:Gb5YEgxqiSSVrXKWQxDcKoCM94NO5QAwOwTaVmIUAMI= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= @@ -42,7 +42,6 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= diff --git a/interface/README.md b/interface/README.md new file mode 100644 index 0000000..f411c47 --- /dev/null +++ b/interface/README.md @@ -0,0 +1,8 @@ +## 不建议使用这个目录下的代码 +建议使用如下目录 +```go +github.com/guonaihong/gout/middler +``` + +## 这个目录的代码将在未来4-5个版本迭代之后删除 +请做好迁移 v0.3.1版本开始算起，大约在v0.3.6版本移除 diff --git a/middler/do.go b/middler/do.go new file mode 100644 index 0000000..2f03bb1 --- /dev/null +++ b/middler/do.go @@ -0,0 +1,7 @@ +package middler + +import "net/http" + +type Do interface { + Do(*http.Request) (*http.Response, error) +} diff --git a/middler/request_use_interface.go b/middler/request_use_interface.go new file mode 100644 index 0000000..4aebe5e --- /dev/null +++ b/middler/request_use_interface.go @@ -0,0 +1,19 @@ +package middler + +import "net/http" + +type RequestMiddlerFunc func(req *http.Request) error + +type RequestMiddler interface { + ModifyRequest(req *http.Request) error +} + +func (f RequestMiddlerFunc) ModifyRequest(req *http.Request) error { + return f(req) +} + +// WithRequestMiddlerFunc 是创建一个 RequestMiddler 的helper +// 如果我们只需要简单的逻辑，只关注闭包本身，则可以使用这个helper快速创建一个 RequestMiddler +func WithRequestMiddlerFunc(f RequestMiddlerFunc) RequestMiddler { + return f +} diff --git a/middler/response_use_interface.go b/middler/response_use_interface.go new file mode 100644 index 0000000..1a2ba13 --- /dev/null +++ b/middler/response_use_interface.go @@ -0,0 +1,22 @@ +package middler + +import ( + "net/http" +) + +type ResponseMiddlerFunc func(response *http.Response) error + +// ResponseMiddler 响应拦截器 +type ResponseMiddler interface { + ModifyResponse(response *http.Response) error +} + +func (f ResponseMiddlerFunc) ModifyResponse(response *http.Response) error { + return f(response) +} + +// WithResponseMiddlerFunc 是创建一个 ResponseMiddler 的helper +// 如果我们只需要简单的逻辑，只关注闭包本身，则可以使用这个helper快速创建一个 ResponseMiddler +func WithResponseMiddlerFunc(f ResponseMiddlerFunc) ResponseMiddler { + return f +} diff --git a/middleware/rsp/autodecodebody/autodecodebody.go b/middleware/rsp/autodecodebody/autodecodebody.go new file mode 100644 index 0000000..94c24ab --- /dev/null +++ b/middleware/rsp/autodecodebody/autodecodebody.go @@ -0,0 +1,48 @@ +package autodecodebody + +import ( + "bytes" + "compress/zlib" + "errors" + "io" + "io/ioutil" + "net/http" + "strings" + + "github.com/andybalholm/brotli" +) + +func AutoDecodeBody(rsp *http.Response) (err error) { + encoding := rsp.Header.Get("Content-Encoding") + + encoding = strings.ToLower(encoding) + var out bytes.Buffer + var rc io.ReadCloser + switch encoding { + //case "gzip": // net/http包里面已经做了gzip自动解码 + //rc, err = gzip.NewReader(rsp.Body) + case "compress": + // compress 是一种浏览器基本不使用的压缩格式，暂不考虑支持 + return errors.New("gout:There is currently no plan to support the compress format") + case "deflate": + rc, err = zlib.NewReader(rsp.Body) + case "br": + rc = ioutil.NopCloser(brotli.NewReader(rsp.Body)) + default: + return nil + } + + if err != nil { + return err + } + + if _, err = io.Copy(&out, rc); err != nil { + return err + } + + if err = rc.Close(); err != nil { + return err + } + rsp.Body = ioutil.NopCloser(&out) + return nil +} diff --git a/middleware/rsp/autodecodebody/autodecodebody_test.go b/middleware/rsp/autodecodebody/autodecodebody_test.go new file mode 100644 index 0000000..4bfc99a --- /dev/null +++ b/middleware/rsp/autodecodebody/autodecodebody_test.go @@ -0,0 +1,76 @@ +package autodecodebody + +import ( + "bytes" + "compress/zlib" + "io/ioutil" + "net/http" + "testing" + + "github.com/andybalholm/brotli" + "github.com/stretchr/testify/assert" +) + +func TestAutoDecodeBody(t *testing.T) { + // https://developer.mozilla.org/zh-CN/docs/web/http/headers/content-encoding + data := "test auto decode boyd function" + for _, rsp := range []http.Response{ + // gzip + /* + func() (rv http.Response) { + var buf bytes.Buffer + + rv.Header = make(map[string][]string) + rv.Header.Set("Content-Encoding", "gzip") + zw := gzip.NewWriter(&buf) + // Setting the Header fields is optional. + zw.Name = "a-new-hope.txt" + zw.Comment = "an epic space opera by George Lucas" + zw.ModTime = time.Date(1977, time.May, 25, 0, 0, 0, 0, time.UTC) + _, err := zw.Write([]byte(data)) + if err != nil { + log.Fatal(err) + } + + if err := zw.Close(); err != nil { + log.Fatal(err) + } + + rv.Body = ioutil.NopCloser(&buf) + return + }(), + */ + // deflate + func() (rv http.Response) { + + rv.Header = make(map[string][]string) + rv.Header.Set("Content-Encoding", "deflate") + var b bytes.Buffer + w := zlib.NewWriter(&b) + w.Write([]byte(data)) + w.Close() + + rv.Body = ioutil.NopCloser(&b) + return + }(), + // br + func() (rv http.Response) { + rv.Header = make(map[string][]string) + rv.Header.Set("Content-Encoding", "br") + var b bytes.Buffer + w := brotli.NewWriter(&b) + w.Write([]byte(data)) + w.Flush() + w.Close() + rv.Body = ioutil.NopCloser(&b) + return + }(), + } { + + err := AutoDecodeBody(&rsp) + assert.NoError(t, err) + all, err := ioutil.ReadAll(rsp.Body) + assert.Equal(t, all, []byte(data)) + assert.NoError(t, err) + } +} diff --git a/setting/setting.go b/setting/setting.go index 6f295d5..f5e71fe 100644 --- a/setting/setting.go +++ b/setting/setting.go @@ -15,7 +15,6 @@ type Setting struct { //是否自动加ContentType NoAutoContentType bool - //超时时间 Timeout time.Duration diff --git a/version.go b/version.go index decbcba..7dfe918 100644 --- a/version.go +++ b/version.go @@ -1,4 +1,4 @@ package gout // Version show version -const Version = "v0.3.1" +const Version = "v0.3.2"