From e0585cc412a9f37ce92d48444cc2f2861965efab Mon Sep 17 00:00:00 2001 From: Carlana Johnson Date: Thu, 8 Aug 2024 13:38:13 -0400 Subject: [PATCH] Add reqtest, reqhtml; deprecate requests.ToHTML and testing transports (#115) * Add reqtest, reqhtml; deprecate requests.ToHTML and testing transports * reqtest: Add package doc * reqtest: Rename to reqtest.Server and improve docs * reqhtml.Body: Better doc string * Docs: Better docs for reqtest.Replay. --- README.md | 4 +- config.go | 2 + handler.go | 2 + recorder.go | 8 ++++ recorder_test.go | 6 ++- reqhtml/html.go | 27 ++++++++++++ reqhtml/html_test.go | 75 ++++++++++++++++++++++++++++++++ reqtest/doc.go | 2 + reqtest/recorder_example_test.go | 53 ++++++++++++++++++++++ reqtest/recorder_test.go | 69 +++++++++++++++++++++++++++++ reqtest/server.go | 14 ++++++ reqtest/server_example_test.go | 54 +++++++++++++++++++++++ reqtest/transport.go | 47 ++++++++++++++++++++ reqxml/to_example_test.go | 5 ++- transport.go | 2 + 15 files changed, 365 insertions(+), 5 deletions(-) create mode 100644 reqhtml/html.go create mode 100644 reqhtml/html_test.go create mode 100644 reqtest/doc.go create mode 100644 reqtest/recorder_example_test.go create mode 100644 reqtest/recorder_test.go create mode 100644 reqtest/server.go create mode 100644 reqtest/server_example_test.go create mode 100644 reqtest/transport.go diff --git a/README.md b/README.md index 5682088..82adc8e 100644 --- a/README.md +++ b/README.md @@ -225,14 +225,14 @@ fmt.Println(u.String()) // https://dev1.example.com/get?a=1&b=3&c=4 // record a request to the file system var s1, s2 string err := requests.URL("http://example.com"). - Transport(requests.Record(nil, "somedir")). + Transport(reqtest.Record(nil, "somedir")). ToString(&s1). Fetch(ctx) check(err) // now replay the request in tests err = requests.URL("http://example.com"). - Transport(requests.Replay("somedir")). + Transport(reqtest.Replay("somedir")). ToString(&s2). Fetch(ctx) check(err) diff --git a/config.go b/config.go index ad166db..8cf7c2c 100644 --- a/config.go +++ b/config.go @@ -34,6 +34,8 @@ func GzipConfig(level int, h func(gw *gzip.Writer) error) Config { // TestServerConfig returns a Config // which sets the Builder's BaseURL to s.URL // and the Builder's Client to s.Client(). +// +// Deprecated: Use reqtest.Server. func TestServerConfig(s *httptest.Server) Config { return func(rb *Builder) { rb. diff --git a/handler.go b/handler.go index 7aa6278..c7230c9 100644 --- a/handler.go +++ b/handler.go @@ -94,6 +94,8 @@ func ToBufioScanner(f func(r *bufio.Scanner) error) ResponseHandler { } // ToHTML parses the page with x/net/html.Parse. +// +// Deprecated: Use reqhtml.To. func ToHTML(n *html.Node) ResponseHandler { return ToBufioReader(func(r *bufio.Reader) error { n2, err := html.Parse(r) diff --git a/recorder.go b/recorder.go index f84a535..a493ead 100644 --- a/recorder.go +++ b/recorder.go @@ -18,6 +18,8 @@ import ( // requests and their responses to text files in basepath. // Requests are named according to a hash of their contents. // Responses are named according to the request that made them. +// +// Deprecated: Use reqtest.Record. func Record(rt http.RoundTripper, basepath string) Transport { if rt == nil { rt = http.DefaultTransport @@ -56,6 +58,8 @@ func Record(rt http.RoundTripper, basepath string) Transport { // Replay returns an http.RoundTripper that reads its // responses from text files in basepath. // Responses are looked up according to a hash of the request. +// +// Deprecated: Use reqtest.Replay. func Replay(basepath string) Transport { return ReplayFS(os.DirFS(basepath)) } @@ -66,6 +70,8 @@ var errNotFound = errors.New("response not found") // responses from text files in the fs.FS. // Responses are looked up according to a hash of the request. // Response file names may optionally be prefixed with comments for better human organization. +// +// Deprecated: Use reqtest.ReplayFS. func ReplayFS(fsys fs.FS) Transport { return RoundTripFunc(func(req *http.Request) (res *http.Response, err error) { defer func() { @@ -110,6 +116,8 @@ func buildName(b []byte) (reqname, resname string) { // it caches the result of issuing the request with rt in basepath. // Requests are named according to a hash of their contents. // Responses are named according to the request that made them. +// +// Deprecated: Use reqtest.Caching. func Caching(rt http.RoundTripper, basepath string) Transport { replay := Replay(basepath).RoundTrip record := Record(rt, basepath).RoundTrip diff --git a/recorder_test.go b/recorder_test.go index cb1d453..52ba562 100644 --- a/recorder_test.go +++ b/recorder_test.go @@ -13,11 +13,14 @@ import ( ) func TestRecordReplay(t *testing.T) { + baseTrans := requests.ReplayString(`HTTP/1.1 200 OK + +Test Document 1`) dir := t.TempDir() var s1, s2 string err := requests.URL("http://example.com"). - Transport(requests.Record(http.DefaultTransport, dir)). + Transport(requests.Record(baseTrans, dir)). ToString(&s1). Fetch(context.Background()) be.NilErr(t, err) @@ -28,6 +31,7 @@ func TestRecordReplay(t *testing.T) { Fetch(context.Background()) be.NilErr(t, err) be.Equal(t, s1, s2) + be.Equal(t, "Test Document 1", s1) } func TestCaching(t *testing.T) { diff --git a/reqhtml/html.go b/reqhtml/html.go new file mode 100644 index 0000000..6e73d9e --- /dev/null +++ b/reqhtml/html.go @@ -0,0 +1,27 @@ +// Package reqhtml contains utilities for sending and receiving x/net/html objects. +package reqhtml + +import ( + "io" + + "github.com/carlmjohnson/requests" + "golang.org/x/net/html" +) + +// To decodes a response as an html document. +func To(n *html.Node) requests.ResponseHandler { + return requests.ToHTML(n) +} + +// Body sets the requests.Builder's request body to the HTML document. +// It also sets ContentType to "text/html" +// if it is not otherwise set. +func Body(n *html.Node) requests.Config { + return func(rb *requests.Builder) { + rb. + Body(requests.BodyWriter(func(w io.Writer) error { + return html.Render(w, n) + })). + HeaderOptional("context-type", "text/html") + } +} diff --git a/reqhtml/html_test.go b/reqhtml/html_test.go new file mode 100644 index 0000000..eb04e94 --- /dev/null +++ b/reqhtml/html_test.go @@ -0,0 +1,75 @@ +package reqhtml_test + +import ( + "context" + "fmt" + "net/http" + "net/http/httputil" + + "github.com/carlmjohnson/requests" + "github.com/carlmjohnson/requests/reqhtml" + "github.com/carlmjohnson/requests/reqtest" + "golang.org/x/net/html" + "golang.org/x/net/html/atom" +) + +func init() { + http.DefaultTransport = reqtest.ReplayString(`HTTP/1.1 200 OK + + `) +} + +func ExampleTo() { + var doc html.Node + err := requests. + URL("http://example.com"). + Handle(reqhtml.To(&doc)). + Fetch(context.Background()) + if err != nil { + fmt.Println("could not connect to example.com:", err) + } + var f func(*html.Node) + f = func(n *html.Node) { + if n.DataAtom == atom.A { + for _, attr := range n.Attr { + if attr.Key == "href" { + fmt.Println("link:", attr.Val) + } + } + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + f(c) + } + } + f(&doc) + // Output: + // link: https://www.iana.org/domains/example +} + +func ExampleBody() { + link := html.Node{ + Type: html.ElementNode, + Data: "a", + Attr: []html.Attribute{ + {Key: "href", Val: "http://example.com"}, + }, + } + text := html.Node{ + Type: html.TextNode, + Data: "Hello, World!", + } + link.AppendChild(&text) + + req, err := requests. + URL("http://example.com"). + Config(reqhtml.Body(&link)). + Request(context.Background()) + b, err := httputil.DumpRequest(req, true) + if err != nil { + panic(err) + } + fmt.Printf("%q\n", b) + + // Output: + // "POST / HTTP/1.1\r\nHost: example.com\r\nContext-Type: text/html\r\n\r\nHello, World!" +} diff --git a/reqtest/doc.go b/reqtest/doc.go new file mode 100644 index 0000000..48de1f0 --- /dev/null +++ b/reqtest/doc.go @@ -0,0 +1,2 @@ +// Package reqtest contains helpers for writing tests of HTTP clients and servers. +package reqtest diff --git a/reqtest/recorder_example_test.go b/reqtest/recorder_example_test.go new file mode 100644 index 0000000..eb7ec25 --- /dev/null +++ b/reqtest/recorder_example_test.go @@ -0,0 +1,53 @@ +package reqtest_test + +import ( + "context" + "fmt" + "testing/fstest" + + "github.com/carlmjohnson/requests" + "github.com/carlmjohnson/requests/reqtest" +) + +func ExampleReplayString() { + const res = `HTTP/1.1 200 OK + +An example response.` + + var s string + const expected = `An example response.` + if err := requests. + URL("http://response.example"). + Transport(reqtest.ReplayString(res)). + ToString(&s). + Fetch(context.Background()); err != nil { + panic(err) + } + fmt.Println(s == expected) + // Output: + // true +} + +func ExampleReplayFS() { + fsys := fstest.MapFS{ + "fsys.example - MKIYDwjs.res.txt": &fstest.MapFile{ + Data: []byte(`HTTP/1.1 200 OK +Content-Type: text/plain; charset=UTF-8 +Date: Mon, 24 May 2021 18:48:50 GMT + +An example response.`), + }, + } + var s string + const expected = `An example response.` + if err := requests. + URL("http://fsys.example"). + Transport(reqtest.ReplayFS(fsys)). + ToString(&s). + Fetch(context.Background()); err != nil { + panic(err) + } + fmt.Println(s == expected) + // Output: + // true +} diff --git a/reqtest/recorder_test.go b/reqtest/recorder_test.go new file mode 100644 index 0000000..7d1453b --- /dev/null +++ b/reqtest/recorder_test.go @@ -0,0 +1,69 @@ +package reqtest_test + +import ( + "context" + "io" + "net/http" + "os" + "strings" + "testing" + + "github.com/carlmjohnson/requests" + "github.com/carlmjohnson/requests/internal/be" + "github.com/carlmjohnson/requests/reqtest" +) + +func TestRecordReplay(t *testing.T) { + baseTrans := requests.ReplayString(`HTTP/1.1 200 OK + +Test Document 1`) + dir := t.TempDir() + + var s1, s2 string + err := requests.URL("http://example.com"). + Transport(reqtest.Record(baseTrans, dir)). + ToString(&s1). + Fetch(context.Background()) + be.NilErr(t, err) + + err = requests.URL("http://example.com"). + Transport(reqtest.Replay(dir)). + ToString(&s2). + Fetch(context.Background()) + be.NilErr(t, err) + be.Equal(t, s1, s2) + be.Equal(t, "Test Document 1", s1) +} + +func TestCaching(t *testing.T) { + dir := t.TempDir() + hasRun := false + content := "some content" + var onceTrans requests.RoundTripFunc = func(req *http.Request) (res *http.Response, err error) { + be.False(t, hasRun) + hasRun = true + res = &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(strings.NewReader(content)), + } + return + } + trans := reqtest.Caching(onceTrans, dir) + var s1, s2 string + err := requests.URL("http://example.com"). + Transport(trans). + ToString(&s1). + Fetch(context.Background()) + be.NilErr(t, err) + err = requests.URL("http://example.com"). + Transport(trans). + ToString(&s2). + Fetch(context.Background()) + be.NilErr(t, err) + be.Equal(t, content, s1) + be.Equal(t, s1, s2) + + entries, err := os.ReadDir(dir) + be.NilErr(t, err) + be.Equal(t, 2, len(entries)) +} diff --git a/reqtest/server.go b/reqtest/server.go new file mode 100644 index 0000000..d404e74 --- /dev/null +++ b/reqtest/server.go @@ -0,0 +1,14 @@ +package reqtest + +import ( + "net/http/httptest" + + "github.com/carlmjohnson/requests" +) + +// Server takes an httptest.Server and returns a requests.Config +// which sets the requests.Builder's BaseURL to s.URL +// and the requests.Builder's Client to s.Client(). +func Server(s *httptest.Server) requests.Config { + return requests.TestServerConfig(s) +} diff --git a/reqtest/server_example_test.go b/reqtest/server_example_test.go new file mode 100644 index 0000000..700c845 --- /dev/null +++ b/reqtest/server_example_test.go @@ -0,0 +1,54 @@ +package reqtest_test + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + + "github.com/carlmjohnson/requests" + "github.com/carlmjohnson/requests/reqtest" +) + +func ExampleServer() { + // Create an httptest.Server for your project's router + mux := http.NewServeMux() + mux.HandleFunc("/greeting", func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, "Hello, world!") + }) + mux.HandleFunc("/salutation", func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, "Howdy, planet!") + }) + + srv := httptest.NewServer(mux) + defer srv.Close() + + // Now test that the handler has the expected return values + { + var s string + err := requests. + New(reqtest.Server(srv)). + Path("/greeting"). + ToString(&s). + Fetch(context.Background()) + if err != nil { + fmt.Println("Error!", err) + } + fmt.Println(s) // Hello, world! + } + { + var s string + err := requests. + New(reqtest.Server(srv)). + Path("/salutation"). + ToString(&s). + Fetch(context.Background()) + if err != nil { + fmt.Println("Error!", err) + } + fmt.Println(s) // Howdy, planet! + } + // Output: + // Hello, world! + // Howdy, planet! +} diff --git a/reqtest/transport.go b/reqtest/transport.go new file mode 100644 index 0000000..cfdefea --- /dev/null +++ b/reqtest/transport.go @@ -0,0 +1,47 @@ +package reqtest + +import ( + "io/fs" + "net/http" + + "github.com/carlmjohnson/requests" +) + +// ReplayString returns an http.RoundTripper that always responds with a +// request built from rawResponse. It is intended for use in one-off tests. +func ReplayString(rawResponse string) requests.Transport { + return requests.ReplayString(rawResponse) +} + +// Record returns an http.RoundTripper that writes out its +// requests and their responses to text files in basepath. +// Requests are named according to a hash of their contents. +// Responses are named according to the request that made them. +func Record(rt http.RoundTripper, basepath string) requests.Transport { + return requests.Record(rt, basepath) +} + +// Replay returns an http.RoundTripper that reads its +// responses from text files in basepath. +// Responses are looked up according to a hash of the request. +// Response file names may optionally be prefixed with comments for better human organization. +func Replay(basepath string) requests.Transport { + return requests.Replay(basepath) +} + +// ReplayFS returns an http.RoundTripper that reads its +// responses from text files in the fs.FS. +// Responses are looked up according to a hash of the request. +// Response file names may optionally be prefixed with comments for better human organization. +func ReplayFS(fsys fs.FS) requests.Transport { + return requests.ReplayFS(fsys) +} + +// Caching returns an http.RoundTripper that attempts to read its +// responses from text files in basepath. If the response is absent, +// it caches the result of issuing the request with rt in basepath. +// Requests are named according to a hash of their contents. +// Responses are named according to the request that made them. +func Caching(rt http.RoundTripper, basepath string) requests.Transport { + return requests.Caching(rt, basepath) +} diff --git a/reqxml/to_example_test.go b/reqxml/to_example_test.go index d24690d..913fe4b 100644 --- a/reqxml/to_example_test.go +++ b/reqxml/to_example_test.go @@ -6,12 +6,13 @@ import ( "net/http" "github.com/carlmjohnson/requests" + "github.com/carlmjohnson/requests/reqtest" "github.com/carlmjohnson/requests/reqxml" ) func init() { - http.DefaultClient.Transport = requests.Replay("testdata") - // http.DefaultClient.Transport = requests.Caching(nil, "testdata") + http.DefaultClient.Transport = reqtest.Replay("testdata") + // http.DefaultClient.Transport = reqtest.Caching(nil, "testdata") } func ExampleTo() { diff --git a/transport.go b/transport.go index 51dd2c0..48bd317 100644 --- a/transport.go +++ b/transport.go @@ -25,6 +25,8 @@ var _ Transport = RoundTripFunc(nil) // ReplayString returns an http.RoundTripper that always responds with a // request built from rawResponse. It is intended for use in one-off tests. +// +// Deprecated: Use reqtest.ReplayString. func ReplayString(rawResponse string) Transport { return RoundTripFunc(func(req *http.Request) (res *http.Response, err error) { r := bufio.NewReader(strings.NewReader(rawResponse))