Skip to content

Commit

Permalink
chore(component,http): auto-detect file type when the response is bin…
Browse files Browse the repository at this point in the history 
…ary (#804)

Because

- Previously, HTTP response data could include files, documents, images,
videos, or audio, and assigning them to the correct Instill format was
necessary for pipeline usability.

This commit:

- Automatically detects the file type when the response is in binary
format.
  • Loading branch information
@donch1989
donch1989 authored Nov 3, 2024
1 parent 04b1252 commit 2368f76
Show file tree
Hide file tree
Showing 2 changed files with 81 additions and 2 deletions.
81 changes: 80 additions & 1 deletion pkg/component/generic/http/v0/component_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,24 @@ func TestComponent(t *testing.T) {
c.Assert(err, qt.IsNil)
case "/error":
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
case "/auth":
user, pass, ok := r.BasicAuth()
if !ok || user != username || pass != password {
http.Error(w, "Unauthorized", http.StatusUnauthorized)
return
}
w.Header().Set("Content-Type", "application/json")
err := json.NewEncoder(w).Encode(map[string]string{"message": "authenticated"})
c.Assert(err, qt.IsNil)
case "/bearer":
authHeader := r.Header.Get("Authorization")
if authHeader != "Bearer "+token {
http.Error(w, "Unauthorized", http.StatusUnauthorized)
return
}
w.Header().Set("Content-Type", "application/json")
err := json.NewEncoder(w).Encode(map[string]string{"message": "authenticated"})
c.Assert(err, qt.IsNil)
}
}))
defer ts.Close()
Expand All @@ -70,6 +88,7 @@ func TestComponent(t *testing.T) {
name string
task string
input httpInput
setup authType
expected httpOutput
}{
{
Expand All @@ -78,6 +97,7 @@ func TestComponent(t *testing.T) {
input: httpInput{
EndpointURL: ts.URL + "/json",
},
setup: noAuthType,
expected: httpOutput{
Body: data.Map{"message": data.NewString("hello")},
Header: map[string][]string{"Content-Type": {"application/json"}},
Expand All @@ -90,6 +110,7 @@ func TestComponent(t *testing.T) {
input: httpInput{
EndpointURL: ts.URL + "/text",
},
setup: noAuthType,
expected: httpOutput{
Body: data.NewString("hello"),
Header: map[string][]string{"Content-Type": {"text/plain"}},
Expand All @@ -102,6 +123,7 @@ func TestComponent(t *testing.T) {
input: httpInput{
EndpointURL: ts.URL + "/file",
},
setup: noAuthType,
expected: httpOutput{
Body: func() format.Value {
v, _ := data.NewFileFromBytes([]byte("hello"), "application/octet-stream", "test.bin")
Expand All @@ -119,6 +141,7 @@ func TestComponent(t *testing.T) {
Header: map[string][]string{"Content-Type": {"application/json"}},
Body: data.Map{"message": data.NewString("hello")},
},
setup: noAuthType,
expected: httpOutput{
Body: data.Map{"message": data.NewString("hello")},
Header: map[string][]string{"Content-Type": {"application/json"}},
Expand All @@ -133,6 +156,7 @@ func TestComponent(t *testing.T) {
Header: map[string][]string{"Content-Type": {"text/plain"}},
Body: data.NewString("hello"),
},
setup: noAuthType,
expected: httpOutput{
Body: data.NewString("hello"),
Header: map[string][]string{"Content-Type": {"text/plain"}},
Expand All @@ -145,6 +169,7 @@ func TestComponent(t *testing.T) {
input: httpInput{
EndpointURL: ts.URL + "/json",
},
setup: noAuthType,
expected: httpOutput{
Body: data.Map{"message": data.NewString("hello")},
Header: map[string][]string{"Content-Type": {"application/json"}},
Expand All @@ -158,6 +183,7 @@ func TestComponent(t *testing.T) {
EndpointURL: ts.URL + "/json",
Body: data.Map{"message": data.NewString("hello")},
},
setup: noAuthType,
expected: httpOutput{
Body: data.Map{"message": data.NewString("hello")},
Header: map[string][]string{"Content-Type": {"application/json"}},
Expand All @@ -170,12 +196,65 @@ func TestComponent(t *testing.T) {
input: httpInput{
EndpointURL: ts.URL + "/error",
},
setup: noAuthType,
expected: httpOutput{
Body: data.NewString("Internal Server Error\n"),
Header: map[string][]string{"Content-Type": {"text/plain; charset=utf-8"}},
StatusCode: 500,
},
},
{
name: "GET with basic auth",
task: "TASK_GET",
input: httpInput{
EndpointURL: ts.URL + "/auth",
},
setup: basicAuthType,
expected: httpOutput{
Body: data.Map{"message": data.NewString("authenticated")},
Header: map[string][]string{"Content-Type": {"application/json"}},
StatusCode: 200,
},
},
{
name: "GET with invalid basic auth",
task: "TASK_GET",
input: httpInput{
EndpointURL: ts.URL + "/auth",
},
setup: noAuthType,
expected: httpOutput{
Body: data.NewString("Unauthorized\n"),
Header: map[string][]string{"Content-Type": {"text/plain; charset=utf-8"}},
StatusCode: 401,
},
},
{
name: "GET with bearer token",
task: "TASK_GET",
input: httpInput{
EndpointURL: ts.URL + "/bearer",
},
setup: bearerTokenType,
expected: httpOutput{
Body: data.Map{"message": data.NewString("authenticated")},
Header: map[string][]string{"Content-Type": {"application/json"}},
StatusCode: 200,
},
},
{
name: "GET with invalid bearer token",
task: "TASK_GET",
input: httpInput{
EndpointURL: ts.URL + "/bearer",
},
setup: noAuthType,
expected: httpOutput{
Body: data.NewString("Unauthorized\n"),
Header: map[string][]string{"Content-Type": {"text/plain; charset=utf-8"}},
StatusCode: 401,
},
},
}

for _, tc := range testCases {
Expand All @@ -186,7 +265,7 @@ func TestComponent(t *testing.T) {
execution, err := component.CreateExecution(base.ComponentExecution{
Component: component,
Task: tc.task,
Setup: cfg(noAuthType),
Setup: cfg(tc.setup),
})
c.Assert(err, qt.IsNil)

Expand Down
2 changes: 1 addition & 1 deletion pkg/component/generic/http/v0/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -203,7 +203,7 @@ func (e *execution) executeHTTP(ctx context.Context, job *base.Job) error {
}
}
// For other content types, return raw bytes wrapped in format.Value
value, err := data.NewFileFromBytes(resp.Body(), contentType, filename)
value, err := data.NewBinaryFromBytes(resp.Body(), contentType, filename)
if err != nil {
return fmt.Errorf("failed to convert response to format.Value: %w", err)
}
Expand Down

0 comments on commit 2368f76

Please sign in to comment.