Skip to content
This repository has been archived by the owner on Mar 28, 2022. It is now read-only.

Commit

Permalink
Merge pull request #43 from httprunner/refactor-case-runner
Browse files Browse the repository at this point in the history
v0.3.1

- feat: set ulimit to 10240 before load testing
- fix: concurrent map writes in load testing
debugtalk authored Dec 30, 2021
2 parents c1e239d + 17657d7 commit e98b1c8
Showing 20 changed files with 170 additions and 82 deletions.
13 changes: 7 additions & 6 deletions boomer.go
Original file line number Diff line number Diff line change
@@ -52,19 +52,20 @@ func (b *hrpBoomer) Run(testcases ...ITestCase) {
}

func (b *hrpBoomer) convertBoomerTask(testcase *TestCase) *boomer.Task {
hrpRunner := NewRunner(nil).SetDebug(b.debug)
config := testcase.Config.ToStruct()
return &boomer.Task{
Name: config.Name,
Weight: config.Weight,
Fn: func() {
runner := NewRunner(nil).SetDebug(b.debug).Reset()
runner := hrpRunner.newCaseRunner(testcase)

testcaseSuccess := true // flag whole testcase result
var transactionSuccess = true // flag current transaction result

startTime := time.Now()
for _, step := range testcase.TestSteps {
stepData, err := runner.runStep(step, testcase.Config)
for index, step := range testcase.TestSteps {
stepData, err := runner.runStep(index)
if err != nil {
// step failed
var elapsed int64
@@ -77,8 +78,8 @@ func (b *hrpBoomer) convertBoomerTask(testcase *TestCase) *boomer.Task {
testcaseSuccess = false
transactionSuccess = false

if runner.failfast {
log.Error().Err(err).Msg("abort running due to failfast setting")
if runner.hrpRunner.failfast {
log.Error().Msg("abort running due to failfast setting")
break
}
log.Warn().Err(err).Msg("run step failed, continue next step")
@@ -107,7 +108,7 @@ func (b *hrpBoomer) convertBoomerTask(testcase *TestCase) *boomer.Task {
for name, transaction := range runner.transactions {
if len(transaction) == 1 {
// if transaction end time not exists, use testcase end time instead
duration := endTime.Sub(transaction[TransactionStart])
duration := endTime.Sub(transaction[transactionStart])
b.RecordTransaction(name, transactionSuccess, duration.Milliseconds(), 0)
}
}
14 changes: 9 additions & 5 deletions docs/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,15 +1,19 @@
# Release History

## v0.3.0 (2021-12-22)
## v0.3.1 (2021-12-30)

- feat: set ulimit to 10240 before load testing
- fix: concurrent map writes in load testing

## v0.3.0 (2021-12-24)

- feat: implement `transaction` mechanism for load test
- feat: continue running next step when failure occurs with `--continue-on-failure` flag, default to failfast
- feat: spawn workers with `--spawn-rate` flag
- refactor: fork [boomer] as sub module
- feat: report GA events with version
- feat: run load test with the given limit and burst as rate limiter
- feat: run load test with the given limit and burst as rate limiter, use `--spawn-count`, `--spawn-rate` and `--request-increase-rate` flag
- feat: report runner state to prometheus
- refactor: fork [boomer] as submodule initially and made a lot of changes
- change: update API models
- feat: report runner state

## v0.2.2 (2021-12-07)

14 changes: 12 additions & 2 deletions docs/cmd/hrp.md
Original file line number Diff line number Diff line change
@@ -4,9 +4,19 @@ One-stop solution for HTTP(S) testing.

### Synopsis

hrp (HttpRunner+) aims to be a one-stop solution for HTTP(S) testing, covering API testing, load testing and digital experience monitoring (DEM). Enjoy! ✨ 🚀 ✨

██╗ ██╗████████╗████████╗██████╗ ██████╗ ██╗ ██╗███╗ ██╗███╗ ██╗███████╗██████╗
██║ ██║╚══██╔══╝╚══██╔══╝██╔══██╗██╔══██╗██║ ██║████╗ ██║████╗ ██║██╔════╝██╔══██╗
███████║ ██║ ██║ ██████╔╝██████╔╝██║ ██║██╔██╗ ██║██╔██╗ ██║█████╗ ██████╔╝
██╔══██║ ██║ ██║ ██╔═══╝ ██╔══██╗██║ ██║██║╚██╗██║██║╚██╗██║██╔══╝ ██╔══██╗
██║ ██║ ██║ ██║ ██║ ██║ ██║╚██████╔╝██║ ╚████║██║ ╚████║███████╗██║ ██║
╚═╝ ╚═╝ ╚═╝ ╚═╝ ╚═╝ ╚═╝ ╚═╝ ╚═════╝ ╚═╝ ╚═══╝╚═╝ ╚═══╝╚══════╝╚═╝ ╚═╝

hrp (HttpRunner+) aims to be a one-stop solution for HTTP(S) testing, covering API testing,
load testing and digital experience monitoring (DEM). Enjoy! ✨ 🚀 ✨

License: Apache-2.0
Website: https://httprunner.com
Github: https://github.com/httprunner/hrp
Copyright 2021 debugtalk

@@ -22,4 +32,4 @@ Copyright 2021 debugtalk
* [hrp har2case](hrp_har2case.md) - Convert HAR to json/yaml testcase files
* [hrp run](hrp_run.md) - run API test

###### Auto generated by spf13/cobra on 24-Dec-2021
###### Auto generated by spf13/cobra on 30-Dec-2021
2 changes: 1 addition & 1 deletion docs/cmd/hrp_boom.md
Original file line number Diff line number Diff line change
@@ -38,4 +38,4 @@ hrp boom [flags]

* [hrp](hrp.md) - One-stop solution for HTTP(S) testing.

###### Auto generated by spf13/cobra on 24-Dec-2021
###### Auto generated by spf13/cobra on 30-Dec-2021
2 changes: 1 addition & 1 deletion docs/cmd/hrp_har2case.md
Original file line number Diff line number Diff line change
@@ -23,4 +23,4 @@ hrp har2case harPath... [flags]

* [hrp](hrp.md) - One-stop solution for HTTP(S) testing.

###### Auto generated by spf13/cobra on 24-Dec-2021
###### Auto generated by spf13/cobra on 30-Dec-2021
2 changes: 1 addition & 1 deletion docs/cmd/hrp_run.md
Original file line number Diff line number Diff line change
@@ -31,4 +31,4 @@ hrp run path... [flags]

* [hrp](hrp.md) - One-stop solution for HTTP(S) testing.

###### Auto generated by spf13/cobra on 24-Dec-2021
###### Auto generated by spf13/cobra on 30-Dec-2021
3 changes: 2 additions & 1 deletion hrp/cmd/boom.go
Original file line number Diff line number Diff line change
@@ -19,7 +19,8 @@ var boomCmd = &cobra.Command{
$ hrp boom examples/ # run testcases in specified folder`,
Args: cobra.MinimumNArgs(1),
PreRun: func(cmd *cobra.Command, args []string) {
setLogLevel("WARN") // disable info logs for load testing
boomer.SetUlimit(10240) // ulimit -n 10240
setLogLevel("WARN") // disable info logs for load testing
},
Run: func(cmd *cobra.Command, args []string) {
var paths []hrp.ITestCase
2 changes: 1 addition & 1 deletion hrp/cmd/har2case.go
Original file line number Diff line number Diff line change
@@ -4,7 +4,7 @@ import (
"github.com/rs/zerolog/log"
"github.com/spf13/cobra"

"github.com/httprunner/hrp/har2case"
"github.com/httprunner/hrp/internal/har2case"
)

// har2caseCmd represents the har2case command
12 changes: 11 additions & 1 deletion hrp/cmd/root.go
Original file line number Diff line number Diff line change
@@ -15,9 +15,19 @@ import (
var RootCmd = &cobra.Command{
Use: "hrp",
Short: "One-stop solution for HTTP(S) testing.",
Long: `hrp (HttpRunner+) aims to be a one-stop solution for HTTP(S) testing, covering API testing, load testing and digital experience monitoring (DEM). Enjoy! ✨ 🚀 ✨
Long: `
██╗ ██╗████████╗████████╗██████╗ ██████╗ ██╗ ██╗███╗ ██╗███╗ ██╗███████╗██████╗
██║ ██║╚══██╔══╝╚══██╔══╝██╔══██╗██╔══██╗██║ ██║████╗ ██║████╗ ██║██╔════╝██╔══██╗
███████║ ██║ ██║ ██████╔╝██████╔╝██║ ██║██╔██╗ ██║██╔██╗ ██║█████╗ ██████╔╝
██╔══██║ ██║ ██║ ██╔═══╝ ██╔══██╗██║ ██║██║╚██╗██║██║╚██╗██║██╔══╝ ██╔══██╗
██║ ██║ ██║ ██║ ██║ ██║ ██║╚██████╔╝██║ ╚████║██║ ╚████║███████╗██║ ██║
╚═╝ ╚═╝ ╚═╝ ╚═╝ ╚═╝ ╚═╝ ╚═╝ ╚═════╝ ╚═╝ ╚═══╝╚═╝ ╚═══╝╚══════╝╚═╝ ╚═╝
hrp (HttpRunner+) aims to be a one-stop solution for HTTP(S) testing, covering API testing,
load testing and digital experience monitoring (DEM). Enjoy! ✨ 🚀 ✨
License: Apache-2.0
Website: https://httprunner.com
Github: https://github.com/httprunner/hrp
Copyright 2021 debugtalk`,
PersistentPreRun: func(cmd *cobra.Command, args []string) {
32 changes: 32 additions & 0 deletions internal/boomer/ulimit.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
// +build !windows

package boomer

import (
"syscall"

"github.com/rs/zerolog/log"
)

// set resource limit
// ulimit -n 10240
func SetUlimit(limit uint64) {
var rLimit syscall.Rlimit
err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
if err != nil {
log.Error().Err(err).Msg("get ulimit failed")
return
}
log.Info().Uint64("limit", rLimit.Cur).Msg("get current ulimit")
if rLimit.Cur >= limit {
return
}

rLimit.Cur = limit
log.Info().Uint64("limit", rLimit.Cur).Msg("set current ulimit")
err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit)
if err != nil {
log.Error().Err(err).Msg("set ulimit failed")
return
}
}
12 changes: 12 additions & 0 deletions internal/boomer/ulimit_windows.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
// +build windows

package boomer

import (
"github.com/rs/zerolog/log"
)

// set resource limit
func SetUlimit(limit uint64) {
log.Warn().Msg("windows does not support setting ulimit")
}
File renamed without changes.
File renamed without changes.
4 changes: 2 additions & 2 deletions har2case/core_test.go → internal/har2case/core_test.go
Original file line number Diff line number Diff line change
@@ -7,8 +7,8 @@ import (
)

var (
harPath = "../examples/har/demo.har"
harPath2 = "../examples/har/postman-echo.har"
harPath = "../../examples/har/demo.har"
harPath2 = "../../examples/har/postman-echo.har"
)

func TestGenJSON(t *testing.T) {
File renamed without changes.
2 changes: 1 addition & 1 deletion internal/version/init.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
package version

const VERSION = "v0.3.0"
const VERSION = "v0.4.0"
8 changes: 4 additions & 4 deletions models.go
Original file line number Diff line number Diff line change
@@ -69,16 +69,16 @@ const (
stepTypeRendezvous stepType = "rendezvous"
)

type TransactionType string
type transactionType string

const (
TransactionStart TransactionType = "start"
TransactionEnd TransactionType = "end"
transactionStart transactionType = "start"
transactionEnd transactionType = "end"
)

type Transaction struct {
Name string `json:"name" yaml:"name"`
Type TransactionType `json:"type" yaml:"type"`
Type transactionType `json:"type" yaml:"type"`
}
type Rendezvous struct {
Name string `json:"name" yaml:"name"` // required
115 changes: 65 additions & 50 deletions runner.go
Original file line number Diff line number Diff line change
@@ -42,30 +42,14 @@ func NewRunner(t *testing.T) *hrpRunner {
},
Timeout: 30 * time.Second,
},
sessionVariables: make(map[string]interface{}),
transactions: make(map[string]map[TransactionType]time.Time),
}
}

type hrpRunner struct {
t *testing.T
failfast bool
debug bool
client *http.Client
sessionVariables map[string]interface{}
// transactions stores transaction timing info.
// key is transaction name, value is map of transaction type and time, e.g. start time and end time.
transactions map[string]map[TransactionType]time.Time
startTime time.Time // record start time of the testcase
}

// Reset clears runner session variables.
func (r *hrpRunner) Reset() *hrpRunner {
log.Info().Msg("[init] Reset session variables")
r.sessionVariables = make(map[string]interface{})
r.transactions = make(map[string]map[TransactionType]time.Time)
r.startTime = time.Now()
return r
t *testing.T
failfast bool
debug bool
client *http.Client
}

// SetFailfast configures whether to stop running when one step fails.
@@ -108,35 +92,63 @@ func (r *hrpRunner) Run(testcases ...ITestCase) error {
// report execution timing event
defer ga.SendEvent(event.StartTiming("execution"))

r.Reset()
for _, iTestCase := range testcases {
testcase, err := iTestCase.ToTestCase()
if err != nil {
log.Error().Err(err).Msg("[Run] convert ITestCase interface to TestCase struct failed")
return err
}
if err := r.runCase(testcase); err != nil {
if err := r.newCaseRunner(testcase).run(); err != nil {
log.Error().Err(err).Msg("[Run] run testcase failed")
return err
}
}
return nil
}

func (r *hrpRunner) runCase(testcase *TestCase) error {
config := testcase.Config
func (r *hrpRunner) newCaseRunner(testcase *TestCase) *caseRunner {
caseRunner := &caseRunner{
TestCase: testcase,
hrpRunner: r,
}
caseRunner.reset()
return caseRunner
}

// caseRunner is used to run testcase and its steps.
// each testcase has its own caseRunner instance and share session variables.
type caseRunner struct {
*TestCase
hrpRunner *hrpRunner
sessionVariables map[string]interface{}
// transactions stores transaction timing info.
// key is transaction name, value is map of transaction type and time, e.g. start time and end time.
transactions map[string]map[transactionType]time.Time
startTime time.Time // record start time of the testcase
}

// reset clears runner session variables.
func (r *caseRunner) reset() *caseRunner {
log.Info().Msg("[init] Reset session variables")
r.sessionVariables = make(map[string]interface{})
r.transactions = make(map[string]map[transactionType]time.Time)
r.startTime = time.Now()
return r
}

func (r *caseRunner) run() error {
config := r.TestCase.Config
if err := r.parseConfig(config); err != nil {
return err
}

log.Info().Str("testcase", config.Name()).Msg("run testcase start")
r.startTime = time.Now()
for _, step := range testcase.TestSteps {
_, err := r.runStep(step, config)
for index := range r.TestCase.TestSteps {
_, err := r.runStep(index)
if err != nil {
if r.failfast {
log.Error().Err(err).Msg("abort running due to failfast setting")
return err
if r.hrpRunner.failfast {
return errors.Wrap(err, "abort running due to failfast setting")
}
log.Warn().Err(err).Msg("run step failed, continue next step")
}
@@ -146,7 +158,10 @@ func (r *hrpRunner) runCase(testcase *TestCase) error {
return nil
}

func (r *hrpRunner) runStep(step IStep, config IConfig) (stepResult *stepData, err error) {
func (r *caseRunner) runStep(index int) (stepResult *stepData, err error) {
config := r.TestCase.Config
step := r.TestCase.TestSteps[index]

// step type priority order: transaction > rendezvous > testcase > request
if stepTran, ok := step.(*StepTransaction); ok {
// transaction step
@@ -218,7 +233,7 @@ func (r *hrpRunner) runStep(step IStep, config IConfig) (stepResult *stepData, e
return stepResult, nil
}

func (r *hrpRunner) runStepTransaction(transaction *Transaction) (stepResult *stepData, err error) {
func (r *caseRunner) runStepTransaction(transaction *Transaction) (stepResult *stepData, err error) {
log.Info().
Str("name", transaction.Name).
Str("type", string(transaction.Type)).
@@ -234,33 +249,33 @@ func (r *hrpRunner) runStepTransaction(transaction *Transaction) (stepResult *st

// create transaction if not exists
if _, ok := r.transactions[transaction.Name]; !ok {
r.transactions[transaction.Name] = make(map[TransactionType]time.Time)
r.transactions[transaction.Name] = make(map[transactionType]time.Time)
}

// record transaction start time, override if already exists
if transaction.Type == TransactionStart {
r.transactions[transaction.Name][TransactionStart] = time.Now()
if transaction.Type == transactionStart {
r.transactions[transaction.Name][transactionStart] = time.Now()
}
// record transaction end time, override if already exists
if transaction.Type == TransactionEnd {
r.transactions[transaction.Name][TransactionEnd] = time.Now()
if transaction.Type == transactionEnd {
r.transactions[transaction.Name][transactionEnd] = time.Now()

// if transaction start time not exists, use testcase start time instead
if _, ok := r.transactions[transaction.Name][TransactionStart]; !ok {
r.transactions[transaction.Name][TransactionStart] = r.startTime
if _, ok := r.transactions[transaction.Name][transactionStart]; !ok {
r.transactions[transaction.Name][transactionStart] = r.startTime
}

// calculate transaction duration
duration := r.transactions[transaction.Name][TransactionEnd].Sub(
r.transactions[transaction.Name][TransactionStart])
duration := r.transactions[transaction.Name][transactionEnd].Sub(
r.transactions[transaction.Name][transactionStart])
stepResult.elapsed = duration.Milliseconds()
log.Info().Str("name", transaction.Name).Dur("elapsed", duration).Msg("transaction")
}

return stepResult, nil
}

func (r *hrpRunner) runStepRendezvous(rend *Rendezvous) (stepResult *stepData, err error) {
func (r *caseRunner) runStepRendezvous(rend *Rendezvous) (stepResult *stepData, err error) {
log.Info().
Str("name", rend.Name).
Float32("percent", rend.Percent).
@@ -275,7 +290,7 @@ func (r *hrpRunner) runStepRendezvous(rend *Rendezvous) (stepResult *stepData, e
return stepResult, nil
}

func (r *hrpRunner) runStepRequest(step *TStep) (stepResult *stepData, err error) {
func (r *caseRunner) runStepRequest(step *TStep) (stepResult *stepData, err error) {
stepResult = &stepData{
name: step.Name,
stepType: stepTypeRequest,
@@ -388,7 +403,7 @@ func (r *hrpRunner) runStepRequest(step *TStep) (stepResult *stepData, err error
req.Host = u.Host

// log & print request
if r.debug {
if r.hrpRunner.debug {
reqDump, err := httputil.DumpRequest(req, true)
if err != nil {
return nil, errors.Wrap(err, "dump request failed")
@@ -399,15 +414,15 @@ func (r *hrpRunner) runStepRequest(step *TStep) (stepResult *stepData, err error

// do request action
start := time.Now()
resp, err := r.client.Do(req)
resp, err := r.hrpRunner.client.Do(req)
stepResult.elapsed = time.Since(start).Milliseconds()
if err != nil {
return nil, errors.Wrap(err, "do request failed")
}
defer resp.Body.Close()

// log & print response
if r.debug {
if r.hrpRunner.debug {
fmt.Println("==================== response ===================")
respDump, err := httputil.DumpResponse(resp, true)
if err != nil {
@@ -418,7 +433,7 @@ func (r *hrpRunner) runStepRequest(step *TStep) (stepResult *stepData, err error
}

// new response object
respObj, err := newResponseObject(r.t, resp)
respObj, err := newResponseObject(r.hrpRunner.t, resp)
if err != nil {
err = errors.Wrap(err, "init ResponseObject error")
return
@@ -443,15 +458,15 @@ func (r *hrpRunner) runStepRequest(step *TStep) (stepResult *stepData, err error
return stepResult, nil
}

func (r *hrpRunner) runStepTestCase(step *TStep) (stepResult *stepData, err error) {
func (r *caseRunner) runStepTestCase(step *TStep) (stepResult *stepData, err error) {
stepResult = &stepData{
name: step.Name,
stepType: stepTypeTestCase,
success: false,
}
testcase := step.TestCase
start := time.Now()
err = r.runCase(testcase)
err = r.hrpRunner.newCaseRunner(testcase).run()
stepResult.elapsed = time.Since(start).Milliseconds()
if err != nil {
return stepResult, err
@@ -460,7 +475,7 @@ func (r *hrpRunner) runStepTestCase(step *TStep) (stepResult *stepData, err erro
return stepResult, nil
}

func (r *hrpRunner) parseConfig(config IConfig) error {
func (r *caseRunner) parseConfig(config IConfig) error {
cfg := config.ToStruct()
// parse config variables
parsedVariables, err := parseVariables(cfg.Variables)
@@ -487,7 +502,7 @@ func (r *hrpRunner) parseConfig(config IConfig) error {
return nil
}

func (r *hrpRunner) getSummary() *testCaseSummary {
func (r *caseRunner) getSummary() *testCaseSummary {
return &testCaseSummary{}
}

4 changes: 2 additions & 2 deletions step.go
Original file line number Diff line number Diff line change
@@ -178,7 +178,7 @@ func (s *StepRequest) CallRefCase(tc *TestCase) *StepTestCaseWithOptionalArgs {
func (s *StepRequest) StartTransaction(name string) *StepTransaction {
s.step.Transaction = &Transaction{
Name: name,
Type: TransactionStart,
Type: transactionStart,
}
return &StepTransaction{
step: s.step,
@@ -189,7 +189,7 @@ func (s *StepRequest) StartTransaction(name string) *StepTransaction {
func (s *StepRequest) EndTransaction(name string) *StepTransaction {
s.step.Transaction = &Transaction{
Name: name,
Type: TransactionEnd,
Type: transactionEnd,
}
return &StepTransaction{
step: s.step,
11 changes: 7 additions & 4 deletions step_test.go
Original file line number Diff line number Diff line change
@@ -74,12 +74,15 @@ func TestRunRequestPostDataToStruct(t *testing.T) {
}

func TestRunRequestRun(t *testing.T) {
config := NewConfig("test").SetBaseURL("https://postman-echo.com")
runner := NewRunner(t).SetDebug(true)
if _, err := runner.runStep(stepGET, config); err != nil {
testcase := &TestCase{
Config: NewConfig("test").SetBaseURL("https://postman-echo.com"),
TestSteps: []IStep{stepGET, stepPOSTData},
}
runner := NewRunner(t).SetDebug(true).newCaseRunner(testcase)
if _, err := runner.runStep(0); err != nil {
t.Fatalf("tStep.Run() error: %s", err)
}
if _, err := runner.runStep(stepPOSTData, config); err != nil {
if _, err := runner.runStep(1); err != nil {
t.Fatalf("tStepPOSTData.Run() error: %s", err)
}
}

0 comments on commit e98b1c8

Please sign in to comment.