diff --git a/cmd_rd.go b/cmd_rd.go new file mode 100644 index 0000000..57fbada --- /dev/null +++ b/cmd_rd.go @@ -0,0 +1,12 @@ +//////////////////////////////////////////////////////////////////////////// +// Porgram: cmd_rd - Result Dump handling +// Authors: Antonio Sun (c) 2015, All rights reserved +//////////////////////////////////////////////////////////////////////////// + +package main + +func cmd_rd(options Options) error { + PerfCounterExport(options, options.Rd.Id, + options.Rd.MachineNameFilter, options.Rd.PathOut) + return nil +} diff --git a/config.go b/config.go index 5299c2f..0a29fd7 100644 --- a/config.go +++ b/config.go @@ -5,7 +5,10 @@ package main -import "io/ioutil" +import ( + "io/ioutil" + "strings" +) import ( "gopkg.in/yaml.v2" @@ -56,7 +59,7 @@ var config struct { func configGet(args0 string) { if len(options.ConfigFile) == 0 { - options.ConfigFile = args0[:len(args0)-4] + options.ConfigExt + options.ConfigFile = Basename(args0) + options.ConfigExt } cfgStr, err := ioutil.ReadFile(options.ConfigFile) @@ -64,3 +67,11 @@ func configGet(args0 string) { check(err) //fmt.Printf("] %#v\r\n", config) } + +func Basename(s string) string { + n := strings.LastIndexByte(s, '.') + if n > 0 { + return s[:n] + } + return s +} diff --git a/lta-main.go b/lta-main.go index 6dacfd9..e449d4c 100644 --- a/lta-main.go +++ b/lta-main.go @@ -32,29 +32,38 @@ type Options struct { SqlConnectionString string `goptions:"--conn, description='ConnectionString of Go MSSQL Odbc to MS SQL Server\n\t\t\t\tTo override the above --cs/cd setting. Sample:\n\t\t\t\tdriver=sql server;server=(local);database=LoadTest2010;uid=user;pwd=pass\n'"` - Verbosity []bool "goptions:\"-v, --verbose, description='Be verbose'\"" - Help goptions.Help `goptions:"-h, --help, description='Show this help\n\nSub-commands (Verbs):\n\n\tcgl\t\tConfig Group List\n\t\t\tList machine groups defined in config file\n\n\trd\t\tResult Dump\n\t\t\tDump load test result, all machine counters\n\trdg\t\tResult Dump Group\n\t\t\tDump load test results, for the machine group\n\n\trbg\t\tReBoot Group\n\t\t\tReboot the machine group'"` + Step int `goptions:"-s, --step, description='Number of record outputed after which indicator is shown\n\t\t\t\t'"` + NoClobber bool `goptions:"--nc, description='No clobber, do not overwrite existing files\n\t\t\t\tDefault: overwrite them\n'"` + + Verbosity []bool `goptions:"-v, --verbose, description='Be verbose'"` + Help goptions.Help `goptions:"-h, --help, description='Show this help\n\nSub-commands (Verbs):\n\n\tcgl\t\tConfig Group List\n\t\t\tList machine groups defined in config file\n\n\trd\t\tResult Dump\n\t\t\tDump load test result, standalone\n\trdg\t\tResult Dump Group\n\t\t\tDump load test results, for the machine group\n\n\trbg\t\tReBoot Group\n\t\t\tReboot the machine group'"` goptions.Verbs - cgl struct{} `goptions:"cgl"` + Cgl struct{} `goptions:"cgl"` - rd struct{} `goptions:"rd"` - rdg struct{} `goptions:"rdg"` + Rd struct { + Id int `goptions:"-n, --id, obligatory, description='Loadtest RunId'"` + PathOut string `goptions:"-p, --path, obligatory, description='Path to where dumps are saved'"` + MachineNameFilter string `goptions:"-m, --mfilter, description='machine name filter for exporting the counters\n\t\t\t\tDefault: export all machines\n'"` + } `goptions:"rd"` + Rdg struct{} `goptions:"rdg"` - rbg struct{} `goptions:"rbg"` + Rbg struct{} `goptions:"rbg"` } var options = Options{ // Default values goes here ConfigExt: ".conf", Server: "(local)", PerfDb: "LoadTest2010", + Step: 50, } type Command func(Options) error var commands = map[goptions.Verbs]Command{ "cgl": cmd_cgl, + "rd": cmd_rd, } var ( diff --git a/perf_export.go b/perf_export.go new file mode 100644 index 0000000..ee46efb --- /dev/null +++ b/perf_export.go @@ -0,0 +1,234 @@ +//////////////////////////////////////////////////////////////////////////// +// Porgram: perf_export.go - Perf Counter Export +// Authors: Antonio Sun (c) 2015, All rights reserved +// Purpose: Export performance counters collected from MS load test to .csv +// files for perfmon to view +//////////////////////////////////////////////////////////////////////////// + +// Translated to GO from C#, http://blogs.msdn.com/b/geoffgr/archive/2013/09/09/ + +package main + +import ( + "database/sql" + "fmt" + "log" + "os" + "path/filepath" + "time" + + "bitbucket.org/kardianos/table" + _ "github.com/alexbrainman/odbc" +) + +var progname string = "lta:dump" + +/* +PerfCounterExport will wxport performance counters collected from MS load +test to .csv files for perfmon to view +*/ +func PerfCounterExport(options Options, ltRunId int, machineNameFilter string, pathOut string) { + + conn := getConnection(getConnectionString(options)) + defer conn.Close() + log.Printf("[%s] Program started\n", progname) + + if 1 == 1 { + // No Loadtest specified. Use Max RunId. + runId, err := table.Get(conn, + "SELECT MAX(LoadTestRunId) AS RunId from LoadTestRun") + if err != nil { + log.Fatal(err) + } + + maxRunId := runId.MustGetScaler(0, "RunId") + fLoadTestRunId := int(maxRunId.(int32)) + log.Printf("[%s] RunId : %d, %d\n", progname, ltRunId, fLoadTestRunId) + } + + // get TraceName according to LoadTestRunId + r, err := table.Get(conn, + "SELECT TraceName from LoadTestRun WHERE LoadTestRunId = ?", ltRunId) + check(err) + ltTraceName := U8ToGoString(r.MustGetScaler(0, "TraceName").([]uint8)) + + resultFilePre := pathOut + string(os.PathSeparator) + resultFilePre = filepath.Dir(resultFilePre) + + string(os.PathSeparator) + ltTraceName + os.Mkdir(resultFilePre, os.ModePerm) + // so far path only, now append folder name as file prefix + resultFilePre += string(os.PathSeparator) + ltTraceName + + log.Printf("[%s] Exporting LoadTest %d\n to %s-...\n with step of %d\n", + progname, ltRunId, resultFilePre, options.Step) + + if machineNameFilter != "" { + fmt.Printf(" limiting to only export machine %s\n\n", machineNameFilter) + savePerfmonAsCsv(options.NoClobber, conn, machineNameFilter, ltRunId, resultFilePre) + os.Exit(0) + } + + /* + Get all machine names + + SELECT category.MachineName + FROM LoadTestPerformanceCounterCategory AS category + JOIN LoadTestPerformanceCounterInstance AS instance + ON category.LoadTestRunId = instance.LoadTestRunId + AND instance.LoadTestRunId = ( + SELECT MAX(LoadTestRunId) from LoadTestRun ) + GROUP BY MachineName + + */ + + machines, err := table.Get(conn, + "SELECT category.MachineName"+ + " FROM LoadTestPerformanceCounterCategory AS category"+ + " JOIN LoadTestPerformanceCounterInstance AS instance"+ + " ON category.LoadTestRunId = instance.LoadTestRunId"+ + " AND instance.LoadTestRunId = ?"+ + " GROUP BY MachineName", ltRunId) + if err != nil { + log.Fatal(err) + } + + for i, machine := range machines.Rows { + // machine.MustGet("MachineName").(string) + machineName := U8ToGoString(machine.MustGet("MachineName").([]uint8)) + if i == 0 { + log.Printf("[%s] (Controller %s skipped)\n", + progname, machineName) + continue + } + savePerfmonAsCsv(options.NoClobber, conn, machineName, ltRunId, resultFilePre) + } + + log.Printf("[%s] Exporting finished correctly.\n", progname) + return +} + +func getConnection(connectionString string) *sql.DB { + conn, err := sql.Open("odbc", connectionString) + check(err) + return conn +} + +func getConnectionString(options Options) string { + // Construct the Go MSSQL odbc SqlConnectionString + // https://code.google.com/p/odbc/source/browse/mssql_test.go + var c string + if options.SqlConnectionString == "" { + var params map[string]string + params = map[string]string{ + "driver": "sql server", + "server": options.Server, + "database": options.PerfDb, + "trusted_connection": "yes", + } + + for n, v := range params { + c += n + "=" + v + ";" + } + } else { + c = options.SqlConnectionString + } + log.Println("Connection string: " + c) + return c +} + +func savePerfmonAsCsv(fNoClobber bool, conn *sql.DB, machineName string, _runId int, resultFilePre string) { + // Only use right(5) + const keep = 5 + if len(machineName) > keep { + machineName = machineName[len(machineName)-keep:] + } + + log.Printf("[%s] Collecting data for %s...\n", progname, machineName) + + // if no clobber and the destination file exists, skip + if options.NoClobber { + if _, err := os.Stat(resultFilePre + "-" + machineName + ".csv"); err == nil { + log.Printf("[%s] (Host %s skipped for no clobbering)\n", + progname, machineName) + return + } + } + + sql := fmt.Sprintf("exec TSL_prc_PerfCounterCollectionInCsvFormat"+ + " @RunId = %d, @InstanceName=N'\\\\%%%s\\%%'", _runId, machineName) + //log.Println("] sql string: " + sql) + table, err := table.Get(conn, sql) + if err != nil { + log.Printf("[%s] Skipping it for the fatal error:\n\t\t %v\n", + progname, err.Error()) + return + } + + log.Printf("[%s] Exporting %s data...\n", progname, machineName) + + // open the output file + file, err := os.Create(resultFilePre + "-" + machineName + ".csv") + if err != nil { + panic(err) + } + // close file on exit and check for its returned error + defer func() { + if err := file.Close(); err != nil { + panic(err) + } + }() + + // output header + for i, element := range table.ColumnName { + if i != 0 { + fmt.Fprintf(file, ",") + } + fmt.Fprintf(file, "\"%s\"", element) + } + fmt.Fprintf(file, "\n") + + // output body + const layout = "01/02/2006 15:04:05.999" + for j, row := range table.Rows { + for i, colname := range table.ColumnName { + if i != 0 { + fmt.Fprintf(file, ",") + } + switch x := row.MustGet(colname).(type) { + case string: // x is a string + fmt.Fprintf(file, "\"%s\"", x) + case int: // now x is an int + fmt.Fprintf(file, "\"%d\"", x) + case int32: // now x is an int32 + fmt.Fprintf(file, "\"%d\"", x) + case int64: // now x is an int64 + fmt.Fprintf(file, "\"%d\"", x) + case float32: // now x is an float32 + fmt.Fprintf(file, "\"%f\"", x) + case float64: // now x is an float64 + fmt.Fprintf(file, "\"%f\"", x) + case time.Time: // now x is a time.Time + fmt.Fprintf(file, "\"%s\"", x.Format(layout)) + default: + fmt.Fprintf(file, "\"%s\"", x) + } + } + fmt.Fprintf(file, "\n") + if j%options.Step == 0 { + fmt.Fprintf(os.Stderr, ".") + } + } + fmt.Fprintf(os.Stderr, "\n") + +} + +func U8ToGoString(c []uint8) string { + n := -1 + for i, b := range c { + if b == 0 { + break + } + n = i + } + return string(c[:n+1]) +} diff --git a/perf_export.sql b/perf_export.sql new file mode 100644 index 0000000..deecbd4 --- /dev/null +++ b/perf_export.sql @@ -0,0 +1,112 @@ +-- To obtain Performance Counters to view within Perfmon +-- From http://blogs.msdn.com/b/geoffgr/archive/2013/09/09/how-to-export-perfmon-counter-values-from-the-visual-studio-load-test-results-database.aspx + +--START CODE-- +CREATE PROCEDURE TSL_prc_PerfCounterCollectionInCsvFormat + @RunId nvarchar(10), + @InstanceName nvarchar(1024) +AS + DECLARE @CounterName nvarchar(max), @CounterNameColumns nvarchar(max) + --Get List of columns to use in query. Shove them into a single long XML string + SELECT @CounterNameColumns = ( + SELECT ', [' + REPLACE(InstanceName, ']', ']]') + ']' FROM MTSL_View_PerfmonInstanceNamesAndIds + WHERE LoadTestRunId = @RunId + AND InstanceName LIKE @InstanceName + FOR XML PATH('')) + --Make a copy of the list WITHOUT the comma at the very beginning of the string + SELECT @CounterName = RIGHT(@CounterNameColumns, LEN(@CounterNameColumns) - 1) + -- Use the previous strings to build the query string that can be pivoted + DECLARE @SQL nvarchar(max) + SELECT @SQL = N' + select + IntervalStartTime AS [(PDH-CSV 4.0) (Eastern Daylight Time)(240)]' + + --IntervalStartTime' + + @CounterNameColumns + ' + from ( + select + interval.IntervalStartTime, + MTSL_View_PerfmonInstanceNamesAndIds.InstanceName, + countersample.ComputedValue + FROM + MTSL_View_PerfmonInstanceNamesAndIds + INNER JOIN LoadTestPerformanceCounterSample AS countersample + ON countersample.InstanceId = MTSL_View_PerfmonInstanceNamesAndIds.InstanceId + AND countersample.LoadTestRunId = MTSL_View_PerfmonInstanceNamesAndIds.LoadTestRunId + INNER JOIN LoadTestRunInterval AS interval + ON interval.LoadTestRunId = countersample.LoadTestRunId + AND interval.TestRunIntervalId = countersample.TestRunIntervalId + WHERE + MTSL_View_PerfmonInstanceNamesAndIds.LoadTestRunId = ' + @RunId + ' + AND + MTSL_View_PerfmonInstanceNamesAndIds.InstanceName LIKE '''+@InstanceName+''' + ) Data + PIVOT ( + SUM(ComputedValue) + FOR InstanceName + IN ( + ' + @CounterName + ' + ) + ) PivotTable + ORDER BY IntervalStartTime ASC + ' + -- print @SQL + -- Execute the generated query + exec sp_executesql @SQL +GO +--START CODE-- + +GRANT EXECUTE ON TSL_prc_PerfCounterCollectionInCsvFormat TO PUBLIC +GO + +/*=============================================================================== +MTSL_View_PerfmonInstanceNamesAndIds +===============================================================================*/ + +CREATE VIEW MTSL_View_PerfmonInstanceNamesAndIds AS + SELECT + instance.LoadTestRunId + ,instance.InstanceId + ,( + '\\' + RIGHT(category.MachineName,6) + + '\' + category.CategoryName + + case instance.InstanceName when 'systemdiagnosticsperfcounterlibsingleinstance' + then '' + else '(' + instance.InstanceName + ')' + end + + '\' + counter.CounterName + ) AS InstanceName + FROM LoadTestPerformanceCounterCategory AS category + INNER JOIN LoadTestPerformanceCounter AS counter + ON category.LoadTestRunId = counter.LoadTestRunId + AND category.CounterCategoryId = counter.CounterCategoryId + INNER JOIN LoadTestPerformanceCounterInstance AS instance + ON counter.CounterId = instance.CounterId + AND counter.LoadTestRunId = instance.LoadTestRunId +GO + +/*=============================================================================== +LoadTestRuns +Describe: Returns LoadTestRuns stored in the database +Example: SELECT * FROM LoadTestRuns ORDER BY LoadTestRunId DESC +===============================================================================*/ + +IF OBJECT_ID ('LoadTestRuns', 'V') IS NOT NULL +DROP VIEW LoadTestRuns +GO + +CREATE VIEW LoadTestRuns +AS +SELECT LoadTestRunId, + LoadTestName + ,StartTime + ,EndTime + ,RunDuration + ,Outcome + FROM LoadTestRun LTR +GROUP BY LoadTestRunId, + LoadTestName + ,StartTime + ,EndTime + ,RunDuration + ,Outcome +--ORDER BY StartTime DESC