Merge pull request #214 from elliotchance/future/158-coverage

Run all integrations tests internally with “go test". Fixes #158

main.go

@@ -50,6 +50,9 @@ type ProgramArgs struct {
 	inputFile   string
 	outputFile  string
 	packageName string
+
+	// A private option to output the Go as a *_test.go file.
+	outputAsTest bool
 }
 
 func readAST(data []byte) []string {
@@ -212,6 +215,7 @@ func Start(args ProgramArgs) error {
 
 	p := program.NewProgram()
 	p.Verbose = args.verbose
+	p.OutputAsTest = true // args.outputAsTest
 
 	err = transpiler.TranspileAST(args.inputFile, args.packageName, p, tree[0].(ast.Node))
 	if err != nil {

main_test.go

@@ -34,7 +34,7 @@ type programOut struct {
 //
 // You can also run a single file with:
 //
-//     go test -tags=integration -run=TestIntegrationScripts/tests/ctype/isalnum.c
+//     go test -tags=integration -run=TestIntegrationScripts/tests/ctype.c
 //
 func TestIntegrationScripts(t *testing.T) {
 	testFiles, err := filepath.Glob("tests/*.c")
@@ -55,7 +55,6 @@ func TestIntegrationScripts(t *testing.T) {
 	var (
 		buildFolder  = "build"
 		cFileName    = "a.out"
-		goFileName   = "go.out"
 		mainFileName = "main.go"
 		stdin        = "7"
 		args         = []string{"some", "args"}
@@ -72,7 +71,6 @@ func TestIntegrationScripts(t *testing.T) {
 			// create subfolders for test
 			subFolder := buildFolder + separator + strings.Split(file, ".")[0] + separator
 			cPath := subFolder + cFileName
-			goPath := subFolder + goFileName
 
 			// Create build folder
 			err = os.MkdirAll(subFolder, os.ModePerm)
@@ -94,10 +92,16 @@ func TestIntegrationScripts(t *testing.T) {
 			err = cmd.Run()
 			cProgram.isZero = err == nil
 
+			mainFileName = "main_test.go"
+
 			programArgs := ProgramArgs{
 				inputFile:   file,
-				outputFile:  subFolder + separator + mainFileName,
+				outputFile:  subFolder + mainFileName,
 				packageName: "main",
+
+				// This appends a TestApp function to the output source so we
+				// can run "go test" against the produced binary.
+				outputAsTest: true,
 			}
 
 			// Compile Go
@@ -106,14 +110,27 @@ func TestIntegrationScripts(t *testing.T) {
 				t.Fatalf("error: %s\n%s", err, out)
 			}
 
-			buildErr, err := exec.Command("go", "build", "-o", goPath, subFolder+mainFileName).CombinedOutput()
-			if err != nil {
-				t.Fatal(string(buildErr), err)
+			// Run Go program. The "-v" option is important; without it most or
+			// all of the fmt.* output would be suppressed.
+			args := []string{
+				"test",
+				programArgs.outputFile,
+				"-v",
 			}
+			if strings.Index(file, "examples/") == -1 {
+				testName := strings.Split(file, ".")[0][6:]
+				args = append(
+					args,
+					"-race",
+					"-covermode=count",
+					"-coverprofile="+testName+".coverprofile",
+					"-coverpkg=./noarch,./linux,./darwin",
+				)
+			}
+			args = append(args, "--", "some", "args")
 
-			// Run Go program
-			cmd = exec.Command(goPath, args...)
-			cmd.Stdin = strings.NewReader(stdin)
+			cmd = exec.Command("go", args...)
+			cmd.Stdin = strings.NewReader("7")
 			cmd.Stdout = &goProgram.stdout
 			cmd.Stderr = &goProgram.stderr
 			err = cmd.Run()
@@ -127,35 +144,102 @@ func TestIntegrationScripts(t *testing.T) {
 				}
 			}
 
-			// Check if both exit codes are zero (or non-zero)
-			if cProgram.isZero != goProgram.isZero {
-				t.Fatalf("Exit statuses did not match.\n" +
-					util.ShowDiff(cProgram.stdout.String(),
-						goProgram.stdout.String()),
-				)
-			}
+			// Check stderr. "go test" will produce warnings when packages are
+			// not referenced as dependencies. We need to strip out these
+			// warnings so it doesn't effect the comparison.
+			cProgramStderr := cProgram.stderr.String()
+			goProgramStderr := goProgram.stderr.String()
+
+			r := regexp.MustCompile("warning: no packages being tested depend on .+\n")
+			goProgramStderr = r.ReplaceAllString(goProgramStderr, "")
 
-			// Check stderr
-			if cProgram.stderr.String() != goProgram.stderr.String() {
-				t.Fatalf("Expected %q, Got: %q",
-					cProgram.stderr.String(),
-					goProgram.stderr.String())
+			if cProgramStderr != goProgramStderr {
+				t.Fatalf("Expected %q, Got: %q", cProgramStderr, goProgramStderr)
 			}
 
 			// Check stdout
-			if cProgram.stdout.String() != goProgram.stdout.String() {
-				t.Fatalf(util.ShowDiff(cProgram.stdout.String(),
-					goProgram.stdout.String()))
+			cOut := cProgram.stdout.String()
+			goOutLines := strings.Split(goProgram.stdout.String(), "\n")
+
+			// An out put should look like this:
+			//
+			//     === RUN   TestApp
+			//     1..3
+			//     1 ok - argc == 3 + offset
+			//     2 ok - argv[1 + offset] == "some"
+			//     3 ok - argv[2 + offset] == "args"
+			//     --- PASS: TestApp (0.03s)
+			//     PASS
+			//     coverage: 0.0% of statements
+			//     ok  	command-line-arguments	1.050s
+			//
+			// The first line and 4 of the last lines can be ignored as they are
+			// part of the "go test" runner and not part of the program output.
+			//
+			// Note: There is a blank line at the end of the output so when we
+			// say the last line we are really talking about the second last
+			// line. Rather than trimming the whitespace off the C and Go output
+			// we will just make note of the different line index.
+			//
+			// Some tests are designed to fail, like assert.c. In this case the
+			// result output is slightly different:
+			//
+			//     === RUN   TestApp
+			//     1..0
+			//     10
+			//     # FAILED: There was 1 failed tests.
+			//     exit status 101
+			//     FAIL	command-line-arguments	0.041s
+			//
+			// The last three lines need to be removed.
+			//
+			// Before we proceed comparing the raw output we should check that
+			// the header and footer of the output fits one of the two formats
+			// in the examples above.
+			if goOutLines[0] != "=== RUN   TestApp" {
+				t.Fatalf("The header of the output cannot be understood:\n%s",
+					strings.Join(goOutLines, "\n"))
+			}
+			if !strings.HasPrefix(goOutLines[len(goOutLines)-2], "ok  \tcommand-line-arguments") &&
+				!strings.HasPrefix(goOutLines[len(goOutLines)-2], "FAIL\tcommand-line-arguments") {
+				t.Fatalf("The footer of the output cannot be understood:\n%v",
+					strings.Join(goOutLines, "\n"))
+			}
+
+			// A failure will cause (always?) "go test" to output the exit code
+			// before the final line. We should also ignore this as its not part
+			// of our output.
+			//
+			// There is a separate check to see that both the C and Go programs
+			// return the same exit code.
+			removeLinesFromEnd := 5
+			if strings.Index(file, "examples/") >= 0 {
+				removeLinesFromEnd = 4
+			} else if strings.HasPrefix(goOutLines[len(goOutLines)-3], "exit status") {
+				removeLinesFromEnd = 3
+			}
+
+			goOut := strings.Join(goOutLines[1:len(goOutLines)-removeLinesFromEnd], "\n") + "\n"
+
+			// Check if both exit codes are zero (or non-zero)
+			if cProgram.isZero != goProgram.isZero {
+				t.Fatalf("Exit statuses did not match.\n%s", util.ShowDiff(cOut, goOut))
+			}
+
+			if cOut != goOut {
+				t.Fatalf(util.ShowDiff(cOut, goOut))
 			}
 
-			// If this is not an example we will extact the number of tests run.
+			// If this is not an example we will extract the number of tests
+			// run.
 			if strings.Index(file, "examples/") == -1 && isVerbose {
-				firstLine := strings.Split(goProgram.stdout.String(), "\n")[0]
+				firstLine := strings.Split(goOut, "\n")[0]
 
 				matches := regexp.MustCompile(`1\.\.(\d+)`).
 					FindStringSubmatch(firstLine)
 				if len(matches) == 0 {
-					t.Fatalf("Test did not output tap: %s", file)
+					t.Fatalf("Test did not output tap: %s, got:\n%s", file,
+						goProgram.stdout.String())
 				}
 
 				fmt.Printf("TAP: # %s: %s tests\n", file, matches[1])

noarch/stdio.go

@@ -8,6 +8,15 @@ import (
 	"reflect"
 )
 
+// Programs generated by c2go will reference noarch.Stdin instead of os.Stdin
+// directly so that under test these can be replaced. This is required because
+// "go test" does not redirect the stdin to the executable it is testing.
+var (
+	Stdin  = NewFile(os.Stdin)
+	Stdout = NewFile(os.Stdout)
+	Stderr = NewFile(os.Stderr)
+)
+
 // File represents the definition has been translated from the original
 // definition for __sFILE, which is an alias for FILE. Not all of the attributes
 // have been translated. They should be turned on as needed.
@@ -470,7 +479,7 @@ func Fputc(c int, f *File) int {
 //
 // It is equivalent to calling getc with stdin as argument.
 func Getchar() int {
-	return getc(os.Stdin)
+	return getc(Stdin.OsFile)
 }
 
 // Fseek handles fseek().
@@ -659,7 +668,10 @@ func Puts(str []byte) int {
 // string.
 func Scanf(format []byte, args ...interface{}) int {
 	realArgs := prepareArgsForScanf(args)
-	n, _ := fmt.Scanf(CStringToString(format), realArgs...)
+
+	// We cannot use fmt.Scanf() here because that would use the real stdin
+	// which does not work under test. See docs for noarch.Stdin.
+	n, _ := fmt.Fscanf(Stdin.OsFile, CStringToString(format), realArgs...)
 	finalizeArgsForScanf(realArgs, args)
 
 	return n

program/program.go

@@ -58,7 +58,7 @@ type Program struct {
 	Unions  StructRegistry
 
 	// If verbose is on progress messages will be printed immediately as code
-	// comments (so that they do not intefere with the program output).
+	// comments (so that they do not interfere with the program output).
 	Verbose bool
 
 	// Contains the messages (for example, "// Warning") generated when
@@ -69,6 +69,11 @@ type Program struct {
 	// A map of all the global variables (variables that exist outside of a
 	// function) and their types.
 	GlobalVariables map[string]string
+
+	// This option is not available through the command line. It is to allow the
+	// internal integration testing to generate the output in the form of a
+	// Go-test rather than a standalone Go file.
+	OutputAsTest bool
 }
 
 // NewProgram creates a new blank program.

tests/argv.c

@@ -5,16 +5,26 @@
 
 int main(int argc, const char **argv)
 {
-    plan(3);
+    plan(2);
 
-    is_eq(argc, 3);
+    // When this file is converted to go it is run through "go test" that needs
+    // some extra arguments before the standard C arguments. We need to adjust
+    // an offset so that the C program and the Go program read the same index
+    // for the first index of the real arguments.
+    int offset = 0;
 
-    // We cannot compare the zeroth argument becuase it will be different for C
+    // More than three arguments means it must be run under "go test". If not
+    // the assertion immediately below will fail.
+    if (argc > 3) {
+        offset = 3;
+    }
+
+    // We cannot compare the zeroth argument because it will be different for C
     // and Go.
     // is_streq(argv[0], "build/go.out");
 
-    is_streq(argv[1], "some");
-    is_streq(argv[2], "args");
+    is_streq(argv[1 + offset], "some");
+    is_streq(argv[2 + offset], "args");
 
     done_testing();
 }

tests/getchar.c

@@ -8,11 +8,8 @@ int main()
     plan(1);
 
     int c;
-    diag("Enter text. Include a dot ('.') in a sentence to exit:");
     c = getchar();
     is_eq(c, '7');
 
-    putchar(c);
-
     done_testing();
 }

tests/stdio.c

@@ -298,7 +298,7 @@ void test_fread()
     // obtain file size:
     fseek(pFile, 0, SEEK_END);
     lSize = ftell(pFile);
-    is_eq(lSize, 301);
+    is_eq(lSize, 216);
 
     rewind(pFile);
 
@@ -309,7 +309,7 @@ void test_fread()
     // See issue #107
     buffer[lSize - 1] = 0;
 
-    is_eq(strlen(buffer), 300);
+    is_eq(strlen(buffer), 215);
 
     // terminate
     fclose(pFile);

tests/tests.h

@@ -270,7 +270,11 @@ static int last_test_was_ok = 1;
         diag("FAILED: Expected %d tests, but ran %d.", total_tests, current_test); \
         exit_status = 102;                                                         \
     }                                                                              \
-    return exit_status;
+    /* If we exit (with any status) the Go code coverage will not be generated. */ \
+    if (exit_status != 0) {                                                        \
+        return exit_status;                                                        \
+    }                                                                              \
+    return 0;
 
 // or_return will return (with an optional value provided) if the check failed.
 //

transpiler/declarations.go

@@ -227,10 +227,7 @@ func transpileVarDecl(p *program.Program, n *ast.VarDecl) (
 				util.NewBinaryExpr(
 					goast.NewIdent(name),
 					token.ASSIGN,
-					util.NewCallExpr(
-						"noarch.NewFile",
-						util.NewTypeIdent("os."+util.Ucfirst(name[2:len(name)-1])),
-					),
+					util.NewTypeIdent("noarch."+util.Ucfirst(name[2:len(name)-1])),
 					"*noarch.File",
 					true,
 				),
@@ -244,10 +241,7 @@ func transpileVarDecl(p *program.Program, n *ast.VarDecl) (
 				util.NewBinaryExpr(
 					goast.NewIdent(name),
 					token.ASSIGN,
-					util.NewCallExpr(
-						"noarch.NewFile",
-						util.NewTypeIdent("os."+util.Ucfirst(name)),
-					),
+					util.NewTypeIdent("noarch."+util.Ucfirst(name)),
 					theType,
 					true,
 				),