From d2c08d487491d4951c6bd1a40e160fa7892dfe01 Mon Sep 17 00:00:00 2001 From: Markus Zimmermann Date: Wed, 3 Apr 2024 14:31:42 +0200 Subject: [PATCH 1/3] Problems are already printed, so we do not need to print them again --- cmd/eval-symflower-codegen-testing/cmd/evaluate.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/cmd/eval-symflower-codegen-testing/cmd/evaluate.go b/cmd/eval-symflower-codegen-testing/cmd/evaluate.go index 90646c76..1c042a61 100644 --- a/cmd/eval-symflower-codegen-testing/cmd/evaluate.go +++ b/cmd/eval-symflower-codegen-testing/cmd/evaluate.go @@ -164,14 +164,6 @@ func (command *Evaluate) Execute(args []string) (err error) { } for _, modelID := range command.Models { - ps := problemsPerModel[modelID] - if len(ps) > 0 { - log.Printf("Problems for %q:", modelID) - for _, p := range ps { - log.Printf("%+v:", p) - } - } - log.Printf("Evaluation score for %q: %s", modelID, metricsPerModel[modelID]) } From 22e46e8deec9145557d35c00182739e5c2f9c0f9 Mon Sep 17 00:00:00 2001 From: Markus Zimmermann Date: Wed, 3 Apr 2024 14:34:05 +0200 Subject: [PATCH 2/3] refactor, Request/response pair naming --- model/llm/llm.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/model/llm/llm.go b/model/llm/llm.go index 49853609..f1a88af5 100644 --- a/model/llm/llm.go +++ b/model/llm/llm.go @@ -86,7 +86,7 @@ func (m *llm) GenerateTestsForFile(language language.Language, repositoryPath st importPath := filepath.Join(filepath.Base(repositoryPath), filepath.Dir(filePath)) - message, err := llmGenerateTestForFilePrompt(&llmGenerateTestForFilePromptContext{ + request, err := llmGenerateTestForFilePrompt(&llmGenerateTestForFilePromptContext{ Language: language, Code: fileContent, @@ -97,11 +97,11 @@ func (m *llm) GenerateTestsForFile(language language.Language, repositoryPath st return err } - response, err := m.provider.Query(context.Background(), m.model, message) + response, err := m.provider.Query(context.Background(), m.model, request) if err != nil { return err } - log.Printf("Model %q responded to query %q with: %q", m.ID(), message, response) + log.Printf("Model %q responded to query %q with: %q", m.ID(), request, response) testContent := prompt.ParseResponse(response) From 87af3c5dd3eaa78586c0dce4418d5d86c7643071 Mon Sep 17 00:00:00 2001 From: Markus Zimmermann Date: Wed, 3 Apr 2024 14:36:28 +0200 Subject: [PATCH 3/3] Make logging requests/responses of LLM queries more human-readable for now --- model/llm/llm.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model/llm/llm.go b/model/llm/llm.go index f1a88af5..a4ba2ccb 100644 --- a/model/llm/llm.go +++ b/model/llm/llm.go @@ -101,7 +101,7 @@ func (m *llm) GenerateTestsForFile(language language.Language, repositoryPath st if err != nil { return err } - log.Printf("Model %q responded to query %q with: %q", m.ID(), request, response) + log.Printf("Model %q responded to query %s with: %s", m.ID(), string(bytesutil.PrefixLines([]byte(request), []byte("\t"))), string(bytesutil.PrefixLines([]byte(response), []byte("\t")))) testContent := prompt.ParseResponse(response)