VS Code: add characters logger metadata to chat code-gen events (#6019)

- Adds new fields to the `cody.characters:flush` analytics event. This
way, we can identify all code generation events coming from chat and
always include them once in our PCW calculation.
- `cody_chat` — to identify document change events originating from
`insert at cursor` and `copy-paste` chat actions.
    - `cody_chat_inserted`
    - `cody_chat_deleted`

vscode/src/main.ts

@@ -100,7 +100,6 @@ import { VSCodeSecretStorage, secretStorage } from './services/SecretStorageProv
 import { registerSidebarCommands } from './services/SidebarCommands'
 import { CodyStatusBar } from './services/StatusBar'
 import { createOrUpdateTelemetryRecorderProvider } from './services/telemetry-v2'
-import { onTextDocumentChange } from './services/utils/codeblock-action-tracker'
 import {
     enableVerboseDebugMode,
     exportOutputLog,
@@ -233,7 +232,6 @@ const register = async (
     disposables.push(await initVSCodeGitApi())
 
     registerParserListeners(disposables)
-    registerChatListeners(disposables)
 
     // Initialize external services
     const {
@@ -340,20 +338,6 @@ function registerParserListeners(disposables: vscode.Disposable[]) {
     disposables.push(vscode.workspace.onDidChangeTextDocument(updateParseTreeOnEdit))
 }
 
-function registerChatListeners(disposables: vscode.Disposable[]) {
-    // Enable tracking for pasting chat responses into editor text
-    disposables.push(
-        vscode.workspace.onDidChangeTextDocument(async e => {
-            const changedText = e.contentChanges[0]?.text
-            // Skip if the document is not a file or if the copied text is from insert
-            if (!changedText || e.document.uri.scheme !== 'file') {
-                return
-            }
-            await onTextDocumentChange(changedText)
-        })
-    )
-}
-
 async function registerOtherCommands(disposables: vscode.Disposable[]) {
     disposables.push(
         // Account links

vscode/src/non-stop/FixupController.ts

@@ -25,7 +25,6 @@ import {
     DEFAULT_EVENT_SOURCE,
     EventSourceTelemetryMetadataMapping,
 } from '@sourcegraph/cody-shared/src/chat/transcript/messages'
-import omit from 'lodash/omit'
 import type { SmartApplyResult } from '../chat/protocol'
 import { PersistenceTracker } from '../common/persistence-tracker'
 import { lines } from '../completions/text-processing'
@@ -590,7 +589,7 @@ export class FixupController
     }
 
     private logTaskCompletion(task: FixupTask, document: vscode.TextDocument, editOk: boolean): void {
-        const charactersLoggerMetadata = charactersLogger.getChangeEventMetadata({
+        const charactersLoggerMetadata = charactersLogger.getChangeEventMetadataForCodyCodeGenEvents({
             document,
             contentChanges: task.getContentChanges(),
             reason: undefined,
@@ -604,13 +603,7 @@ export class FixupController
             model: task.model,
             ...this.countEditInsertions(task),
             ...task.telemetryMetadata,
-            ...omit(charactersLoggerMetadata, [
-                'changeSize',
-                'changeType',
-                'isRedo',
-                'isUndo',
-                'isRapidChange',
-            ]),
+            ...charactersLoggerMetadata,
         }
         const { metadata, privateMetadata } = splitSafeMetadata(legacyMetadata)
         if (!editOk) {

vscode/src/services/CharactersLogger.test.ts

@@ -15,6 +15,7 @@ import {
     RAPID_CHANGE_TIMEOUT,
     SELECTION_TIMEOUT,
 } from './CharactersLogger'
+import * as codeBlockUtils from './utils/codeblock-action-tracker'
 
 const testDocument = document('foo')
 
@@ -23,7 +24,7 @@ describe('CharactersLogger', () => {
     let tracker: CharactersLogger
 
     let onDidChangeActiveTextEditor: (event: vscode.TextEditor | undefined) => void
-    let onDidChangeTextDocument: (event: vscode.TextDocumentChangeEvent) => void
+    let onDidChangeTextDocument: (event: vscode.TextDocumentChangeEvent) => Promise<void>
     let onDidCloseTextDocument: (document: vscode.TextDocument) => void
     let onDidChangeWindowState: (state: vscode.WindowState) => void
     let onDidChangeTextEditorSelection: (event: vscode.TextEditorSelectionChangeEvent) => void
@@ -117,27 +118,29 @@ describe('CharactersLogger', () => {
         return { ...DEFAULT_COUNTERS, ...expected }
     }
 
-    it('should handle insertions, deletions, rapid and stale changes, and changes outside of visible range', () => {
+    it('should handle insertions, deletions, rapid and stale changes, and changes outside of visible range', async () => {
         // Simulate user typing in the active text editor
         onDidChangeActiveTextEditor(mockActiveTextEditor)
         onDidChangeTextEditorSelection(mockTextEditorSelectionEvent)
 
         // Scenario 1: User types 'hello' (insertion)
-        onDidChangeTextDocument(
+        await onDidChangeTextDocument(
             createChange({ text: 'hello', range: range(0, 0, 0, 0), rangeLength: 0 })
         )
 
         // Advance time less than RAPID_CHANGE_TIMEOUT to simulate rapid change
         vi.advanceTimersByTime(RAPID_CHANGE_TIMEOUT - 5)
 
         // Scenario 2: User deletes 'he' (deletion)
-        onDidChangeTextDocument(createChange({ text: '', range: range(0, 0, 0, 2), rangeLength: 2 }))
+        await onDidChangeTextDocument(
+            createChange({ text: '', range: range(0, 0, 0, 2), rangeLength: 2 })
+        )
 
         // Now, advance time beyond SELECTION_TIMEOUT to make selection stale
         vi.advanceTimersByTime(SELECTION_TIMEOUT + 1000)
 
         // Scenario 3: User types 'there' (stale insertion)
-        onDidChangeTextDocument(
+        await onDidChangeTextDocument(
             createChange({ text: 'there', range: range(0, 3, 0, 3), rangeLength: 0 })
         )
         // Should be counted as an insertion, stale change
@@ -148,7 +151,7 @@ describe('CharactersLogger', () => {
         onDidChangeActiveTextEditor(mockActiveTextEditor)
 
         // User types 'hidden' at line 50 (outside visible range)
-        onDidChangeTextDocument(
+        await onDidChangeTextDocument(
             createChange({
                 text: 'hidden',
                 range: range(50, 0, 50, 0), // Line 50, start
@@ -180,23 +183,23 @@ describe('CharactersLogger', () => {
         })
     })
 
-    it('should handle undo, redo, window not focused, no active editor, outside of active editor, and document closing', () => {
+    it('should handle undo, redo, window not focused, no active editor, outside of active editor, and document closing', async () => {
         onDidChangeActiveTextEditor(mockActiveTextEditor)
         onDidChangeTextEditorSelection(mockTextEditorSelectionEvent)
 
         const changeReasons = { Undo: 1, Redo: 2 } as const
 
         const xxsChangeEvent = createChange({ text: 'test', range: range(0, 0, 0, 0), rangeLength: 0 })
-        onDidChangeTextDocument(xxsChangeEvent)
+        await onDidChangeTextDocument(xxsChangeEvent)
 
         const disjointChange = createChange({ text: 'test', range: range(2, 0, 0, 0), rangeLength: 0 })
-        onDidChangeTextDocument({
+        await onDidChangeTextDocument({
             ...xxsChangeEvent,
             contentChanges: [xxsChangeEvent.contentChanges[0], disjointChange.contentChanges[0]],
         })
 
         // Simulate undo
-        onDidChangeTextDocument(
+        await onDidChangeTextDocument(
             createChange({
                 text: '',
                 range: range(0, 4, 0, 0),
@@ -206,7 +209,7 @@ describe('CharactersLogger', () => {
         )
 
         // Simulate redo
-        onDidChangeTextDocument(
+        await onDidChangeTextDocument(
             createChange({
                 text: 'test',
                 range: range(0, 0, 0, 0),
@@ -215,11 +218,29 @@ describe('CharactersLogger', () => {
             })
         )
 
+        const codeFromChat = 'insert_from_chat'
+        vi.spyOn(codeBlockUtils, 'isCodeFromChatCodeBlockAction').mockResolvedValueOnce({
+            operation: 'insert',
+            code: codeFromChat,
+            lineCount: 1,
+            charCount: codeFromChat.length,
+            eventName: 'insert',
+            source: 'chat',
+        })
+
+        await onDidChangeTextDocument(
+            createChange({
+                text: codeFromChat,
+                range: range(0, 0, 0, 0),
+                rangeLength: 0,
+            })
+        )
+
         mockWindowState.focused = false
         onDidChangeWindowState(mockWindowState)
 
         // User types ' window not focused' when window not focused
-        onDidChangeTextDocument(
+        await onDidChangeTextDocument(
             createChange({ text: 'window not focused', range: range(0, 4, 0, 4), rangeLength: 0 })
         )
 
@@ -229,7 +250,7 @@ describe('CharactersLogger', () => {
 
         // Simulate no active editor
         onDidChangeActiveTextEditor(undefined)
-        onDidChangeTextDocument(
+        await onDidChangeTextDocument(
             createChange({ text: 'no active editor', range: range(0, 21, 0, 21), rangeLength: 0 })
         )
 
@@ -246,7 +267,7 @@ describe('CharactersLogger', () => {
         onDidChangeActiveTextEditor(anotherEditor)
 
         // User types in original document (not the active editor's document)
-        onDidChangeTextDocument(
+        await onDidChangeTextDocument(
             createChange({
                 text: 'outside active editor',
                 range: range(0, 21, 0, 21),
@@ -256,7 +277,7 @@ describe('CharactersLogger', () => {
         )
 
         onDidCloseTextDocument(testDocument)
-        onDidChangeTextDocument(
+        await onDidChangeTextDocument(
             createChange({
                 text: '!',
                 range: range(0, 50, 0, 50),
@@ -270,6 +291,9 @@ describe('CharactersLogger', () => {
         // Expected counters:
         expect(recordSpy).toHaveBeenCalledWith('cody.characters', 'flush', {
             metadata: expectedCharCounters({
+                cody_chat: 1,
+                cody_chat_inserted: 16, // 'insert_from_chat'
+
                 xxs_change: 1,
                 xxs_change_inserted: 4, // 'test'