Skip to content

Commit

Permalink
telemetry(amazonq): cleanup amazonq_utgGenerateTests logic #6212
Browse files Browse the repository at this point in the history 
## Problem
- Repetitive code when emitting amazonq_utgGenerateTests telemetry
events
- Metric has already been migrated to aws-toolkit-common

## Solution
- Created sendTestGenerationToolkitEvent helper to extract common fields
- Removed metric override in this repo
- Tested and can confirm that metrics are same as previous method
  • Loading branch information
@chungjac
chungjac authored Dec 11, 2024
1 parent 2af8b45 commit e63079d
Show file tree
Hide file tree
Showing 6 changed files with 119 additions and 220 deletions.
8 changes: 4 additions & 4 deletions package-lock.json
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
"generateNonCodeFiles": "npm run generateNonCodeFiles -w packages/ --if-present"
},
"devDependencies": {
"@aws-toolkits/telemetry": "^1.0.284",
"@aws-toolkits/telemetry": "^1.0.287",
"@playwright/browser-chromium": "^1.43.1",
"@stylistic/eslint-plugin": "^2.11.0",
"@types/he": "^1.2.3",
Expand Down
110 changes: 52 additions & 58 deletions packages/core/src/amazonqTest/chat/controller/controller.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -242,22 +242,20 @@ export class TestController {
this.messenger.sendUpdatePromptProgress(data.tabID, null)
const session = this.sessionStorage.getSession()
const isCancel = data.error.message === unitTestGenerationCancelMessage
telemetry.amazonq_utgGenerateTests.emit({
cwsprChatProgrammingLanguage: session.fileLanguage ?? 'plaintext',
jobId: session.listOfTestGenerationJobId[0], // For RIV, UTG does only one StartTestGeneration API call
jobGroup: session.testGenerationJobGroupName,
requestId: session.startTestGenerationRequestId,
hasUserPromptSupplied: session.hasUserPromptSupplied,
isCodeBlockSelected: session.isCodeBlockSelected,
buildPayloadBytes: session.srcPayloadSize,
buildZipFileBytes: session.srcZipFileSize,
artifactsUploadDuration: session.artifactsUploadDuration,
perfClientLatency: performance.now() - session.testGenerationStartTime,
result: isCancel ? 'Cancelled' : 'Failed',
reasonDesc: getTelemetryReasonDesc(data.error),
isSupportedLanguage: true,
credentialStartUrl: AuthUtil.instance.startUrl,
})

TelemetryHelper.instance.sendTestGenerationToolkitEvent(
session,
true,
isCancel ? 'Cancelled' : 'Failed',
session.startTestGenerationRequestId,
performance.now() - session.testGenerationStartTime,
getTelemetryReasonDesc(data.error),
session.isCodeBlockSelected,
session.artifactsUploadDuration,
session.srcPayloadSize,
session.srcZipFileSize
)

if (session.stopIteration) {
// Error from Science
this.messenger.sendMessage(data.error.message.replaceAll('```', ''), data.tabID, 'answer')
Expand Down Expand Up @@ -716,27 +714,25 @@ export class TestController {
// TODO: send the message once again once build is enabled
// this.messenger.sendMessage('Accepted', message.tabID, 'prompt')
telemetry.ui_click.emit({ elementId: 'unitTestGeneration_acceptDiff' })
telemetry.amazonq_utgGenerateTests.emit({
generatedCount: session.numberOfTestsGenerated,
acceptedCount: session.numberOfTestsGenerated,
generatedCharactersCount: session.charsOfCodeGenerated,
acceptedCharactersCount: session.charsOfCodeAccepted,
generatedLinesCount: session.linesOfCodeGenerated,
acceptedLinesCount: session.linesOfCodeAccepted,
cwsprChatProgrammingLanguage: session.fileLanguage ?? 'plaintext',
jobId: session.listOfTestGenerationJobId[0], // For RIV, UTG does only one StartTestGeneration API call so jobId = session.listOfTestGenerationJobId[0]
jobGroup: session.testGenerationJobGroupName,
requestId: session.startTestGenerationRequestId,
buildPayloadBytes: session.srcPayloadSize,
buildZipFileBytes: session.srcZipFileSize,
artifactsUploadDuration: session.artifactsUploadDuration,
hasUserPromptSupplied: session.hasUserPromptSupplied,
isCodeBlockSelected: session.isCodeBlockSelected,
perfClientLatency: session.latencyOfTestGeneration,
isSupportedLanguage: true,
credentialStartUrl: AuthUtil.instance.startUrl,
result: 'Succeeded',
})

TelemetryHelper.instance.sendTestGenerationToolkitEvent(
session,
true,
'Succeeded',
session.startTestGenerationRequestId,
session.latencyOfTestGeneration,
undefined,
session.isCodeBlockSelected,
session.artifactsUploadDuration,
session.srcPayloadSize,
session.srcZipFileSize,
session.charsOfCodeAccepted,
session.numberOfTestsGenerated,
session.linesOfCodeAccepted,
session.charsOfCodeGenerated,
session.numberOfTestsGenerated,
session.linesOfCodeGenerated
)

await this.endSession(message, FollowUpTypes.SkipBuildAndFinish)
await this.sessionCleanUp()
Expand Down Expand Up @@ -840,27 +836,25 @@ export class TestController {
private async endSession(data: any, step: FollowUpTypes) {
const session = this.sessionStorage.getSession()
if (step === FollowUpTypes.RejectCode) {
telemetry.amazonq_utgGenerateTests.emit({
generatedCount: session.numberOfTestsGenerated,
acceptedCount: 0,
generatedCharactersCount: session.charsOfCodeGenerated,
acceptedCharactersCount: 0,
generatedLinesCount: session.linesOfCodeGenerated,
acceptedLinesCount: 0,
cwsprChatProgrammingLanguage: session.fileLanguage ?? 'plaintext',
jobId: session.listOfTestGenerationJobId[0], // For RIV, UTG does only one StartTestGeneration API call so jobId = session.listOfTestGenerationJobId[0]
jobGroup: session.testGenerationJobGroupName,
requestId: session.startTestGenerationRequestId,
buildPayloadBytes: session.srcPayloadSize,
buildZipFileBytes: session.srcZipFileSize,
artifactsUploadDuration: session.artifactsUploadDuration,
hasUserPromptSupplied: session.hasUserPromptSupplied,
isCodeBlockSelected: session.isCodeBlockSelected,
perfClientLatency: session.latencyOfTestGeneration,
isSupportedLanguage: true,
credentialStartUrl: AuthUtil.instance.startUrl,
result: 'Succeeded',
})
TelemetryHelper.instance.sendTestGenerationToolkitEvent(
session,
true,
'Succeeded',
session.startTestGenerationRequestId,
session.latencyOfTestGeneration,
undefined,
session.isCodeBlockSelected,
session.artifactsUploadDuration,
session.srcPayloadSize,
session.srcZipFileSize,
0,
0,
0,
session.charsOfCodeGenerated,
session.numberOfTestsGenerated,
session.linesOfCodeGenerated
)

telemetry.ui_click.emit({ elementId: 'unitTestGeneration_rejectDiff' })
}

Expand Down
38 changes: 17 additions & 21 deletions packages/core/src/amazonqTest/chat/controller/messenger/messenger.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,8 @@ import { CodeReference } from '../../../../amazonq/webview/ui/apps/amazonqCommon
import { getHttpStatusCode, getRequestId, getTelemetryReasonDesc, ToolkitError } from '../../../../shared/errors'
import { sleep, waitUntil } from '../../../../shared/utilities/timeoutUtils'
import { keys } from '../../../../shared/utilities/tsUtils'
import { AuthUtil, testGenState } from '../../../../codewhisperer'
import { TelemetryHelper, testGenState } from '../../../../codewhisperer'
import { cancellingProgressField, testGenCompletedField } from '../../../models/constants'
import { telemetry } from '../../../../shared/telemetry/telemetry'

export type UnrecoverableErrorType = 'no-project-found' | 'no-open-file-found' | 'invalid-file-type'

Expand Down Expand Up @@ -275,31 +274,28 @@ export class Messenger {
.finally(async () => {
if (testGenState.isCancelling()) {
this.sendMessage(CodeWhispererConstants.unitTestGenerationCancelMessage, tabID, 'answer')
telemetry.amazonq_utgGenerateTests.emit({
cwsprChatProgrammingLanguage: session.fileLanguage ?? 'plaintext',
hasUserPromptSupplied: session.hasUserPromptSupplied,
perfClientLatency: performance.now() - session.testGenerationStartTime,
result: 'Cancelled',
reasonDesc: getTelemetryReasonDesc(CodeWhispererConstants.unitTestGenerationCancelMessage),
isSupportedLanguage: false,
credentialStartUrl: AuthUtil.instance.startUrl,
requestId: messageId,
})
TelemetryHelper.instance.sendTestGenerationToolkitEvent(
session,
false,
'Cancelled',
messageId,
performance.now() - session.testGenerationStartTime,
getTelemetryReasonDesc(CodeWhispererConstants.unitTestGenerationCancelMessage)
)

this.dispatcher.sendUpdatePromptProgress(
new UpdatePromptProgressMessage(tabID, cancellingProgressField)
)
await sleep(500)
} else {
telemetry.amazonq_utgGenerateTests.emit({
cwsprChatProgrammingLanguage: session.fileLanguage ?? 'plaintext',
hasUserPromptSupplied: session.hasUserPromptSupplied,
perfClientLatency: performance.now() - session.testGenerationStartTime,
result: 'Succeeded',
isSupportedLanguage: false,
credentialStartUrl: AuthUtil.instance.startUrl,
requestId: messageId,
})
TelemetryHelper.instance.sendTestGenerationToolkitEvent(
session,
false,
'Succeeded',
messageId,
performance.now() - session.testGenerationStartTime
)

this.dispatcher.sendUpdatePromptProgress(
new UpdatePromptProgressMessage(tabID, testGenCompletedField)
)
Expand Down
45 changes: 45 additions & 0 deletions packages/core/src/codewhisperer/util/telemetryHelper.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import { CodeWhispererSupplementalContext } from '../models/model'
import { FeatureConfigProvider } from '../../shared/featureConfig'
import { CodeScanRemediationsEventType } from '../client/codewhispereruserclient'
import { CodeAnalysisScope as CodeAnalysisScopeClientSide } from '../models/constants'
import { Session } from '../../amazonqTest/chat/session/session'

export class TelemetryHelper {
// Some variables for client component latency
Expand Down Expand Up @@ -57,6 +58,50 @@ export class TelemetryHelper {
return (this.#instance ??= new this())
}

public sendTestGenerationToolkitEvent(
session: Session,
isSupportedLanguage: boolean,
result: 'Succeeded' | 'Failed' | 'Cancelled',
requestId?: string,
perfClientLatency?: number,
reasonDesc?: string,
isCodeBlockSelected?: boolean,
artifactsUploadDuration?: number,
buildPayloadBytes?: number,
buildZipFileBytes?: number,
acceptedCharactersCount?: number,
acceptedCount?: number,
acceptedLinesCount?: number,
generatedCharactersCount?: number,
generatedCount?: number,
generatedLinesCount?: number,
reason?: string
) {
telemetry.amazonq_utgGenerateTests.emit({
cwsprChatProgrammingLanguage: session.fileLanguage ?? 'plaintext',
hasUserPromptSupplied: session.hasUserPromptSupplied,
isSupportedLanguage: isSupportedLanguage,
result: result,
artifactsUploadDuration: artifactsUploadDuration,
buildPayloadBytes: buildPayloadBytes,
buildZipFileBytes: buildZipFileBytes,
credentialStartUrl: AuthUtil.instance.startUrl,
acceptedCharactersCount: acceptedCharactersCount,
acceptedCount: acceptedCount,
acceptedLinesCount: acceptedLinesCount,
generatedCharactersCount: generatedCharactersCount,
generatedCount: generatedCount,
generatedLinesCount: generatedLinesCount,
isCodeBlockSelected: isCodeBlockSelected,
jobGroup: session.testGenerationJobGroupName,
jobId: session.listOfTestGenerationJobId[0],
perfClientLatency: perfClientLatency,
requestId: requestId,
reasonDesc: reasonDesc,
reason: reason,
})
}

public recordServiceInvocationTelemetry(
requestId: string,
sessionId: string,
Expand Down
Loading

0 comments on commit e63079d

Please sign in to comment.