Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

move to logging; check for safety exceptions #269

Merged
merged 1 commit into from
Jun 11, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions pkgs/sdk_triage_bot/bin/triage.dart
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ void main(List<String> arguments) async {
force: force,
githubService: githubService,
geminiService: geminiService,
logger: Logger(),
);

client.close();
Expand Down
6 changes: 6 additions & 0 deletions pkgs/sdk_triage_bot/lib/src/common.dart
Original file line number Diff line number Diff line change
Expand Up @@ -41,3 +41,9 @@ String get geminiKey {
String trimmedBody(String body) {
return body.length > 4096 ? body = body.substring(0, 4096) : body;
}

class Logger {
void log(String message) {
print(message);
}
}
6 changes: 6 additions & 0 deletions pkgs/sdk_triage_bot/lib/src/gemini.dart
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,16 @@ class GeminiService {
httpClient: httpClient,
);

/// Call the summarize model with the given prompt.
///
/// On failures, this will throw a `GenerativeAIException`.
Future<String> summarize(String prompt) {
return _query(_summarizeModel, prompt);
}

/// Call the classify model with the given prompt.
///
/// On failures, this will throw a `GenerativeAIException`.
Future<List<String>> classify(String prompt) async {
final result = await _query(_classifyModel, prompt);
final labels = result.split(',').map((l) => l.trim()).toList();
Expand Down
94 changes: 58 additions & 36 deletions pkgs/sdk_triage_bot/lib/triage.dart
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,10 @@
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'dart:io';

import 'package:github/github.dart';
import 'package:google_generative_ai/google_generative_ai.dart';

import 'src/common.dart';
import 'src/gemini.dart';
Expand All @@ -17,66 +20,85 @@ Future<void> triage(
bool force = false,
required GithubService githubService,
required GeminiService geminiService,
required Logger logger,
}) async {
print('Triaging $sdkSlug...');
print('');
logger.log('Triaging $sdkSlug...');
logger.log('');

// retrieve the issue
final issue = await githubService.fetchIssue(sdkSlug, issueNumber);
print('## issue ${issue.url}');
print('');
print('title: ${issue.title}');
logger.log('## issue "${issue.htmlUrl}"');
logger.log('');
final labels = issue.labels.map((l) => l.name).toList();
if (labels.isNotEmpty) {
print('labels: ${labels.join(', ')}');
logger.log('labels: ${labels.join(', ')}');
logger.log('');
}
logger.log('"${issue.title}"');
logger.log('');
final bodyLines =
issue.body.split('\n').where((l) => l.trim().isNotEmpty).toList();
print('');
for (final line in bodyLines.take(4)) {
print(' $line');
logger.log(line);
}
if (bodyLines.length > 4) {
logger.log('...');
}
print('');
logger.log('');

// decide if we should triage
final alreadyTriaged = labels.any((l) => l.startsWith('area-'));
if (alreadyTriaged && !force) {
print('Exiting (issue is already triaged).');
logger.log('Exiting (issue is already triaged).');
return;
}

// ask for the summary
var bodyTrimmed = trimmedBody(issue.body);
// TODO(devoncarew): handle safety failures
final summary = await geminiService.summarize(
summarizeIssuePrompt(title: issue.title, body: bodyTrimmed),
);
print('## gemini summary');
print('');
print(summary);
print('');
String summary;
try {
// Failures here can include things like gemini safety issues, ...
summary = await geminiService.summarize(
summarizeIssuePrompt(title: issue.title, body: bodyTrimmed),
);
} on GenerativeAIException catch (e) {
stderr.writeln('gemini: $e');
exit(1);
}

logger.log('## gemini summary');
logger.log('');
logger.log(summary);
logger.log('');

// ask for the 'area-' classification
// TODO(devoncarew): handle safety failures
final classification = await geminiService.classify(
assignAreaPrompt(title: issue.title, body: bodyTrimmed),
);
print('## gemini classification');
print('');
print(classification);
print('');
List<String> classification;
try {
// Failures here can include things like gemini safety issues, ...
classification = await geminiService.classify(
assignAreaPrompt(title: issue.title, body: bodyTrimmed),
);
} on GenerativeAIException catch (e) {
stderr.writeln('gemini: $e');
exit(1);
}

logger.log('## gemini classification');
logger.log('');
logger.log(classification.toString());
logger.log('');

if (dryRun) {
print('Exiting (dry run mode - not applying changes).');
logger.log('Exiting (dry run mode - not applying changes).');
return;
}

// perform changes
print('## github comment');
print('');
print(summary);
print('');
print('labels: $classification');
logger.log('## github comment');
logger.log('');
logger.log(summary);
logger.log('');
logger.log('labels: $classification');

var comment = '';
if (classification.isNotEmpty) {
Expand All @@ -101,10 +123,10 @@ Future<void> triage(
await githubService.addLabelsToIssue(sdkSlug, issueNumber, newLabels);
}

print('');
print('---');
print('');
print('Triaged ${issue.url}.');
logger.log('');
logger.log('---');
logger.log('');
logger.log('Triaged ${issue.htmlUrl}.');
}

List<String> filterExistingLabels(
Expand Down
9 changes: 9 additions & 0 deletions pkgs/sdk_triage_bot/test/fakes.dart
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,10 @@
// BSD-style license that can be found in the LICENSE file.

import 'package:github/github.dart';
import 'package:sdk_triage_bot/src/common.dart';
import 'package:sdk_triage_bot/src/gemini.dart';
import 'package:sdk_triage_bot/src/github.dart';
import 'package:test/test.dart';

const int mockIssueNumber = 123;

Expand Down Expand Up @@ -55,3 +57,10 @@ class GeminiServiceStub implements GeminiService {
return ['area-vm', 'type-bug'];
}
}

class TestLogger implements Logger {
@override
void log(String message) {
printOnFailure(message);
}
}
5 changes: 4 additions & 1 deletion pkgs/sdk_triage_bot/test/triage_test.dart
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ void main() {
mockIssueNumber,
githubService: githubService,
geminiService: geminiService,
logger: TestLogger(),
);

expect(githubService.updatedComment, isNotEmpty);
Expand All @@ -41,6 +42,7 @@ void main() {
mockIssueNumber,
githubService: githubService,
geminiService: geminiService,
logger: TestLogger(),
);

expect(githubService.updatedComment, isNull);
Expand All @@ -61,9 +63,10 @@ void main() {

await triage(
mockIssueNumber,
force: true,
githubService: githubService,
geminiService: geminiService,
force: true,
logger: TestLogger(),
);

expect(githubService.updatedComment, isNotEmpty);
Expand Down