Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Development: Improve execution speed of Playwright tests #9817

Merged
merged 13 commits into from
Nov 21, 2024
Merged

Development: Improve execution speed of Playwright tests #9817

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
75b8656
Tag more tests as @fast to speed up their execution
muradium Nov 16, 2024
01cfb9d
Define custom timeouts for slow parallel tests
muradium Nov 16, 2024
56a943c
Define some of the slow tests as part of @slow tests project again
muradium Nov 16, 2024
a4b31b0
Revert "Define some of the slow tests as part of @slow tests project …
muradium Nov 16, 2024
c58dbeb
Increase timeouts for some of the slow-running tests
muradium Nov 16, 2024
c2d79e9
Add custom timeout to test exam student exams
muradium Nov 16, 2024
7521755
Extended timeout for exam results tests
muradium Nov 17, 2024
75d6d34
Decrease number of @slow tests
muradium Nov 17, 2024
462d0c0
Switch to @slow tests-only to measure their performance
muradium Nov 18, 2024
92f0d1f
Revert "Switch to @slow tests-only to measure their performance"
muradium Nov 18, 2024
460b074
Try increasing timeouts of slow tests that run in parallel
muradium Nov 18, 2024
18beef5
Run slow parallel tests as a separate project to assign higher timeou…
muradium Nov 18, 2024
b5caaf4
Merge branch 'develop' into feature/playwright/improve-test-execution…
muradium Nov 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 8 additions & 7 deletions src/test/playwright/e2e/exam/ExamAssessment.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,12 +53,13 @@ test.beforeAll('Create course', async ({ browser }) => {
studentOneName = (await users.getUserInfo(studentOne.username, page)).name!;
});

test.describe('Exam assessment', { tag: '@slow' }, () => {
test.describe('Exam assessment', () => {
test.describe.configure({ mode: 'serial' });
let programmingAssessmentSuccessful = false;
let modelingAssessmentSuccessful = false;
let textAssessmentSuccessful = false;

test.describe.serial('Programming exercise assessment', () => {
test.describe.serial('Programming exercise assessment', { tag: '@sequential' }, () => {
test.beforeAll('Prepare exam', async ({ browser }) => {
examEnd = dayjs().add(2, 'minutes');
const page = await newBrowserPage(browser);
Expand All @@ -85,7 +86,7 @@ test.describe('Exam assessment', { tag: '@slow' }, () => {
});
});

test.describe.serial('Modeling exercise assessment', () => {
test.describe.serial('Modeling exercise assessment', { tag: '@slow' }, () => {
muradium marked this conversation as resolved.
Show resolved Hide resolved
test.beforeAll('Prepare exam', async ({ browser }) => {
examEnd = dayjs().add(45, 'seconds');
const page = await newBrowserPage(browser);
Expand Down Expand Up @@ -126,7 +127,7 @@ test.describe('Exam assessment', { tag: '@slow' }, () => {
});
});

test.describe.serial('Text exercise assessment', () => {
test.describe.serial('Text exercise assessment', { tag: '@slow' }, () => {
muradium marked this conversation as resolved.
Show resolved Hide resolved
test.beforeAll('Prepare exam', async ({ browser }) => {
examEnd = dayjs().add(40, 'seconds');
const page = await newBrowserPage(browser);
Expand All @@ -153,7 +154,7 @@ test.describe('Exam assessment', { tag: '@slow' }, () => {
});
});

test.describe('Quiz exercise assessment', () => {
test.describe('Quiz exercise assessment', { tag: '@slow' }, () => {
let resultDate: Dayjs;

test.beforeAll('Prepare exam', async ({ browser }) => {
Expand Down Expand Up @@ -183,7 +184,7 @@ test.describe('Exam assessment', { tag: '@slow' }, () => {
});
});

test.describe('Exam grading', () => {
test.describe('Exam grading', { tag: '@fast' }, () => {
test.describe.serial('Instructor sets grades and student receives a grade', () => {
let exam: Exam;

Expand Down Expand Up @@ -222,7 +223,7 @@ test.describe('Exam grading', () => {
});
});

test.describe('Exam statistics', { tag: '@slow' }, () => {
test.describe('Exam statistics', { tag: '@sequential' }, () => {
let exercise: Exercise;
const students = [studentOne, studentTwo, studentThree, studentFour];

Expand Down
11 changes: 5 additions & 6 deletions src/test/playwright/e2e/exam/ExamParticipation.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import { ExamParticipationActions } from '../../support/pageobjects/exam/ExamPar
const textFixture = 'loremIpsum.txt';
const textFixtureShort = 'loremIpsum-short.txt';

test.describe('Exam participation', { tag: '@slow' }, () => {
test.describe('Exam participation', () => {
let course: Course;
let exerciseArray: Array<Exercise> = [];
let studentTwoName: string;
Expand All @@ -43,7 +43,7 @@ test.describe('Exam participation', { tag: '@slow' }, () => {
studentFourName = studentFourInfo.name!;
});

test.describe('Early Hand-in', () => {
test.describe('Early Hand-in', { tag: '@slow' }, () => {
let exam: Exam;
const examTitle = 'exam' + generateUUID();

Expand Down Expand Up @@ -107,7 +107,6 @@ test.describe('Exam participation', { tag: '@slow' }, () => {
await examParticipation.startParticipation(studentFour, course, exam);
for (let j = 0; j < exerciseArray.length; j++) {
const exercise = exerciseArray[j];
// await
await examNavigation.openFromOverviewByTitle(exercise.exerciseGroup!.title!);
await examNavigation.openOverview();
}
Expand All @@ -118,7 +117,7 @@ test.describe('Exam participation', { tag: '@slow' }, () => {
});
});

test.describe('Early hand-in with continue and reload page', () => {
test.describe('Early hand-in with continue and reload page', { tag: '@slow' }, () => {
let exam: Exam;
const examTitle = 'exam' + generateUUID();

Expand Down Expand Up @@ -222,7 +221,7 @@ test.describe('Exam participation', { tag: '@slow' }, () => {
});
});

test.describe('Normal Hand-in', () => {
test.describe('Normal Hand-in', { tag: '@sequential' }, () => {
let exam: Exam;
const examTitle = 'exam' + generateUUID();

Expand Down Expand Up @@ -259,7 +258,7 @@ test.describe('Exam participation', { tag: '@slow' }, () => {
});
});

test.describe('Exam announcements', () => {
test.describe('Exam announcements', { tag: '@slow' }, () => {
let exam: Exam;
const students = [studentOne, studentTwo];

Expand Down
80 changes: 42 additions & 38 deletions src/test/playwright/e2e/exam/ExamResults.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import { ProgrammingExerciseTaskStatus } from '../../support/pageobjects/exam/Ex
import { Page } from '@playwright/test';
import { StudentExam } from 'app/entities/student-exam.model';

test.describe('Exam Results', { tag: '@slow' }, () => {
test.describe('Exam Results', () => {
let course: Course;

test.beforeEach('Create course', async ({ browser }) => {
Expand Down Expand Up @@ -126,44 +126,48 @@ test.describe('Exam Results', { tag: '@slow' }, () => {
},
);

test(`Check exam ${exerciseTypeString} exercise results`, async ({ page, login, examParticipation, examResultsPage }) => {
await login(studentOne);
await waitForExamEnd(examEndDate, page);
await page.goto(`/courses/${course.id}/exams/${exam.id}`);
await examParticipation.checkResultScore(testCase.resultScore, exercise.id!);
test(
`Check exam ${exerciseTypeString} exercise results`,
{ tag: testCase.exerciseType === ExerciseType.PROGRAMMING ? '@sequential' : '@slow' },
async ({ page, login, examParticipation, examResultsPage }) => {
await login(studentOne);
await waitForExamEnd(examEndDate, page);
await page.goto(`/courses/${course.id}/exams/${exam.id}`);
await examParticipation.checkResultScore(testCase.resultScore, exercise.id!);

switch (testCase.exerciseType) {
case ExerciseType.TEXT:
await examResultsPage.checkTextExerciseContent(exercise.id!, exercise.additionalData!.textFixture!);
await examResultsPage.checkAdditionalFeedback(exercise.id!, 7, 'Good job');
break;
case ExerciseType.PROGRAMMING:
await examResultsPage.checkProgrammingExerciseAssessments(exercise.id!, 'Wrong', 7);
await examResultsPage.checkProgrammingExerciseAssessments(exercise.id!, 'Correct', 6);
const taskStatuses: ProgrammingExerciseTaskStatus[] = [
ProgrammingExerciseTaskStatus.SUCCESS,
ProgrammingExerciseTaskStatus.SUCCESS,
ProgrammingExerciseTaskStatus.SUCCESS,
ProgrammingExerciseTaskStatus.FAILURE,
ProgrammingExerciseTaskStatus.FAILURE,
ProgrammingExerciseTaskStatus.FAILURE,
ProgrammingExerciseTaskStatus.FAILURE,
];
await examResultsPage.checkProgrammingExerciseTasks(exercise.id!, taskStatuses);
break;
case ExerciseType.QUIZ:
await examResultsPage.checkQuizExerciseScore(exercise.id!, 5, 10);
const studentAnswers = [true, false, true, false];
const correctAnswers = [true, true, false, false];
await examResultsPage.checkQuizExerciseAnswers(exercise.id!, studentAnswers, correctAnswers);
break;
case ExerciseType.MODELING:
await examResultsPage.checkAdditionalFeedback(exercise.id!, 5, 'Good');
await examResultsPage.checkModellingExerciseAssessment(exercise.id!, 'class Class', 'Wrong', -1);
await examResultsPage.checkModellingExerciseAssessment(exercise.id!, 'abstract class Abstract', 'Neutral', 0);
break;
}
});
switch (testCase.exerciseType) {
case ExerciseType.TEXT:
await examResultsPage.checkTextExerciseContent(exercise.id!, exercise.additionalData!.textFixture!);
await examResultsPage.checkAdditionalFeedback(exercise.id!, 7, 'Good job');
break;
case ExerciseType.PROGRAMMING:
await examResultsPage.checkProgrammingExerciseAssessments(exercise.id!, 'Wrong', 7);
await examResultsPage.checkProgrammingExerciseAssessments(exercise.id!, 'Correct', 6);
const taskStatuses: ProgrammingExerciseTaskStatus[] = [
ProgrammingExerciseTaskStatus.SUCCESS,
ProgrammingExerciseTaskStatus.SUCCESS,
ProgrammingExerciseTaskStatus.SUCCESS,
ProgrammingExerciseTaskStatus.FAILURE,
ProgrammingExerciseTaskStatus.FAILURE,
ProgrammingExerciseTaskStatus.FAILURE,
ProgrammingExerciseTaskStatus.FAILURE,
];
await examResultsPage.checkProgrammingExerciseTasks(exercise.id!, taskStatuses);
break;
case ExerciseType.QUIZ:
await examResultsPage.checkQuizExerciseScore(exercise.id!, 5, 10);
const studentAnswers = [true, false, true, false];
const correctAnswers = [true, true, false, false];
await examResultsPage.checkQuizExerciseAnswers(exercise.id!, studentAnswers, correctAnswers);
break;
case ExerciseType.MODELING:
await examResultsPage.checkAdditionalFeedback(exercise.id!, 5, 'Good');
await examResultsPage.checkModellingExerciseAssessment(exercise.id!, 'class Class', 'Wrong', -1);
await examResultsPage.checkModellingExerciseAssessment(exercise.id!, 'abstract class Abstract', 'Neutral', 0);
break;
}
},
);

if (testCase.exerciseType === ExerciseType.TEXT) {
test('Check exam result overview', async ({ page, login, examAPIRequests, examResultsPage }) => {
Expand Down
2 changes: 1 addition & 1 deletion src/test/playwright/e2e/exam/ExamTestRun.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import { expect } from '@playwright/test';
const textFixture = 'loremIpsum-short.txt';
const examTitle = 'exam' + generateUUID();

test.describe('Exam test run', { tag: '@slow' }, () => {
test.describe('Exam test run', { tag: '@fast' }, () => {
muradium marked this conversation as resolved.
Show resolved Hide resolved
let course: Course;
let exam: Exam;
let exerciseArray: Array<Exercise> = [];
Expand Down
2 changes: 1 addition & 1 deletion src/test/playwright/e2e/exam/test-exam/TestExamCreationDeletion.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ const examData = {
confirmationEndText: 'Exam confirmation end text',
};

test.describe('Test Exam creation/deletion', { tag: '@slow' }, () => {
test.describe('Test Exam creation/deletion', { tag: '@fast' }, () => {
let course: Course;
let exam: Exam;

Expand Down
2 changes: 1 addition & 1 deletion src/test/playwright/e2e/exam/test-exam/TestExamManagement.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import { expect } from '@playwright/test';
const uid = generateUUID();
const examTitle = 'test-exam' + uid;

test.describe('Test Exam management', { tag: '@slow' }, () => {
test.describe('Test Exam management', { tag: '@fast' }, () => {
let course: Course;
let exam: Exam;

Expand Down
Loading
Loading