Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

P95 #1008

Draft
wants to merge 9 commits into
base: main
Choose a base branch
from
Draft

P95 #1008

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
072ba20
wip
k2xl Sep 30, 2023
5b305b8
speed up calcPlayAttempts and also calc p95s
k2xl Sep 30, 2023
661ebee
forgot to commit
k2xl Oct 2, 2023
0424893
Merge branch 'main' into p95
sspenst Oct 3, 2023
dc0a98c
merge
sspenst Oct 3, 2023
f9b7938
Merge branch 'main' into p95
k2xl Oct 3, 2023
8049a19
should group by user first then do the p95
k2xl Oct 3, 2023
5a34345
tests
k2xl Oct 3, 2023
f1e9604
merge
sspenst Oct 4, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions models/db/level.d.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,9 @@ interface Level {
authorNote?: string;
calc_difficulty_estimate: number;
calc_playattempts_duration_sum: number;
calc_playattempts_duration_sum_p95: number;
calc_playattempts_just_beaten_count: number;
calc_playattempts_just_beaten_count_p95: number;
calc_playattempts_unique_users: Types.ObjectId[];
calc_reviews_count: number;
calc_reviews_score_avg: number;
Expand Down
190 changes: 133 additions & 57 deletions models/schemas/levelSchema.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,18 @@ const LevelSchema = new mongoose.Schema<Level>(
type: Number,
default: 0,
},
calc_playattempts_duration_sum_p95: {
type: Number,
default: 0,
},
calc_playattempts_just_beaten_count: {
type: Number,
default: 0,
},
calc_playattempts_just_beaten_count_p95: {
type: Number,
default: 0,
},
calc_playattempts_unique_users: {
type: [mongoose.Schema.Types.ObjectId],
default: [],
Expand Down Expand Up @@ -125,6 +133,8 @@ LevelSchema.index({ isDraft: 1 });
LevelSchema.index({ leastMoves: 1 });
LevelSchema.index({ calc_difficulty_estimate: 1 });
LevelSchema.index({ calc_playattempts_duration_sum: 1 });
LevelSchema.index({ calc_playattempts_duration_sum_p95: 1 });
LevelSchema.index({ calc_playattempts_just_beaten_count_p95: 1 });
LevelSchema.index({ calc_playattempts_just_beaten_count: 1 });
LevelSchema.index({ calc_playattempts_unique_users: 1 });
LevelSchema.index({ calc_reviews_count: 1 });
Expand Down Expand Up @@ -199,78 +209,144 @@ async function calcStats(lvl: Level) {

// eslint-disable-next-line @typescript-eslint/no-explicit-any
export async function calcPlayAttempts(levelId: Types.ObjectId, options: any = {}) {
const countJustBeaten = await PlayAttemptModel.countDocuments({
levelId: levelId,
attemptContext: AttemptContext.JUST_BEATEN,
});

// sumDuration is all of the sum(endTime-startTime) within the playAttempts
const sumDuration = await PlayAttemptModel.aggregate([
{
$match: {
levelId: levelId,
attemptContext: { $ne: AttemptContext.BEATEN },
}
},
{
$group: {
_id: null,
sumDuration: {
$sum: {
$subtract: ['$endTime', '$startTime']
const [countJustBeaten, sumDuration, sumDurationP95, uniqueUsersList] = await Promise.all([
PlayAttemptModel.countDocuments({
levelId: levelId,
attemptContext: AttemptContext.JUST_BEATEN,
}),
PlayAttemptModel.aggregate([
{
$match: {
levelId: levelId,
attemptContext: { $ne: AttemptContext.BEATEN },
}
},
{
$group: {
_id: null,
sumDuration: {
$sum: {
$subtract: ['$endTime', '$startTime']
}
}
}
}
}
], options);

// get array of unique userIds from playattempt calc_playattempts_unique_users
const uniqueUsersList = await PlayAttemptModel.aggregate([
{
$match: {
$or: [
{
$and: [
], options),
PlayAttemptModel.aggregate([
{
$match: {
levelId: levelId,
attemptContext: { $ne: AttemptContext.BEATEN },
// where endTime and startTime are not equal
$expr: {
$gt: [
{
$expr: {
$gt: [
{
$subtract: ['$endTime', '$startTime']
},
0
]
$subtract: ['$endTime', '$startTime']
},
0
]
}
}
},
{
$project: {
userId: 1,
duration: { $subtract: ['$endTime', '$startTime'] }
}
},
{
$group: {
_id: '$userId',
userTotalDuration: { $sum: '$duration' }
}
},
{
$sort: { userTotalDuration: 1 }
},
{
$group: {
_id: null,
userDurations: { $push: '$userTotalDuration' },
count: { $sum: 1 }
}
},
{
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we combine sumDuration and sumDurationP95? Should have all the data for sumDuration at this point

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yeah good idea

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

actually i realized when trying to do this that this is risky and not actually as straightforward since i do a bunch of groups. probably safer for now to keep it separate

$project: {
lowerIndex: { $ceil: { $multiply: ['$count', 0.025] } },
upperIndex: { $floor: { $multiply: ['$count', 0.975] } },
userDurations: 1,
count: 1
}
},
{
$project: {
countExcluded: { $subtract: ['$count', { $add: ['$lowerIndex', { $subtract: ['$count', '$upperIndex'] }] }] },
sumExcluded: {
$cond: {
if: { $gt: [{ $subtract: ['$upperIndex', '$lowerIndex'] }, 0] },
then: {
$reduce: {
input: { $slice: ['$userDurations', '$lowerIndex', { $subtract: ['$upperIndex', '$lowerIndex'] }] },
initialValue: 0,
in: { $add: ['$$value', '$$this'] }
}
},
{
attemptContext: AttemptContext.UNBEATEN,
}
],
},
{
attemptContext: AttemptContext.JUST_BEATEN,
else: 0
}
}
}
}
]),
PlayAttemptModel.aggregate([
{
$match: {
$or: [
{
$and: [
{
$expr: {
$gt: [
{
$subtract: ['$endTime', '$startTime']
},
0
]
}
},
{
attemptContext: AttemptContext.UNBEATEN,
}
],
},
{
attemptContext: AttemptContext.JUST_BEATEN,
},
],
levelId: levelId,
},
},
{
$group: {
_id: null,
userId: {
$addToSet: '$userId',
},
],
levelId: levelId,
}
},
},
{
$group: {
_id: null,
userId: {
$addToSet: '$userId',
{
$unwind: {
path: '$userId',
preserveNullAndEmptyArrays: true,
},
}
},
{
$unwind: {
path: '$userId',
preserveNullAndEmptyArrays: true,
},
},
])
]);

const update = {
calc_playattempts_duration_sum: sumDuration[0]?.sumDuration ?? 0,
calc_playattempts_duration_sum_p95: sumDurationP95[0]?.sumExcluded ?? 0,
calc_playattempts_just_beaten_count_p95: sumDurationP95[0]?.countExcluded ?? 0,
k2xl marked this conversation as resolved.
Show resolved Hide resolved
calc_playattempts_just_beaten_count: countJustBeaten,
calc_playattempts_unique_users: uniqueUsersList.map(u => u?.userId.toString()),
} as Partial<Level>;
Expand Down
52 changes: 51 additions & 1 deletion tests/pages/api/play-attempt/play-attempt.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -816,9 +816,10 @@ describe('Testing stats api', () => {
});
test('calcDifficultyEstimate', async () => {
const level = await initLevel(TestId.USER, 'calcDifficultyEstimate', {}, false);
const toInsert = [];

for (let i = 0; i < 9; i++) {
await PlayAttemptModel.create({
toInsert.push({
_id: new Types.ObjectId(),
// half beaten
attemptContext: i % 2 === 0 ? AttemptContext.JUST_BEATEN : AttemptContext.UNBEATEN,
Expand All @@ -830,6 +831,8 @@ describe('Testing stats api', () => {
});
}

await PlayAttemptModel.insertMany(toInsert);

await queueCalcPlayAttempts(level._id);
await processQueueMessages();

Expand All @@ -844,6 +847,7 @@ describe('Testing stats api', () => {
const unbeatenUserId = new Types.ObjectId();

// create a playattempt for the 10th unique user

await PlayAttemptModel.create({
_id: new Types.ObjectId(),
attemptContext: AttemptContext.UNBEATEN,
Expand All @@ -864,17 +868,63 @@ describe('Testing stats api', () => {
ts: 0,
});

const unbeatenUserId2 = new Types.ObjectId();

// create a playattempt for the 11th unique user But make it have 0 play time so it shouldn't affect calculations
await PlayAttemptModel.create({
_id: new Types.ObjectId(),
attemptContext: AttemptContext.UNBEATEN,
endTime: 0,
levelId: level._id,
startTime: 0,
updateCount: 0,
userId: unbeatenUserId,
});
await UserModel.create({
_id: unbeatenUserId2,
calc_records: 0,
email: '[email protected]',
last_visited_at: 0,
name: 'unbeaten2',
password: 'unbeaten',
score: 0,
ts: 0,
});

await queueCalcPlayAttempts(level._id);
await processQueueMessages();

const levelUpdated2 = await LevelModel.findById(level._id);

expect(levelUpdated2).toBeDefined();
expect(levelUpdated2?.calc_difficulty_estimate).toBeCloseTo(29.2 * 1.47629);

/**
* User 1 has 10
* User 2 has 11
* User 3 has 12
* User 4 has 13
* User 5 has 14
* User 6 has 15
* User 7 has 16
* User 8 has 17
* User 9 has 18
* User 10 (created above manually): 20
* Sum = 146
*/
expect(levelUpdated2?.calc_playattempts_duration_sum).toBe(146);
expect(levelUpdated2?.calc_playattempts_just_beaten_count).toBe(5);

expect(levelUpdated2?.calc_playattempts_unique_users?.length).toBe(10);

/** Take away top 2.5% and bottom 2.5% for p95
* User 1 and User 10 are the outliers (10 and 20)
* 146 - 10 - 20 = 116
*/

expect(levelUpdated2?.calc_playattempts_just_beaten_count_p95).toBe(8); // removing user 1 and 10
expect(levelUpdated2?.calc_playattempts_duration_sum_p95).toBe(116);

jest.spyOn(TimerUtil, 'getTs').mockReturnValue(30);
await testApiHandler({
handler: async (_, res) => {
Expand Down