From 12fec6de740ee57d85311513cdb5e17f0e782d37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Guid=C3=A9e?= <git@arra.red> Date: Tue, 27 Feb 2024 20:37:18 -0500 Subject: [PATCH] performances: Get top tracks/artists/albums 50 times faster MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Quentin Guidée <git@arra.red> --- apps/server/src/database/queries/stats.ts | 140 +----------------- .../server/src/database/queries/statsTools.ts | 66 +++++++++ 2 files changed, 70 insertions(+), 136 deletions(-) diff --git a/apps/server/src/database/queries/stats.ts b/apps/server/src/database/queries/stats.ts index e531a895..c43dfd07 100644 --- a/apps/server/src/database/queries/stats.ts +++ b/apps/server/src/database/queries/stats.ts @@ -3,6 +3,7 @@ import { InfosModel } from "../Models"; import { User } from "../schemas/user"; import { basicMatch, + getBestInfos, getGroupByDateProjection, getGroupingByTimeSplit, getTrackSumType, @@ -557,50 +558,7 @@ export const getBestSongsNbOffseted = ( end: Date, nb: number, offset: number, -) => - InfosModel.aggregate([ - { $match: basicMatch(user._id, start, end) }, - { - $project: { ...getGroupByDateProjection(user.settings.timezone), id: 1 }, - }, - { - $lookup: lightTrackLookupPipeline(), - }, - { $unwind: "$track" }, - - // Adding the sum of the duration of all musics - { - $group: { - _id: null, - track: { $push: "$track" }, - total_duration_ms: { $sum: "$track.duration_ms" }, - total_count: { $sum: 1 }, - }, - }, - { $unwind: "$track" }, - { - $group: { - _id: "$track.id", - track: { $last: "$track" }, - total_count: { $last: "$total_count" }, - total_duration_ms: { $last: "$total_duration_ms" }, - duration_ms: { $sum: "$track.duration_ms" }, - count: { $sum: 1 }, - }, - }, - { $sort: { count: -1, "track.name": 1 } }, - { $skip: offset }, - { $limit: nb }, - { $addFields: { "track.artist": { $first: "$track.artists" } } }, - { - $lookup: lightAlbumLookupPipeline(), - }, - { $unwind: "$album" }, - { - $lookup: lightArtistLookupPipeline(), - }, - { $unwind: "$artist" }, - ]); +) => getBestInfos("id", user, start, end, nb, offset); export const getBestArtistsNbOffseted = ( user: User, @@ -608,52 +566,7 @@ export const getBestArtistsNbOffseted = ( end: Date, nb: number, offset: number, -) => - InfosModel.aggregate([ - { $match: basicMatch(user._id, start, end) }, - { - $project: { ...getGroupByDateProjection(user.settings.timezone), id: 1 }, - }, - { - $lookup: lightTrackLookupPipeline(), - }, - { $unwind: "$track" }, - - // Adding the sum of the duration of all musics - { - $group: { - _id: null, - track: { $push: "$track" }, - total_duration_ms: { $sum: "$track.duration_ms" }, - total_count: { $sum: 1 }, - }, - }, - { $unwind: "$track" }, - { $addFields: { "track.artist": { $first: "$track.artists" } } }, - { - $group: { - _id: "$track.artist", - track: { $last: "$track" }, - total_count: { $last: "$total_count" }, - total_duration_ms: { $last: "$total_duration_ms" }, - duration_ms: { $sum: "$track.duration_ms" }, - count: { $sum: 1 }, - differents: { $addToSet: "$track.id" }, - }, - }, - { $addFields: { differents: { $size: "$differents" } } }, - { $sort: { count: -1, _id: 1 } }, - { $skip: offset }, - { $limit: nb }, - { - $lookup: lightAlbumLookupPipeline(), - }, - { $unwind: "$album" }, - { - $lookup: lightArtistLookupPipeline(), - }, - { $unwind: "$artist" }, - ]); +) => getBestInfos("primaryArtistId", user, start, end, nb, offset); export const getBestAlbumsNbOffseted = ( user: User, @@ -661,52 +574,7 @@ export const getBestAlbumsNbOffseted = ( end: Date, nb: number, offset: number, -) => - InfosModel.aggregate([ - { $match: basicMatch(user._id, start, end) }, - { - $project: { ...getGroupByDateProjection(user.settings.timezone), id: 1 }, - }, - { - $lookup: lightTrackLookupPipeline(), - }, - { $unwind: "$track" }, - - // Adding the sum of the duration of all musics - { - $group: { - _id: null, - track: { $push: "$track" }, - total_duration_ms: { $sum: "$track.duration_ms" }, - total_count: { $sum: 1 }, - }, - }, - { $unwind: "$track" }, - { $addFields: { "track.artist": { $first: "$track.artists" } } }, - { - $group: { - _id: "$track.album", - track: { $last: "$track" }, - total_count: { $last: "$total_count" }, - total_duration_ms: { $last: "$total_duration_ms" }, - duration_ms: { $sum: "$track.duration_ms" }, - count: { $sum: 1 }, - differents: { $addToSet: "$track.id" }, - }, - }, - { $addFields: { differents: { $size: "$differents" } } }, - { $sort: { count: -1, _id: 1 } }, - { $skip: offset }, - { $limit: nb }, - { - $lookup: lightAlbumLookupPipeline(), - }, - { $unwind: "$album" }, - { - $lookup: lightArtistLookupPipeline(), - }, - { $unwind: "$artist" }, - ]); +) => getBestInfos("albumId", user, start, end, nb, offset); export const getBestSongsOfHour = (user: User, start: Date, end: Date) => { return InfosModel.aggregate([ diff --git a/apps/server/src/database/queries/statsTools.ts b/apps/server/src/database/queries/statsTools.ts index e10ee59a..94a11648 100644 --- a/apps/server/src/database/queries/statsTools.ts +++ b/apps/server/src/database/queries/statsTools.ts @@ -2,6 +2,7 @@ import { PipelineStage, Types } from "mongoose"; import { getWithDefault } from "../../tools/env"; import { Timesplit } from "../../tools/types"; import { User } from "../schemas/user"; +import { InfosModel } from "../Models"; export const basicMatch = ( userId: string | Types.ObjectId, @@ -182,3 +183,68 @@ export const lightArtistLookupPipeline = ( from: "artists", as: "artist", }); + +export const getBestInfos = ( + idField: string, + user: User, + start: Date, + end: Date, + nb: number, + offset: number, +) => + InfosModel.aggregate([ + { $match: basicMatch(user._id, start, end) }, + { + $group: { + _id: `$${idField}`, + duration_ms: { $sum: "$durationMs" }, + count: { $sum: 1 }, + trackId: { $first: "$id" }, + albumId: { $first: "$albumId" }, + primaryArtistId: { $first: "$primaryArtistId" }, + trackIds: { $addToSet: "$id" }, + }, + }, + { $addFields: { differents: { $size: "$trackIds" } } }, + { + $facet: { + infos: [ + { $sort: { count: -1, _id: 1 } }, + { $skip: offset }, + { $limit: nb }, + ], + computations: [ + { + $group: { + _id: null, + total_duration_ms: { $sum: "$duration_ms" }, + total_count: { $sum: "$count" }, + }, + }, + ], + }, + }, + { $unwind: "$infos" }, + { $unwind: "$computations" }, + { + $project: { + _id: "$infos._id", + result: { + $mergeObjects: ["$infos", "$computations"], + }, + }, + }, + { + $replaceRoot: { + newRoot: { + $mergeObjects: ["$result", { _id: "$_id" }], + }, + }, + }, + { $lookup: lightTrackLookupPipeline("trackId") }, + { $unwind: "$track" }, + { $lookup: lightAlbumLookupPipeline("albumId") }, + { $unwind: "$album" }, + { $lookup: lightArtistLookupPipeline("primaryArtistId", false) }, + { $unwind: "$artist" }, + ]);