♻️ Rewrite data storage

Androz2091 · Jun 19, 2021 · d1496b5 · d1496b5
1 parent fe9c214
commit d1496b5
Show file tree

Hide file tree

Showing 6 changed files with 102 additions and 72 deletions.
diff --git a/src/App.svelte b/src/App.svelte
@@ -6,6 +6,7 @@
 	
 	import Stats from './views/Stats.svelte';
 	import Loader from './views/Loader.svelte';
+	import Help from "./views/Help.svelte";
 
 	import Modal from 'svelte-simple-modal';
 
@@ -30,7 +31,7 @@
 			<Header />
 			<div>
 				<Route path="/stats" component={Stats} />
-				<Route path="/help" />
+				<Route path="/help" component={Help} />
 				<Route path="/" component={Loader} />
 				<Route>
 					404

diff --git a/src/app/extractor.js b/src/app/extractor.js
@@ -112,11 +112,14 @@ export const extractData = async (files) => {
 
     const extractedData = {
         user: null,
-        channels: [],
-        guilds: [],
 
         topDMs: [],
+        topChannels: [],
+        guildCount: 0,
+        dmChannelCount: 0,
+        channelCount: 0,
         messageCount: 0,
+        characterCount: 0,
         totalSpent: 0,
         hoursValues: [],
         favoriteWords: null,
@@ -148,13 +151,13 @@ export const extractData = async (files) => {
     // Parse and load current user informations
     console.log('[debug] Loading user info...');
     loadTask.set('Loading user information...');
+
     extractedData.user = JSON.parse(await readFile('account/user.json'));
-    await fetchUser(extractedData.user.id).then((fetchedUser) => {
-        extractedData.user.username = fetchedUser.username;
-        extractedData.user.discriminator = fetchedUser.discriminator;
-        extractedData.user.avatar_hash = fetchedUser.avatar;
+    const fetchedUser = await fetchUser(extractedData.user.id);
+    extractedData.user.username = fetchedUser.username;
+    extractedData.user.discriminator = fetchedUser.discriminator;
+    extractedData.user.avatar_hash = fetchedUser.avatar;
 
-    }).catch(() => {});
     const confirmedPayments = extractedData.user.payments.filter((p) => p.status === 1);
     if (confirmedPayments.length) {
         extractedData.payments.total += confirmedPayments.map((p) => p.amount / 100).reduce((p, c) => p + c);
@@ -175,6 +178,7 @@ export const extractData = async (files) => {
     const isOldPackage = channelsIDsFile[0].name.match(/messages\/(c)?([0-9]{16,32})\/$/)[1] === undefined;
     const channelsIDs = channelsIDsFile.map((file) => file.name.match(messagesPathRegex)[1]);
 
+    const channels = [];
     let messagesRead = 0;
 
     await Promise.all(channelsIDs.map((channelID) => {
@@ -198,7 +202,7 @@ export const extractData = async (files) => {
                 const name = messagesIndex[data.id];
                 const isDM = data.recipients && data.recipients.length === 2;
                 const dmUserID = isDM ? data.recipients.find((userID) => userID !== extractedData.user.id) : undefined;
-                extractedData.channels.push({
+                channels.push({
                     data,
                     messages,
                     name,
@@ -214,18 +218,30 @@ export const extractData = async (files) => {
 
     if (messagesRead === 0) throw new Error('invalid_package_missing_messages');
 
-    console.log(`[debug] ${extractedData.channels.length} channels loaded.`);
+    extractedData.channelCount = channels.filter(c => !c.isDM).length;
+    extractedData.dmChannelCount = channels.length - extractedData.channelCount;
+    extractedData.topChannels = channels.filter(c => c.data && c.data.guild).sort((a, b) => b.messages.length - a.messages.length).slice(0, 10).map((channel) => ({
+        name: channel.name,
+        messageCount: channel.messages.length,
+        guildName: channel.data.guild.name
+    }));
+    extractedData.characterCount = channels.map((channel) => channel.messages).flat().map((message) => message.length).reduce((p, c) => p + c);
+
+    for (let i = 0; i < 24; i++) {
+        extractedData.hoursValues.push(channels.map((c) => c.messages).flat().filter((m) => new Date(m.timestamp).getHours() === i).length);
+    }
+
+    console.log(`[debug] ${channels.length} channels loaded.`);
 
     console.log('[debug] Loading guilds...');
     loadTask.set('Loading joined servers...');
 
     const guildIndex = JSON.parse(await readFile('servers/index.json'));
-    const guilds = Object.entries(guildIndex).map(g => ({ id: g[0], name: g[1] }));
-    extractedData.guilds = guilds;
+    extractedData.guildCount = Object.keys(guildIndex).length;
 
-    console.log(`[debug] ${guilds.length} guilds loaded`);
+    console.log(`[debug] ${extractedData.guildCount} guilds loaded`);
 
-    const words = extractedData.channels.map((channel) => channel.messages).flat().map((message) => message.words).flat().filter((w) => w.length > 5);
+    const words = channels.map((channel) => channel.messages).flat().map((message) => message.words).flat().filter((w) => w.length > 5);
     extractedData.favoriteWords = getFavoriteWords(words);
     for (let wordData of extractedData.favoriteWords) {
         const userID = parseMention(wordData.word);
@@ -241,14 +257,20 @@ export const extractData = async (files) => {
     console.log('[debug] Fetching top DMs...');
     loadTask.set('Loading user activity...');
 
-    extractedData.topDMs = extractedData.channels
+    extractedData.topDMs = channels
         .filter((channel) => channel.isDM)
         .sort((a, b) => b.messages.length - a.messages.length)
-        .slice(0, 10);
+        .slice(0, 10)
+        .map((channel) => ({
+            id: channel.data.id,
+            dmUserID: channel.dmUserID,
+            messageCount: channel.messages.length,
+            userData: null
+        }));
     await Promise.all(extractedData.topDMs.map((channel) => {
         return new Promise((resolve) => {
             fetchUser(channel.dmUserID).then((userData) => {
-                const channelIndex = extractedData.topDMs.findIndex((c) => c.data.id === channel.data.id);
+                const channelIndex = extractedData.topDMs.findIndex((c) => c.id === channel.id);
                 extractedData.topDMs[channelIndex].userData = userData;
                 resolve();
             });
@@ -276,9 +298,7 @@ export const extractData = async (files) => {
 
     loadTask.set('Calculating statistics...');
 
-    for (let i = 0; i < 24; i++) {
-        extractedData.hoursValues.push(extractedData.channels.map((c) => c.messages).flat().filter((m) => new Date(m.timestamp).getHours() === i).length);
-    }
+    console.log(extractedData);
 
     return extractedData;
 };
diff --git a/src/app/store.js b/src/app/store.js
@@ -1,54 +1,41 @@
 
 import { writable } from 'svelte/store';
 
-let loadedValue = false;
-let dataValue = null;
+const storedData = localStorage.getItem('data') || null;
+
+let dataValue = storedData && JSON.parse(storedData);
+
 const isDemo = window.location.href.includes('demo');
 const removeAnalytics = window.location.href.includes('noanalytics');
 if (isDemo) {
     const randomNumber = (min, max) => Math.floor(Math.random() * (max - min + 1) + min);
-    loadedValue = true;
+
     const demoUserObject = {
         id: 422820341791064085,
         username: 'Wumpus',
         discriminator: '0000',
         avatar: null
     };
-    const demoMessageObject = {
-        id: 422820341791064085,
-        timestamp: 1613810737577,
-        length: randomNumber(200, 600)
-    };
-    const analytics = removeAnalytics ? {} : {
-        openCount: randomNumber(200, 300),
-        averageOpenCountPerDay: randomNumber(3, 5),
-        notificationCount: randomNumber(200, 400),
-        joinVoiceChannelCount: randomNumber(40, 100), 
-        joinCallCount: randomNumber(20, 30),
-        addReactionCount: randomNumber(100, 200),
-        messageEditedCount: randomNumber(50, 70),
-        sentMessageCount: randomNumber(200, 600),
-        averageMessageCountPerDay: randomNumber(20, 30),
-        slashCommandUsedCount: randomNumber(10, 20)
-    };
     dataValue = {
+        isDemo: true,
+
         user: demoUserObject,
-        channels: new Array(randomNumber(200, 600)).fill({}).map(() => ({
-            isDM: true,
-            name: 'just-chatting',
-            data: {
-                guild: {
-                    name: 'Cool Discord'
-                }
-            },
-            messages: new Array(randomNumber(200, 600)).fill(demoMessageObject)
-        })),
-        guilds: new Array(randomNumber(10, 100)),
-        applications: [],
+
         topDMs: new Array(10).fill({}).map(() => ({
-            userData: demoUserObject,
-            messages: new Array(randomNumber(200, 10000)).fill(demoMessageObject)
-        })).sort((a, b) => b.messages.length - a.messages.length),
+            messageCount: randomNumber(200, 600),
+            userData: demoUserObject
+        })),
+        topChannels: new Array(10).fill({}).map(() => ({
+            messageCount: randomNumber(200, 600),
+            name: 'awesome',
+            guildName: 'AndrozDev'
+        })),
+        guildCount: randomNumber(10, 200),
+        dmChannelCount: randomNumber(30, 50),
+        channelCount: randomNumber(50, 100),
+        messageCount: randomNumber(300, 600),
+        characterCount: randomNumber(4000, 10000),
+        totalSpent: randomNumber(100, 200),
         hoursValues: new Array(24).fill(0).map(() => Math.floor(Math.random() * 300) + 1),
         favoriteWords: [
             {
@@ -61,14 +48,30 @@ if (isDemo) {
             }
         ],
         payments: {
-            total: 500,
-            list: 'Super Mega Nitro ($500)'
+            total: 0,
+            list: ''
         },
-        ...analytics
+
+        ...(!removeAnalytics && {
+            openCount: randomNumber(200, 300),
+            averageOpenCountPerDay: randomNumber(3, 5),
+            notificationCount: randomNumber(200, 400),
+            joinVoiceChannelCount: randomNumber(40, 100), 
+            joinCallCount: randomNumber(20, 30),
+            addReactionCount: randomNumber(100, 200),
+            messageEditedCount: randomNumber(50, 70),
+            sentMessageCount: randomNumber(200, 600),
+            averageMessageCountPerDay: randomNumber(20, 30),
+            slashCommandUsedCount: randomNumber(10, 20)
+        })
     };
 }
 
-export const loaded = writable(loadedValue);
 export const loadTask = writable(null);
 export const loadEstimatedTime = writable(null);
 export const data = writable(dataValue);
+
+data.subscribe((value) => {
+    if (!value) localStorage.removeItem('data');
+    else if (!value.isDemo) localStorage.setItem('data', JSON.stringify(value));
+});
diff --git a/src/views/Help.svelte b/src/views/Help.svelte
@@ -0,0 +1,7 @@
+<script>
+    import { link } from 'svelte-routing';
+</script>
+
+<main>
+    To get 
+</main>
diff --git a/src/views/Loader.svelte b/src/views/Loader.svelte
@@ -1,7 +1,8 @@
 <script>
+    import { link } from 'svelte-routing';
 	import { Unzip, AsyncUnzipInflate } from 'fflate';
     import { navigate } from "svelte-routing";
-    import { loaded, loadTask, loadEstimatedTime, data } from '../app/store';
+    import { loadTask, loadEstimatedTime, data } from '../app/store';
     import { extractData } from '../app/extractor';
 
     let loading = false;
@@ -55,7 +56,6 @@
         extractData(files).then((extractedData) => {
             loading = false;
             data.set(extractedData)
-            loaded.set(true);
             loadTask.set(null);
             console.log(`[debug] Data extracted in ${(Date.now() - extractStartAt) / 1000} seconds.`);
             navigate('/stats');
@@ -98,7 +98,7 @@
         <div class="app-loader-boxes">
             <p class="app-loader-description"><a href="https://ddpe.me">DDPE</a> is a site that generates stats from your Discord Data Package. It is your device that processes the data, nothing is sent to any server!</p>
             <p>
-                <a class="app-loader-tuto" href="/get-data" target="_blank">
+                <a class="app-loader-tuto" href="/get-data" use:link>
                     <small class="app-loader-tag tag">1</small>
                     Get my Discord JSON data 👆
                     <br>

diff --git a/src/views/Stats.svelte b/src/views/Stats.svelte
@@ -1,6 +1,6 @@
 <script>
     import { fly } from 'svelte/transition';
-    import { loaded, data } from "../app/store";
+    import { data } from "../app/store";
     import { generateAvatarURL, getGitHubContributors } from '../app/helpers';
     import Chart from 'svelte-frappe-charts';
     import Modal from '../components/Modal.svelte';
@@ -33,7 +33,6 @@
     };
     
     const hoursLabels = $data && new Array(24).fill(0).map((v, i) => i == 0 ? '12am' : i < 12 ? `${i}am` : i == 12 ? '12pm' : `${i-12}pm`);
-    const channels = $data && $data.channels.filter(c => !c.isDM);
     </script>
 
     <div class="statistics" transition:fly="{{ y: 200, duration: 1000 }}">
@@ -50,7 +49,7 @@
                     <FunFact
                         svg="M18 9v3m0 0v3m0-3h3m-3 0h-3m-2-5a4 4 0 11-8 0 4 4 0 018 0zM3 20a6 6 0 0112 0v1H3v-1z"
                         content="You talked to % distinct users"
-                        count="{ $data.channels.filter((c) => c.isDM).length }"
+                        count="{ $data.dmChannelCount }"
                         explanation="Well, you know a lot of people!"
                     />
                     <FunFact
@@ -83,7 +82,7 @@
                     <FunFact
                         svg="M16 12a4 4 0 10-8 0 4 4 0 008 0zm0 0v1.5a2.5 2.5 0 005 0V12a9 9 0 10-9 9m4.5-1.206a8.959 8.959 0 01-4.5 1.207"
                         content="You sent % characters through Discord"
-                        count="{ $data.channels.map((channel) => channel.messages).flat().map((message) => message.length).reduce((p, c) => p + c) }"
+                        count="{ $data.characterCount }"
                     />
                     <FunFact svg="M4.318 6.318a4.5 4.5 0 000 6.364L12 20.364l7.682-7.682a4.5 4.5 0 00-6.364-6.364L12 7.636l-1.318-1.318a4.5 4.5 0 00-6.364 0z">
                         <h3 slot="content">Your favorite words are
@@ -145,8 +144,8 @@
                         strokeLinecap="square"
                         strokeLinejoin="square"
                         content="You have spoken in % different text channels"
-                        count="{ channels.length }"
-                        explanation="That's ~{ Math.round(channels.length / $data.guilds.length) } per guild!"
+                        count="{ $data.channelCount }"
+                        explanation="That's ~{ Math.round($data.channelCount / $data.guildCount) } per guild!"
                     />
                 </Card>
                 <Card name="top-users">
@@ -157,19 +156,19 @@
                                 avatarURL={generateAvatarURL(channel.userData.avatar, channel.userData.id, channel.userData.discriminator)}
                                 name={channel.userData.username}
                                 discriminator={channel.userData.discriminator}
-                                count={channel.messages.length.toLocaleString('en-US')}
+                                count={channel.messageCount.toLocaleString('en-US')}
                             />
                         {/each}
                     </Leaderboard>
                 </Card>
                 <Card name="top-channels">
                     <Leaderboard title="Top Channels" description="The channels you chat the most in!">
-                        {#each $data.channels.filter(c => c.data && c.data.guild).sort((a, b) => b.messages.length - a.messages.length).slice(0, 10) as channel, i}
+                        {#each $data.topChannels as channel, i}
                             <LeaderboardItem
                                 position={i}
                                 name={channel.name}
-                                guild={channel.data.guild.name}
-                                count={channel.messages.length}
+                                guild={channel.guildName}
+                                count={channel.messageCount}
                                 channel
                             />
                         {/each}