Add tag count analytics with Prisma

env/.env.sample

@@ -16,3 +16,6 @@ SCAN_API_URL=https://mastodon.social/api/v1/
 
 # An access token valid for the Mastodon instance at SCAN_API_URL
 SCAN_ACCESS_TOKEN=abcdefghijklmnopqrstuvwxyz0123456789ABCDEFG
+
+# A connection string where the tag counts are saved to the "TagCount" table
+DATABASE_URL=postgresql://user:pass@host:port/database

index.ts

@@ -2,14 +2,20 @@ import sleep from "sleep-promise";
 import Followers from "./lib/followers";
 import Tags from "./lib/tags";
 import spicyTags from "./spicyTags.json";
-import { HOME_API_URL, HOME_ACCESS_TOKEN, SCAN_API_URL, SCAN_ACCESS_TOKEN, ACCOUNT_ID, SLEEP_MS } from "./lib/env";
+import {
+  HOME_API_URL,
+  HOME_ACCESS_TOKEN,
+  SCAN_API_URL,
+  SCAN_ACCESS_TOKEN,
+  ACCOUNT_ID,
+  SLEEP_MS,
+} from "./lib/env";
 
 const Mastodon = require("mastodon-api");
 
 const tags = new Tags(spicyTags.spicyTags);
 
 const main = async () => {
-
   /*
     Watch the federated stream for key hashtags.  For each post:
     IF
@@ -22,17 +28,24 @@ const main = async () => {
   */
 
   const allTagsSeen = new Set();
-  const homeInstance = new Mastodon({ api_url: HOME_API_URL, access_token: HOME_ACCESS_TOKEN });
+
+  const homeInstance = new Mastodon({
+    api_url: HOME_API_URL,
+    access_token: HOME_ACCESS_TOKEN,
+  });
   const following = new Followers(homeInstance, ACCOUNT_ID);
   const follows = await following.load();
   console.log(`Loaded ${follows.size} following`);
 
-  const M = new Mastodon({ api_url: SCAN_API_URL, access_token: SCAN_ACCESS_TOKEN });
+  const M = new Mastodon({
+    api_url: SCAN_API_URL,
+    access_token: SCAN_ACCESS_TOKEN,
+  });
   do {
-
     const result = await M.get("timelines/public", {});
     const newTags = new Set();
-    const batchTags = new Set();
+    const batchTagCounts = new Map<string, number>();
+
     for (let item of result.data) {
       const user = item.account.acct;
 
@@ -43,11 +56,10 @@ const main = async () => {
 
       // Collect tags from the batch
       item.tags.forEach((t: any) => {
-        if (!allTagsSeen.has(t.name)) {
-          allTagsSeen.add(t.name);
+        if (!batchTagCounts.has(t.name)) {
           newTags.add(t.name);
         }
-        batchTags.add(t.name);
+        batchTagCounts.set(t.name, (batchTagCounts.get(t.name) ?? 0) + 1);
       });
 
       // If the batch has a post with a spicy tag, log the user & spicy tags
@@ -58,8 +70,11 @@ const main = async () => {
       }
     }
 
+    await tags.insertTags(batchTagCounts);
+
     // Show timestamped tags for this batch/iteration
     console.log(`${new Date().toISOString()} #${Array.from(newTags)}`);
+
     await sleep(SLEEP_MS);
   } while (true);
 };

lib/tags.ts

@@ -1,8 +1,12 @@
+import { PrismaClient } from "@prisma/client";
+
 export default class Tags {
   private spicyTags: Set<string>;
+  private prisma: PrismaClient;
 
   constructor(spicyTags: string[]) {
     this.spicyTags = new Set(spicyTags);
+    this.prisma = new PrismaClient();
   }
 
   /*
@@ -18,6 +22,15 @@ export default class Tags {
     }
     return found;
   }
+
+  async insertTags(tags: Map<string, number>) {
+    var today = new Date();
+    today.setHours(0, 0, 0, 0);
+
+    await this.prisma.tagCount.createMany({
+      data: Array.from(tags, ([tag, count]) => ({ tag, count, date: today })),
+    });
+  }
 }
 
 interface NameUrl {

package.json

@@ -10,6 +10,7 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
+    "@prisma/client": "^4.8.0",
     "dotenv": "^16.0.3",
     "mastodon-api": "^1.3.0",
     "sleep-promise": "^9.1.0"

prisma/schema.prisma

@@ -0,0 +1,19 @@
+
+datasource db {
+  provider = "postgresql"
+  url      = env("DATABASE_URL")
+}
+
+generator client {
+  provider = "prisma-client-js"
+  engineType = "binary"
+}
+
+model TagCount {
+  id    Int @id @default(autoincrement())
+  date  DateTime @db.Date
+  tag   String
+  count Int
+
+  @@index([date, tag, count])
+}