chore: add dashboard automation

.eslintignore

@@ -1 +1,2 @@
 dist
+reports

.github/workflows/metrics-update.yml

@@ -0,0 +1,25 @@
+name: Update metrics data in google spreadsheet
+
+on:
+  workflow_dispatch:
+  schedule:
+  # Every 2 hours: "At minute 0 past every 2nd hour."
+  - cron: '0 */2 * * *'
+
+jobs:
+  update-metrics:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./reports
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-node@v3
+      with:
+        node-version: lts/*
+    - uses: ipfs/aegir/actions/cache-node-modules@master # npm install at the root
+    - run: npm install && npm run update-dashboards # npm install & run in reports directory
+      env:
+        GOOGLE_CREDENTIALS: '${{ secrets.GOOGLE_CREDENTIALS }}'
+        COUNTLY_USERNAME: '${{ secrets.COUNTLY_USERNAME }}'
+        COUNTLY_PASSWORD: '${{ secrets.COUNTLY_PASSWORD }}'

.gitignore

@@ -30,3 +30,7 @@ typings/
 storybook-static
 .envrc
 .tool-versions
+reports/data
+reports/node_modules
+ignite-metrics-dashboard-d99dde383c4c.json
+.env

reports/README.md

@@ -0,0 +1,46 @@
+# Explanation
+
+The code in this folder (`reports`) is used to generate CSV files for getting daily/weekly/monthly active users for all Ignite team (ipfs-gui) projects.
+
+The CSV content is then loaded into it's respective sheet at https://docs.google.com/spreadsheets/d/1xq36kjThObEaRKzb3VRtXEs9RgM-bfgfjGbi1vbPUiE/edit#gid=755468744
+
+The charts in the "Charts" sheet are loaded in our Notion page at https://www.notion.so/pl-strflt/Ignite-IPFS-GUI-Tools-3bc1c1bf54d74f928bf11ef59c876b74#b6970aa92e914114848fbddd84eab2ba
+
+## With google sheets authentication
+
+Just run `npm run update-dashboards`. This will download all data from countly and then automatically update the google sheets.
+
+## Without google sheets authenticaiton
+
+### How to get the data from countly
+
+Inside the `./reports` folder, run
+
+```bash
+npm install
+npm run get-csv
+```
+
+### How to copy the data to google spreadsheets
+
+If you have a valid keyfile for google sheets authentication
+
+1. Open up the relevant `./reports/output/*.csv` daily/weekly/monthly file and copy its contents.
+1. Paste that content into the relevant google sheet, cell A1, at https://docs.google.com/spreadsheets/d/1xq36kjThObEaRKzb3VRtXEs9RgM-bfgfjGbi1vbPUiE/edit#gid=755468744
+1. Select "Data->Split Text to columns"
+
+The charts and everything should automatically update.
+
+## How to embed into Notion
+
+This is already done and should automatically update, but if it needs redone, it's somewhat like follows:
+
+***NOTE:*** DO NOT CHANGE THE Published Content & Settings unless you know what you're doing. You will break existing embeds if you change this.
+
+1. click the three dots in the top right of the chart.
+1. select "publish chart"
+1. Select the chart you wish to get the link for (in the first dropdown). Leave "Interactive" selected (in the second dropdown).
+1. Copy the link
+1. Go to Notion where you want to embed. Type `/embed` and select the generic embed "for PDFs, google maps, and more"
+1. Paste the link you copied from google sheets.
+

reports/constants.ts

@@ -0,0 +1,66 @@
+export const hostname = 'countly.ipfs.tech'
+export const authorizationHeader = `Basic ${Buffer.from(`${process.env.COUNTLY_USERNAME}:${process.env.COUNTLY_PASSWORD}`).toString('base64')}`
+export const baseOptions = {
+  method: 'GET',
+  headers: {
+    accept: 'application/json',
+    authorization: authorizationHeader
+  }
+}
+
+async function getApiKey (): Promise<string> {
+  const response = await fetch(`https://${hostname}/api-key`, {
+    ...baseOptions,
+    headers: {
+      ...baseOptions.headers,
+      accept: 'text/plain'
+    }
+  })
+
+  return await response.text()
+}
+
+export const apiKey = await getApiKey()
+
+/**
+ * 90 days of data
+ */
+export const daysOfDataInMs = 1000 * 60 * 60 * 24 * 90
+
+export const appIds = {
+  // Webui.ipfs.io
+  'ipfs-webui': '5c6e72803fd4432348b8119c',
+
+  // webui-kubo
+  'ipfs-webui-kubo': '63c596762a7760344a6b2cfd',
+
+  // ipfs-desktop
+  'ipfs-desktop': '5c6ec2b13fd4432348b811a0',
+
+  // ipfs-companion
+  'ipfs-companion': '639cbbcf8e6f3439c3796738',
+
+  // public gateway checker
+  'public-gateway-checker': '6345a52a31fdc11369a2f2db',
+
+  // starmap.site
+  'starmap.site': '639915ff21fd4330c469a191',
+
+  // cid-utils-website
+  'cid-utils-website': '63cf2d6ed09125d219d3d86c',
+
+  // explore.ipld.io
+  'explore.ipld.io': '63cef029d09125d219d3d69a',
+
+  // ipfs-check
+  'ipfs-check': '63d039e622fb279599709b09',
+
+  // ipfs-dag-builder-vis
+  'ipfs-dag-builder-vis': '63cee76ad09125d219d3d640',
+
+  // pinning-service-compliance
+  'pinning-service-compliance': '63cf08ccd09125d219d3d776',
+
+  // pl-diagnose
+  'pl-diagnose': '63cef095d09125d219d3d6a6'
+}

reports/doCountlyFetch.ts

@@ -0,0 +1,6 @@
+import { apiKey, baseOptions, hostname } from './constants.js';
+
+export async function doCountlyFetch({ path = '/o', fetchOptions = {}, appId, extraParams }: { path: string, fetchOptions?: RequestInit, appId: string, extraParams: string }) {
+  const response = await fetch(`https://${hostname}${path}?api_key=${apiKey}&app_id=${appId}&${extraParams}`, {...baseOptions, ...fetchOptions});
+  return await response.json();
+}

reports/downloadDashboardData.ts

@@ -0,0 +1,75 @@
+import { appIds, daysOfDataInMs } from './constants.js'
+import { doCountlyFetch } from './doCountlyFetch.js'
+import { writeFile } from 'node:fs/promises'
+
+interface ActiveUsersResponse {
+  calculating: boolean
+  data: Record<string, {
+    d: number
+    w: number
+    m: number
+  }>
+}
+
+/**
+ * Print out a CSV of the number of unique users per day for each app
+ */
+export async function downloadDashboardData (): Promise<void> {
+  // let dailyOutputCsv: string = ''
+  // let weeklyOutputCsv: string = ''
+  // let monthlyOutputCsv: string = ''
+  let printedHeaders = false
+  const dailyArray = []
+  const weeklyArray = []
+  const monthlyArray = []
+  for (const [appName, appId] of Object.entries(appIds)) {
+    // let calculating = true
+    let response: ActiveUsersResponse = { calculating: true, data: {} }
+    while (response.calculating) {
+      /**
+       * @see https://api.count.ly/reference/oanalyticssessions
+       */
+      response = await doCountlyFetch({ path: '/o/active_users', appId, extraParams: `period=[${new Date().getTime() - daysOfDataInMs}, ${new Date().getTime()}]` })
+      // eslint-disable-next-line no-console
+      console.log(`${appName} calculating? `, response.calculating)
+      if (response.calculating) {
+        await new Promise((resolve) => setTimeout(resolve, 4000))
+      }
+    }
+    const activeUserData = []
+    for (const [key, value] of Object.entries(response.data)) {
+      activeUserData.push({
+        _id: key,
+        ...value
+      })
+    }
+
+    activeUserData.sort((a, b) => new Date(a._id).getTime() - new Date(b._id).getTime())
+
+    // output the name of the app as row headers and the date labels as column headers
+    if (!printedHeaders) {
+      const dateLabels = activeUserData.map((day: any) => new Date(day._id).toISOString().split('T')[0])
+      dailyArray.push(['App Name', ...dateLabels])
+      // dailyOutputCsv = `App Name, ${dateLabels.join(', ')}\n`
+      weeklyArray.push(['App Name', ...dateLabels])
+      // weeklyOutputCsv = `App Name, ${dateLabels.join(', ')}\n`
+      monthlyArray.push(['App Name', ...dateLabels])
+      // monthlyOutputCsv = `App Name, ${dateLabels.join(', ')}\n`
+      printedHeaders = true
+    }
+
+    dailyArray.push([appName, ...activeUserData.map((day: any) => day.d)])
+    // dailyOutputCsv = `${dailyOutputCsv}${appName}, ${activeUserData.map((day: any) => day.d).join(', ')}\n`
+    weeklyArray.push([appName, ...activeUserData.map((day: any) => day.w)])
+    // weeklyOutputCsv = `${weeklyOutputCsv}${appName}, ${activeUserData.map((day: any) => day.w).join(', ')}\n`
+    monthlyArray.push([appName, ...activeUserData.map((day: any) => day.m)])
+    // monthlyOutputCsv = `${monthlyOutputCsv}${appName}, ${activeUserData.map((day: any) => day.m).join(', ')}\n`
+  }
+  // Write the outputs to their appropriate Csv files
+  await writeFile('./output/activeUsers-daily.json', JSON.stringify(dailyArray, null, 2))
+  await writeFile('./output/activeUsers-daily.csv', dailyArray.join('\n').toString())
+  await writeFile('./output/activeUsers-weekly.json', JSON.stringify(weeklyArray, null, 2))
+  await writeFile('./output/activeUsers-weekly.csv', weeklyArray.join('\n').toString())
+  await writeFile('./output/activeUsers-monthly.json', JSON.stringify(monthlyArray, null, 2))
+  await writeFile('./output/activeUsers-monthly.csv', monthlyArray.join('\n').toString())
+}

reports/getApiKey.ts

@@ -0,0 +1,13 @@
+import { baseOptions, hostname } from './constants.js'
+
+export async function getApiKey (): Promise<string> {
+  const response = await fetch(`https://${hostname}/api-key`, {
+    ...baseOptions,
+    headers: {
+      ...baseOptions.headers,
+      accept: 'text/plain'
+    }
+  })
+
+  return await response.text()
+}

reports/global.d.ts

@@ -0,0 +1 @@
+declare module 'netrc';

reports/index.ts

@@ -0,0 +1,5 @@
+import { downloadDashboardData } from './downloadDashboardData.js'
+import { updateGoogleSheets } from './updateGoogleSheets.js'
+
+await downloadDashboardData()
+await updateGoogleSheets()