Add code for the cache_following scheduled lambda

.gitignore

@@ -2,4 +2,8 @@ build
 node_modules
 *.js
 .env.*
-!.env.sample
+!.env.sample
+.terraform
+.terraform.*
+terraform.tfstate*
+terraform/terraform.tfvars

functions/Dockerfile

@@ -0,0 +1,16 @@
+FROM public.ecr.aws/lambda/nodejs:16.2022.05.31.10
+
+# Assumes your function is named "app.js", and there is a package.json file in the app directory 
+COPY build /var/task/
+
+RUN mkdir /var/task/schema
+COPY src/prisma /var/task/prisma
+COPY package.json package-lock.json /var/task/
+
+# Install NPM dependencies for function
+WORKDIR /var/task
+RUN npm ci
+RUN npx prisma generate --schema=/var/task/prisma/schema.prisma
+
+# Set the CMD to your handler (could also be done as a parameter override outside of the Dockerfile)
+CMD [ "app.handler" ]

functions/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "spicytags-lambda",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "jest --verbose"
+  },
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "@aws-sdk/client-cloudwatch": "^3.241.0",
+    "@aws-sdk/client-ssm": "^3.241.0",
+    "@jest/globals": "^29.3.1",
+    "@prisma/client": "^4.8.0",
+    "mastodon-api": "=1.3.0",
+    "redis": "^4.5.1",
+    "ts-node": "^10.9.1"
+  },
+  "devDependencies": {
+    "@types/jest": "^29.2.5",
+    "ts-jest": "^29.0.3"
+  }
+}

functions/pu.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+
+region=ca-central-1
+image_name=spicytags_prod_core
+
+set -e
+
+if [[ "$AWS_PROFILE" == "" ]]; then
+    echo "AWS_PROFILE is not set"
+    exit 1
+else
+    echo "Using AWS_PROFILE $AWS_PROFILE"
+fi
+
+if [ "$1" == "" ]; then
+    echo "Missing <function_name>"
+    echo
+    echo "Usage: pu.sh <function_name>"
+    echo
+    echo "Example: ./pu.sh spicytags_prod_cache_following"
+    echo
+    exit 1
+fi
+
+
+account_id=`aws sts get-caller-identity --query "Account" --output text`
+
+
+echo "Building image"
+npx tsc && docker build -t ${account_id}.dkr.ecr.${region}.amazonaws.com/${image_name}:latest .
+
+echo "Docker logging in"
+aws ecr get-login-password --region ${region} | docker login --username AWS --password-stdin ${account_id}.dkr.ecr.${region}.amazonaws.com
+
+echo "Pushing image"
+docker push ${account_id}.dkr.ecr.${region}.amazonaws.com/${image_name}:latest
+
+echo "Publishing function"
+aws lambda update-function-code --function-name $1 --image-uri ${account_id}.dkr.ecr.${region}.amazonaws.com/${image_name}:latest --publish --query 'FunctionArn'
+
+
+if [[ $? == 0 ]]
+then 
+    status=""
+    until [ "$status" == "\"Successful\"" ]
+    do
+        status=`aws lambda get-function --function-name $1 --query "Configuration.LastUpdateStatus"`
+        echo $status
+        sleep 1
+    done
+    echo "Deployed."
+
+else
+    echo "Failed."
+fi

functions/src/app.ts

@@ -0,0 +1,40 @@
+import Config from './config';
+
+export const handler = async function(event:any) {
+
+    let config = await Config.get()
+
+    if (event.Records) {
+        for (const rec of event.Records) {
+            let msg = rec
+            if (rec.body) {
+                const body = JSON.parse(rec.body);
+                msg = JSON.parse(body.Message);
+            }
+
+            await route(msg, config);
+        }
+    } else {
+        await route(event, config)
+    }
+
+    return {
+        "statusCode": 200, 
+        "body": ""
+    };
+}
+
+
+const route = async (payload: any, config: Config) => {
+
+    let commandId = process.env.COMMAND!;
+
+    console.log(`Received ${commandId} command with payload`, payload);
+    if (commandId) {
+
+        const commandImpl = await import(`./commands/${commandId}/handler`);
+        await commandImpl.default(payload, config)
+    } else {
+        throw Error(`unrecognized command ${commandId}`)
+    }
+}

functions/src/commands/cache_following/CacheFollowingCommand.ts

@@ -0,0 +1,58 @@
+import Following from "../../lib/mastodon/following";
+import FollowsCache from "../../svc/cache/follows";
+const Mastodon = require("mastodon-api");
+
+export default class CacheFollowingCommand {
+  private cache!: FollowsCache;
+  private following!: Following;
+  private input: CacheFollowingCommandInput;
+
+  constructor(input: CacheFollowingCommandInput) {
+    this.input = input
+  }
+
+  private async setup() {
+    this.cache = new FollowsCache(this.input.redisURL);
+    const homeInstance = new Mastodon({
+      api_url: this.input.mastodonApi,
+      access_token: this.input.mastodonApiKey,
+    });
+    this.following = new Following(homeInstance, this.input.mastodonAccountId);
+  }
+
+  async send() {
+    await this.setup();
+    const follows = await this.loadFollowing();
+    console.log(`Caching ${follows.size} follows`)
+
+    await this.cacheFollowing(follows);
+  }
+
+  async loadFollowing(): Promise<Set<string>> {
+    return await this.following.load();
+  }
+
+  async cacheFollowing(follows: Set<string>) {
+    try {
+      await this.cache.connect();
+      await this.cache.addFollows(...follows);
+    } finally {
+      await this.cache.disconnect();
+    }
+  }
+}
+
+
+export interface CacheFollowingCommandInput {
+  // A Mastodon Account ID who owns the following
+  mastodonAccountId: string;
+
+  // Base URL for the Mastodon API of the server that owns `mastodonAccountId`
+  mastodonApi: string;
+
+  // API key to access `mastodonApi`
+  mastodonApiKey: string;
+
+  // The URL of the Redis instance where the followers will be cached
+  redisURL: string;
+}

functions/src/commands/cache_following/handler.ts

@@ -0,0 +1,11 @@
+import Config from "../../config";
+import CacheFollowingCommand from "./CacheFollowingCommand";
+
+export default async (payload: {}, config: Config) => {
+  return await new CacheFollowingCommand({
+    mastodonAccountId: config.getAccountId(),
+    mastodonApi: config.getHomeApiUrl(),
+    mastodonApiKey: config.getHomeApiKey(),
+    redisURL: config.getRedisUrl(),
+  }).send();
+};

functions/src/config.ts

@@ -0,0 +1,68 @@
+import { getJsonSecret } from "./lib/aws/ssm/ssm";
+
+export type ConfigType = { [key: string]: string };
+
+let config: Config;
+
+const KEY_DATABASE_URL = "DATABASE_URL";
+const KEY_REDIS_URL = "REDIS_URL";
+
+const KEY_HOME_ACCOUNT_ID = "HOME_ACCOUNT_ID";
+const KEY_HOME_API_URL = "HOME_API_URL";
+const KEY_HOME_API_KEY = "HOME_API_KEY";
+
+const KEY_SCAN_API_URL = "SCAN_API_URL";
+const KEY_SCAN_API_KEY = "SCAN_API_KEY";
+
+export default class Config {
+  config: ConfigType;
+
+  private constructor(config?: any) {
+    if (config) {
+      this.config = config;
+    } else {
+      throw new Error("Undefined config");
+    }
+  }
+
+  getAccountId() {
+    return this.config[KEY_HOME_ACCOUNT_ID];
+  }
+
+  getHomeApiUrl() {
+    return this.config[KEY_HOME_API_URL];
+  }
+
+  getHomeApiKey() {
+    return this.config[KEY_HOME_API_KEY];
+  }
+
+  getScanApiUrl() {
+    return this.config[KEY_SCAN_API_URL];
+  }
+
+  getScanApiKey() {
+    return this.config[KEY_SCAN_API_KEY];
+  }
+
+  getDatabaseUrl() {
+    return this.config[KEY_DATABASE_URL];
+  }
+
+  getRedisUrl() {
+    return this.config[KEY_REDIS_URL];
+  }
+
+  static get = async () => {
+    if (!config) {
+      try {
+        // fetches `/${env.PRODUCT}/${env.ENV}/config`
+        config = new Config(await getJsonSecret("config"));
+      } catch (e) {
+        console.error("Error loading config", e);
+        throw e;
+      }
+    }
+    return config;
+  };
+}

functions/src/lib/aws/ssm/ssm.ts

@@ -0,0 +1,25 @@
+import { GetParameterCommand, SSMClient } from "@aws-sdk/client-ssm";
+
+const client = new SSMClient({});
+
+/*
+    Fetch a secret from AWS Parameter Store and decode with JSON.parse
+*/
+export const getJsonSecret = async (key: string) => {
+    const keyPath = createKeyPath(key);
+
+    const secret = await client.send(new GetParameterCommand({
+        Name: keyPath,
+        WithDecryption: true
+    })).then(output => output.Parameter?.Value)
+
+    console.log(`ssm:getSecret(): Loaded ${keyPath}.  Secret length: ${secret?.length}`);
+    return secret ? JSON.parse(secret) : undefined;
+}
+
+/*
+    Returns a path scoped to the current product & environment: /<env.PRODUCT>/<env.ENV>/<key>
+*/
+export const createKeyPath = (key: string) => {
+    return `/${process.env.PRODUCT}/${process.env.ENV}/${key}`
+}