Add local DynamoDB support and update configuration

docker-compose.yml

@@ -117,6 +117,18 @@ services:
     networks:
       - "polis-net"
 
+  dynamodb:
+    image: amazon/dynamodb-local
+    container_name: polis-dynamodb
+    ports:
+      - "8000:8000"
+    networks:
+      - "polis-net"
+    # Running DynamoDB in memory for ephemeral storage (no persistence)
+    command: "-jar DynamoDBLocal.jar -sharedDb -inMemory"
+    labels:
+      polis_tag: ${TAG:-dev}
+
 networks:
   polis-net:
 

docs/configuration.md

@@ -77,6 +77,10 @@ If you are deploying to a custom domain (not `pol.is`) then you need to update b
 - **`DATABASE_URL`** should be the combination of above values, `postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/${POSTGRES_DB}`
 - **`POSTGRES_DOCKER`** Set to `false` if using a postgres database outside of docker. Defaults to `true`. Read by Makefile.
 
+#### DynamoDB
+
+- **`DYNAMODB_ENDPOINT`** (optional) DynamoDB endpoint. If not set, the default AWS SDK endpoint will be used.
+
 ### Docker Concerns
 
 - **`TAG`** used by **`COMPOSE_PROJECT_NAME`** below. Defaults to `dev`.
@@ -126,15 +130,18 @@ If you are deploying to a custom domain (not `pol.is`) then you need to update b
 (All are optional, and omitting them will disable the related feature.)
 
 - **`AKISMET_ANTISPAM_API_KEY`** Comment spam detection and filtering.
-- **`AWS_REGION`** Used for S3 data import/export.
 - **`ENABLE_TWITTER_WIDGETS`** set to `true` to enable twitter widgets on the client-admin authentication pages.
 - **`FB_APP_ID`** Must register with Facebook to get an ID to enable Facebook App connectivity.
 - **`GA_TRACKING_ID`** For using Google Analytics on client pages.
 - **`GOOGLE_CREDENTIALS_BASE64`** Required if using Google Translate API. (See below).
 - **`GOOGLE_CREDS_STRINGIFIED`** Alternative to **`GOOGLE_CREDENTIALS_BASE64`** (See below).
 - **`MAILGUN_API_KEY`**, **`MAILGUN_DOMAIN`** If using Mailgun as an email transport.
 - **`TWITTER_CONSUMER_KEY`**, **`TWITTER_CONSUMER_SECRET`** For Twitter integration.
-- **`AWS_ACCESS_KEY_ID`**, **`AWS_SECRET_ACCESS_KEY`** If using Amazon SES as an email transport.
+- **`AWS_REGION`** Used for some data import/export.
+- **`AWS_ACCESS_KEY_ID`**, **`AWS_SECRET_ACCESS_KEY`** Useful for AWS SDK operations.
+- **`ANTHROPIC_API_KEY`** For using Anthropic as a generative AI model.
+- **`GEMINI_API_KEY`** For using Gemini as a generative AI model.
+- **`OPENAI_API_KEY`** For using OpenAI as a generative AI model.
 
 ### Deprecated
 

example.env

@@ -102,8 +102,6 @@ STATIC_FILES_HOST=file-server
 ###### THIRD PARTY API CREDENTIALS ######
 # These are all optional, but some features will not work without them.
 AKISMET_ANTISPAM_API_KEY=
-# Used for S3 data import/export:
-AWS_REGION=
 # Set to true to enable the twitter widgets in the client:
 ENABLE_TWITTER_WIDGETS=
 # Must register with facebook and get a facebook app id to use the facebook auth features:
@@ -117,24 +115,28 @@ MAILGUN_API_KEY=
 MAILGUN_DOMAIN=
 TWITTER_CONSUMER_KEY=
 TWITTER_CONSUMER_SECRET=
-# Read from process.env by aws-sdk, if using SES for email transport
+# Read from process.env by aws-sdk
 # https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/loading-node-credentials-environment.html
+AWS_REGION=
 AWS_ACCESS_KEY_ID=
 AWS_SECRET_ACCESS_KEY=
 # This value is written by the server app if SHOULD_USE_TRANSLATION_API is true.
-
-
 GOOGLE_APPLICATION_CREDENTIALS=
+# Optional API keys for other services:
 GOOGLE_JIGSAW_PERSPECTIVE_API_KEY=
 ANTHROPIC_API_KEY=
 GEMINI_API_KEY=
-OPEN_ROUTER_API_KEY=
 OPENAI_API_KEY=
-
-
 # A value in miliseconds for caching AI responses for narrativeReport
 MAX_REPORT_CACHE_DURATION=
 
+
+###### DYNAMODB ######
+# When using local DynamoDB, this should be http://dynamodb:8000.
+# In production, set DYNAMODB_ENDPOINT to the cloud endpoint (e.g. https://dynamodb.us-west-2.amazonaws.com),
+# or simply leave it blank.
+DYNAMODB_ENDPOINT=http://dynamodb:8000
+
 ###### DEPRECATED ######
 # (Deprecated) Used internally by Node.Crypto to encrypt/decrypt IP addresses.
 ENCRYPTION_PASSWORD_00001=

server/src/config.ts

@@ -67,25 +67,33 @@ export default {
   adminEmails: process.env.ADMIN_EMAILS || "[]",
   adminUIDs: process.env.ADMIN_UIDS || "[]",
   akismetAntispamApiKey: process.env.AKISMET_ANTISPAM_API_KEY || null,
+  anthropicApiKey: process.env.ANTHROPIC_API_KEY || null,
   applicationName: process.env.APPLICATION_NAME || null,
-  awsRegion: process.env.AWS_REGION as string,
+  awsAccessKeyId: process.env.AWS_ACCESS_KEY_ID || 'local' as string,
+  awsRegion: process.env.AWS_REGION || 'local' as string,
+  awsSecretAccessKey: process.env.AWS_SECRET_ACCESS_KEY || 'local' as string,
   backfillCommentLangDetection: isTrue(
     process.env.BACKFILL_COMMENT_LANG_DETECTION
   ),
   cacheMathResults: isTrueOrBlank(process.env.CACHE_MATH_RESULTS),
   databaseSSL: isTrue(process.env.DATABASE_SSL),
   databaseURL: process.env.DATABASE_URL as string,
+  dynamoDbEndpoint: process.env.DYNAMODB_ENDPOINT || null,
   emailTransportTypes: process.env.EMAIL_TRANSPORT_TYPES || null,
   encryptionPassword: process.env.ENCRYPTION_PASSWORD_00001 as string,
   fbAppId: process.env.FB_APP_ID || null,
+  geminiApiKey: process.env.GEMINI_API_KEY || null,
+  googleApiKey: process.env.GOOGLE_API_KEY || null,
   googleJigsawPerspectiveApiKey:
     process.env.GOOGLE_JIGSAW_PERSPECTIVE_API_KEY || null,
   logLevel: process.env.SERVER_LOG_LEVEL as string,
   logToFile: isTrue(process.env.SERVER_LOG_TO_FILE),
   mailgunApiKey: process.env.MAILGUN_API_KEY || null,
   mailgunDomain: process.env.MAILGUN_DOMAIN || null,
+  maxReportCacheDuration: parseInt(process.env.MAX_REPORT_CACHE_DURATION || "3600000", 10),
   mathEnv: process.env.MATH_ENV as string,
   nodeEnv: process.env.NODE_ENV as string,
+  openaiApiKey: process.env.OPENAI_API_KEY || null,
   polisFromAddress: process.env.POLIS_FROM_ADDRESS as string,
   readOnlyDatabaseURL:
     process.env.READ_ONLY_DATABASE_URL || (process.env.DATABASE_URL as string),

server/src/report_experimental/topics-example/index.ts

@@ -9,6 +9,8 @@ import { Sensemaker } from "@tevko/sensemaking-tools/src/sensemaker";
 import { GoogleAIModel } from "@tevko/sensemaking-tools/src/models/aiStudio_model";
 import { Comment, VoteTally, Topic } from "@tevko/sensemaking-tools/src/types";
 import { parse } from "csv-parse";
+import config from "../../config";
+import logger from "../../utils/logger";
 
 async function parseCsvString(csvString: string) {
   return new Promise((resolve, reject) => {
@@ -53,20 +55,23 @@ async function parseCsvString(csvString: string) {
 
 export async function getTopicsFromRID(zId: number) {
   try {
+    if (!config.geminiApiKey) {
+      throw new Error("polis_err_gemini_api_key_not_set");
+    }
     const resp = await sendCommentGroupsSummary(zId, undefined, false);
     const modified = (resp as string).split("\n");
     modified[0] = `comment-id,comment_text,total-votes,total-agrees,total-disagrees,total-passes,group-a-votes,group-0-agree-count,group-0-disagree-count,group-0-pass-count,group-b-votes,group-1-agree-count,group-1-disagree-count,group-1-pass-count`;
 
     const comments = await parseCsvString(modified.join("\n"));
     const topics = await new Sensemaker({
       defaultModel: new GoogleAIModel(
-        process.env.GEMINI_API_KEY as string,
+        config.geminiApiKey,
         "gemini-exp-1206"
       ),
     }).learnTopics(comments as Comment[], false);
     const categorizedComments = await new Sensemaker({
       defaultModel: new GoogleAIModel(
-        process.env.GEMINI_API_KEY as string,
+        config.geminiApiKey,
         "gemini-1.5-flash-8b"
       ),
     }).categorizeComments(comments as Comment[], false, topics);
@@ -89,7 +94,7 @@ export async function getTopicsFromRID(zId: number) {
       citations: value.citations,
     }));
   } catch (error) {
-    console.error(error);
+    logger.error(error);
     return [];
   }
 }

server/src/routes/reportNarrative.ts

@@ -17,7 +17,10 @@ import { sendCommentGroupsSummary } from "./export";
 import { getTopicsFromRID } from "../report_experimental/topics-example";
 import DynamoStorageService from "../utils/storage";
 import { PathLike } from "fs";
+import config from "../config";
+import logger from "../utils/logger";
 
+// eslint-disable-next-line @typescript-eslint/no-var-requires
 const js2xmlparser = require("js2xmlparser");
 
 interface PolisRecord {
@@ -108,11 +111,11 @@ export class PolisConverter {
   }
 }
 
-const anthropic = new Anthropic({
-  apiKey: process.env.ANTHROPIC_API_KEY,
-});
+const anthropic = config.anthropicApiKey ? new Anthropic({
+  apiKey: config.anthropicApiKey,
+}) : null;
 
-const genAI = new GoogleGenerativeAI(process.env.GEMINI_API_KEY as string);
+const genAI = config.geminiApiKey ? new GoogleGenerativeAI(config.geminiApiKey) : null;
 
 const getCommentsAsXML = async (
   id: number,
@@ -131,10 +134,10 @@ const getCommentsAsXML = async (
     const xml = PolisConverter.convertToXml(resp as string);
     // eslint-disable-next-line no-console
     if (xml.trim().length === 0)
-      console.error("No data has been returned by sendCommentGroupsSummary");
+      logger.error("No data has been returned by sendCommentGroupsSummary");
     return xml;
   } catch (e) {
-    console.error("Error in getCommentsAsXML:", e);
+    logger.error("Error in getCommentsAsXML:", e);
     throw e; // Re-throw instead of returning empty string
   }
 };
@@ -147,10 +150,7 @@ const isFreshData = (timestamp: string) => {
   const now = new Date().getTime();
   const then = new Date(timestamp).getTime();
   const elapsed = Math.abs(now - then);
-  return (
-    elapsed <
-    (((process.env.MAX_REPORT_CACHE_DURATION as unknown) as number) || 3600000)
-  );
+  return elapsed < config.maxReportCacheDuration;
 };
 
 const getModelResponse = async (
@@ -160,7 +160,7 @@ const getModelResponse = async (
   modelVersion?: string
 ) => {
   try {
-    const gemeniModel = genAI.getGenerativeModel({
+    const gemeniModel = genAI?.getGenerativeModel({
       // model: "gemini-1.5-pro-002",
       model: modelVersion || "gemini-2.0-pro-exp-02-05",
       generationConfig: {
@@ -201,15 +201,23 @@ const getModelResponse = async (
       ],
       systemInstruction: system_lore,
     };
-    const openai = new OpenAI();
+    const openai = config.openaiApiKey ? new OpenAI({
+      apiKey: config.openaiApiKey,
+    }) : null;
 
     switch (model) {
       case "gemini": {
+        if (!gemeniModel) {
+          throw new Error("polis_err_gemini_api_key_not_set");
+        }
         const respGem = await gemeniModel.generateContent(gemeniModelprompt);
         const result = await respGem.response.text();
         return result;
       }
       case "claude": {
+        if (!anthropic) {
+          throw new Error("polis_err_anthropic_api_key_not_set");
+        }
         const responseClaude = await anthropic.messages.create({
           model: modelVersion || "claude-3-5-sonnet-20241022",
           max_tokens: 3000,
@@ -230,6 +238,9 @@ const getModelResponse = async (
         return `{${responseClaude?.content[0]?.text}`;
       }
       case "openai": {
+        if (!openai) {
+          throw new Error("polis_err_openai_api_key_not_set");
+        }
         const responseOpenAI = await openai.chat.completions.create({
           model: modelVersion || "gpt-4o",
           messages: [
@@ -243,7 +254,7 @@ const getModelResponse = async (
         return "";
     }
   } catch (error) {
-    console.error("ERROR IN GETMODELRESPONSE", error);
+    logger.error("ERROR IN GETMODELRESPONSE", error);
     return `{
       "id": "polis_narrative_error_message",
       "title": "Narrative Error Message",
@@ -608,9 +619,9 @@ export async function handle_GET_topics(
 
   sections.forEach(
     async (
-      section: { name: any; templatePath: PathLike | fs.FileHandle },
+      section: { name: string; templatePath: PathLike | fs.FileHandle },
       i: number,
-      arr: any
+      arr: { name: string; templatePath: PathLike | fs.FileHandle }[]
     ) => {
       const cachedResponse = await storage?.queryItemsByRidSectionModel(
         `${rid}#${section.name}#${model}`
@@ -687,12 +698,12 @@ export async function handle_GET_topics(
               },
             }) + `|||`
           );
-          console.log("topic over");
+          logger.debug("topic over");
           // @ts-expect-error flush - calling due to use of compression
           res.flush();
 
           if (arr.length - 1 === i) {
-            console.log("all promises completed");
+            logger.debug("all promises completed");
             res.end();
           }
         }, 3000 * i);
@@ -705,14 +716,12 @@ export async function handle_GET_reportNarrative(
   req: { p: { rid: string }; query: QueryParams },
   res: Response
 ) {
-  let storage;
-  if (process.env.AWS_REGION && process.env.AWS_REGION?.trim().length > 0) {
-    storage = new DynamoStorageService(
-      process.env.AWS_REGION,
-      "report_narrative_store",
-      req.query.noCache === "true"
-    );
-  }
+  const storage = new DynamoStorageService(
+    "report_narrative_store",
+    req.query.noCache === "true"
+  );
+  await storage.initTable();
+
   const modelParam = req.query.model || "openai";
   const modelVersionParam = req.query.modelVersion;
 
@@ -790,7 +799,7 @@ export async function handle_GET_reportNarrative(
   } catch (err) {
     // @ts-expect-error flush - calling due to use of compression
     res.flush();
-    console.log(err);
+    logger.error(err);
     const msg =
       err instanceof Error && err.message && err.message.startsWith("polis_")
         ? err.message

server/src/server.ts

@@ -1897,7 +1897,7 @@ Feel free to reply to this email if you need help.`;
       // Type '[unknown, unknown]' is not assignable to type '{ location: any; source: any; }'.ts(2345)
       // @ts-ignore
       .then(function (locationData: { location: any; source: any }) {
-        if (!locationData || !process.env.GOOGLE_API_KEY) {
+        if (!locationData || !Config.googleApiKey) {
           return;
         }
         geoCode(locationData.location)
@@ -9646,9 +9646,13 @@ Thanks for using Polis!
   }
 
   function geoCodeWithGoogleApi(locationString: string) {
-    let googleApiKey = process.env.GOOGLE_API_KEY;
+    let googleApiKey = Config.googleApiKey;
     let address = encodeURI(locationString);
 
+    if (!googleApiKey) {
+      return Promise.reject("polis_err_geocoding_no_api_key");
+    }
+
     return new Promise(function (
       resolve: (arg0: any) => void,
       reject: (arg0: string) => void