Merge branch 'master' of github.com:khoj-ai/khoj into features/add-su…

…pport-for-mermaidjs

.github/workflows/desktop.yml

@@ -60,39 +60,39 @@ jobs:
 
       - name: ⏫ Upload Mac ARM App
         if: startsWith(github.ref, 'refs/tags/')
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           if-no-files-found: warn
           name: khoj-${{ github.ref_name }}-arm64.dmg
           path: src/interface/desktop/khoj-${{ github.ref_name }}-arm64.dmg
 
       - name: ⏫ Upload Mac x64 App
         if: startsWith(github.ref, 'refs/tags/')
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           if-no-files-found: warn
           name: khoj-${{ github.ref_name }}-x64.dmg
           path: src/interface/desktop/khoj-${{ github.ref_name }}-x64.dmg
 
       - name: ⏫ Upload Windows App
         if: startsWith(github.ref, 'refs/tags/')
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           if-no-files-found: warn
           name: khoj-${{ github.ref_name }}-x64.exe
           path: src/interface/desktop/khoj-${{ github.ref_name }}-x64.exe
 
       - name: ⏫ Upload Debian App
         if: startsWith(github.ref, 'refs/tags/')
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           if-no-files-found: warn
           name: khoj-${{ github.ref_name }}-x64.deb
           path: src/interface/desktop/khoj-${{ github.ref_name }}-x64.deb
 
       - name: ⏫ Upload Linux App Image
         if: startsWith(github.ref, 'refs/tags/')
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           if-no-files-found: warn
           name: khoj-${{ github.ref_name }}-x64.AppImage

.github/workflows/github_pages_deploy.yml

@@ -17,10 +17,10 @@ jobs:
       runs-on: ubuntu-latest
       steps:
         - name: Checkout
-          uses: actions/checkout@v3
+          uses: actions/checkout@v4
         # 👇 Build steps
         - name: Set up Node.js
-          uses: actions/setup-node@v3
+          uses: actions/setup-node@v4
           with:
             node-version: 18.x
             cache: yarn
@@ -35,12 +35,12 @@ jobs:
             yarn build
         # 👆 Build steps
         - name: Setup Pages
-          uses: actions/configure-pages@v3
+          uses: actions/configure-pages@v5
         - name: Upload artifact
-          uses: actions/upload-pages-artifact@v2
+          uses: actions/upload-pages-artifact@v3
           with:
             # 👇 Specify build output path
             path: documentation/build
         - name: Deploy to GitHub Pages
           id: deployment
-          uses: actions/deploy-pages@v2
+          uses: actions/deploy-pages@v4

.github/workflows/release.yml

@@ -35,21 +35,21 @@ jobs:
           yarn run build --if-present
 
       - name: ⏫ Upload Obsidian Plugin main.js
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           if-no-files-found: error
           name: main.js
           path: src/interface/obsidian/main.js
 
       - name: ⏫ Upload Obsidian Plugin manifest.json
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           if-no-files-found: error
           name: manifest.json
           path: src/interface/obsidian/manifest.json
 
       - name: ⏫ Upload Obsidian Plugin styles.css
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           if-no-files-found: error
           name: styles.css

.github/workflows/run_evals.yml

@@ -133,7 +133,7 @@ jobs:
 
       - name: Upload Results
         if: always()  # Upload results even if tests fail
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: eval-results-${{ steps.hatch.outputs.version }}-${{ matrix.khoj_mode }}-${{ matrix.dataset }}
           path: |

docker-compose.yml

@@ -6,8 +6,6 @@ services:
       POSTGRES_USER: postgres
       POSTGRES_PASSWORD: postgres
       POSTGRES_DB: postgres
-    networks:
-      - default
     volumes:
       - khoj_db:/var/lib/postgresql/data/
     healthcheck:
@@ -17,14 +15,10 @@ services:
       retries: 5
   sandbox:
     image: ghcr.io/khoj-ai/terrarium:latest
-    restart: always
-    networks:
-      - default
+    restart: unless-stopped
   search:
     image: docker.io/searxng/searxng:latest
-    restart: always
-    networks:
-      - default
+    restart: unless-stopped
     volumes:
       - khoj_search:/etc/searxng
     environment:
@@ -35,7 +29,7 @@ services:
         condition: service_healthy
     # Use the following line to use the latest version of khoj. Otherwise, it will build from source. Set this to ghcr.io/khoj-ai/khoj-cloud:latest if you want to use the prod image.
     image: ghcr.io/khoj-ai/khoj:latest
-    restart: always
+    restart: unless-stopped
     # Uncomment the following line to build from source. This will take a few minutes. Comment the next two lines out if you want to use the official image.
     # build:
       # context: .
@@ -45,11 +39,9 @@ services:
       #   change the port in the args in the build section,
       #   as well as the port in the command section to match
       - "42110:42110"
-    working_dir: /app
-    networks:
-      - default
     extra_hosts:
       - "host.docker.internal:host-gateway"
+    working_dir: /app
     volumes:
       - khoj_config:/root/.khoj/
       - khoj_models:/root/.cache/torch/sentence_transformers

documentation/docs/advanced/admin.md

@@ -47,7 +47,7 @@ For each AI Model API you [add](http://localhost:42110/server/admin/database/aim
   ![example configuration for ai model api](/img/example_openai_processor_config.png)
 
 ### Search Model Configs
-Search models are used to generate vector embeddings of your documents for natural language search and chat. You can choose any [embeddings models on HuggingFace](https://huggingface.co/models?pipeline_tag=sentence-similarity) to try, use for your to create vector embeddings of your documents for natural language search and chat.
+Search models are used to generate vector embeddings of your documents for natural language search and chat. You can choose any [embeddings models on HuggingFace](https://huggingface.co/models?pipeline_tag=sentence-similarity) to create vector embeddings of your documents for natural language search and chat.
 
 <img src="/img/example_search_model_admin_settings.png" alt="Example Search Model Settings" style={{width: 500}} />
 
@@ -64,11 +64,14 @@ Add speech to text models with these settings. Khoj currently only supports whis
 ### Voice Model Options
 Add text to speech models with these settings. Khoj currently supports models from [ElevenLabs](https://elevenlabs.io/).
 
+### Reflective Questions
+This is a static list of starter question suggestions for each user. It is not current used in any client app. It used to be shown on the web app home page. We may turn it into a dynamic list of starter questions personalized to each users, say based on their recent conversations or synced knowledge base.
+
 ## User Data
 - Users, Entrys, Conversations, Subscriptions, Github configs, Notion configs, User search configs, User conversation configs, User voice configs
 
 ## Miscellaneous Data
 - Process Locks: Persistent Locks for Automations
 - Client Applications:
 
-Client applications allow you to setup third party applications that can query your Khoj server using a client application ID + secret. The secret would go in a bearer token.
+  Client applications allow you to setup third party applications that can query your Khoj server using a client application ID + secret. The secret would go in a bearer token.

documentation/docs/data-sources/github_integration.md

@@ -1,6 +1,10 @@
 # Github integration
 
-The Github integration allows you to index as many repositories as you want. It's currently default configured to index Issues, Commits, and all Markdown/Org files in each repository. For large repositories, this takes a fairly long time, but it works well for smaller projects.
+:::warning[Unmaintained]
+The Github integration is not maintained. We are considering deprecating it. It doesn't seem used by many folks and its cumbersome for us to maintain.
+:::
+
+The Github integration allows you to index as many repositories as you want. It's currently default configured to index all Markdown/Org/Text files in each repository. For large repositories, this takes a fairly long time, but it works well for smaller projects.
 
 # Configure your settings
 
@@ -9,6 +13,6 @@ The Github integration allows you to index as many repositories as you want. It'
 ## Use the Github plugin
 
 1. Generate a [classic PAT (personal access token)](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens) from [Github](https://github.com/settings/tokens) with `repo` and `admin:org` scopes at least.
-2. Navigate to [https://app.khoj.dev/settings#github](https://app.khoj.dev/settings#github) to configure your Github settings. Enter in your PAT, along with details for each repository you want to index.
+2. Navigate to [https://app.khoj.dev/settings#github](https://app.khoj.dev/settings/content/github) to configure your Github settings. Enter in your PAT, along with details for each repository you want to index.
 3. Click `Save`. Go back to the settings page and click `Configure`.
 4. Go to [https://app.khoj.dev/](https://app.khoj.dev/) and start searching!

src/interface/obsidian/src/main.ts

@@ -34,6 +34,21 @@ export default class Khoj extends Plugin {
             callback: () => { this.activateView(KhojView.CHAT); }
         });
 
+        // Add sync command to manually sync new changes
+        this.addCommand({
+            id: 'sync',
+            name: 'Sync new changes',
+            callback: async () => {
+                this.settings.lastSync = await updateContentIndex(
+                    this.app.vault,
+                    this.settings,
+                    this.settings.lastSync,
+                    false,
+                    true
+                );
+            }
+        });
+
         this.registerView(KhojView.CHAT, (leaf) => new KhojChatView(leaf, this.settings));
 
         // Create an icon in the left ribbon.
@@ -44,12 +59,32 @@ export default class Khoj extends Plugin {
         // Add a settings tab so the user can configure khoj
         this.addSettingTab(new KhojSettingTab(this.app, this));
 
-        // Add scheduled job to update index every 60 minutes
+        // Start the sync timer
+        this.startSyncTimer();
+    }
+
+    // Method to start the sync timer
+    private startSyncTimer() {
+        // Clean up the old timer if it exists
+        if (this.indexingTimer) {
+            clearInterval(this.indexingTimer);
+        }
+
+        // Start a new timer with the configured interval
         this.indexingTimer = setInterval(async () => {
             if (this.settings.autoConfigure) {
-                this.settings.lastSync = await updateContentIndex(this.app.vault, this.settings, this.settings.lastSync);
+                this.settings.lastSync = await updateContentIndex(
+                    this.app.vault,
+                    this.settings,
+                    this.settings.lastSync
+                );
             }
-        }, 60 * 60 * 1000);
+        }, this.settings.syncInterval * 60 * 1000); // Convert minutes to milliseconds
+    }
+
+    // Public method to restart the timer (called from settings)
+    public restartSyncTimer() {
+        this.startSyncTimer();
     }
 
     async loadSettings() {
@@ -62,7 +97,7 @@ export default class Khoj extends Plugin {
     }
 
     async saveSettings() {
-        this.saveData(this.settings);
+        await this.saveData(this.settings);
     }
 
     async onunload() {