Merge from main

.devcontainer/devcontainer.json

@@ -12,7 +12,7 @@
 
 	// The optional 'workspaceFolder' property is the path VS Code should open by default when
 	// connected. This is typically a file mount in .devcontainer/docker-compose.yml
-	"workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
+	"workspaceFolder": "/workspaces/langchain",
 
 	// Prevent the container from shutting down
 	"overrideCommand": true

.devcontainer/docker-compose.yaml

@@ -6,7 +6,7 @@ services:
       context: ..
     volumes:
    # Update this to wherever you want VS Code to mount the folder of your project
-      - ..:/workspaces:cached
+      - ..:/workspaces/langchain:cached
     networks:
       - langchain-network 
   #   environment:

.github/scripts/check_diff.py

@@ -19,6 +19,7 @@
         "test": set(),
         "extended-test": set(),
     }
+    docs_edited = False
 
     if len(files) == 300:
         # max diff length is 300 files - there are likely files missing
@@ -47,6 +48,17 @@
                     found = True
                 if found:
                     dirs_to_run["extended-test"].add(dir_)
+        elif file.startswith("libs/standard-tests"):
+            # TODO: update to include all packages that rely on standard-tests (all partner packages)
+            # note: won't run on external repo partners
+            dirs_to_run["lint"].add("libs/standard-tests")
+            dirs_to_run["test"].add("libs/partners/mistralai")
+            dirs_to_run["test"].add("libs/partners/openai")
+            dirs_to_run["test"].add("libs/partners/anthropic")
+            dirs_to_run["test"].add("libs/partners/ai21")
+            dirs_to_run["test"].add("libs/partners/fireworks")
+            dirs_to_run["test"].add("libs/partners/groq")
+
         elif file.startswith("libs/cli"):
             # todo: add cli makefile
             pass
@@ -65,6 +77,8 @@
                 "an update for this new library!"
             )
         elif any(file.startswith(p) for p in ["docs/", "templates/", "cookbook/"]):
+            if file.startswith("docs/"):
+                docs_edited = True
             dirs_to_run["lint"].add(".")
 
     outputs = {
@@ -73,6 +87,7 @@
         ),
         "dirs-to-test": list(dirs_to_run["test"] | dirs_to_run["extended-test"]),
         "dirs-to-extended-test": list(dirs_to_run["extended-test"]),
+        "docs-edited": "true" if docs_edited else "",
     }
     for key, value in outputs.items():
         json_output = json.dumps(value)

.github/scripts/get_min_versions.py

@@ -4,25 +4,33 @@
 from packaging.version import parse as parse_version
 import re
 
-MIN_VERSION_LIBS = ["langchain-core", "langchain-community", "langchain", "langchain-text-splitters"]
+MIN_VERSION_LIBS = [
+    "langchain-core",
+    "langchain-community",
+    "langchain",
+    "langchain-text-splitters",
+]
 
 
 def get_min_version(version: str) -> str:
+    # base regex for x.x.x with cases for rc/post/etc
+    # valid strings: https://peps.python.org/pep-0440/#public-version-identifiers
+    vstring = r"\d+(?:\.\d+){0,2}(?:(?:a|b|rc|\.post|\.dev)\d+)?"
     # case ^x.x.x
-    _match = re.match(r"^\^(\d+(?:\.\d+){0,2})$", version)
+    _match = re.match(f"^\\^({vstring})$", version)
     if _match:
         return _match.group(1)
 
     # case >=x.x.x,<y.y.y
-    _match = re.match(r"^>=(\d+(?:\.\d+){0,2}),<(\d+(?:\.\d+){0,2})$", version)
+    _match = re.match(f"^>=({vstring}),<({vstring})$", version)
     if _match:
         _min = _match.group(1)
         _max = _match.group(2)
         assert parse_version(_min) < parse_version(_max)
         return _min
 
     # case x.x.x
-    _match = re.match(r"^(\d+(?:\.\d+){0,2})$", version)
+    _match = re.match(f"^({vstring})$", version)
     if _match:
         return _match.group(1)
 
@@ -47,6 +55,9 @@ def get_min_version_from_toml(toml_path: str):
             # Get the version string
             version_string = dependencies[lib]
 
+            if isinstance(version_string, dict):
+                version_string = version_string["version"]
+
             # Use parse_version to get the minimum supported version from version_string
             min_version = get_min_version(version_string)
 
@@ -56,12 +67,13 @@ def get_min_version_from_toml(toml_path: str):
     return min_versions
 
 
-# Get the TOML file path from the command line argument
-toml_file = sys.argv[1]
+if __name__ == "__main__":
+    # Get the TOML file path from the command line argument
+    toml_file = sys.argv[1]
 
-# Call the function to get the minimum versions
-min_versions = get_min_version_from_toml(toml_file)
+    # Call the function to get the minimum versions
+    min_versions = get_min_version_from_toml(toml_file)
 
-print(
-    " ".join([f"{lib}=={version}" for lib, version in min_versions.items()])
-)  # noqa: T201
+    print(
+        " ".join([f"{lib}=={version}" for lib, version in min_versions.items()])
+    )  # noqa: T201

.github/workflows/_integration_test.yml

@@ -58,6 +58,7 @@ jobs:
           MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}
           TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
           NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
           GOOGLE_SEARCH_API_KEY: ${{ secrets.GOOGLE_SEARCH_API_KEY }}
           GOOGLE_CSE_ID: ${{ secrets.GOOGLE_CSE_ID }}
@@ -76,6 +77,8 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # for airbyte
           MONGODB_ATLAS_URI: ${{ secrets.MONGODB_ATLAS_URI }}
           VOYAGE_API_KEY: ${{ secrets.VOYAGE_API_KEY }}
+          COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
+          UPSTAGE_API_KEY: ${{ secrets.UPSTAGE_API_KEY }}
         run: |
           make integration_tests
 

.github/workflows/_release.yml

@@ -13,14 +13,19 @@ on:
         required: true
         type: string
         default: 'libs/langchain'
+      dangerous-nonmaster-release:
+        required: false
+        type: boolean
+        default: false
+        description: "Release from a non-master branch (danger!)"
 
 env:
   PYTHON_VERSION: "3.11"
   POETRY_VERSION: "1.7.1"
 
 jobs:
   build:
-    if: github.ref == 'refs/heads/master'
+    if: github.ref == 'refs/heads/master' || inputs.dangerous-nonmaster-release
     environment: Scheduled testing
     runs-on: ubuntu-latest
 
@@ -75,6 +80,7 @@ jobs:
       ./.github/workflows/_test_release.yml
     with:
       working-directory: ${{ inputs.working-directory }}
+      dangerous-nonmaster-release: ${{ inputs.dangerous-nonmaster-release }}
     secrets: inherit
 
   pre-release-checks:
@@ -112,7 +118,7 @@ jobs:
           PKG_NAME: ${{ needs.build.outputs.pkg-name }}
           VERSION: ${{ needs.build.outputs.version }}
         # Here we use:
-        # - The default regular PyPI index as the *primary* index, meaning 
+        # - The default regular PyPI index as the *primary* index, meaning
         #   that it takes priority (https://pypi.org/simple)
         # - The test PyPI index as an extra index, so that any dependencies that
         #   are not found on test PyPI can be resolved and installed anyway.
@@ -157,6 +163,24 @@ jobs:
         run: make tests
         working-directory: ${{ inputs.working-directory }}
 
+      - name: Get minimum versions
+        working-directory: ${{ inputs.working-directory }}
+        id: min-version
+        run: |
+          poetry run pip install packaging
+          min_versions="$(poetry run python $GITHUB_WORKSPACE/.github/scripts/get_min_versions.py pyproject.toml)"
+          echo "min-versions=$min_versions" >> "$GITHUB_OUTPUT"
+          echo "min-versions=$min_versions"
+
+      - name: Run unit tests with minimum dependency versions
+        if: ${{ steps.min-version.outputs.min-versions != '' }}
+        env:
+          MIN_VERSIONS: ${{ steps.min-version.outputs.min-versions }}
+        run: |
+          poetry run pip install --force-reinstall $MIN_VERSIONS
+          make tests
+        working-directory: ${{ inputs.working-directory }}
+
       - name: 'Authenticate to Google Cloud'
         id: 'auth'
         uses: google-github-actions/auth@v2
@@ -197,27 +221,10 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # for airbyte
           MONGODB_ATLAS_URI: ${{ secrets.MONGODB_ATLAS_URI }}
           VOYAGE_API_KEY: ${{ secrets.VOYAGE_API_KEY }}
+          UPSTAGE_API_KEY: ${{ secrets.UPSTAGE_API_KEY }}
         run: make integration_tests
         working-directory: ${{ inputs.working-directory }}
 
-      - name: Get minimum versions
-        working-directory: ${{ inputs.working-directory }}
-        id: min-version
-        run: |
-          poetry run pip install packaging
-          min_versions="$(poetry run python $GITHUB_WORKSPACE/.github/scripts/get_min_versions.py pyproject.toml)"
-          echo "min-versions=$min_versions" >> "$GITHUB_OUTPUT"
-          echo "min-versions=$min_versions"
-
-      - name: Run unit tests with minimum dependency versions
-        if: ${{ steps.min-version.outputs.min-versions != '' }}
-        env:
-          MIN_VERSIONS: ${{ steps.min-version.outputs.min-versions }}
-        run: |
-          poetry run pip install $MIN_VERSIONS
-          make tests
-        working-directory: ${{ inputs.working-directory }}
-
   publish:
     needs:
       - build
@@ -290,14 +297,13 @@ jobs:
         with:
           name: dist
           path: ${{ inputs.working-directory }}/dist/
-
-      - name: Create Release
+          
+      - name: Create Tag
         uses: ncipollo/release-action@v1
-        if: ${{ inputs.working-directory == 'libs/langchain' }}
         with:
           artifacts: "dist/*"
           token: ${{ secrets.GITHUB_TOKEN }}
-          draft: false
-          generateReleaseNotes: true
-          tag: v${{ needs.build.outputs.version }}
-          commit: master
+          generateReleaseNotes: false
+          tag: ${{needs.build.outputs.pkg-name}}==${{ needs.build.outputs.version }}
+          body: "# Release ${{needs.build.outputs.pkg-name}}==${{ needs.build.outputs.version }}\n\nPackage-specific release note generation coming soon."
+          commit: ${{ github.sha }}

.github/workflows/_test_doc_imports.yml

@@ -0,0 +1,50 @@
+name: test_doc_imports
+
+on:
+  workflow_call:
+
+env:
+  POETRY_VERSION: "1.7.1"
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version:
+          - "3.11"
+    name: "check doc imports #${{ matrix.python-version }}"
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
+        uses: "./.github/actions/poetry_setup"
+        with:
+          python-version: ${{ matrix.python-version }}
+          poetry-version: ${{ env.POETRY_VERSION }}
+          cache-key: core
+
+      - name: Install dependencies
+        shell: bash
+        run: poetry install --with test
+
+      - name: Install langchain editable
+        run: |
+          poetry run pip install -e libs/core libs/langchain libs/community libs/experimental
+
+      - name: Check doc imports
+        shell: bash
+        run: |
+          poetry run python docs/scripts/check_imports.py
+
+      - name: Ensure the test did not create any additional files
+        shell: bash
+        run: |
+          set -eu
+
+          STATUS="$(git status)"
+          echo "$STATUS"
+
+          # grep will exit non-zero if the target message isn't found,
+          # and `set -e` above will cause the step to fail.
+          echo "$STATUS" | grep 'nothing to commit, working tree clean'

.github/workflows/_test_release.yml

@@ -7,14 +7,19 @@ on:
         required: true
         type: string
         description: "From which folder this pipeline executes"
+      dangerous-nonmaster-release:
+        required: false
+        type: boolean
+        default: false
+        description: "Release from a non-master branch (danger!)"
 
 env:
   POETRY_VERSION: "1.7.1"
   PYTHON_VERSION: "3.10"
 
 jobs:
   build:
-    if: github.ref == 'refs/heads/master'
+    if: github.ref == 'refs/heads/master' || inputs.dangerous-nonmaster-release
     runs-on: ubuntu-latest
 
     outputs:

.github/workflows/check_diffs.yml

@@ -36,6 +36,7 @@ jobs:
       dirs-to-lint: ${{ steps.set-matrix.outputs.dirs-to-lint }}
       dirs-to-test: ${{ steps.set-matrix.outputs.dirs-to-test }}
       dirs-to-extended-test: ${{ steps.set-matrix.outputs.dirs-to-extended-test }}
+      docs-edited: ${{ steps.set-matrix.outputs.docs-edited }}
   lint:
     name: cd ${{ matrix.working-directory }}
     needs: [ build ]
@@ -60,6 +61,12 @@ jobs:
       working-directory: ${{ matrix.working-directory }}
     secrets: inherit
 
+  test-doc-imports:
+    needs: [ build ]
+    if: ${{ needs.build.outputs.dirs-to-test != '[]' || needs.build.outputs.docs-edited }}
+    uses: ./.github/workflows/_test_doc_imports.yml
+    secrets: inherit
+
   compile-integration-tests:
     name: cd ${{ matrix.working-directory }}
     needs: [ build ]
@@ -116,8 +123,7 @@ jobs:
         shell: bash
         run: |
           echo "Running extended tests, installing dependencies with poetry..."
-          poetry install --with test
-          poetry run pip install -r extended_requirements.txt
+          poetry install -E extended_testing --with test
 
       - name: Run extended tests
         run: make extended_tests
@@ -135,7 +141,7 @@ jobs:
           echo "$STATUS" | grep 'nothing to commit, working tree clean'
   ci_success:
     name: "CI Success"
-    needs: [build, lint, test, compile-integration-tests, dependencies, extended-tests]
+    needs: [build, lint, test, compile-integration-tests, dependencies, extended-tests, test-doc-imports]
     if: |
       always()
     runs-on: ubuntu-latest