Merge branch 'develop' into feature/BCI-2981-logpoller-resolve-reorg

.changeset/brown-penguins-grin.md

@@ -0,0 +1,5 @@
+---
+"chainlink": patch
+---
+
+Fix in memory data source cache changes/bug that only allowed pipeline results where none of the data sources failed. #bugfix

.changeset/chilled-bikes-unite.md

@@ -0,0 +1,5 @@
+---
+"chainlink": minor
+---
+
+#internal Add script to create test database user and update docs

.changeset/dull-dingos-remember.md

@@ -0,0 +1,5 @@
+---
+"chainlink": patch
+---
+
+use safe lib for approve #bugfix

.changeset/four-shoes-trade.md

@@ -0,0 +1,5 @@
+---
+"chainlink": patch
+---
+
+Removed AppConfig from Evm config #internal

.changeset/fresh-moles-explode.md

@@ -0,0 +1,5 @@
+---
+"chainlink": minor
+---
+
+core/chains/evm/logpoller: Stricter finality checks in LogPoller, to be more robust during rpc failover events #updated

.changeset/fuzzy-pans-destroy.md

@@ -0,0 +1,5 @@
+---
+"chainlink": minor
+---
+
+Use sqlutil instead of pg.Opts/Q/Queryer #internal

.changeset/kind-deers-leave.md

@@ -0,0 +1,5 @@
+---
+"chainlink": patch
+---
+
+bump mockery in makefile #updated

.changeset/lovely-jeans-confess.md

@@ -0,0 +1,6 @@
+---
+"chainlink": minor
+---
+
+#nops : Enable configurable client error regexes for error classification
+#added : New toml configuration options for [EVM.NodePool.Errors] to pass regexes on NonceTooLow, NonceTooHigh, ReplacementTransactionUnderpriced, LimitReached, TransactionAlreadyInMempool, TerminallyUnderpriced, InsufficientEth, TxFeeExceedsCap, L2FeeTooLow, L2FeeTooHigh, L2Full, TransactionAlreadyMined, Fatal, and ServiceUnavailable.

.changeset/lucky-ghosts-give.md

@@ -0,0 +1,5 @@
+---
+"chainlink": patch
+---
+
+core/services/keystore: switch to sqlutil.DataStore #internal

.changeset/new-forks-grab.md

@@ -1,5 +1,5 @@
 ---
-"chainlink": removed
+"chainlink": patch
 ---
 
 Drop unused queryTimeout config from TXM strategy #internal

.changeset/olive-knives-happen.md

@@ -0,0 +1,5 @@
+---
+"chainlink": patch
+---
+
+#internal Generic Plugin `onchainSigningStrategy` support

.changeset/plenty-wombats-grab.md

@@ -0,0 +1,5 @@
+---
+"chainlink": patch
+---
+
+#wip Regenerate Keystone wrappers

.changeset/pretty-kangaroos-tell.md

@@ -0,0 +1,5 @@
+---
+"chainlink": patch
+---
+
+Add check for valid semvar value for changeset file #internal

.changeset/smooth-comics-love.md

@@ -0,0 +1,5 @@
+---
+"chainlink": patch
+---
+
+update keystone gethwrapper with remove operator function #internal

.changeset/sour-needles-collect.md

@@ -0,0 +1,5 @@
+---
+"chainlink": patch
+---
+
+cleanup set #bugfix

.changeset/stale-terms-march.md

@@ -0,0 +1,7 @@
+---
+"chainlink": patch
+---
+
+Bump libocr => fd3cab206b2ca3b7ff207996b95673b2d6303ec4
+
+#internal

.changeset/tasty-lions-rhyme.md

@@ -0,0 +1,5 @@
+---
+"chainlink": patch
+---
+
+add getters #internal

.changeset/tricky-bats-exist.md

@@ -0,0 +1,5 @@
+---
+"chainlink": patch
+---
+
+minor fixes #bugfix

.changeset/two-countries-lay.md

@@ -0,0 +1,5 @@
+---
+"chainlink": patch
+---
+
+update keystone gethwrapper #internal

.changeset/warm-impalas-return.md

@@ -0,0 +1,5 @@
+---
+"chainlink": patch
+---
+
+#added JuelsPerFeeCoinCache is enabled by default for OCR2 jobs, added `Disable` field under [pluginConfig.JuelsPerFeeCoinCache] tag to disable this feature (e.g. Disable=true)

.changeset/witty-icons-rhyme.md

@@ -0,0 +1,5 @@
+---
+"chainlink": patch
+---
+
+offchain settlement fix #bugfix

.github/actions/build-chainlink-image/action.yml

@@ -33,14 +33,15 @@ runs:
         AWS_ROLE_TO_ASSUME: ${{ inputs.AWS_ROLE_TO_ASSUME }}
     - name: Build Image
       if: steps.check-image.outputs.exists != 'true'
-      uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11
+      uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@519851800779323566b7b7c22cc21bff95dbb639 # v2.3.14
       with:
         cl_repo: smartcontractkit/chainlink
         cl_ref: ${{ inputs.git_commit_sha }}
         cl_dockerfile: ${{ inputs.dockerfile }}
         push_tag: ${{ env.CHAINLINK_IMAGE }}:${{ inputs.git_commit_sha }}${{ inputs.tag_suffix }}
         QA_AWS_REGION: ${{ inputs.AWS_REGION }}
         QA_AWS_ROLE_TO_ASSUME: ${{ inputs.AWS_ROLE_TO_ASSUME }}
+        GO_COVER_FLAG: true
         dep_evm_sha: ${{ inputs.dep_evm_sha }}
     - name: Print Chainlink Image Built
       shell: sh

.github/actions/setup-create-base64-upgrade-config/action.yml

@@ -17,6 +17,23 @@ inputs:
     default: "public.ecr.aws/chainlink/chainlink"
   upgradeVersion:
     description: The git commit sha to use for the image tag
+  runId:
+    description: The run id
+  testLogCollect:
+    description: Whether to always collect logs, even for passing tests
+    default: "false"
+  lokiEndpoint:
+    description: Loki push endpoint
+  lokiTenantId:
+    description: Loki tenant id
+  lokiBasicAuth:
+    description: Loki basic auth
+  logstreamLogTargets:
+    description: Where to send logs (e.g. file, loki)
+  grafanaUrl:
+    description: Grafana URL
+  grafanaDashboardUrl:
+    description: Grafana dashboard URL
 
 runs:
   using: composite
@@ -31,6 +48,14 @@ runs:
         CHAINLINK_POSTGRES_VERSION: ${{ inputs.chainlinkPostgresVersion }}
         UPGRADE_IMAGE: ${{ inputs.upgradeImage }}
         UPGRADE_VERSION: ${{ inputs.upgradeVersion }}
+        RUN_ID: ${{ inputs.runId }}
+        TEST_LOG_COLLECT: ${{ inputs.testLogCollect }}
+        LOKI_ENDPOINT: ${{ inputs.lokiEndpoint }}
+        LOKI_TENANT_ID: ${{ inputs.lokiTenantId }}
+        LOKI_BASIC_AUTH: ${{ inputs.lokiBasicAuth }}
+        LOGSTREAM_LOG_TARGETS: ${{ inputs.logstreamLogTargets }}
+        GRAFANA_URL: ${{ inputs.grafanaUrl }}
+        GRAFANA_DASHBOARD_URL: ${{ inputs.grafanaDashboardUrl }}
       run: |
         function convert_to_toml_array() {
           local IFS=','
@@ -47,6 +72,14 @@ runs:
 
         selected_networks=$(convert_to_toml_array "$SELECTED_NETWORKS")
 
+        if [ -n "$TEST_LOG_COLLECT" ]; then
+          test_log_collect=true
+        else
+          test_log_collect=false
+        fi
+
+        log_targets=$(convert_to_toml_array "$LOGSTREAM_LOG_TARGETS")
+
         cat << EOF > config.toml
         [Network]
         selected_networks=$selected_networks
@@ -59,6 +92,22 @@ runs:
         [ChainlinkUpgradeImage]
         image="$UPGRADE_IMAGE"
         version="$UPGRADE_VERSION"
+
+        [Logging]
+        test_log_collect=$test_log_collect
+        run_id="$RUN_ID"
+
+        [Logging.LogStream]
+        log_targets=$log_targets
+
+        [Logging.Loki]
+        tenant_id="$LOKI_TENANT_ID"
+        endpoint="$LOKI_ENDPOINT"
+        basic_auth_secret="$LOKI_BASIC_AUTH"
+
+        [Logging.Grafana]
+        base_url="$GRAFANA_URL"
+        dashboard_url="$GRAFANA_DASHBOARD_URL"
         EOF
 
         BASE64_CONFIG_OVERRIDE=$(cat config.toml | base64 -w 0)

.github/scripts/check-changeset-tags.sh

@@ -29,6 +29,14 @@ if [[ ! -f "$CHANGESET_FILE_PATH" ]]; then
   exit 1
 fi
 
+changeset_content=$(sed -n '/^---$/,/^---$/{ /^---$/!p; }' $CHANGESET_FILE_PATH)
+semvar_value=$(echo "$changeset_content" | awk -F": " '/"chainlink"/ {print $2}')
+
+if [[ "$semvar_value" != "major" && "$semvar_value" != "minor" && "$semvar_value" != "patch" ]]; then
+  echo "Invalid changeset semvar value for 'chainlink'. Must be 'major', 'minor', or 'patch'."
+  exit 1
+fi
+
 while IFS= read -r line; do
   for tag in "${tags_list[@]}"; do
     if [[ "$line" == *"$tag"* ]]; then

.github/workflows/automation-nightly-tests.yml

@@ -95,6 +95,14 @@ jobs:
           chainlinkVersion: "latest"
           upgradeImage: ${{ env.CHAINLINK_IMAGE }}
           upgradeVersion: ${{ github.sha }}
+          runId: ${{ github.run_id }}
+          testLogCollect: "true"
+          lokiEndpoint: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
+          lokiTenantId: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+          lokiBasicAuth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+          logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
+          grafanaUrl: ${{ vars.GRAFANA_URL }}
+          grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
       - name: Run Tests
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11
         env:

.github/workflows/automation-ondemand-tests.yml

@@ -226,7 +226,15 @@ jobs:
           UPGRADE_IMAGE: ${{ steps.determine-build.outputs.upgrade_image }}
           PYROSCOPE_SERVER: ${{ matrix.tests.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
           PYROSCOPE_ENVIRONMENT: ${{ matrix.tests.pyroscope_env }}
-          PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}          
+          PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
+          RUN_ID: ${{ github.run_id }}
+          TEST_LOG_COLLECT: "true"
+          LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
+          LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+          LOKI_BASIC_AUTH:  ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+          LOGSTREAM_LOG_TARGETS: ${{ vars.LOGSTREAM_LOG_TARGETS }}
+          GRAFANA_URL: ${{ vars.GRAFANA_URL }}
+          GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
         run: |
           echo ::add-mask::$UPGRADE_IMAGE
           echo ::add-mask::$OLD_IMAGE
@@ -242,6 +250,14 @@ jobs:
             pyroscope_enabled=false
           fi          
 
+          if [ -n "$TEST_LOG_COLLECT" ]; then
+          test_log_collect=true
+          else
+          test_log_collect=false
+          fi
+
+          log_targets=$(convert_to_toml_array "$LOGSTREAM_LOG_TARGETS")
+
           cat << EOF > config.toml
           [Network]
           selected_networks=$selected_networks
@@ -254,6 +270,22 @@ jobs:
           image="$UPGRADE_IMAGE"
           version="$UPGRADE_VERSION"
 
+          [Logging]
+          test_log_collect=$test_log_collect
+          run_id="$RUN_ID"
+
+          [Logging.LogStream]
+          log_targets=$log_targets
+
+          [Logging.Loki]
+          tenant_id="$LOKI_TENANT_ID"
+          endpoint="$LOKI_ENDPOINT"
+          basic_auth_secret="$LOKI_BASIC_AUTH"
+
+          [Logging.Grafana]
+          base_url="$GRAFANA_URL"
+          dashboard_url="$GRAFANA_DASHBOARD_URL"
+
           [Pyroscope]
           enabled=$pyroscope_enabled
           server_url="$PYROSCOPE_SERVER"

.github/workflows/build-publish-develop.yml

@@ -56,44 +56,6 @@ jobs:
           dockerhub_password: ${{ secrets.DOCKERHUB_READONLY_PASSWORD }}
           git-commit-sha: ${{ steps.git-ref.outputs.checked-out || github.sha }}
 
-      - name: Setup GAP
-        # Don't run for plugins.
-        if: matrix.image.name == ''
-        uses: smartcontractkit/.github/actions/setup-gap@1bc7ce34fa81fffcb4a6eb0e4e12e59d94d0fc8f # [email protected]
-        with:
-          aws-region: ${{ secrets.AWS_REGION }}
-          aws-role-arn: ${{ secrets.AWS_OIDC_CRIB_ROLE_ARN_SAND }}
-          api-gateway-host: ${{ secrets.AWS_API_GW_HOST_K8S_SAND }}
-          use-k8s: "true"
-          k8s-cluster-name: ${{ secrets.AWS_EKS_CLUSTER_NAME_SAND }}
-          use-private-ecr-registry: true
-          ecr-private-registry: ${{ secrets.AWS_ACCOUNT_ID_PROD }}
-          metrics-job-name: push-chainlink-develop ${{ matrix.image.name }}
-          gc-host: ${{ secrets.GRAFANA_INTERNAL_HOST }}
-          gc-basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
-          gc-org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-
-      # A mutable image tag is used for these CRIBs and it was just built/published
-      # from this workflow. The deployment has an `imagePullPolicy: Always` set, so
-      # we need to restart the deployments to pick up the new image.
-      - name: Restart K8s Deployments for CRIBs
-        # Don't run for plugins.
-        if: matrix.image.name == ''
-        shell: bash
-        run: |
-          set -euo pipefail
-          # Removes the "smartcontractkit/" (org name) prefix.
-          REPO_NAME_ONLY="${GITHUB_REPOSITORY##*/}"
-          K8S_NAMESPACE="crib-${REPO_NAME_ONLY}-develop"
-
-          deployment_node_names=$(kubectl --namespace "${K8S_NAMESPACE}" \
-              get deployments \
-              -l "app=${K8S_NAMESPACE}" \
-              -o custom-columns=:metadata.name --no-headers)
-
-          echo "Restarting deployments: $deployment_node_names"
-          kubectl rollout restart deployment $deployment_node_names --namespace "${K8S_NAMESPACE}"
-
       - name: Collect Metrics
         if: always()
         id: collect-gha-metrics

.github/workflows/ci-core.yml

@@ -109,6 +109,7 @@ jobs:
         if: ${{ needs.filter.outputs.changes == 'true' }}
         uses: ./.github/actions/setup-go
       - name: Run short tests
+        if: ${{ needs.filter.outputs.changes == 'true' }}
         run: go test -short ./...
       - name: Setup Solana
         if: ${{ needs.filter.outputs.changes == 'true' }}
@@ -170,8 +171,16 @@ jobs:
         run: |
           cat output.txt | gotestloghelper -ci
       - name: Print Races
+        id: print-races
         if: ${{ failure() && matrix.type.cmd == 'go_core_race_tests' && needs.filter.outputs.changes == 'true' }}
-        run: find race.* | xargs cat
+        run: |
+          find race.* | xargs cat > race.txt
+          if [[ -s race.txt ]]; then
+            cat race.txt
+            echo "post_to_slack=true" >> $GITHUB_OUTPUT
+          else
+            echo "post_to_slack=false" >> $GITHUB_OUTPUT
+          fi
       - name: Print postgres logs
         if: ${{ always() &&  needs.filter.outputs.changes == 'true' }}
         run: docker compose logs postgres | tee ../../../postgres_logs.txt
@@ -188,7 +197,7 @@ jobs:
             ./coverage.txt
             ./postgres_logs.txt
       - name: Notify Slack
-        if: ${{ failure() && matrix.type.cmd == 'go_core_race_tests' && (github.event_name == 'merge_group' || github.event.branch == 'develop') &&  needs.filter.outputs.changes == 'true' }}
+        if: ${{ failure() && steps.print-races.outputs.post_to_slack == 'true' && matrix.type.cmd == 'go_core_race_tests' && (github.event_name == 'merge_group' || github.event.branch == 'develop') &&  needs.filter.outputs.changes == 'true' }}
         uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
         env:
           SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}