diff --git a/.changes/header.tpl.md b/.changes/header.tpl.md index 8bee7b4a..4dbd7204 100644 --- a/.changes/header.tpl.md +++ b/.changes/header.tpl.md @@ -1,4 +1,4 @@ -# dbt Core Changelog +# dbt-common Changelog - This file provides a full account of all changes to `dbt-common` - Changes are listed under the (pre)release in which they first appear. Subsequent releases include changes from previous releases. diff --git a/.changes/unreleased/Dependencies-20240425-112704.yaml b/.changes/unreleased/Dependencies-20240425-112704.yaml deleted file mode 100644 index 1805a413..00000000 --- a/.changes/unreleased/Dependencies-20240425-112704.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Dependencies -body: Pin agate `>=1.7.0,<1.10` -time: 2024-04-25T11:27:04.91767-05:00 -custom: - Author: emmyoop - PR: "119" diff --git a/.changes/unreleased/Dependencies-20240501-140032.yaml b/.changes/unreleased/Dependencies-20240501-140032.yaml deleted file mode 100644 index bc21f82f..00000000 --- a/.changes/unreleased/Dependencies-20240501-140032.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Dependencies -body: update pathspec upper version limitation -time: 2024-05-01T14:00:32.257513+10:00 -custom: - Author: jerry153fish - PR: "120" diff --git a/.changes/unreleased/Features-20240131-152923.yaml b/.changes/unreleased/Features-20240131-152923.yaml deleted file mode 100644 index fc649269..00000000 --- a/.changes/unreleased/Features-20240131-152923.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Cache static objects between tests to accelerate integration runs. -time: 2024-01-31T15:29:23.951857-05:00 -custom: - Author: peterallenwebb - Issue: "55" diff --git a/.changes/unreleased/Features-20240201-101851.yaml b/.changes/unreleased/Features-20240201-101851.yaml deleted file mode 100644 index fdd88f5c..00000000 --- a/.changes/unreleased/Features-20240201-101851.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Add dataclass utility `Mergeable` -time: 2024-02-01T10:18:51.474231-08:00 -custom: - Author: QMalcolm - Issue: "58" diff --git a/.changes/unreleased/Features-20240201-104437.yaml b/.changes/unreleased/Features-20240201-104437.yaml deleted file mode 100644 index ff251faf..00000000 --- a/.changes/unreleased/Features-20240201-104437.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Added a new InvocationContext class for application context management. -time: 2024-02-01T10:44:37.161298-05:00 -custom: - Author: peterallenwebb - Issue: "57" diff --git a/.changes/unreleased/Features-20240410-090810.yaml b/.changes/unreleased/Features-20240410-090810.yaml deleted file mode 100644 index e9473165..00000000 --- a/.changes/unreleased/Features-20240410-090810.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Support environment variables which dbt can see but user code cannot. -time: 2024-04-10T09:08:10.068999-04:00 -custom: - Author: peterallenwebb - Issue: "103" diff --git a/.changes/unreleased/Features-20240412-105245.yaml b/.changes/unreleased/Features-20240412-105245.yaml deleted file mode 100644 index 2712c277..00000000 --- a/.changes/unreleased/Features-20240412-105245.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Add `error_tag` util to the `ui` module -time: 2024-04-12T10:52:45.887334-07:00 -custom: - Author: QMalcolm - Issue: "107" diff --git a/.changes/unreleased/Features-20240419-232030.yaml b/.changes/unreleased/Features-20240419-232030.yaml deleted file mode 100644 index bd4e621c..00000000 --- a/.changes/unreleased/Features-20240419-232030.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Add ability to silence warnings via `WarnErrorOptions` -time: 2024-04-19T23:20:30.014054-07:00 -custom: - Author: QMalcolm - Issue: "111" diff --git a/.changes/unreleased/Features-20240506-170058.yaml b/.changes/unreleased/Features-20240506-170058.yaml deleted file mode 100644 index d378e228..00000000 --- a/.changes/unreleased/Features-20240506-170058.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Enable serialization contexts -time: 2024-05-06T17:00:58.272104-04:00 -custom: - Author: gshank - Issue: "126" diff --git a/.changes/unreleased/Features-20240514-162052.yaml b/.changes/unreleased/Features-20240514-162052.yaml deleted file mode 100644 index 1503c479..00000000 --- a/.changes/unreleased/Features-20240514-162052.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Support adding callbacks to the event manager -time: 2024-05-14T16:20:52.120336-07:00 -custom: - Author: QMalcolm - Issue: "131" diff --git a/.changes/unreleased/Features-20240515-161106.yaml b/.changes/unreleased/Features-20240515-161106.yaml deleted file mode 100644 index ef4b7afb..00000000 --- a/.changes/unreleased/Features-20240515-161106.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Add a print event to support firing event to console in --quiet mode -time: 2024-05-15T16:11:06.815526-07:00 -custom: - Author: ChenyuLInx - Issue: "8756" diff --git a/.changes/unreleased/Features-20240716-102457.yaml b/.changes/unreleased/Features-20240716-102457.yaml deleted file mode 100644 index 096b4259..00000000 --- a/.changes/unreleased/Features-20240716-102457.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Add to and to_columns to ColumnLevelConstraint and ModelLevelConstraint contracts -time: 2024-07-16T10:24:57.11251-04:00 -custom: - Author: michelleark - Issue: "168" diff --git a/.changes/unreleased/Fixes-20240206-160231.yaml b/.changes/unreleased/Fixes-20240206-160231.yaml deleted file mode 100644 index a00205b3..00000000 --- a/.changes/unreleased/Fixes-20240206-160231.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Make invocation contexts more reliable in testing scenarios. -time: 2024-02-06T16:02:31.81842-05:00 -custom: - Author: peterallenwebb - Issue: "52" diff --git a/.changes/unreleased/Fixes-20240221-200204.yaml b/.changes/unreleased/Fixes-20240221-200204.yaml deleted file mode 100644 index 8df4eb14..00000000 --- a/.changes/unreleased/Fixes-20240221-200204.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Fix the unset_contextvars function in events/contextvars.py -time: 2024-02-21T20:02:04.341681-05:00 -custom: - Author: gshank - Issue: "82" diff --git a/.changes/unreleased/Fixes-20240715-205355.yaml b/.changes/unreleased/Fixes-20240715-205355.yaml deleted file mode 100644 index 780a6ad9..00000000 --- a/.changes/unreleased/Fixes-20240715-205355.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Fix case-insensitive env vars for Windows -time: 2024-07-15T20:53:55.946355+01:00 -custom: - Author: peterallenwebb aranke - Issue: "166" diff --git a/.changes/unreleased/Under the Hood-20240122-163546.yaml b/.changes/unreleased/Under the Hood-20240122-163546.yaml deleted file mode 100644 index 0d32e2e3..00000000 --- a/.changes/unreleased/Under the Hood-20240122-163546.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Clean up macro contexts. -time: 2024-01-22T16:35:46.907999-05:00 -custom: - Author: peterallenwebb - Issue: "35" diff --git a/.changes/unreleased/Under the Hood-20240123-161107.yaml b/.changes/unreleased/Under the Hood-20240123-161107.yaml deleted file mode 100644 index 68a83ab4..00000000 --- a/.changes/unreleased/Under the Hood-20240123-161107.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Inject TagIterator into BlockIterator for greater flexibility. -time: 2024-01-23T16:11:07.24321-05:00 -custom: - Author: peterallenwebb - Issue: "38" diff --git a/.changes/unreleased/Under the Hood-20240123-194242.yaml b/.changes/unreleased/Under the Hood-20240123-194242.yaml deleted file mode 100644 index a0cb7431..00000000 --- a/.changes/unreleased/Under the Hood-20240123-194242.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Change metadata_vars \`if not\` to \`if ... is None\` -time: 2024-01-23T19:42:42.95727089Z -custom: - Author: truls-p - Issue: "6073" diff --git a/.changes/unreleased/Under the Hood-20240206-081323.yaml b/.changes/unreleased/Under the Hood-20240206-081323.yaml deleted file mode 100644 index f5565b06..00000000 --- a/.changes/unreleased/Under the Hood-20240206-081323.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Remove `fire_event_if_test` method -time: 2024-02-06T08:13:23.722615-06:00 -custom: - Author: emmyoop - Issue: "66" diff --git a/.changes/unreleased/Under the Hood-20240208-100709.yaml b/.changes/unreleased/Under the Hood-20240208-100709.yaml deleted file mode 100644 index 2c824b39..00000000 --- a/.changes/unreleased/Under the Hood-20240208-100709.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Add Invocation Context Support to MultiThreadedExecutor -time: 2024-02-08T10:07:09.584747-05:00 -custom: - Author: peterallenwebb - Issue: "75" diff --git a/.changes/unreleased/Under the Hood-20240221-104344.yaml b/.changes/unreleased/Under the Hood-20240221-104344.yaml deleted file mode 100644 index bd1423a9..00000000 --- a/.changes/unreleased/Under the Hood-20240221-104344.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Restrict protobuf library to major version 4. -time: 2024-02-21T10:43:44.642458-05:00 -custom: - Author: peterallenwebb - Issue: "9566" diff --git a/.changes/unreleased/Under the Hood-20240227-145400.yaml b/.changes/unreleased/Under the Hood-20240227-145400.yaml deleted file mode 100644 index baf3b0fa..00000000 --- a/.changes/unreleased/Under the Hood-20240227-145400.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Implement record/replay mechanism -time: 2024-02-27T14:54:00.94815-05:00 -custom: - Author: peterallenwebb - Issue: "9689" diff --git a/.changes/unreleased/Under the Hood-20240314-161737.yaml b/.changes/unreleased/Under the Hood-20240314-161737.yaml deleted file mode 100644 index d5d58059..00000000 --- a/.changes/unreleased/Under the Hood-20240314-161737.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Stream JSON on read/write instead of holding it in memory -time: 2024-03-14T16:17:37.570328-04:00 -custom: - Author: peterallenwebb - Issue: "96" diff --git a/.changes/unreleased/Under the Hood-20240319-122548.yaml b/.changes/unreleased/Under the Hood-20240319-122548.yaml deleted file mode 100644 index 15640369..00000000 --- a/.changes/unreleased/Under the Hood-20240319-122548.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Improve startup performance by lazy-compiling mashumaro serialization functions -time: 2024-03-19T12:25:48.615365-04:00 -custom: - Author: peterallenwebb - Issue: "97" diff --git a/.changes/unreleased/Under the Hood-20240423-080648.yaml b/.changes/unreleased/Under the Hood-20240423-080648.yaml deleted file mode 100644 index 448630c8..00000000 --- a/.changes/unreleased/Under the Hood-20240423-080648.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Remove the final underscore from secret and private environment variable constants. -time: 2024-04-23T08:06:48.857514-05:00 -custom: - Author: emmyoop - Issue: "109" diff --git a/.changes/unreleased/Under the Hood-20240528-110518.yaml b/.changes/unreleased/Under the Hood-20240528-110518.yaml deleted file mode 100644 index 58b243c7..00000000 --- a/.changes/unreleased/Under the Hood-20240528-110518.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Allow dynamic selection of record types when recording. -time: 2024-05-28T11:05:18.290107-05:00 -custom: - Author: emmyoop - Issue: "140" diff --git a/.changes/unreleased/Under the Hood-20240529-143154.yaml b/.changes/unreleased/Under the Hood-20240529-143154.yaml deleted file mode 100644 index e9664254..00000000 --- a/.changes/unreleased/Under the Hood-20240529-143154.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Move StatsItem, StatsDict, TableMetadata to dbt-common -time: 2024-05-29T14:31:54.854468+01:00 -custom: - Author: aranke - Issue: "141" diff --git a/.changes/unreleased/Under the Hood-20240603-123631.yaml b/.changes/unreleased/Under the Hood-20240603-123631.yaml deleted file mode 100644 index 18a649df..00000000 --- a/.changes/unreleased/Under the Hood-20240603-123631.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Move CatalogKey, ColumnMetadata, ColumnMap, CatalogTable to dbt-common -time: 2024-06-03T12:36:31.542118+02:00 -custom: - Author: aranke - Issue: "147" diff --git a/.changes/unreleased/Under the Hood-20240617-204541.yaml b/.changes/unreleased/Under the Hood-20240617-204541.yaml deleted file mode 100644 index b4e33c36..00000000 --- a/.changes/unreleased/Under the Hood-20240617-204541.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Add support for basic diff of run recordings. -time: 2024-06-17T20:45:41.123374-05:00 -custom: - Author: emmyoop - Issue: "144" diff --git a/.changes/unreleased/Under the Hood-20240618-155025.yaml b/.changes/unreleased/Under the Hood-20240618-155025.yaml deleted file mode 100644 index b540d3d7..00000000 --- a/.changes/unreleased/Under the Hood-20240618-155025.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Deserialize Record objects on a just-in-time basis. -time: 2024-06-18T15:50:25.985387-04:00 -custom: - Author: peterallenwebb - Issue: "151" diff --git a/.changes/unreleased/Under the Hood-20240716-125753.yaml b/.changes/unreleased/Under the Hood-20240716-125753.yaml deleted file mode 100644 index 55b36cb3..00000000 --- a/.changes/unreleased/Under the Hood-20240716-125753.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Add record grouping mechanism to record/replay. -time: 2024-07-16T12:57:53.434099-04:00 -custom: - Author: peterallenwebb - Issue: "169" diff --git a/.changes/unreleased/Under the Hood-20240730-111902.yaml b/.changes/unreleased/Under the Hood-20240730-111902.yaml new file mode 100644 index 00000000..0c2e4cdf --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240730-111902.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Automate changelog and version bump during release +time: 2024-07-30T11:19:02.277008-04:00 +custom: + Author: peterallenwebb + Issue: "171" diff --git a/.github/workflows/release-prep.yml b/.github/workflows/release-prep.yml new file mode 100644 index 00000000..b3986069 --- /dev/null +++ b/.github/workflows/release-prep.yml @@ -0,0 +1,496 @@ +# **what?** +# This is a lightly modified copy/paste of the release-prep.yml workflow defined +# in the dbt-release repo. We should eventually generalize and reuse more of +# the logic. +# +# Perform the version bump, generate the changelog and run tests. +# +# Inputs: +# sha: The commit to attach to this release +# version_number: The release version number (i.e. 1.0.0b1, 1.2.3rc2, 1.0.0) +# target_branch: The branch that we will release from +# test_run: Test run (The temp branch will be used for release) +# +# +# Branching strategy: +# - During execution workflow execution the temp branch will be generated. +# - For normal runs the temp branch will be removed once changes were merged to target branch; +# - For test runs we will keep temp branch and will use it for release; +# Naming strategy: +# - For normal runs: prep-release/${{ inputs.version_number }}_$GITHUB_RUN_ID +# - For test runs: prep-release/test-run/${{ inputs.version_number }}_$GITHUB_RUN_ID + + +name: Version Bump and Changelog Generation + +on: + workflow_call: + inputs: + sha: + required: true + type: string + version_number: + required: true + type: string + target_branch: + required: true + type: string + test_run: + required: false + default: true + type: boolean + outputs: + final_sha: + description: The new commit that includes the changelog and version bump. + value: ${{ jobs.determine-release-sha.outputs.final_sha }} + changelog_path: + description: The path to the changelog for this version + value: ${{ jobs.audit-changelog.outputs.changelog_path }} + secrets: + FISHTOWN_BOT_PAT: + description: "Token to commit/merge changes into branches" + required: true + IT_TEAM_MEMBERSHIP: + description: "Token that can view org level teams" + required: true + +permissions: + contents: write + +defaults: + run: + shell: bash + +env: + PYTHON_TARGET_VERSION: 3.8 + NOTIFICATION_PREFIX: "[Release Preparation]" + +jobs: + log-inputs: + runs-on: ubuntu-latest + + steps: + - name: "[DEBUG] Print Variables" + run: | + # WORKFLOW INPUTS + echo The last commit sha in the release: ${{ inputs.sha }} + echo The release version number: ${{ inputs.version_number }} + echo The branch that we will release from: ${{ inputs.target_branch }} + echo Test run: ${{ inputs.test_run }} + # ENVIRONMENT VARIABLES + echo Python target version: ${{ env.PYTHON_TARGET_VERSION }} + echo Notification prefix: ${{ env.NOTIFICATION_PREFIX }} + + audit-changelog: + runs-on: ubuntu-latest + + outputs: + changelog_path: ${{ steps.set_path.outputs.changelog_path }} + exists: ${{ steps.set_existence.outputs.exists }} + base_version: ${{ steps.semver.outputs.base-version }} + prerelease: ${{ steps.semver.outputs.pre-release }} + is_prerelease: ${{ steps.semver.outputs.is-pre-release }} + + steps: + - name: "Checkout ${{ github.repository }} Commit ${{ inputs.sha }}" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.sha }} + + - name: "Audit Version And Parse Into Parts" + id: semver + uses: dbt-labs/actions/parse-semver@v1.1.0 + with: + version: ${{ inputs.version_number }} + + - name: "Set Changelog Path" + id: set_path + run: | + path=".changes/" + if [[ ${{ steps.semver.outputs.is-pre-release }} -eq 1 ]] + then + path+="${{ steps.semver.outputs.base-version }}-${{ steps.semver.outputs.pre-release }}.md" + else + path+="${{ steps.semver.outputs.base-version }}.md" + fi + # Send notification + echo "changelog_path=$path" >> $GITHUB_OUTPUT + title="Changelog path" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$changelog_path" + + - name: "Set Changelog Existence For Subsequent Jobs" + id: set_existence + run: | + does_exist=false + if test -f ${{ steps.set_path.outputs.changelog_path }} + then + does_exist=true + fi + echo "exists=$does_exist">> $GITHUB_OUTPUT + + - name: "[Notification] Set Changelog Existence For Subsequent Jobs" + run: | + title="Changelog exists" + if [[ ${{ steps.set_existence.outputs.exists }} == true ]] + then + message="Changelog file ${{ steps.set_path.outputs.changelog_path }} already exists" + else + message="Changelog file ${{ steps.set_path.outputs.changelog_path }} doesn't exist" + fi + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + - name: "[DEBUG] Print Outputs" + run: | + echo changelog_path: ${{ steps.set_path.outputs.changelog_path }} + echo exists: ${{ steps.set_existence.outputs.exists }} + echo base_version: ${{ steps.semver.outputs.base-version }} + echo prerelease: ${{ steps.semver.outputs.pre-release }} + echo is_prerelease: ${{ steps.semver.outputs.is-pre-release }} + + audit-version-in-code: + runs-on: ubuntu-latest + + outputs: + up_to_date: ${{ steps.version-check.outputs.up_to_date }} + + steps: + - name: "Checkout ${{ github.repository }} Commit ${{ inputs.sha }}" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.sha }} + + - name: "Check Current Version In Code" + id: version-check + run: | + is_updated=false + if grep -Fxq "version = \"${{ inputs.version_number }}\"" ./dbt_common/__about__.py + then + is_updated=true + fi + echo "up_to_date=$is_updated" >> $GITHUB_OUTPUT + + - name: "[Notification] Check Current Version In Code" + run: | + title="Version check" + if [[ ${{ steps.version-check.outputs.up_to_date }} == true ]] + then + message="The version in the codebase is equal to the provided version" + else + message="The version in the codebase differs from the provided version" + fi + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + - name: "[DEBUG] Print Outputs" + run: | + echo up_to_date: ${{ steps.version-check.outputs.up_to_date }} + + skip-generate-changelog: + runs-on: ubuntu-latest + needs: [audit-changelog] + if: needs.audit-changelog.outputs.exists == 'true' + + steps: + - name: "Changelog Exists, Skip Generating New Changelog" + run: | + # Send notification + title="Skip changelog generation" + message="A changelog file already exists at ${{ needs.audit-changelog.outputs.changelog_path }}, skipping generating changelog" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + skip-version-bump: + runs-on: ubuntu-latest + needs: [audit-version-in-code] + if: needs.audit-version-in-code.outputs.up_to_date == 'true' + + steps: + - name: "Version Already Bumped" + run: | + # Send notification + title="Skip version bump" + message="The version has already been bumped to ${{ inputs.version_number }}, skipping version bump" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + create-temp-branch: + runs-on: ubuntu-latest + needs: [audit-changelog, audit-version-in-code] + if: needs.audit-changelog.outputs.exists == 'false' || needs.audit-version-in-code.outputs.up_to_date == 'false' + + outputs: + branch_name: ${{ steps.variables.outputs.branch_name }} + + steps: + - name: "Checkout ${{ github.repository }} Commit ${{ inputs.sha }}" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.sha }} + + - name: "Generate Branch Name" + id: variables + run: | + name="prep-release/" + if [[ ${{ inputs.test_run }} == true ]] + then + name+="test-run/" + fi + name+="${{ inputs.version_number }}_$GITHUB_RUN_ID" + echo "branch_name=$name" >> $GITHUB_OUTPUT + + - name: "Create Branch - ${{ steps.variables.outputs.branch_name }}" + run: | + git checkout -b ${{ steps.variables.outputs.branch_name }} + git push -u origin ${{ steps.variables.outputs.branch_name }} + + - name: "[Notification] Temp branch created" + run: | + # Send notification + title="Temp branch generated" + message="The ${{ steps.variables.outputs.branch_name }} branch created" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + - name: "[DEBUG] Print Outputs" + run: | + echo branch_name ${{ steps.variables.outputs.branch_name }} + + generate-changelog-bump-version: + runs-on: ubuntu-latest + needs: [audit-changelog, audit-version-in-code, create-temp-branch] + + steps: + - name: "Checkout ${{ github.repository }} Branch ${{ needs.create-temp-branch.outputs.branch_name }}" + uses: actions/checkout@v4 + with: + ref: ${{ needs.create-temp-branch.outputs.branch_name }} + + - name: "Add Homebrew To PATH" + run: | + echo "/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin" >> $GITHUB_PATH + + - name: "Install Homebrew Packages" + run: | + brew install pre-commit + brew tap miniscruff/changie https://github.com/miniscruff/changie + brew install changie + + - name: "Set json File Name" + id: json_file + run: | + echo "name=output_$GITHUB_RUN_ID.json" >> $GITHUB_OUTPUT + + - name: "Get Core Team Membership" + run: | + gh api -H "Accept: application/vnd.github+json" orgs/dbt-labs/teams/core-group/members > ${{ steps.json_file.outputs.name }} + env: + GH_TOKEN: ${{ secrets.IT_TEAM_MEMBERSHIP }} + + - name: "Set Core Team Membership for Changie Contributors exclusion" + id: set_team_membership + run: | + team_list=$(jq -r '.[].login' ${{ steps.json_file.outputs.name }}) + echo $team_list + team_list_single=$(echo $team_list | tr '\n' ' ') + echo "CHANGIE_CORE_TEAM=$team_list_single" >> $GITHUB_ENV + + - name: "Delete the json File" + run: | + rm ${{ steps.json_file.outputs.name }} + + - name: "Generate Release Changelog" + if: needs.audit-changelog.outputs.exists == 'false' + run: | + if [[ ${{ needs.audit-changelog.outputs.is_prerelease }} -eq 1 ]] + then + changie batch ${{ needs.audit-changelog.outputs.base_version }} --move-dir '${{ needs.audit-changelog.outputs.base_version }}' --prerelease ${{ needs.audit-changelog.outputs.prerelease }} + elif [[ -d ".changes/${{ needs.audit-changelog.outputs.base_version }}" ]] + then + changie batch ${{ needs.audit-changelog.outputs.base_version }} --include '${{ needs.audit-changelog.outputs.base_version }}' --remove-prereleases + else # releasing a final patch with no prereleases + changie batch ${{ needs.audit-changelog.outputs.base_version }} + fi + changie merge + git status + + - name: "Check Changelog Created Successfully" + if: needs.audit-changelog.outputs.exists == 'false' + run: | + title="Changelog" + if [[ -f ${{ needs.audit-changelog.outputs.changelog_path }} ]] + then + message="Changelog file created successfully" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + else + message="Changelog failed to generate" + echo "::error title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + exit 1 + fi + + - name: "Set up Python - ${{ env.PYTHON_TARGET_VERSION }}" + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_TARGET_VERSION }} + + - name: "Install Python Dependencies" + if: needs.audit-version-in-code.outputs.up_to_date == 'false' + run: | + python3 -m venv env + source env/bin/activate + python -m pip install --upgrade pip + + - name: Install Hatch + uses: pypa/hatch@install + + - name: "Bump Version To ${{ inputs.version_number }}" + if: needs.audit-version-in-code.outputs.up_to_date == 'false' + run: | + hatch version ${{ inputs.version_number }} + git status + + - name: "[Notification] Bump Version To ${{ inputs.version_number }}" + if: needs.audit-version-in-code.outputs.up_to_date == 'false' + run: | + title="Version bump" + message="Version successfully bumped in codebase to ${{ inputs.version_number }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + # this step will fail on whitespace errors but also correct them + - name: "Remove Trailing Whitespace Via Pre-commit" + continue-on-error: true + run: | + pre-commit run trailing-whitespace --files .bumpversion.cfg CHANGELOG.md .changes/* + git status + + # this step will fail on newline errors but also correct them + - name: "Removing Extra Newlines Via Pre-commit" + continue-on-error: true + run: | + pre-commit run end-of-file-fixer --files .bumpversion.cfg CHANGELOG.md .changes/* + git status + + - name: "Commit & Push Changes" + run: | + #Data for commit + user="Github Build Bot" + email="buildbot@fishtownanalytics.com" + commit_message="Bumping version to ${{ inputs.version_number }} and generate changelog" + #Commit changes to branch + git config user.name "$user" + git config user.email "$email" + git pull + git add . + git commit -m "$commit_message" + git push + + merge-changes-into-target-branch: + runs-on: ubuntu-latest + needs: [create-temp-branch, audit-version-in-code, audit-changelog] + if: | + !failure() && !cancelled() && + inputs.test_run == false && + ( + needs.audit-changelog.outputs.exists == 'false' || + needs.audit-version-in-code.outputs.up_to_date == 'false' + ) + + steps: + - name: "[Debug] Print Variables" + run: | + echo target_branch: ${{ inputs.target_branch }} + echo branch_name: ${{ needs.create-temp-branch.outputs.branch_name }} + echo inputs.test_run: ${{ inputs.test_run }} + echo needs.audit-changelog.outputs.exists: ${{ needs.audit-changelog.outputs.exists }} + echo needs.audit-version-in-code.outputs.up_to_date: ${{ needs.audit-version-in-code.outputs.up_to_date }} + + - name: "Checkout Repo ${{ github.repository }}" + uses: actions/checkout@v4 + + - name: "Merge Changes Into ${{ inputs.target_branch }}" + uses: everlytic/branch-merge@1.1.5 + with: + source_ref: ${{ needs.create-temp-branch.outputs.branch_name }} + target_branch: ${{ inputs.target_branch }} + github_token: ${{ secrets.FISHTOWN_BOT_PAT }} + commit_message_template: "[Automated] Merged {source_ref} into target {target_branch} during release process" + + - name: "[Notification] Changes Merged into ${{ inputs.target_branch }}" + run: | + title="Changelog and Version Bump Branch Merge" + message="The ${{ needs.create-temp-branch.outputs.branch_name }} branch was merged into ${{ inputs.target_branch }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + determine-release-sha: + runs-on: ubuntu-latest + needs: + [ + create-temp-branch, + merge-changes-into-target-branch, + audit-changelog, + audit-version-in-code, + ] + # always run this job, regardless of if the dependant jobs were skipped + if: ${{ !failure() && !cancelled() }} + + # Get the sha that will be released. If the changelog already exists on the input sha and the version has already been bumped, + # then it is what we will release. Otherwise we generated a changelog and did the version bump in this workflow and there is a + # new sha to use from the merge we just did. Grab that here instead. + outputs: + final_sha: ${{ steps.resolve_commit_sha.outputs.release_sha }} + + steps: + - name: "[Debug] Print Variables" + run: | + echo target_branch: ${{ inputs.target_branch }} + echo new_branch: ${{ needs.create-temp-branch.outputs.branch_name }} + echo changelog_exists: ${{ needs.audit-changelog.outputs.exists }} + echo up_to_date: ${{ needs.audit-version-in-code.outputs.up_to_date }} + + - name: "Resolve Branch To Checkout" + id: resolve_branch + run: | + branch="" + if [[ ${{ inputs.test_run == true }} ]] + then + branch=${{ needs.create-temp-branch.outputs.branch_name }} + else + branch=${{ inputs.target_branch }} + fi + echo "target_branch=$branch" >> $GITHUB_OUTPUT + + - name: "[Notification] Resolve Branch To Checkout" + run: | + title="Branch pick" + message="The ${{ steps.resolve_branch.outputs.target_branch }} branch will be used for release" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + - name: "Checkout Resolved Branch - ${{ steps.resolve_branch.outputs.target_branch }}" + uses: actions/checkout@v4 + with: + ref: ${{ steps.resolve_branch.outputs.target_branch }} + + - name: "[Debug] Log Branch" + run: git status + + - name: "Resolve Commit SHA For Release" + id: resolve_commit_sha + run: | + commit_sha="" + if [[ ${{ needs.audit-changelog.outputs.exists }} == false ]] || [[ ${{ needs.audit-version-in-code.outputs.up_to_date }} == false ]] + then + commit_sha=$(git rev-parse HEAD) + else + commit_sha=${{ inputs.sha }} + fi + echo "release_sha=$commit_sha" >> $GITHUB_OUTPUT + + - name: "[Notification] Resolve Commit SHA For Release" + run: | + title="Release commit pick" + message="The ${{ steps.resolve_commit_sha.outputs.release_sha }} commit will be used for release" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + - name: "Remove Temp Branch - ${{ needs.create-temp-branch.outputs.branch_name }}" + if: ${{ inputs.test_run == false && needs.create-temp-branch.outputs.branch_name != '' }} + run: | + git push origin -d ${{ needs.create-temp-branch.outputs.branch_name }} + + - name: "[Debug] Print Outputs" + run: | + echo release_sha: ${{ steps.resolve_commit_sha.outputs.release_sha }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ad581c99..e889f717 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,6 +3,19 @@ name: Release on: workflow_dispatch: inputs: + target_branch: + description: "The branch to release from" + type: string + required: true + version_number: + description: "The release version number (i.e. 1.0.0b1)" + type: string + required: true + test_run: + description: "Test run (Publish release as draft)" + type: boolean + default: true + required: false deploy-to: type: choice description: Choose where to publish (test/prod) @@ -24,25 +37,28 @@ concurrency: jobs: - release: - name: PyPI - ${{ inputs.deploy-to }} + test-and-check: + name: Test and Check runs-on: ubuntu-latest environment: name: ${{ inputs.deploy-to }} - permissions: - id-token: write # IMPORTANT: this permission is mandatory for trusted publishing steps: - - name: Check out repository + - name: Check out target branch uses: actions/checkout@v4 with: persist-credentials: false + ref: ${{ inputs.target_branch }} - name: "Set up Python & Hatch - 3.11" uses: ./.github/actions/setup-python-hatch with: python-version: "3.11" + - name: Run tests + run: hatch test + shell: bash + - name: Build artifacts run: hatch build shell: bash @@ -51,12 +67,42 @@ jobs: run: hatch run build:check-all shell: bash - - name: Publish artifacts to PyPI Test - if: inputs.deploy-to == 'PypiTest' + + update-version-and-create-changelog: + name: Update Version and Create Changelog + needs: [test-and-check] + permissions: + contents: write # this is the permission that allows creating a new release + secrets: inherit + uses: ./.github/workflows/release-prep.yml + with: + sha: ${{ github.sha }} + version_number: ${{ inputs.version_number }} + target_branch: ${{ inputs.target_branch }} + test_run: ${{ inputs.test_run }} + + + publish: + name: Publish to PyPI + if: ${{ !inputs.test_run }} + needs: [test-and-check, update-version-and-create-changelog] + runs-on: ubuntu-latest + environment: + name: ${{ inputs.deploy-to }} + permissions: + id-token: write # IMPORTANT: this permission is mandatory for trusted publishing + + steps: + - name: Build Artifacts + run: hatch build + shell: bash + + - name: Publish Artifacts to PyPI Test + if: inputs.deploy-to == 'PypiTest' && !inputs.test_run uses: pypa/gh-action-pypi-publish@release/v1 with: repository-url: https://test.pypi.org/legacy/ - name: Publish artifacts to PyPI Prod - if: inputs.deploy-to == 'PypiProd' + if: inputs.deploy-to == 'PypiProd' && !inputs.test_run uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ca9d33a..c70909f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,148 @@ -# Changelog -All notable changes to this project will be documented in this file. +# dbt-common Changelog -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), -adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), -and is generated by [Changie](https://github.com/miniscruff/changie). +- This file provides a full account of all changes to `dbt-common` +- Changes are listed under the (pre)release in which they first appear. Subsequent releases include changes from previous releases. +- "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version. +- Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-common/blob/main/CONTRIBUTING.md#adding-changelog-entry) +## dbt-common 1.6.0 - July 16, 2024 -No releases yet, this file will be updated when generating your first release. +### Fixes + +- Fix case-insensitive env vars for Windows ([#166](https://github.com/dbt-labs/dbt-common/issues/166)) + +### Under the Hood + +- Add record grouping mechanism to record/replay. ([#169](https://github.com/dbt-labs/dbt-common/issues/169)) + +## dbt-common 1.6.0 - July 16, 2024 + +### Features + +- Add to and to_columns to ColumnLevelConstraint and ModelLevelConstraint contracts ([#168](https://github.com/dbt-labs/dbt-common/issues/168)) + +## dbt-common 1.5.0 - July 2, 2024 + +## dbt-common 1.4.0 - June 18, 2024 + +### Under the Hood + +- Add support for basic diff of run recordings. ([#144](https://github.com/dbt-labs/dbt-common/issues/144)) +- Deserialize Record objects on a just-in-time basis. ([#151](https://github.com/dbt-labs/dbt-common/issues/151)) + +## dbt-common 1.3.0 - June 5, 2024 + +### Under the Hood + +- Add automation workflow for changelog updates and version bump ([#92](https://github.com/dbt-labs/dbt-common/issues/92)) +- Move CatalogKey, ColumnMetadata, ColumnMap, CatalogTable to dbt-common ([#147](https://github.com/dbt-labs/dbt-common/issues/147)) + +## dbt-common 1.2.0 - May 29, 2024 + +### Under the Hood + +- Allow dynamic selection of record types when recording. ([#140](https://github.com/dbt-labs/dbt-common/issues/140)) +- Move StatsItem, StatsDict, TableMetadata to dbt-common ([#141](https://github.com/dbt-labs/dbt-common/issues/141)) + +## dbt-common 1.1.0 - May 21, 2024 + +### Features + +- Support adding callbacks to the event manager ([#131](https://github.com/dbt-labs/dbt-common/issues/131)) +- Add a print event to support firing event to console in --quiet mode ([#8756](https://github.com/dbt-labs/dbt-common/issues/8756)) + +## dbt-common 1.0.4 - May 7, 2024 + +### Features + +- Enable serialization contexts ([#126](https://github.com/dbt-labs/dbt-common/issues/126)) + +## dbt-common 1.0.3 - May 1, 2024 + +### Dependencies + +- Pin agate `>=1.7.0,<1.10` ([#119](https://github.com/dbt-labs/dbt-common/pull/119)) +- update pathspec upper version limitation ([#120](https://github.com/dbt-labs/dbt-common/pull/120)) + +### Contributors +- [@jerry153fish](https://github.com/jerry153fish) ([#120](https://github.com/dbt-labs/dbt-common/pull/120)) + +## dbt-common 1.0.2 - April 24, 2024 + +### Features + +- Add ability to silence warnings via `WarnErrorOptions` ([#111](https://github.com/dbt-labs/dbt-common/issues/111)) + +### Under the Hood + +- Remove the final underscore from secret and private environment variable constants. ([#109](https://github.com/dbt-labs/dbt-common/issues/109)) + +## dbt-common 1.0.1 - April 12, 2024 + +### Features + +- Support environment variables which dbt can see but user code cannot. ([#103](https://github.com/dbt-labs/dbt-common/issues/103)) +- Add `error_tag` util to the `ui` module ([#107](https://github.com/dbt-labs/dbt-common/issues/107)) + +## dbt-common 1.0.0 - April 1, 2024 + +### Under the Hood + +- Implement record/replay mechanism ([#9689](https://github.com/dbt-labs/dbt-common/issues/9689)) +- Stream JSON on read/write instead of holding it in memory ([#96](https://github.com/dbt-labs/dbt-common/issues/96)) +- Improve startup performance by lazy-compiling mashumaro serialization functions ([#97](https://github.com/dbt-labs/dbt-common/issues/97)) + + +## dbt-common 0.1.6 - February 22, 2024 + +### Fixes + +- Fix the unset_contextvars function in events/contextvars.py ([#82](https://github.com/dbt-labs/dbt-common/issues/82)) + +### Under the Hood + +- Restrict protobuf library to major version 4. ([#9566](https://github.com/dbt-labs/dbt-common/issues/9566)) + + +## dbt-common 0.1.5 - February 8, 2024 + +### Under the Hood + +- Add Invocation Context Support to MultiThreadedExecutor ([#75](https://github.com/dbt-labs/dbt-common/issues/75)) + + +## dbt-common 0.1.4 - February 7, 2024 + +### Fixes + +- Make invocation contexts more reliable in testing scenarios. ([#52](https://github.com/dbt-labs/dbt-common/issues/52)) + +### Under the Hood + +- Remove `fire_event_if_test` method ([#66](https://github.com/dbt-labs/dbt-common/issues/66)) + + +## dbt-common 0.1.3 - February 1, 2024 + +### Features + +- Added a new InvocationContext class for application context management. ([#57](https://github.com/dbt-labs/dbt-common/issues/57)) +- Add dataclass utility `Mergeable` ([#58](https://github.com/dbt-labs/dbt-common/issues/58)) + +## dbt-common 0.1.1 - January 26, 2024 + +### Features + +- Cache static objects between tests to accelerate integration runs. ([#55](https://github.com/dbt-labs/dbt-common/issues/55)) + +### Under the Hood + +- Clean up macro contexts. ([#35](https://github.com/dbt-labs/dbt-common/issues/35)) +- Change metadata_vars \`if not\` to \`if ... is None\` ([#6073](https://github.com/dbt-labs/dbt-common/issues/6073)) +- Inject TagIterator into BlockIterator for greater flexibility. ([#38](https://github.com/dbt-labs/dbt-common/issues/38)) + +### Contributors +- [@truls-p](https://github.com/truls-p) ([#6073](https://github.com/dbt-labs/dbt-common/issues/6073)) + + +## dbt-common 0.1.0 - January 17, 2024