diff --git a/.github/actions-scripts/msft-create-translation-batch-pr.js b/.github/actions-scripts/msft-create-translation-batch-pr.js new file mode 100755 index 000000000000..1ea57c342db2 --- /dev/null +++ b/.github/actions-scripts/msft-create-translation-batch-pr.js @@ -0,0 +1,142 @@ +#!/usr/bin/env node + +import fs from 'fs' +import github from '@actions/github' + +const OPTIONS = Object.fromEntries( + ['BASE', 'BODY_FILE', 'GITHUB_TOKEN', 'HEAD', 'LANGUAGE', 'TITLE', 'GITHUB_REPOSITORY'].map( + (envVarName) => { + const envVarValue = process.env[envVarName] + if (!envVarValue) { + throw new Error(`You must supply a ${envVarName} environment variable`) + } + return [envVarName, envVarValue] + } + ) +) + +if (!process.env.GITHUB_REPOSITORY) { + throw new Error('GITHUB_REPOSITORY environment variable not set') +} + +const RETRY_STATUSES = [ + 422, // Retry the operation if the PR already exists + 502, // Retry the operation if the API responds with a `502 Bad Gateway` error. +] +const RETRY_ATTEMPTS = 3 +const { + // One of the default environment variables provided by Actions. + GITHUB_REPOSITORY, + + // These are passed in from the step in the workflow file. + TITLE, + BASE, + HEAD, + LANGUAGE, + BODY_FILE, + GITHUB_TOKEN, +} = OPTIONS +const [OWNER, REPO] = GITHUB_REPOSITORY.split('/') + +const octokit = github.getOctokit(GITHUB_TOKEN) + +/** + * @param {object} config Configuration options for finding the PR. + * @returns {Promise} The PR number. + */ +async function findPullRequestNumber(config) { + // Get a list of PRs and see if one already exists. + const { data: listOfPullRequests } = await octokit.rest.pulls.list({ + owner: config.owner, + repo: config.repo, + head: `${config.owner}:${config.head}`, + }) + + return listOfPullRequests[0]?.number +} + +/** + * When this file was first created, we only introduced support for creating a pull request for some translation batch. + * However, some of our first workflow runs failed during the pull request creation due to a timeout error. + * There have been cases where, despite the timeout error, the pull request gets created _anyway_. + * To accommodate this reality, we created this function to look for an existing pull request before a new one is created. + * Although the "find" check is redundant in the first "cycle", it's designed this way to recursively call the function again via its retry mechanism should that be necessary. + * + * @param {object} config Configuration options for creating the pull request. + * @returns {Promise} The PR number. + */ +async function findOrCreatePullRequest(config) { + const found = await findPullRequestNumber(config) + + if (found) { + return found + } + + try { + const { data: pullRequest } = await octokit.rest.pulls.create({ + owner: config.owner, + repo: config.repo, + base: config.base, + head: config.head, + title: config.title, + body: config.body, + draft: false, + }) + + return pullRequest.number + } catch (error) { + if (!error.response || !config.retryCount) { + throw error + } + + if (!config.retryStatuses.includes(error.response.status)) { + throw error + } + + console.error(`Error creating pull request: ${error.message}`) + console.warn(`Retrying in 5 seconds...`) + await new Promise((resolve) => setTimeout(resolve, 5000)) + + config.retryCount -= 1 + + return findOrCreatePullRequest(config) + } +} + +/** + * @param {object} config Configuration options for labeling the PR + * @returns {Promise} + */ +// async function labelPullRequest(config) { +// await octokit.rest.issues.update({ +// owner: config.owner, +// repo: config.repo, +// issue_number: config.issue_number, +// labels: config.labels, +// }) +// } + +async function main() { + const options = { + title: TITLE, + base: BASE, + head: HEAD, + body: fs.readFileSync(BODY_FILE, 'utf8'), + labels: ['translation-batch', `translation-batch-${LANGUAGE}`], + owner: OWNER, + repo: REPO, + retryStatuses: RETRY_STATUSES, + retryCount: RETRY_ATTEMPTS, + } + + options.issue_number = await findOrCreatePullRequest(options) + const pr = `${GITHUB_REPOSITORY}#${options.issue_number}` + console.log(`Created PR ${pr}`) + + // metadata parameters aren't currently available in `github.rest.pulls.create`, + // but they are in `github.rest.issues.update`. + // await labelPullRequest(options) + // console.log(`Updated ${pr} with these labels: ${options.labels.join(', ')}`) +} + +main() diff --git a/.github/workflows/msft-create-translation-batch-pr.yml b/.github/workflows/msft-create-translation-batch-pr.yml new file mode 100644 index 000000000000..5e3bebefdbc7 --- /dev/null +++ b/.github/workflows/msft-create-translation-batch-pr.yml @@ -0,0 +1,204 @@ +name: Create translation Batch Pull Request + +# **What it does**: +# - Creates one pull request per language after running a series of automated checks, +# removing translations that are broken in any known way +# **Why we have it**: +# - To deploy translations +# **Who does it impact**: It automates what would otherwise be manual work, +# helping docs engineering focus on higher value work + +on: + workflow_dispatch: + # TODO: bring schedule back in + # schedule: + # - cron: '02 17 * * *' # Once a day at 17:02 UTC / 9:02 PST + +permissions: + contents: write + +jobs: + create-translation-batch: + name: Create translation batch + if: github.repository == 'github/docs-internal' + runs-on: ubuntu-latest + # A sync's average run time is ~3.2 hours. + # This sets a maximum execution time of 300 minutes (5 hours) to prevent the workflow from running longer than necessary. + timeout-minutes: 300 + strategy: + fail-fast: false + max-parallel: 1 + matrix: + include: + # TODO: replace language_repos with actual repos once created + # - language: pt + # crowdin_language: pt-BR + # language_dir: translations/pt-BR + # language_repo: github/docs-translations-pt-br + + - language: es + crowdin_language: es-ES + language_dir: translations/es-ES + language_repo: github/docs-localization-test-es-es + + # - language: cn + # crowdin_language: zh-CN + # language_dir: translations/zh-CN + # language_repo: github/docs-translations-zh-cn + + # - language: ja + # crowdin_language: ja + # language_dir: translations/ja-JP + # language_repo: github/docs-translations-ja-jp + + # TODO: replace the branch name + steps: + - name: Set branch name + id: set-branch + run: | + echo "::set-output name=BRANCH_NAME::msft-translation-batch-${{ matrix.language }}-$(date +%Y-%m-%d__%H-%M)" + + - run: git config --global user.name "docubot" + - run: git config --global user.email "67483024+docubot@users.noreply.github.com" + + - name: Checkout the docs-internal repo + uses: actions/checkout@dcd71f646680f2efd8db4afa5ad64fdcba30e748 + with: + fetch-depth: 0 + lfs: true + + - name: Create a branch for the current language + run: git checkout -b ${{ steps.set-branch.outputs.BRANCH_NAME }} + + - name: Remove unwanted git hooks + run: rm .git/hooks/post-checkout + + - name: Remove all language translations + run: | + git rm -rf --quiet ${{ matrix.language_dir }}/content + git rm -rf --quiet ${{ matrix.language_dir }}/data + + - name: Checkout the language-specific repo + uses: actions/checkout@dcd71f646680f2efd8db4afa5ad64fdcba30e748 + with: + repository: ${{ matrix.language_repo }} + token: ${{ secrets.DOCUBOT_READORG_REPO_WORKFLOW_SCOPES }} + path: ${{ matrix.language_dir }} + + - name: Remove .git from the language-specific repo + run: rm -rf ${{ matrix.language_dir }}/.git + + # TODO: Rename this step + - name: Commit crowdin sync + run: | + git add ${{ matrix.language_dir }} + git commit -m "Add crowdin translations" || echo "Nothing to commit" + + - name: 'Setup node' + uses: actions/setup-node@1f8c6b94b26d0feae1e387ca63ccbdc44d27b561 + with: + node-version: 16.14.x + + - run: npm ci + + # step 6 in docs-engineering/crowdin.md + - name: Homogenize frontmatter + run: | + node script/i18n/homogenize-frontmatter.js + git add ${{ matrix.language_dir }} && git commit -m "Run script/i18n/homogenize-frontmatter.js" || echo "Nothing to commit" + + # step 7 in docs-engineering/crowdin.md + - name: Fix translation errors + run: | + node script/i18n/fix-translation-errors.js + git add ${{ matrix.language_dir }} && git commit -m "Run script/i18n/fix-translation-errors.js" || echo "Nothing to commit" + + # step 8a in docs-engineering/crowdin.md + - name: Check parsing + run: | + node script/i18n/lint-translation-files.js --check parsing | tee -a /tmp/batch.log | cat + git add ${{ matrix.language_dir }} && git commit -m "Run script/i18n/lint-translation-files.js --check parsing" || echo "Nothing to commit" + + # step 8b in docs-engineering/crowdin.md + - name: Check rendering + run: | + node script/i18n/lint-translation-files.js --check rendering | tee -a /tmp/batch.log | cat + git add ${{ matrix.language_dir }} && git commit -m "Run script/i18n/lint-translation-files.js --check rendering" || echo "Nothing to commit" + + - name: Reset files with broken liquid tags + run: | + node script/i18n/reset-files-with-broken-liquid-tags.js --language=${{ matrix.language }} | tee -a /tmp/batch.log | cat + git add ${{ matrix.language_dir }} && git commit -m "run script/i18n/reset-files-with-broken-liquid-tags.js --language=${{ matrix.language }}" || echo "Nothing to commit" + + # step 5 in docs-engineering/crowdin.md using script from docs-internal#22709 + - name: Reset known broken files + run: | + node script/i18n/reset-known-broken-translation-files.js | tee -a /tmp/batch.log | cat + git add ${{ matrix.language_dir }} && git commit -m "run script/i18n/reset-known-broken-translation-files.js" || echo "Nothing to commit" + env: + GITHUB_TOKEN: ${{ secrets.DOCUBOT_REPO_PAT }} + + - name: Check in CSV report + run: | + mkdir -p translations/log + csvFile=translations/log/${{ matrix.language }}-resets.csv + script/i18n/report-reset-files.js --report-type=csv --language=${{ matrix.language }} --log-file=/tmp/batch.log > $csvFile + git add -f $csvFile && git commit -m "Check in ${{ matrix.language }} CSV report" || echo "Nothing to commit" + + - name: Write the reported files that were reset to /tmp/pr-body.txt + run: script/i18n/report-reset-files.js --report-type=pull-request-body --language=${{ matrix.language }} --log-file=/tmp/batch.log > /tmp/pr-body.txt + + - name: Push filtered translations + run: git push origin ${{ steps.set-branch.outputs.BRANCH_NAME }} + + # TODO: bring this step back + # - name: Close existing stale batches + # uses: lee-dohm/close-matching-issues@e9e43aad2fa6f06a058cedfd8fb975fd93b56d8f + # with: + # token: ${{ secrets.OCTOMERGER_PAT_WITH_REPO_AND_WORKFLOW_SCOPE }} + # query: 'type:pr label:translation-batch-${{ matrix.language }}' + + # TODO: bring labels back into the PR script + - name: Create translation batch pull request + env: + GITHUB_TOKEN: ${{ secrets.DOCUBOT_REPO_PAT }} + TITLE: '[DO NOT MERGE] Msft: New translation batch for ${{ matrix.language }}' + BASE: 'main' + HEAD: ${{ steps.set-branch.outputs.BRANCH_NAME }} + LANGUAGE: ${{ matrix.language }} + BODY_FILE: '/tmp/pr-body.txt' + run: .github/actions-scripts/msft-create-translation-batch-pr.js + + # TODO: bring back these steps + # - name: Approve PR + # if: github.ref_name == 'main' + # env: + # GITHUB_TOKEN: ${{ secrets.OCTOMERGER_PAT_WITH_REPO_AND_WORKFLOW_SCOPE }} + # run: gh pr review --approve || echo "Nothing to approve" + + # - name: Set auto-merge + # if: github.ref_name == 'main' + # env: + # GITHUB_TOKEN: ${{ secrets.OCTOMERGER_PAT_WITH_REPO_AND_WORKFLOW_SCOPE }} + # run: gh pr merge ${{ steps.set-branch.outputs.BRANCH_NAME }} --auto --squash || echo "Nothing to merge" + + # # When the maximum execution time is reached for this job, Actions cancels the workflow run. + # # This emits a notification for the first responder to triage. + # - name: Send Slack notification if workflow is cancelled + # uses: someimportantcompany/github-actions-slack-message@f8d28715e7b8a4717047d23f48c39827cacad340 + # if: cancelled() + # with: + # channel: ${{ secrets.DOCS_ALERTS_SLACK_CHANNEL_ID }} + # bot-token: ${{ secrets.SLACK_DOCS_BOT_TOKEN }} + # color: failure + # text: 'The new translation batch for ${{ matrix.language }} was cancelled.' + + # # Emit a notification for the first responder to triage if the workflow failed. + # - name: Send Slack notification if workflow failed + # uses: someimportantcompany/github-actions-slack-message@f8d28715e7b8a4717047d23f48c39827cacad340 + # if: failure() + # with: + # channel: ${{ secrets.DOCS_ALERTS_SLACK_CHANNEL_ID }} + # bot-token: ${{ secrets.SLACK_DOCS_BOT_TOKEN }} + # color: failure + # text: 'The new translation batch for ${{ matrix.language }} failed.'