repo sync

.github/actions-scripts/msft-create-translation-batch-pr.js

@@ -0,0 +1,142 @@
+#!/usr/bin/env node
+
+import fs from 'fs'
+import github from '@actions/github'
+
+const OPTIONS = Object.fromEntries(
+  ['BASE', 'BODY_FILE', 'GITHUB_TOKEN', 'HEAD', 'LANGUAGE', 'TITLE', 'GITHUB_REPOSITORY'].map(
+    (envVarName) => {
+      const envVarValue = process.env[envVarName]
+      if (!envVarValue) {
+        throw new Error(`You must supply a ${envVarName} environment variable`)
+      }
+      return [envVarName, envVarValue]
+    }
+  )
+)
+
+if (!process.env.GITHUB_REPOSITORY) {
+  throw new Error('GITHUB_REPOSITORY environment variable not set')
+}
+
+const RETRY_STATUSES = [
+  422, // Retry the operation if the PR already exists
+  502, // Retry the operation if the API responds with a `502 Bad Gateway` error.
+]
+const RETRY_ATTEMPTS = 3
+const {
+  // One of the default environment variables provided by Actions.
+  GITHUB_REPOSITORY,
+
+  // These are passed in from the step in the workflow file.
+  TITLE,
+  BASE,
+  HEAD,
+  LANGUAGE,
+  BODY_FILE,
+  GITHUB_TOKEN,
+} = OPTIONS
+const [OWNER, REPO] = GITHUB_REPOSITORY.split('/')
+
+const octokit = github.getOctokit(GITHUB_TOKEN)
+
+/**
+ * @param {object} config Configuration options for finding the PR.
+ * @returns {Promise<number | undefined>} The PR number.
+ */
+async function findPullRequestNumber(config) {
+  // Get a list of PRs and see if one already exists.
+  const { data: listOfPullRequests } = await octokit.rest.pulls.list({
+    owner: config.owner,
+    repo: config.repo,
+    head: `${config.owner}:${config.head}`,
+  })
+
+  return listOfPullRequests[0]?.number
+}
+
+/**
+ * When this file was first created, we only introduced support for creating a pull request for some translation batch.
+ * However, some of our first workflow runs failed during the pull request creation due to a timeout error.
+ * There have been cases where, despite the timeout error, the pull request gets created _anyway_.
+ * To accommodate this reality, we created this function to look for an existing pull request before a new one is created.
+ * Although the "find" check is redundant in the first "cycle", it's designed this way to recursively call the function again via its retry mechanism should that be necessary.
+ *
+ * @param {object} config Configuration options for creating the pull request.
+ * @returns {Promise<number>} The PR number.
+ */
+async function findOrCreatePullRequest(config) {
+  const found = await findPullRequestNumber(config)
+
+  if (found) {
+    return found
+  }
+
+  try {
+    const { data: pullRequest } = await octokit.rest.pulls.create({
+      owner: config.owner,
+      repo: config.repo,
+      base: config.base,
+      head: config.head,
+      title: config.title,
+      body: config.body,
+      draft: false,
+    })
+
+    return pullRequest.number
+  } catch (error) {
+    if (!error.response || !config.retryCount) {
+      throw error
+    }
+
+    if (!config.retryStatuses.includes(error.response.status)) {
+      throw error
+    }
+
+    console.error(`Error creating pull request: ${error.message}`)
+    console.warn(`Retrying in 5 seconds...`)
+    await new Promise((resolve) => setTimeout(resolve, 5000))
+
+    config.retryCount -= 1
+
+    return findOrCreatePullRequest(config)
+  }
+}
+
+/**
+ * @param {object} config Configuration options for labeling the PR
+ * @returns {Promise<undefined>}
+ */
+// async function labelPullRequest(config) {
+//   await octokit.rest.issues.update({
+//     owner: config.owner,
+//     repo: config.repo,
+//     issue_number: config.issue_number,
+//     labels: config.labels,
+//   })
+// }
+
+async function main() {
+  const options = {
+    title: TITLE,
+    base: BASE,
+    head: HEAD,
+    body: fs.readFileSync(BODY_FILE, 'utf8'),
+    labels: ['translation-batch', `translation-batch-${LANGUAGE}`],
+    owner: OWNER,
+    repo: REPO,
+    retryStatuses: RETRY_STATUSES,
+    retryCount: RETRY_ATTEMPTS,
+  }
+
+  options.issue_number = await findOrCreatePullRequest(options)
+  const pr = `${GITHUB_REPOSITORY}#${options.issue_number}`
+  console.log(`Created PR ${pr}`)
+
+  // metadata parameters aren't currently available in `github.rest.pulls.create`,
+  // but they are in `github.rest.issues.update`.
+  // await labelPullRequest(options)
+  // console.log(`Updated ${pr} with these labels: ${options.labels.join(', ')}`)
+}
+
+main()

.github/workflows/msft-create-translation-batch-pr.yml

@@ -0,0 +1,204 @@
+name: Create translation Batch Pull Request
+
+# **What it does**:
+#  - Creates one pull request per language after running a series of automated checks,
+#    removing translations that are broken in any known way
+# **Why we have it**:
+#  - To deploy translations
+# **Who does it impact**: It automates what would otherwise be manual work,
+#   helping docs engineering focus on higher value work
+
+on:
+  workflow_dispatch:
+  # TODO: bring schedule back in
+  # schedule:
+  #   - cron: '02 17 * * *' # Once a day at 17:02 UTC / 9:02 PST
+
+permissions:
+  contents: write
+
+jobs:
+  create-translation-batch:
+    name: Create translation batch
+    if: github.repository == 'github/docs-internal'
+    runs-on: ubuntu-latest
+    # A sync's average run time is ~3.2 hours.
+    # This sets a maximum execution time of 300 minutes (5 hours) to prevent the workflow from running longer than necessary.
+    timeout-minutes: 300
+    strategy:
+      fail-fast: false
+      max-parallel: 1
+      matrix:
+        include:
+          # TODO: replace language_repos with actual repos once created
+          # - language: pt
+          #   crowdin_language: pt-BR
+          #   language_dir: translations/pt-BR
+          #   language_repo: github/docs-translations-pt-br
+
+          - language: es
+            crowdin_language: es-ES
+            language_dir: translations/es-ES
+            language_repo: github/docs-localization-test-es-es
+
+          # - language: cn
+          #   crowdin_language: zh-CN
+          #   language_dir: translations/zh-CN
+          #   language_repo: github/docs-translations-zh-cn
+
+          # - language: ja
+          #   crowdin_language: ja
+          #   language_dir: translations/ja-JP
+          #   language_repo: github/docs-translations-ja-jp
+
+    # TODO: replace the branch name
+    steps:
+      - name: Set branch name
+        id: set-branch
+        run: |
+          echo "::set-output name=BRANCH_NAME::msft-translation-batch-${{ matrix.language }}-$(date +%Y-%m-%d__%H-%M)"
+
+      - run: git config --global user.name "docubot"
+      - run: git config --global user.email "[email protected]"
+
+      - name: Checkout the docs-internal repo
+        uses: actions/checkout@dcd71f646680f2efd8db4afa5ad64fdcba30e748
+        with:
+          fetch-depth: 0
+          lfs: true
+
+      - name: Create a branch for the current language
+        run: git checkout -b ${{ steps.set-branch.outputs.BRANCH_NAME }}
+
+      - name: Remove unwanted git hooks
+        run: rm .git/hooks/post-checkout
+
+      - name: Remove all language translations
+        run: |
+          git rm -rf --quiet ${{ matrix.language_dir }}/content
+          git rm -rf --quiet ${{ matrix.language_dir }}/data
+
+      - name: Checkout the language-specific repo
+        uses: actions/checkout@dcd71f646680f2efd8db4afa5ad64fdcba30e748
+        with:
+          repository: ${{ matrix.language_repo }}
+          token: ${{ secrets.DOCUBOT_READORG_REPO_WORKFLOW_SCOPES }}
+          path: ${{ matrix.language_dir }}
+
+      - name: Remove .git from the language-specific repo
+        run: rm -rf ${{ matrix.language_dir }}/.git
+
+      # TODO: Rename this step
+      - name: Commit crowdin sync
+        run: |
+          git add ${{ matrix.language_dir }}
+          git commit -m "Add crowdin translations" || echo "Nothing to commit"
+
+      - name: 'Setup node'
+        uses: actions/setup-node@1f8c6b94b26d0feae1e387ca63ccbdc44d27b561
+        with:
+          node-version: 16.14.x
+
+      - run: npm ci
+
+      # step 6 in docs-engineering/crowdin.md
+      - name: Homogenize frontmatter
+        run: |
+          node script/i18n/homogenize-frontmatter.js
+          git add ${{ matrix.language_dir }} && git commit -m "Run script/i18n/homogenize-frontmatter.js" || echo "Nothing to commit"
+
+      # step 7 in docs-engineering/crowdin.md
+      - name: Fix translation errors
+        run: |
+          node script/i18n/fix-translation-errors.js
+          git add ${{ matrix.language_dir }} && git commit -m "Run script/i18n/fix-translation-errors.js" || echo "Nothing to commit"
+
+      # step 8a in docs-engineering/crowdin.md
+      - name: Check parsing
+        run: |
+          node script/i18n/lint-translation-files.js --check parsing | tee -a /tmp/batch.log | cat
+          git add ${{ matrix.language_dir }} && git commit -m "Run script/i18n/lint-translation-files.js --check parsing" || echo "Nothing to commit"
+
+      # step 8b in docs-engineering/crowdin.md
+      - name: Check rendering
+        run: |
+          node script/i18n/lint-translation-files.js --check rendering | tee -a /tmp/batch.log | cat
+          git add ${{ matrix.language_dir }} && git commit -m "Run script/i18n/lint-translation-files.js --check rendering" || echo "Nothing to commit"
+
+      - name: Reset files with broken liquid tags
+        run: |
+          node script/i18n/reset-files-with-broken-liquid-tags.js --language=${{ matrix.language }} | tee -a /tmp/batch.log | cat
+          git add ${{ matrix.language_dir }} && git commit -m "run script/i18n/reset-files-with-broken-liquid-tags.js --language=${{ matrix.language }}" || echo "Nothing to commit"
+
+      # step 5 in docs-engineering/crowdin.md using script from docs-internal#22709
+      - name: Reset known broken files
+        run: |
+          node script/i18n/reset-known-broken-translation-files.js | tee -a /tmp/batch.log | cat
+          git add ${{ matrix.language_dir }} && git commit -m "run script/i18n/reset-known-broken-translation-files.js" || echo "Nothing to commit"
+        env:
+          GITHUB_TOKEN: ${{ secrets.DOCUBOT_REPO_PAT }}
+
+      - name: Check in CSV report
+        run: |
+          mkdir -p translations/log
+          csvFile=translations/log/${{ matrix.language }}-resets.csv
+          script/i18n/report-reset-files.js --report-type=csv --language=${{ matrix.language }} --log-file=/tmp/batch.log > $csvFile
+          git add -f $csvFile && git commit -m "Check in ${{ matrix.language }} CSV report" || echo "Nothing to commit"
+
+      - name: Write the reported files that were reset to /tmp/pr-body.txt
+        run: script/i18n/report-reset-files.js --report-type=pull-request-body --language=${{ matrix.language }} --log-file=/tmp/batch.log > /tmp/pr-body.txt
+
+      - name: Push filtered translations
+        run: git push origin ${{ steps.set-branch.outputs.BRANCH_NAME }}
+
+      # TODO: bring this step back
+      # - name: Close existing stale batches
+      #   uses: lee-dohm/close-matching-issues@e9e43aad2fa6f06a058cedfd8fb975fd93b56d8f
+      #   with:
+      #     token: ${{ secrets.OCTOMERGER_PAT_WITH_REPO_AND_WORKFLOW_SCOPE }}
+      #     query: 'type:pr label:translation-batch-${{ matrix.language }}'
+
+      # TODO: bring labels back into the PR script
+      - name: Create translation batch pull request
+        env:
+          GITHUB_TOKEN: ${{ secrets.DOCUBOT_REPO_PAT }}
+          TITLE: '[DO NOT MERGE] Msft: New translation batch for ${{ matrix.language }}'
+          BASE: 'main'
+          HEAD: ${{ steps.set-branch.outputs.BRANCH_NAME }}
+          LANGUAGE: ${{ matrix.language }}
+          BODY_FILE: '/tmp/pr-body.txt'
+        run: .github/actions-scripts/msft-create-translation-batch-pr.js
+
+      # TODO: bring back these steps
+      # - name: Approve PR
+      #   if: github.ref_name == 'main'
+      #   env:
+      #     GITHUB_TOKEN: ${{ secrets.OCTOMERGER_PAT_WITH_REPO_AND_WORKFLOW_SCOPE }}
+      #   run: gh pr review --approve || echo "Nothing to approve"
+
+      # - name: Set auto-merge
+      #   if: github.ref_name == 'main'
+      #   env:
+      #     GITHUB_TOKEN: ${{ secrets.OCTOMERGER_PAT_WITH_REPO_AND_WORKFLOW_SCOPE }}
+      #   run: gh pr merge ${{ steps.set-branch.outputs.BRANCH_NAME }} --auto --squash || echo "Nothing to merge"
+
+      # # When the maximum execution time is reached for this job, Actions cancels the workflow run.
+      # # This emits a notification for the first responder to triage.
+      # - name: Send Slack notification if workflow is cancelled
+      #   uses: someimportantcompany/github-actions-slack-message@f8d28715e7b8a4717047d23f48c39827cacad340
+      #   if: cancelled()
+      #   with:
+      #     channel: ${{ secrets.DOCS_ALERTS_SLACK_CHANNEL_ID }}
+      #     bot-token: ${{ secrets.SLACK_DOCS_BOT_TOKEN }}
+      #     color: failure
+      #     text: 'The new translation batch for ${{ matrix.language }} was cancelled.'
+
+      # # Emit a notification for the first responder to triage if the workflow failed.
+      # - name: Send Slack notification if workflow failed
+      #   uses: someimportantcompany/github-actions-slack-message@f8d28715e7b8a4717047d23f48c39827cacad340
+      #   if: failure()
+      #   with:
+      #     channel: ${{ secrets.DOCS_ALERTS_SLACK_CHANNEL_ID }}
+      #     bot-token: ${{ secrets.SLACK_DOCS_BOT_TOKEN }}
+      #     color: failure
+      #     text: 'The new translation batch for ${{ matrix.language }} failed.'