TeamCity: archive debug logs if there are over 1000 of them (#12083)

mmv1/third_party/terraform/.teamcity/components/builds/build_configuration_per_package.kt

@@ -7,6 +7,7 @@
 
 package builds
 
+import ArtifactRules
 import DefaultBuildTimeoutDuration
 import DefaultParallelism
 import generated.ServiceParallelism
@@ -74,6 +75,7 @@ class PackageDetails(private val packageName: String, private val displayName: S
                 downloadTerraformBinary()
                 runAcceptanceTests()
                 saveArtifactsToGCS()
+                archiveArtifactsIfOverLimit() // Must be after push to GCS step, as this step impacts debug log files
             }
 
             features {
@@ -98,7 +100,7 @@ class PackageDetails(private val packageName: String, private val displayName: S
                 workingDirectory(path)
             }
 
-            artifactRules = "%teamcity.build.checkoutDir%/debug*.txt"
+            artifactRules = ArtifactRules
 
             failureConditions {
                 errorMessage = true

mmv1/third_party/terraform/.teamcity/components/builds/build_configuration_sweepers.kt

@@ -7,6 +7,7 @@
 
 package builds
 
+import ArtifactRules
 import DefaultBuildTimeoutDuration
 import DefaultParallelism
 import jetbrains.buildServer.configs.kotlin.BuildType
@@ -102,7 +103,7 @@ class SweeperDetails(private val sweeperName: String, private val parentProjectN
                 workingDirectory(path)
             }
 
-            artifactRules = "%teamcity.build.checkoutDir%/debug*.txt"
+            artifactRules = ArtifactRules
 
             failureConditions {
                 errorMessage = true

mmv1/third_party/terraform/.teamcity/components/builds/build_configuration_vcr_recording.kt

@@ -7,6 +7,7 @@
 
 package builds
 
+import ArtifactRules
 import DefaultBuildTimeoutDuration
 import DefaultParallelism
 import jetbrains.buildServer.configs.kotlin.BuildType
@@ -77,7 +78,7 @@ class VcrDetails(private val providerName: String, private val buildId: String,
                 workingDirectory(path)
             }
 
-            artifactRules = "%teamcity.build.checkoutDir%/debug*.txt"
+            artifactRules = ArtifactRules
 
             failureConditions {
                 errorMessage = true

mmv1/third_party/terraform/.teamcity/components/builds/build_steps.kt

@@ -137,7 +137,7 @@ fun BuildSteps.saveArtifactsToGCS() {
         name = "Tasks after running nightly tests: push artifacts(debug logs) to GCS"
         scriptContent = """
             #!/bin/bash
-            echo "Post-test step - storge artifacts(debug logs) to GCS"
+            echo "Post-test step - storage artifacts(debug logs) to GCS"
 
             # Authenticate gcloud CLI
             echo "${'$'}{GOOGLE_CREDENTIALS_GCS}" > google-account.json
@@ -173,4 +173,56 @@ fun BuildSteps.saveArtifactsToGCS() {
         // parts like ${GIT_HASH_SHORT} without having Kotlin syntax issues. For more info see:
         // https://youtrack.jetbrains.com/issue/KT-2425/Provide-a-way-for-escaping-the-dollar-sign-symbol-in-multiline-strings-and-string-templates
     })
-}
+}
+
+// The S3 plugin we use to upload artifacts to S3 (enabling them to be accessed via the TeamCity UI later) has a limit of
+// 1000 artifacts to be uploaded at a time. To avoid a situation where no artifacts are uploaded as a result of exceeding this
+// limit we archive all the debug logs if they equal or exceed 1000 for a given build.
+fun BuildSteps.archiveArtifactsIfOverLimit() {
+    step(ScriptBuildStep {
+        name = "Tasks after running nightly tests: archive artifacts(debug logs) if there are >=1000 before S3 upload"
+        scriptContent = """
+            #!/bin/bash
+            echo "Post-test step - archive artifacts(debug logs) if there are >=1000 before S3 upload"
+
+            # Get number of artifacts created
+            ARTIFACT_COUNT=$(ls %teamcity.build.checkoutDir%/debug* | wc -l | grep -o -E '[0-9]+')
+
+            if test ${'$'}ARTIFACT_COUNT -lt "1000"; then
+                echo "Found <1000 debug artifacts; we won't archive them before upload to S3"
+                exit 0
+            fi
+
+            echo "Found >=1000 debug artifacts; archiving before upload to S3"
+
+            # Make tarball of all debug logs
+            # Name should look similar to: debug-google-d2503f7-253644-TerraformProviders_GoogleCloud_GOOGLE_NIGHTLYTESTS_GOOGLE_PACKAGE_ACCESSAPPROVAL.tar.gz
+            cd %teamcity.build.checkoutDir%
+            ARCHIVE_NAME=debug-%PROVIDER_NAME%-%env.BUILD_NUMBER%-%system.teamcity.buildType.id%-archive.tar.gz
+            tar -cf ${'$'}ARCHIVE_NAME ./debug*
+
+            # Fail loudly if archive not made as expected
+            if [ ! -f ${'$'}ARCHIVE_NAME ]; then
+                echo "Archive file ${'$'}ARCHIVE_NAME not found!"
+
+                # Allow sanity checking
+                echo "Listing contents of %teamcity.build.checkoutDir%"
+                ls
+
+                exit 1
+            fi
+
+            # Remove all debug logs. These are all .txt files due to the effects of TF_LOG_PATH_MASK.
+            rm ./debug*.txt
+
+            # Allow sanity checking
+            echo "Listing files matching the artifact rule value %teamcity.build.checkoutDir%/debug*"
+            ls debug*
+
+            echo "Finished"
+        """.trimIndent()
+        // ${'$'} is required to allow creating a script in TeamCity that contains
+        // parts like ${GIT_HASH_SHORT} without having Kotlin syntax issues. For more info see:
+        // https://youtrack.jetbrains.com/issue/KT-2425/Provide-a-way-for-escaping-the-dollar-sign-symbol-in-multiline-strings-and-string-templates
+    })
+}

mmv1/third_party/terraform/.teamcity/components/constants.kt

@@ -43,3 +43,8 @@ const val ProjectSweeperName = "Project Sweeper"
 const val NightlyTestsProjectId = "NightlyTests"
 const val MMUpstreamProjectId = "MMUpstreamTests"
 const val VcrRecordingProjectId = "VCRRecording"
+
+// Artifact rules controls which artifacts are uploaded to S3
+// https://www.jetbrains.com/help/teamcity/2024.07/configuring-general-settings.html#Artifact+Paths
+// The value below lacks a file extension, to allow upload of individual .txt files or a single .tar.gz file
+const val ArtifactRules = "%teamcity.build.checkoutDir%/debug*"