Separate out whitesource and create more scans (kyma-project#1849)

* add more whitesource scans, create a separate file for whitesource periodics, whitesource runner is more generic

development/tools/jobs/kyma/kyma_integration_test.go

@@ -306,7 +306,7 @@ func TestKymaIntegrationJobPeriodics(t *testing.T) {
 	require.NoError(t, err)
 
 	periodics := jobConfig.Periodics
-	assert.Len(t, periodics, 16)
+	assert.Len(t, periodics, 15)
 
 	expName := "orphaned-disks-cleaner"
 	disksCleanerPeriodic := tester.FindPeriodicJobByName(periodics, expName)

prow/config.yaml

@@ -425,6 +425,41 @@ presets:
     env:
     - name: CLUSTER_USE_SSD
       value: "true"
+  # whitesource labels
+  - labels:
+      preset-sa-gke-kyma-integration-kyma-incubator-whitesource: "true"
+    env:
+      - name: GOOGLE_APPLICATION_CREDENTIALS
+        value: /etc/credentials/sa-gke-kyma-integration/service-account.json
+      - name: CLOUDSDK_DNS_ZONE_NAME
+        value: "build-kyma-workloads" #GCloud DNS Zone Name (NOT it's DNS name!)
+      - name: GITHUB_ORG_DIR
+        value: "/home/prow/go/src/github.com/kyma-incubator"
+    volumes:
+      - name: sa-gke-kyma-integration
+        secret:
+          secretName: sa-gke-kyma-integration
+    volumeMounts:
+      - name: sa-gke-kyma-integration
+        mountPath: /etc/credentials/sa-gke-kyma-integration
+        readOnly: true
+  - labels:
+      preset-sa-gke-kyma-integration-kyma-project-whitesource: "true"
+    env:
+      - name: GOOGLE_APPLICATION_CREDENTIALS
+        value: /etc/credentials/sa-gke-kyma-integration/service-account.json
+      - name: CLOUDSDK_DNS_ZONE_NAME
+        value: "build-kyma-workloads" #GCloud DNS Zone Name (NOT it's DNS name!)
+      - name: GITHUB_ORG_DIR
+        value: "/home/prow/go/src/github.com/kyma-project"
+    volumes:
+      - name: sa-gke-kyma-integration
+        secret:
+          secretName: sa-gke-kyma-integration
+    volumeMounts:
+      - name: sa-gke-kyma-integration
+        mountPath: /etc/credentials/sa-gke-kyma-integration
+        readOnly: true
 
 branch-protection:
   enforce_admins: false

prow/jobs/kyma/kyma-integration.yaml

@@ -1110,39 +1110,6 @@ periodics:
               memory: 200Mi
               cpu: 80m
 
-  - name: kyma-whitesource-scan
-    decorate: true
-    cron: "0 4 * * *" # At 04:00 am every day
-    labels:
-      preset-kyma-wssagent-config: "true"
-      preset-kyma-keyring: "true"
-      preset-kyma-encryption-key: "true"
-      preset-kms-gc-project-env: "true"
-      preset-sa-gke-kyma-integration: "true"
-      preset-gc-project-env: "true"
-    extra_refs:
-      - <<: *test_infra_ref
-        base_ref: master
-      - <<: *kyma_ref
-        base_ref: master
-    spec:
-      containers:
-        - image: eu.gcr.io/kyma-project/test-infra/wssagent:v20190909-3282dd6
-          command:
-            - "bash"
-          args:
-            - "-c"
-            - "${KYMA_PROJECT_DIR}/test-infra/prow/scripts/cluster-integration/helpers/start-wssagent.sh"
-          env:
-            - name : PROJECTNAME
-              value: "kyma"
-            - name: DRYRUN
-              value: "false"
-          resources:
-            requests:
-              memory: 1Gi
-              cpu: 400m
-
   - name: kyma-components-use-recent-versions
     decorate: true
     cron: "0 4 * * 1" # At 04:00 on every Monday

prow/jobs/scans/whitesource-periodics.yaml

@@ -0,0 +1,225 @@
+# Code generated by rendertemplates. DO NOT EDIT. 
+
+test_infra_ref: &test_infra_ref
+  org: kyma-project
+  repo: test-infra
+  path_alias: github.com/kyma-project/test-infra
+  base_ref: master
+
+periodics:
+
+  - name: kyma-whitesource-scan
+    decorate: true
+    cron: "0 4 * * *" # At 04:00 am every day
+    labels:
+        preset-kyma-wssagent-config: "true"
+        preset-kyma-keyring: "true"
+        preset-kyma-encryption-key: "true"
+        preset-kms-gc-project-env: "true"
+        preset-gc-project-env: "true"
+        preset-sa-gke-kyma-integration-kyma-project-whitesource: "true"
+    extra_refs:
+        - <<: *test_infra_ref
+        - org: kyma-project
+          repo: kyma
+          path_alias: github.com/kyma-project/kyma
+          base_ref: master
+    spec:
+        containers:
+        - image: eu.gcr.io/kyma-project/test-infra/wssagent:v20190909-3282dd6
+          command:
+            - "bash"
+          args:
+            - "-c"
+            - "${KYMA_PROJECT_DIR}/test-infra/prow/scripts/cluster-integration/helpers/start-wssagent.sh"
+          env:
+            - name : PROJECTNAME
+              value: kyma
+            - name: DRYRUN
+              value: "false"
+            - name: SCAN_LANGUAGE
+              value: golang
+          resources:
+            requests:
+              memory: 1Gi
+              cpu: 400m
+
+  - name: console-whitesource-scan
+    decorate: true
+    cron: "0 4 * * *" # At 04:00 am every day
+    labels:
+        preset-kyma-wssagent-config: "true"
+        preset-kyma-keyring: "true"
+        preset-kyma-encryption-key: "true"
+        preset-kms-gc-project-env: "true"
+        preset-gc-project-env: "true"
+        preset-sa-gke-kyma-integration-kyma-project-whitesource: "true"
+    extra_refs:
+        - <<: *test_infra_ref
+        - org: kyma-project
+          repo: console
+          path_alias: github.com/kyma-project/console
+          base_ref: master
+    spec:
+        containers:
+        - image: eu.gcr.io/kyma-project/test-infra/wssagent:v20190909-3282dd6
+          command:
+            - "bash"
+          args:
+            - "-c"
+            - "${KYMA_PROJECT_DIR}/test-infra/prow/scripts/cluster-integration/helpers/start-wssagent.sh"
+          env:
+            - name : PROJECTNAME
+              value: console
+            - name: DRYRUN
+              value: "false"
+            - name: SCAN_LANGUAGE
+              value: javascript
+          resources:
+            requests:
+              memory: 1Gi
+              cpu: 400m
+
+  - name: cli-whitesource-scan
+    decorate: true
+    cron: "0 4 * * *" # At 04:00 am every day
+    labels:
+        preset-kyma-wssagent-config: "true"
+        preset-kyma-keyring: "true"
+        preset-kyma-encryption-key: "true"
+        preset-kms-gc-project-env: "true"
+        preset-gc-project-env: "true"
+        preset-sa-gke-kyma-integration-kyma-project-whitesource: "true"
+    extra_refs:
+        - <<: *test_infra_ref
+        - org: kyma-project
+          repo: cli
+          path_alias: github.com/kyma-project/cli
+          base_ref: master
+    spec:
+        containers:
+        - image: eu.gcr.io/kyma-project/test-infra/wssagent:v20190909-3282dd6
+          command:
+            - "bash"
+          args:
+            - "-c"
+            - "${KYMA_PROJECT_DIR}/test-infra/prow/scripts/cluster-integration/helpers/start-wssagent.sh"
+          env:
+            - name : PROJECTNAME
+              value: cli
+            - name: DRYRUN
+              value: "false"
+            - name: SCAN_LANGUAGE
+              value: golang
+          resources:
+            requests:
+              memory: 1Gi
+              cpu: 400m
+
+  - name: helm-broker-whitesource-scan
+    decorate: true
+    cron: "0 4 * * *" # At 04:00 am every day
+    labels:
+        preset-kyma-wssagent-config: "true"
+        preset-kyma-keyring: "true"
+        preset-kyma-encryption-key: "true"
+        preset-kms-gc-project-env: "true"
+        preset-gc-project-env: "true"
+        preset-sa-gke-kyma-integration-kyma-project-whitesource: "true"
+    extra_refs:
+        - <<: *test_infra_ref
+        - org: kyma-project
+          repo: helm-broker
+          path_alias: github.com/kyma-project/helm-broker
+          base_ref: master
+    spec:
+        containers:
+        - image: eu.gcr.io/kyma-project/test-infra/wssagent:v20190909-3282dd6
+          command:
+            - "bash"
+          args:
+            - "-c"
+            - "${KYMA_PROJECT_DIR}/test-infra/prow/scripts/cluster-integration/helpers/start-wssagent.sh"
+          env:
+            - name : PROJECTNAME
+              value: helm-broker
+            - name: DRYRUN
+              value: "false"
+            - name: SCAN_LANGUAGE
+              value: golang
+          resources:
+            requests:
+              memory: 1Gi
+              cpu: 400m
+
+  - name: rafter-whitesource-scan
+    decorate: true
+    cron: "0 4 * * *" # At 04:00 am every day
+    labels:
+        preset-kyma-wssagent-config: "true"
+        preset-kyma-keyring: "true"
+        preset-kyma-encryption-key: "true"
+        preset-kms-gc-project-env: "true"
+        preset-gc-project-env: "true"
+        preset-sa-gke-kyma-integration-kyma-project-whitesource: "true"
+    extra_refs:
+        - <<: *test_infra_ref
+        - org: kyma-project
+          repo: rafter
+          path_alias: github.com/kyma-project/rafter
+          base_ref: master
+    spec:
+        containers:
+        - image: eu.gcr.io/kyma-project/test-infra/wssagent:v20190909-3282dd6
+          command:
+            - "bash"
+          args:
+            - "-c"
+            - "${KYMA_PROJECT_DIR}/test-infra/prow/scripts/cluster-integration/helpers/start-wssagent.sh"
+          env:
+            - name : PROJECTNAME
+              value: rafter
+            - name: DRYRUN
+              value: "false"
+            - name: SCAN_LANGUAGE
+              value: golang
+          resources:
+            requests:
+              memory: 1Gi
+              cpu: 400m
+
+  - name: compass-whitesource-scan
+    decorate: true
+    cron: "0 4 * * *" # At 04:00 am every day
+    labels:
+        preset-kyma-wssagent-config: "true"
+        preset-kyma-keyring: "true"
+        preset-kyma-encryption-key: "true"
+        preset-kms-gc-project-env: "true"
+        preset-gc-project-env: "true"
+        preset-sa-gke-kyma-integration-kyma-incubator-whitesource: "true"
+    extra_refs:
+        - <<: *test_infra_ref
+        - org: kyma-incubator
+          repo: compass
+          path_alias: github.com/kyma-incubator/compass
+          base_ref: master
+    spec:
+        containers:
+        - image: eu.gcr.io/kyma-project/test-infra/wssagent:v20190909-3282dd6
+          command:
+            - "bash"
+          args:
+            - "-c"
+            - "${KYMA_PROJECT_DIR}/test-infra/prow/scripts/cluster-integration/helpers/start-wssagent.sh"
+          env:
+            - name : PROJECTNAME
+              value: compass
+            - name: DRYRUN
+              value: "false"
+            - name: SCAN_LANGUAGE
+              value: golang
+          resources:
+            requests:
+              memory: 1Gi
+              cpu: 400m

prow/scripts/cluster-integration/helpers/start-wssagent.sh

@@ -8,7 +8,9 @@
 # - APIKEY- Key provided by SAP Whitesource Team
 # - PRODUCTNAME - Product inside whitesource
 # - USERKEY - Users specified key(should be a service account)
-# - PROJECTNAME- Kyma component name, scans that directory and posts the results in whitesource
+# - PROJECTNAME - Kyma component name, scans that directory and posts the results in whitesource
+# - GITHUB_ORG_DIR - Project directory to scan
+# - SCAN_LANGUAGE - Scan language is used to set the correct values in the whitesource config for golang / javascript
 
 set -o errexit
 
@@ -41,7 +43,21 @@ USERKEY=$(cat "whitesource-userkey")
 "${TEST_INFRA_CLUSTER_INTEGRATION_SCRIPTS}/decrypt.sh" "whitesource-apikey" "whitesource-apikey.encrypted"
 APIKEY=$(cat "whitesource-apikey")
 
-sed -i.bak "s|go.dependencyManager=godep|go.dependencyManager=dep|g" /wss/wss-unified-agent.config
+case "${SCANLANG}" in
+    golang)
+        sed -i.bak "s|go.dependencyManager=godep|go.dependencyManager=dep|g" /wss/wss-unified-agent.config
+        ;;
+
+    javascript)
+        sed -i.bak "go.resolveDependencies=true|# go.resolveDependencies=true" /wss/wss-unified-agent.config
+        sed -i.bak "go.collectDependenciesAtRuntime=false|# go.collectDependenciesAtRuntime=false" /wss/wss-unified-agent.config
+        sed -i.bak "go.dependencyManager=godep|# go.dependencyManager=godep" /wss/wss-unified-agent.config
+        ;;
+
+    *)
+        echo "can only be golang or javascript"
+        exit 1
+esac
 
 # backup config for re-use
 /bin/cp /wss/wss-unified-agent.config /wss/wss-unified-agent.config.backup
@@ -50,7 +66,7 @@ echo "***********************************"
 echo "***********Starting Scan***********"
 echo "***********************************"
 
-KYMA_SRC="${KYMA_PROJECT_DIR}/${PROJECTNAME}"
+KYMA_SRC="${GITHUB_ORG_DIR}/${PROJECTNAME}"
 
 function scanFolder() { # expects to get the fqdn of folder passed to scan
     if [[ $1 == "" ]]; then
@@ -64,6 +80,11 @@ function scanFolder() { # expects to get the fqdn of folder passed to scan
     fi
     cd "${FOLDER}" # change to passed parameter
     PROJNAME=$2
+    if [[ $3 == "" ]]; then
+        echo "need language parameter for config"
+        exit 1
+    fi
+    SCANLANG=$3
 
     /bin/cp /wss/wss-unified-agent.config.backup /wss/wss-unified-agent.config
 
@@ -91,22 +112,7 @@ function scanFolder() { # expects to get the fqdn of folder passed to scan
     fi
 }
 
-scanFolder "${KYMA_SRC}" "kyma"
-
-# KYMA_COMMON="${KYMA_SRC}/common"
-# KYMA_INSTALLATION="${KYMA_SRC}/installation"
-# KYMA_COMPONENTS="${KYMA_SRC}/components"
-# scanFolder "${KYMA_COMMON}" "kyma/common"
-# scanFolder "${KYMA_INSTALLATION}" "kyma/installation"
-
-# cd "${KYMA_COMPONENTS}"
-# for comp_dir in */;
-# do
-#     # shellcheck disable=SC2001
-#     VAL=$(echo "${comp_dir}" | sed 's/.$//')
-#     echo "Processing '${VAL}' for scan'"
-#     scanFolder "${KYMA_COMPONENTS}/${VAL}" "${VAL}"
-# done
+scanFolder "${KYMA_SRC}" "${PROJECTNAME}"
 
 echo "***********************************"
 echo "*********Scanning Finished*********"