diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..3229dcc --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +@Hygieia/hygieia-codequality-sonar-collector \ No newline at end of file diff --git a/.github/stale.yml b/.github/stale.yml new file mode 100644 index 0000000..e1e612f --- /dev/null +++ b/.github/stale.yml @@ -0,0 +1,18 @@ +# Number of days of inactivity before an issue becomes stale +daysUntilStale: 60 +# Number of days of inactivity before a stale issue is closed +daysUntilClose: 7 +# Issues with these labels will never be considered stale +exemptLabels: + - bug + - enhancement + - discussion +# Label to use when marking an issue as stale +staleLabel: wontfix +# Comment to post when marking an issue as stale. Set to `false` to disable +markComment: > + This issue has been automatically marked as stale because it has not had + recent activity. It will be closed if no further activity occurs. Thank you + for your contributions. +# Comment to post when closing a stale issue. Set to `false` to disable +closeComment: false \ No newline at end of file diff --git a/.github/workflows/maven-pr-checks.yml b/.github/workflows/maven-pr-checks.yml new file mode 100644 index 0000000..ea11bf0 --- /dev/null +++ b/.github/workflows/maven-pr-checks.yml @@ -0,0 +1,18 @@ +name: Java Compile and Test + +on: + pull_request: + branches: + - master +jobs: + build: + runs-on: ubuntu-18.04 + + steps: + - uses: actions/checkout@v1 + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - name: Build with Maven + run: mvn test --file pom.xml \ No newline at end of file diff --git a/.github/workflows/maven-snapshot-release.yml b/.github/workflows/maven-snapshot-release.yml new file mode 100644 index 0000000..c673fe6 --- /dev/null +++ b/.github/workflows/maven-snapshot-release.yml @@ -0,0 +1,19 @@ +name: Release Snapshot + +on: + push: + branches: + - master + +jobs: + build: + runs-on: ubuntu-18.04 + + steps: + - uses: actions/checkout@v1 + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - name: Release to snapshots repository + run: mvn deploy -q --file pom.xml --settings src/devops/.travis.settings.xml -Denv.OSSRH_USERNAME=${{ secrets.OSSRH_USERNAME }} -Denv.OSSRH_PASSWORD=${{ secrets.OSSRH_PASSWORD }} \ No newline at end of file diff --git a/.sonarcloud.properties b/.sonarcloud.properties new file mode 100644 index 0000000..7a93b07 --- /dev/null +++ b/.sonarcloud.properties @@ -0,0 +1,15 @@ +# Path to sources +#sonar.sources=. +#sonar.exclusions= +#sonar.inclusions= + +# Path to tests +#sonar.tests= +#sonar.test.exclusions= +#sonar.test.inclusions= + +# Source encoding +#sonar.sourceEncoding=UTF-8 + +# Exclusions for copy-paste detection +#sonar.cpd.exclusions= diff --git a/.travis.yml b/.travis.yml index fccf6d9..31e3f24 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,11 +1,11 @@ language: java jdk: - openjdk8 +services: + - docker branches: only: - - master - - gerrit-integration - "/^v[0-9]+\\.[0-9]+\\.[0-9]+.*$/" install: true @@ -14,9 +14,15 @@ before_script: - sudo chown -R $USER:$GROUP $TRAVIS_BUILD_DIR script: - - if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then mvn clean install -q; fi - - if [ "$TRAVIS_PULL_REQUEST" = "false" ] && [ "$TRAVIS_BRANCH" = "master" ]; then cp ./src/devops/deploy-snapshot.sh . && ./deploy-snapshot.sh; fi - if [ "$TRAVIS_PULL_REQUEST" = "false" ] && [[ "$TRAVIS_BRANCH" =~ ^v[0-9]+\.[0-9]+\.[0-9]+.*$ ]]; then cp ./src/devops/release.sh . && ./release.sh; fi + - export TAG=$TRAVIS_BRANCH + - export IMAGE_NAME='hygieiadoc/sonarcodequalitycollector' + - echo $IMAGE_NAME + - echo $TAG + - docker build -t $IMAGE_NAME . + - docker tag $IMAGE_NAME $IMAGE_NAME:$TAG + - docker login -u $DOCKERHUB_USER -p $DOCKERHUB_PASS + - docker push $IMAGE_NAME notifications: webhooks: @@ -30,4 +36,4 @@ notifications: - ragha.vema@capitalone.com - hygieia2@capitalone.com on_success: always - on_failure: always + on_failure: always \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index e031404..8205081 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ ENV PROP_FILE /hygieia/config/application.properties WORKDIR /hygieia -COPY target/*.jar /hygieia +COPY target/*.jar /hygieia/ COPY docker/properties-builder.sh /hygieia/ CMD ./properties-builder.sh &&\ diff --git a/README.md b/README.md index 04fa917..72a3799 100644 --- a/README.md +++ b/README.md @@ -1 +1,12 @@ +## Hygieia Collector to collect static code analysis data from Sonar +[![Build Status](https://travis-ci.com/Hygieia/hygieia-codequality-sonar-collector.svg?branch=master)](https://travis-ci.com/Hygieia/hygieia-codequality-sonar-collector) +[![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=Hygieia_hygieia-codequality-sonar-collector&metric=alert_status)](https://sonarcloud.io/dashboard?id=Hygieia_hygieia-codequality-sonar-collector) +[![Total alerts](https://img.shields.io/lgtm/alerts/g/Hygieia/hygieia-codequality-sonar-collector.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/Hygieia/hygieia-codequality-sonar-collector/alerts/) +[![Language grade: Java](https://img.shields.io/lgtm/grade/java/g/Hygieia/hygieia-codequality-sonar-collector.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/Hygieia/hygieia-codequality-sonar-collector/context:java) +[![Maven Central](https://img.shields.io/maven-central/v/com.capitalone.dashboard/sonar-codequality-collector.svg?label=Maven%20Central)](https://search.maven.org/search?q=g:%22com.capitalone.dashboard%22%20AND%20a:%22sonar-codequality-collector%22) +[![License](https://img.shields.io/badge/license-Apache%202-blue.svg)](https://www.apache.org/licenses/LICENSE-2.0) +[![Gitter Chat](https://badges.gitter.im/Join%20Chat.svg)](https://www.apache.org/licenses/LICENSE-2.0) +
+
+ The README is in the [gh-pages](https://github.com/capitalone/Hygieia/blob/gh-pages/pages/hygieia/collectors/build/sonar.md) branch. Please update it there. diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index 92d15d5..0000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,19 +0,0 @@ - -FROM docker.io/openjdk:8-jre - -MAINTAINER Hygieia@capitalone.com - -RUN mkdir /hygieia /hygieia/config - -COPY *.jar /hygieia/ -COPY properties-builder.sh /hygieia/ - -WORKDIR /hygieia - -VOLUME ["/hygieia/logs"] - -ENV PROP_FILE /hygieia/config/application.properties - -CMD ./properties-builder.sh && \ - java -jar sonar-codequality-collector*.jar --spring.config.location=$PROP_FILE - diff --git a/pom.xml b/pom.xml index f489c55..2688fae 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ CodeQuality Collector Microservices currently collects data from Sonar https://hygieia.github.io/Hygieia/getting_started.html jar - 3.1.1-SNAPSHOT + 3.1.3-SNAPSHOT com.capitalone.dashboard @@ -46,7 +46,7 @@ 0.13 3.0.1 - 3.1.2 + 3.1.11 2.4 3.8.1 4.3.0 @@ -199,6 +199,7 @@ clean verify apache-rat:check japicmp:cmp checkstyle:check spotbugs:check javadoc:javadoc + ${artifactId} org.springframework.boot diff --git a/src/devops/keys.gpg.enc b/src/devops/keys.gpg.enc index 7c82cb4..c4b0b54 100644 Binary files a/src/devops/keys.gpg.enc and b/src/devops/keys.gpg.enc differ diff --git a/src/devops/release.sh b/src/devops/release.sh index 714a08a..295d888 100755 --- a/src/devops/release.sh +++ b/src/devops/release.sh @@ -2,7 +2,7 @@ cp src/devops/.travis.settings.xml $HOME/.m2/settings.xml -openssl aes-256-cbc -K $encrypted_8b6a4661ac23_key -iv $encrypted_8b6a4661ac23_iv -in src/devops/keys.gpg.enc -out keys.gpg -d +openssl aes-256-cbc -K $encrypted_64259960f58b_key -iv $encrypted_64259960f58b_iv -in src/devops/keys.gpg.enc -out keys.gpg -d gpg --fast-import keys.gpg diff --git a/src/main/java/com/capitalone/dashboard/collector/DefaultSonar56Client.java b/src/main/java/com/capitalone/dashboard/collector/DefaultSonar56Client.java index 59f70ea..3789638 100755 --- a/src/main/java/com/capitalone/dashboard/collector/DefaultSonar56Client.java +++ b/src/main/java/com/capitalone/dashboard/collector/DefaultSonar56Client.java @@ -1,5 +1,6 @@ package com.capitalone.dashboard.collector; +import com.capitalone.dashboard.client.RestClient; import com.capitalone.dashboard.model.SonarProject; import com.capitalone.dashboard.util.Supplier; import org.apache.commons.logging.Log; @@ -21,12 +22,12 @@ public class DefaultSonar56Client extends DefaultSonarClient { private static final String URL_PROJECTS = "/api/projects?format=json"; @Autowired - public DefaultSonar56Client(Supplier restOperationsSupplier, SonarSettings settings) { - super(restOperationsSupplier,settings); + public DefaultSonar56Client(RestClient restClient, SonarSettings settings) { + super(restClient,settings); } @Override - public List getProjects(String instanceUrl) { + public List getProjects(String instanceUrl,String token) { List projects = new ArrayList<>(); String url = instanceUrl + URL_PROJECTS; try { diff --git a/src/main/java/com/capitalone/dashboard/collector/DefaultSonar6Client.java b/src/main/java/com/capitalone/dashboard/collector/DefaultSonar6Client.java index fd72b56..ce05acc 100644 --- a/src/main/java/com/capitalone/dashboard/collector/DefaultSonar6Client.java +++ b/src/main/java/com/capitalone/dashboard/collector/DefaultSonar6Client.java @@ -1,10 +1,13 @@ package com.capitalone.dashboard.collector; +import com.capitalone.dashboard.client.RestClient; +import com.capitalone.dashboard.client.RestUserInfo; import com.capitalone.dashboard.model.CodeQuality; import com.capitalone.dashboard.model.CodeQualityMetric; import com.capitalone.dashboard.model.CodeQualityMetricStatus; import com.capitalone.dashboard.model.CodeQualityType; import com.capitalone.dashboard.model.SonarProject; +import com.capitalone.dashboard.model.UserInfo; import com.capitalone.dashboard.util.SonarDashboardUrl; import com.capitalone.dashboard.util.Supplier; import org.apache.commons.codec.binary.Base64; @@ -30,12 +33,13 @@ import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.List; +import java.util.Objects; @Component public class DefaultSonar6Client implements SonarClient { private static final Log LOG = LogFactory.getLog(DefaultSonar6Client.class); - private static final String URL_RESOURCES = "/api/components/search?qualifiers=TRK&ps=500"; + private static final String URL_RESOURCES_AUTHENTICATED = "/api/projects/search?ps=500"; private static final String URL_RESOURCE_DETAILS = "/api/measures/component?format=json&componentId=%s&metricKeys=%s&includealerts=true"; private static final String URL_PROJECT_ANALYSES = "/api/project_analyses/search?project=%s"; private static final String URL_QUALITY_PROFILES = "/api/qualityprofiles/search"; @@ -56,8 +60,8 @@ public class DefaultSonar6Client implements SonarClient { private static final String DATE = "date"; private static final String EVENTS = "events"; - private final RestOperations rest; - private final HttpEntity httpHeaders; + private final RestClient restClient; + private final RestUserInfo userInfo; private static final String MINUTES_FORMAT = "%smin"; private static final String HOURS_FORMAT = "%sh"; @@ -66,21 +70,30 @@ public class DefaultSonar6Client implements SonarClient { private static final int PAGE_SIZE=500; @Autowired - public DefaultSonar6Client(Supplier restOperationsSupplier, SonarSettings settings) { - this.httpHeaders = new HttpEntity<>( - this.createHeaders(settings.getUsername(), settings.getPassword()) - ); - this.rest = restOperationsSupplier.get(); + public DefaultSonar6Client(RestClient restClient, SonarSettings settings) { + if(StringUtils.isEmpty(settings.getUsername())|| StringUtils.isEmpty(settings.getPassword())){ + this.userInfo= new RestUserInfo("",""); + }else{ + this.userInfo = new RestUserInfo(settings.getUsername(),settings.getPassword()); + } + + this.restClient = restClient; } @Override - public List getProjects(String instanceUrl) { + public List getProjects(String instanceUrl,String token) { List projects = new ArrayList<>(); - String url = instanceUrl + URL_RESOURCES; + String url = ""; + // take authenticated route + if(Objects.nonNull(token)){ + url = instanceUrl + URL_RESOURCES_AUTHENTICATED; + userInfo.setToken(token); + }else{ + url = instanceUrl + URL_RESOURCES; + } try { - String key = "components"; - JSONArray jsonArray = getProjects(url, key); + JSONArray jsonArray = getProjects(url); for (Object obj : jsonArray) { JSONObject prjData = (JSONObject) obj; @@ -100,7 +113,8 @@ public List getProjects(String instanceUrl) { return projects; } - private JSONArray getProjects(String url, String key) throws ParseException { + private JSONArray getProjects(String url) throws ParseException { + String key = "components"; Long totalRecords = getTotalCount(parseJsonObject(url, "paging")); int pages = (int) Math.ceil((double)totalRecords / PAGE_SIZE); JSONArray jsonArray = new JSONArray(); @@ -114,14 +128,24 @@ private JSONArray getProjects(String url, String key, JSONArray jsonArray) throw } private JSONArray getProjects(String url, String key, int pages, JSONArray jsonArray) throws ParseException { + if(Objects.isNull(userInfo.getToken())){ + pagingUnAuthenticated(url, key, pages, jsonArray); + }else{ + for (int start=1;start<=pages;start++){ + getProjects(url, key, jsonArray, start); + } + } + return jsonArray; + } + + private void pagingUnAuthenticated(String url, String key, int pages, JSONArray jsonArray) throws ParseException { int maxPages = 20; if(pages <= maxPages) { maxPages = pages; } - for (int start=1;start<=maxPages;start++){ + for (int start=1;start<=maxPages;start++){ getProjects(url, key, jsonArray, start); } - return jsonArray; } private void getProjects(String url, String key, JSONArray jsonArray, int pageNumber) throws ParseException { @@ -135,7 +159,7 @@ public CodeQuality currentCodeQuality(SonarProject project, String metrics) { project.getInstanceUrl() + URL_RESOURCE_DETAILS, project.getProjectId(), metrics); try { - ResponseEntity response = rest.exchange(url, HttpMethod.GET, this.httpHeaders, String.class); + ResponseEntity response = restClient.makeRestCallGet(url,setHeaders(userInfo) ); JSONParser jsonParser = new JSONParser(); JSONObject jsonObject = (JSONObject) jsonParser.parse(response.getBody()); String key = "component"; @@ -258,7 +282,7 @@ private JSONObject parseJsonObject(String url, String key) throws ParseException } private JSONObject getResponse(String url) throws ParseException { - ResponseEntity response = rest.exchange(url, HttpMethod.GET, this.httpHeaders, String.class); + ResponseEntity response = restClient.makeRestCallGet(url, setHeaders(userInfo)); JSONParser jsonParser = new JSONParser(); JSONObject jsonObject = (JSONObject) jsonParser.parse(response.getBody()); LOG.debug(url); @@ -366,6 +390,10 @@ private CodeQualityMetricStatus metricStatus(String status) { } } + private Long getTotalCount(JSONObject pagingObject) { + return (Long) pagingObject.get("total"); + } + private HttpHeaders createHeaders(String username, String password){ HttpHeaders headers = new HttpHeaders(); if (username != null && !username.isEmpty()) { @@ -379,8 +407,27 @@ private HttpHeaders createHeaders(String username, String password){ return headers; } - private Long getTotalCount(JSONObject pagingObject) { - return (Long) pagingObject.get("total"); + private HttpHeaders createHeaders(String token) { + String auth = token.trim(); + auth = auth+":"; + byte[] encodedAuth = Base64.encodeBase64( + auth.getBytes(Charset.forName("US-ASCII")) + ); + String authHeader = "Basic " + new String(encodedAuth); + HttpHeaders headers = new HttpHeaders(); + headers.set("Authorization", authHeader); + return headers; + } + + private HttpHeaders setHeaders(RestUserInfo userInfo){ + if(Objects.isNull(userInfo)) return null; + if(StringUtils.isNotBlank(userInfo.getUserId())&& StringUtils.isNotBlank(userInfo.getPassCode())){ + return createHeaders(userInfo.getUserId(),userInfo.getPassCode()); + }else if(StringUtils.isNotBlank(userInfo.getToken())){ + return createHeaders(userInfo.getToken()); + } + return null; } + } diff --git a/src/main/java/com/capitalone/dashboard/collector/DefaultSonarClient.java b/src/main/java/com/capitalone/dashboard/collector/DefaultSonarClient.java index 79f3457..8272676 100644 --- a/src/main/java/com/capitalone/dashboard/collector/DefaultSonarClient.java +++ b/src/main/java/com/capitalone/dashboard/collector/DefaultSonarClient.java @@ -1,12 +1,13 @@ package com.capitalone.dashboard.collector; +import com.capitalone.dashboard.client.RestClient; +import com.capitalone.dashboard.client.RestUserInfo; import com.capitalone.dashboard.model.CodeQuality; import com.capitalone.dashboard.model.CodeQualityMetric; import com.capitalone.dashboard.model.CodeQualityMetricStatus; import com.capitalone.dashboard.model.CodeQualityType; import com.capitalone.dashboard.model.SonarProject; import com.capitalone.dashboard.util.SonarDashboardUrl; -import com.capitalone.dashboard.util.Supplier; import org.apache.commons.codec.binary.Base64; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -17,13 +18,10 @@ import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.HttpEntity; import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpMethod; import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Component; import org.springframework.web.client.RestClientException; -import org.springframework.web.client.RestOperations; import java.math.BigDecimal; import java.nio.charset.Charset; @@ -55,19 +53,17 @@ public class DefaultSonarClient implements SonarClient { protected static final String STATUS_ALERT = "ALERT"; protected static final String DATE = "date"; - protected final RestOperations rest; - protected final HttpEntity httpHeaders; + protected final RestClient restClient; + protected final RestUserInfo userInfo; @Autowired - public DefaultSonarClient(Supplier restOperationsSupplier, SonarSettings settings) { - this.httpHeaders = new HttpEntity<>( - this.createHeaders(settings.getUsername(), settings.getPassword()) - ); - this.rest = restOperationsSupplier.get(); + public DefaultSonarClient(RestClient restClient, SonarSettings settings) { + userInfo = settings.getUsername()==null?null:new RestUserInfo(settings.getUsername(), settings.getPassword()); + this.restClient = restClient; } @Override - public List getProjects(String instanceUrl) { + public List getProjects(String instanceUrl,String token) { List projects = new ArrayList<>(); String url = instanceUrl + URL_RESOURCES; @@ -186,12 +182,12 @@ public JSONArray getQualityProfileConfigurationChanges(String instanceUrl,String } protected JSONArray parseAsArray(String url) throws ParseException { - ResponseEntity response = rest.exchange(url, HttpMethod.GET, this.httpHeaders, String.class); + ResponseEntity response = restClient.makeRestCallGet(url, this.userInfo); return (JSONArray) new JSONParser().parse(response.getBody()); } protected JSONArray parseAsArray(String url, String key) throws ParseException { - ResponseEntity response = rest.exchange(url, HttpMethod.GET, this.httpHeaders, String.class); + ResponseEntity response = restClient.makeRestCallGet(url, this.userInfo); JSONParser jsonParser = new JSONParser(); JSONObject jsonObject = (JSONObject) jsonParser.parse(response.getBody()); LOG.debug(url); @@ -244,17 +240,4 @@ protected CodeQualityMetricStatus metricStatus(String status) { } } - private final HttpHeaders createHeaders(String username, String password){ - HttpHeaders headers = new HttpHeaders(); - if (username != null && !username.isEmpty() && - password != null && !password.isEmpty()) { - String auth = username + ":" + password; - byte[] encodedAuth = Base64.encodeBase64( - auth.getBytes(Charset.forName("US-ASCII")) - ); - String authHeader = "Basic " + new String(encodedAuth); - headers.set("Authorization", authHeader); - } - return headers; - } } diff --git a/src/main/java/com/capitalone/dashboard/collector/SonarClient.java b/src/main/java/com/capitalone/dashboard/collector/SonarClient.java index 0f3cbe4..4694f60 100644 --- a/src/main/java/com/capitalone/dashboard/collector/SonarClient.java +++ b/src/main/java/com/capitalone/dashboard/collector/SonarClient.java @@ -10,7 +10,7 @@ public interface SonarClient { - List getProjects(String instanceUrl); + List getProjects(String instanceUrl,String token); CodeQuality currentCodeQuality(SonarProject project, String metrics); JSONArray getQualityProfiles(String instanceUrl) throws ParseException; List retrieveProfileAndProjectAssociation(String instanceUrl,String qualityProfile) throws ParseException; diff --git a/src/main/java/com/capitalone/dashboard/collector/SonarClientSelector.java b/src/main/java/com/capitalone/dashboard/collector/SonarClientSelector.java index 4149e2e..796bdb0 100644 --- a/src/main/java/com/capitalone/dashboard/collector/SonarClientSelector.java +++ b/src/main/java/com/capitalone/dashboard/collector/SonarClientSelector.java @@ -1,20 +1,63 @@ package com.capitalone.dashboard.collector; +import java.net.URI; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; +import org.springframework.web.client.RestClientException; +import org.springframework.web.client.RestOperations; + +import com.capitalone.dashboard.util.Supplier; @Component public class SonarClientSelector { + private static final Log LOG = LogFactory.getLog(SonarClientSelector.class); + + private static final String URL_VERSION_RESOURCE = "/api/server/version"; - @Autowired private DefaultSonar6Client sonar6Client; - @Autowired private DefaultSonar56Client sonar56Client; - @Autowired - @Qualifier("DefaultSonarClient") private DefaultSonarClient sonarClient; + private RestOperations rest; + + @Autowired + public SonarClientSelector( + DefaultSonar6Client sonar6Client, DefaultSonar56Client sonar56Client, + @Qualifier("DefaultSonarClient") DefaultSonarClient sonarClient, + Supplier restOperationsSupplier) { + + this.sonar6Client = sonar6Client; + this.sonar56Client = sonar56Client; + this.sonarClient = sonarClient; + this.rest = restOperationsSupplier.get(); + } + + public Double getSonarVersion(String instanceUrl){ + Double version = 5.0; + try { + ResponseEntity versionResponse = rest.exchange(URI.create(instanceUrl + URL_VERSION_RESOURCE), HttpMethod.GET, new HttpEntity<>(new HttpHeaders()), String.class); + if(!versionResponse.getBody().isEmpty()) { + if(StringUtils.countOccurrencesOf(versionResponse.getBody(), ".") > 1) { + version = Double.parseDouble(versionResponse.getBody().substring(0, 3)); + } else { + version = Double.parseDouble(versionResponse.getBody()); + } + } + } catch (RestClientException e) { + LOG.info("Rest exception occured at fetching sonar version"); + } + + return version; + } public SonarClient getSonarClient(Double version) { if(version != null && version == 5.6){ diff --git a/src/main/java/com/capitalone/dashboard/collector/SonarCollectorTask.java b/src/main/java/com/capitalone/dashboard/collector/SonarCollectorTask.java index 4e1338f..4c1368b 100644 --- a/src/main/java/com/capitalone/dashboard/collector/SonarCollectorTask.java +++ b/src/main/java/com/capitalone/dashboard/collector/SonarCollectorTask.java @@ -34,10 +34,12 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; @Component public class SonarCollectorTask extends CollectorTask { - @SuppressWarnings({ "PMD.UnusedPrivateField", "unused" }) + private static final Log LOG = LogFactory.getLog(SonarCollectorTask.class); private final SonarCollectorRepository sonarCollectorRepository; @@ -69,7 +71,7 @@ public SonarCollectorTask(TaskScheduler taskScheduler, @Override public SonarCollector getCollector() { - return SonarCollector.prototype(sonarSettings.getServers(), sonarSettings.getVersions(), sonarSettings.getMetrics(),sonarSettings.getNiceNames()); + return SonarCollector.prototype(sonarSettings.getServers(), sonarSettings.getMetrics(),sonarSettings.getNiceNames()); } @Override @@ -97,12 +99,13 @@ public void collect(SonarCollector collector) { for (int i = 0; i < collector.getSonarServers().size(); i++) { String instanceUrl = collector.getSonarServers().get(i); - Double version = collector.getSonarVersions().get(i); + Double version = sonarClientSelector.getSonarVersion(instanceUrl); String metrics = collector.getSonarMetrics().get(i); + String token = getToken(sonarSettings.getTokens(),i); logBanner(instanceUrl); SonarClient sonarClient = sonarClientSelector.getSonarClient(version); - List projects = sonarClient.getProjects(instanceUrl); + List projects = sonarClient.getProjects(instanceUrl,token); latestProjects.addAll(projects); int projSize = ((CollectionUtils.isEmpty(projects)) ? 0 : projects.size()); @@ -127,30 +130,37 @@ public void collect(SonarCollector collector) { deleteUnwantedJobs(latestProjects, existingProjects, collector); } + + private String getToken(List tokens,int index){ + if(CollectionUtils.isEmpty(tokens)) return null; + if (CollectionUtils.isNotEmpty(tokens)){ + if(tokens.size()>index){ + return tokens.get(index); + } + } + return null; + } /** * Clean up unused sonar collector items * * @param collector * the {@link SonarCollector} */ - - @SuppressWarnings("PMD.AvoidDeeplyNestedIfStmts") // agreed PMD, fixme private void clean(SonarCollector collector, List existingProjects) { - Set uniqueIDs = new HashSet<>(); - for (com.capitalone.dashboard.model.Component comp : dbComponentRepository - .findAll()) { - if (comp.getCollectorItems() != null && !comp.getCollectorItems().isEmpty()) { - List itemList = comp.getCollectorItems().get( - CollectorType.CodeQuality); - if (itemList != null) { - for (CollectorItem ci : itemList) { - if (ci != null && ci.getCollectorId().equals(collector.getId())) { - uniqueIDs.add(ci.getId()); - } - } - } - } - } + // extract unique collector item IDs from components + // (in this context collector_items are sonar projects) + Set uniqueIDs = StreamSupport.stream(dbComponentRepository.findAll().spliterator(),false) + .filter( comp -> comp.getCollectorItems() != null && !comp.getCollectorItems().isEmpty()) + .map(comp -> comp.getCollectorItems().get(CollectorType.CodeQuality)) + // keep nonNull List + .filter(itemList -> itemList != null ) + // merge all lists (flatten) into a stream + .flatMap(List::stream) + // keep nonNull CollectorItems + .filter(ci -> ci != null && ci.getCollectorId().equals(collector.getId())) + .map(CollectorItem::getId) + .collect(Collectors.toSet()); + List stateChangeJobList = new ArrayList<>(); Set udId = new HashSet<>(); udId.add(collector.getId()); @@ -223,7 +233,6 @@ private void refreshData(List sonarProjects, SonarClient sonarClie log("Updated", start, count); } - @SuppressWarnings("PMD.AvoidDeeplyNestedIfStmts") private void fetchQualityProfileConfigChanges(SonarCollector collector,String instanceUrl,SonarClient sonarClient) throws org.json.simple.parser.ParseException{ JSONArray qualityProfiles = sonarClient.getQualityProfiles(instanceUrl); JSONArray sonarProfileConfigurationChanges = new JSONArray(); diff --git a/src/main/java/com/capitalone/dashboard/collector/SonarSettings.java b/src/main/java/com/capitalone/dashboard/collector/SonarSettings.java index 73d9a1d..8fa181f 100644 --- a/src/main/java/com/capitalone/dashboard/collector/SonarSettings.java +++ b/src/main/java/com/capitalone/dashboard/collector/SonarSettings.java @@ -15,9 +15,9 @@ public class SonarSettings { private String username; private String password; private List servers; - private List versions; private List metrics; private List niceNames; + private List tokens; public String getCron() { return cron; @@ -59,14 +59,6 @@ public void setServers(List servers) { this.servers = servers; } - public List getVersions() { - return versions; - } - - public void setVersions(List versions) { - this.versions = versions; - } - public List getNiceNames() { return niceNames; } @@ -75,4 +67,12 @@ public void setNiceNames(List niceNames) { this.niceNames = niceNames; } + public List getTokens() { + return tokens; + } + + public void setTokens(List tokens) { + this.tokens = tokens; + } + } diff --git a/src/main/java/com/capitalone/dashboard/model/SonarCollector.java b/src/main/java/com/capitalone/dashboard/model/SonarCollector.java index 81e1a9c..e9b9b2c 100644 --- a/src/main/java/com/capitalone/dashboard/model/SonarCollector.java +++ b/src/main/java/com/capitalone/dashboard/model/SonarCollector.java @@ -10,7 +10,6 @@ public class SonarCollector extends Collector { private List sonarServers = new ArrayList<>(); - private List sonarVersions = new ArrayList<>(); private List sonarMetrics = new ArrayList<>(); private List niceNames = new ArrayList<>(); private static final String NICE_NAME = "niceName"; @@ -19,15 +18,11 @@ public class SonarCollector extends Collector { public List getSonarServers() { return sonarServers; } - public List getSonarVersions() { - return sonarVersions; - } public List getSonarMetrics() { return sonarMetrics; } - public List getNiceNames() { return niceNames; } @@ -36,7 +31,7 @@ public void setNiceNames(List niceNames) { this.niceNames = niceNames; } - public static SonarCollector prototype(List servers, List versions, List metrics,List niceNames) { + public static SonarCollector prototype(List servers, List metrics,List niceNames) { SonarCollector protoType = new SonarCollector(); protoType.setName("Sonar"); protoType.setCollectorType(CollectorType.CodeQuality); @@ -45,9 +40,6 @@ public static SonarCollector prototype(List servers, List versio if(servers!=null) { protoType.getSonarServers().addAll(servers); } - if(versions!=null) { - protoType.getSonarVersions().addAll(versions); - } if(metrics!=null) { protoType.getSonarMetrics().addAll(metrics); } diff --git a/src/test/java/com/capitalone/dashboard/collector/DefaultSonar6ClientTest.java b/src/test/java/com/capitalone/dashboard/collector/DefaultSonar6ClientTest.java index b618c55..b5bc4a3 100644 --- a/src/test/java/com/capitalone/dashboard/collector/DefaultSonar6ClientTest.java +++ b/src/test/java/com/capitalone/dashboard/collector/DefaultSonar6ClientTest.java @@ -1,5 +1,6 @@ package com.capitalone.dashboard.collector; +import com.capitalone.dashboard.client.RestClient; import com.capitalone.dashboard.model.CodeQuality; import com.capitalone.dashboard.model.CodeQualityType; import com.capitalone.dashboard.model.SonarProject; @@ -49,7 +50,7 @@ public void init() { when(restOperationsSupplier.get()).thenReturn(rest); settings = new SonarSettings(); settings.setMetrics(Collections.singletonList(METRICS)); - defaultSonar6Client = new DefaultSonar6Client(restOperationsSupplier, settings); + defaultSonar6Client = new DefaultSonar6Client(new RestClient(restOperationsSupplier), settings); } @Test @@ -57,7 +58,7 @@ public void getProjects() throws Exception { String projectJson = getJson("sonar6projects.json"); String projectsUrl = SONAR_URL + URL_RESOURCES; doReturn(new ResponseEntity<>(projectJson, HttpStatus.OK)).when(rest).exchange(eq(projectsUrl), eq(HttpMethod.GET), Matchers.any(HttpEntity.class), eq(String.class)); - List projects = defaultSonar6Client.getProjects(SONAR_URL); + List projects = defaultSonar6Client.getProjects(SONAR_URL,null); assertThat(projects.size(), is(2)); assertThat(projects.get(0).getProjectName(), is("com.capitalone.test:TestProject")); assertThat(projects.get(1).getProjectName(), is("com.capitalone.test:AnotherTestProject")); @@ -82,7 +83,7 @@ public void getProjects500() throws Exception { doReturn(new ResponseEntity<>(projectJson1500, HttpStatus.OK)).when(rest).exchange(eq(projectsUrl3), eq(HttpMethod.GET), Matchers.any(HttpEntity.class), eq(String.class)); doReturn(new ResponseEntity<>(projectJson2000, HttpStatus.OK)).when(rest).exchange(eq(projectsUrl4), eq(HttpMethod.GET), Matchers.any(HttpEntity.class), eq(String.class)); - List projects = defaultSonar6Client.getProjects(SONAR_URL); + List projects = defaultSonar6Client.getProjects(SONAR_URL,null); assertThat(projects.size(), is(2000)); } diff --git a/src/test/java/com/capitalone/dashboard/collector/SonarClientSelectorTest.java b/src/test/java/com/capitalone/dashboard/collector/SonarClientSelectorTest.java index 736150c..2017c8b 100644 --- a/src/test/java/com/capitalone/dashboard/collector/SonarClientSelectorTest.java +++ b/src/test/java/com/capitalone/dashboard/collector/SonarClientSelectorTest.java @@ -5,6 +5,9 @@ import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.runners.MockitoJUnitRunner; +import org.springframework.web.client.RestOperations; + +import com.capitalone.dashboard.util.Supplier; import static org.hamcrest.Matchers.instanceOf; import static org.junit.Assert.assertThat; @@ -21,6 +24,8 @@ public class SonarClientSelectorTest { private DefaultSonar6Client defaultSonar6Client; @Mock private DefaultSonar56Client defaultSonar56Client; + @Mock + private Supplier restOperationsSupplier; @Test public void getSonarClient4() throws Exception { @@ -52,4 +57,4 @@ public void getSonarClient6() throws Exception { assertThat(sonarClient, instanceOf(DefaultSonar6Client.class)); } -} \ No newline at end of file +} diff --git a/src/test/java/com/capitalone/dashboard/collector/SonarCollectorTaskTest.java b/src/test/java/com/capitalone/dashboard/collector/SonarCollectorTaskTest.java index 1438a97..6de58fa 100644 --- a/src/test/java/com/capitalone/dashboard/collector/SonarCollectorTaskTest.java +++ b/src/test/java/com/capitalone/dashboard/collector/SonarCollectorTaskTest.java @@ -19,7 +19,6 @@ import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.runners.MockitoJUnitRunner; -import org.mockito.stubbing.OngoingStubbing; import java.util.ArrayList; import java.util.Arrays; @@ -38,7 +37,6 @@ public class SonarCollectorTaskTest { @Mock private CodeQualityRepository codeQualityRepository; @Mock private SonarProfileRepostory sonarProfileRepostory; - @Mock private SonarSettings sonarSettings; @Mock private ComponentRepository dbComponentRepository; @Mock private SonarClientSelector sonarClientSelector; @@ -82,7 +80,6 @@ public void setup() throws ParseException{ Mockito.doReturn(profileConfigurationChanges).when(defaultSonar6Client).getQualityProfileConfigurationChanges(SERVER1, QUALITYPROFILE); Mockito.doReturn(profileConfigurationChanges).when(defaultSonar6Client).getQualityProfileConfigurationChanges(SERVER2, QUALITYPROFILE); - } @Test @@ -95,42 +92,52 @@ public void collectEmpty() throws Exception { @Test public void collectOneServer43() throws Exception { when(dbComponentRepository.findAll()).thenReturn(components()); + when(sonarClientSelector.getSonarVersion(SERVER1)).thenReturn(VERSION43); when(sonarClientSelector.getSonarClient(VERSION43)).thenReturn(defaultSonarClient); - task.collect(collectorWithOneServer(VERSION43)); - verify(sonarClientSelector).getSonarClient(VERSION43); + task.collect(collectorWithOneServer()); + + verify(sonarClientSelector).getSonarClient(VERSION43); } @Test public void collectOneServer54() throws Exception { when(dbComponentRepository.findAll()).thenReturn(components()); + when(sonarClientSelector.getSonarVersion(SERVER1)).thenReturn(VERSION54); when(sonarClientSelector.getSonarClient(VERSION54)).thenReturn(defaultSonar6Client); - task.collect(collectorWithOneServer(VERSION54)); + + task.collect(collectorWithOneServer()); + verify(sonarClientSelector).getSonarClient(VERSION54); verify(defaultSonar6Client).getQualityProfiles(SERVER1); verify(defaultSonar6Client).retrieveProfileAndProjectAssociation(SERVER1, QUALITYPROFILE); verify(defaultSonar6Client).getQualityProfileConfigurationChanges(SERVER1, QUALITYPROFILE); } - @Test public void collectOneServer63() throws Exception { when(dbComponentRepository.findAll()).thenReturn(components()); + when(sonarClientSelector.getSonarVersion(SERVER1)).thenReturn(VERSION63); when(sonarClientSelector.getSonarClient(VERSION63)).thenReturn(defaultSonar6Client); - task.collect(collectorWithOneServer(VERSION63)); + + task.collect(collectorWithOneServer()); + verify(sonarClientSelector).getSonarClient(VERSION63); verify(defaultSonar6Client).getQualityProfiles(SERVER1); verify(defaultSonar6Client).retrieveProfileAndProjectAssociation(SERVER1, QUALITYPROFILE); verify(defaultSonar6Client).getQualityProfileConfigurationChanges(SERVER1, QUALITYPROFILE); } - @Test public void collectTwoServer43And54() throws Exception { when(dbComponentRepository.findAll()).thenReturn(components()); + when(sonarClientSelector.getSonarVersion(SERVER1)).thenReturn(VERSION43); + when(sonarClientSelector.getSonarVersion(SERVER2)).thenReturn(VERSION54); when(sonarClientSelector.getSonarClient(VERSION54)).thenReturn(defaultSonar6Client); when(sonarClientSelector.getSonarClient(VERSION43)).thenReturn(defaultSonarClient); - task.collect(collectorWithOnTwoServers(VERSION43, VERSION54)); + + task.collect(collectorWithOnTwoServers()); + verify(sonarClientSelector).getSonarClient(VERSION43); verify(sonarClientSelector).getSonarClient(VERSION54); @@ -150,12 +157,12 @@ private ArrayList components() { return cArray; } - private SonarCollector collectorWithOneServer(Double version) { - return SonarCollector.prototype(Collections.singletonList(SERVER1), Collections.singletonList(version), Collections.singletonList(METRICS1),Collections.singletonList(NICENAME1)); + private SonarCollector collectorWithOneServer() { + return SonarCollector.prototype(Collections.singletonList(SERVER1), Collections.singletonList(METRICS1),Collections.singletonList(NICENAME1)); } - private SonarCollector collectorWithOnTwoServers(Double version1, Double version2) { - return SonarCollector.prototype(Arrays.asList(SERVER1, SERVER2), Arrays.asList(version1, version2), Arrays.asList(METRICS1,METRICS2),Arrays.asList(NICENAME1,NICENAME2)); + private SonarCollector collectorWithOnTwoServers() { + return SonarCollector.prototype(Arrays.asList(SERVER1, SERVER2), Arrays.asList(METRICS1,METRICS2),Arrays.asList(NICENAME1,NICENAME2)); } -} \ No newline at end of file +}