SB-21700 - Conflicts Resolve ( Release 4.4.0 Into Master )

.circleci/config.yml

@@ -1,39 +1,71 @@
 version: 2.1
 jobs:
   sunbird-ed-dp-build:
-    machine: true
+    machine:
+      image: ubuntu-2004:202008-01
     steps:
       - checkout
       - run:
           name: clone analytics core
           command: |
             if [ -z $CIRCLE_PR_NUMBER ]; then
-            target_branch=$CIRCLE_BRANCH
-            git clone https://github.com/project-sunbird/sunbird-analytics-core.git -b $target_branch
-            git clone https://github.com/project-sunbird/sunbird-core-dataproducts.git -b $target_branch
+              target_branch=$CIRCLE_BRANCH
             else
-            prdata=$(curl -X GET -u $GITHUB_USER_TOKEN:x-oauth-basic https://api.github.com/repos/Sunbird-Ed/sunbird-data-products/pulls/$CIRCLE_PR_NUMBER)
-            target_branch=$(echo "${prdata}" | jq -r '.base.ref')
-            git clone https://github.com/project-sunbird/sunbird-core-dataproducts.git -b $target_branch
-            git clone https://github.com/project-sunbird/sunbird-analytics-core.git -b $target_branch
+              prdata=$(curl -X GET -u $GITHUB_USER_TOKEN:x-oauth-basic https://api.github.com/repos/Sunbird-Ed/sunbird-data-products/pulls/$CIRCLE_PR_NUMBER)
+              target_branch=$(echo "${prdata}" | jq -r '.base.ref')
+            fi
+            echo $target_branch
+            git clone https://github.com/project-sunbird/sunbird-analytics-core.git
+            branchExists=$(cd sunbird-analytics-core && git ls-remote --heads origin $target_branch)
+            echo $branchExists
+            if [ -z ${branchExists} ]; then
+                latest_branch=$(cd sunbird-analytics-core && git for-each-ref --count=1 --sort=-committerdate 'refs/remotes/**/release*' --format='%(refname:short)' | head -1)
+                echo $latest_branch
+                cd sunbird-analytics-core && git checkout $latest_branch
+            else
+                cd sunbird-analytics-core && git checkout $target_branch
+            fi
+            cd ../ && git clone https://github.com/project-sunbird/sunbird-core-dataproducts.git
+            branchExists=$(cd sunbird-core-dataproducts && git ls-remote --heads origin $target_branch)
+            echo $branchExists
+            if [ -z ${branchExists} ]; then
+               latest_branch=$(cd sunbird-core-dataproducts && git for-each-ref --count=1 --sort=-committerdate 'refs/remotes/**/release*' --format='%(refname:short)' | head -1)
+               echo $latest_branch
+               cd sunbird-core-dataproducts && git checkout $latest_branch
+            else
+               cd sunbird-core-dataproducts && git checkout $target_branch
             fi
       - restore_cache:
           keys:
             - dp-dependency-cache-{{ checksum "data-products/pom.xml" }}
-      - run: cd sunbird-analytics-core && mvn install -DskipTests
-      - run: cd sunbird-core-dataproducts && mvn install -DskipTests
+      - run: 
+          name: Build analytics-core dependency jar
+          command: |
+            export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64
+            export PATH=$JAVA_HOME/bin:$PATH
+            cd sunbird-analytics-core && mvn install -DskipTests
+      - run:
+          name: Build core-data-products dependency jar 
+          command:   |
+            export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64
+            export PATH=$JAVA_HOME/bin:$PATH
+            cd sunbird-core-dataproducts && mvn install -DskipTests
       - run:
           name: Ed-data-products-build
-          command: cd data-products && mvn scoverage:report
+          command: |
+            export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64
+            export PATH=$JAVA_HOME/bin:$PATH
+            cd data-products && mvn scoverage:report
       - save_cache:
           key: dp-dependency-cache-{{ checksum "data-products/pom.xml" }}
           paths: ~/.m2
       - run:
           name: sonar
           command: |
+            export JAVA_HOME=/usr/lib/jvm/java-1.11.0-openjdk-amd64
             cd data-products && mvn -X sonar:sonar -Dsonar.projectKey=Sunbird-Ed_sunbird-data-products  -Dsonar.host.url=https://sonarcloud.io -Dsonar.exclusions=/home/circleci/project/adhoc-scripts/**,/home/circleci/project/replay-scripts/** -Dsonar.scala.coverage.reportPaths=/home/circleci/project/data-products/target/scoverage.xml -Dsonar.organization=sunbird-ed
 workflows:
   version: 2.1
   workflow:
     jobs:
-      - sunbird-ed-dp-build
+      - sunbird-ed-dp-build

.gitignore

@@ -9,4 +9,6 @@ dependency-reduced-pom.xml
 data-products/.settings
 data-products/data-products.iml
 python-scripts/.pybuilder
+anomaly-detection/.pybuilder
+**/bin/*
 etl-jobs/.settings

anomaly-detection/Jenkinsfile

@@ -0,0 +1,50 @@
+@Library('deploy-conf') _
+node() {
+    try {
+        String ANSI_GREEN = "\u001B[32m"
+        String ANSI_NORMAL = "\u001B[0m"
+        String ANSI_BOLD = "\u001B[1m"
+        String ANSI_RED = "\u001B[31m"
+        String ANSI_YELLOW = "\u001B[33m"
+
+        ansiColor('xterm') {
+            stage('Checkout') {
+                cleanWs()
+                checkout scm
+                commit_hash = sh(script: 'git rev-parse --short HEAD', returnStdout: true).trim()
+                artifact_version = sh(script: "echo " + params.github_release_tag.split('/')[-1] + "_" + commit_hash + "_" + env.BUILD_NUMBER, returnStdout: true).trim()
+                echo "artifact_version: "+ artifact_version
+            }
+        }
+
+        stage('Build') {
+            sh '''
+                cd anomaly-detection
+                docker run --rm -v /etc/passwd:/etc/passwd -u $(id -u):$(id -g) -v$(pwd):/home/work -e HOME=/home/work -w /home/work python:3.8 ./build.sh
+            '''
+        }
+
+        stage('Archive artifacts'){
+            sh """
+                    cd anomaly-detection
+                    mkdir anomaly_detection_artifacts
+                    cp bin/anomaly_detection.tar.gz anomaly_detection_artifacts/
+                    zip -j anomaly_detection_artifacts.zip:${artifact_version} anomaly_detection_artifacts/*
+                """
+            archiveArtifacts artifacts: "anomaly-detection/anomaly_detection_artifacts.zip:${artifact_version}", fingerprint: true, onlyIfSuccessful: true
+            sh """echo {\\"artifact_name\\" : \\"anomaly_detection_artifacts.zip\\", \\"artifact_version\\" : \\"${artifact_version}\\", \\"node_name\\" : \\"${env.NODE_NAME}\\"} > metadata.json"""
+            archiveArtifacts artifacts: 'metadata.json', onlyIfSuccessful: true
+            currentBuild.result = "SUCCESS"
+            currentBuild.description = "Artifact: ${artifact_version}, Public: ${params.github_release_tag}"
+        }
+    }
+
+    catch (err) {
+        currentBuild.result = "FAILURE"
+        throw err
+    }
+    finally {
+        slack_notify(currentBuild.result)
+        email_notify()
+    }
+}

anomaly-detection/README.md

@@ -0,0 +1,9 @@
+# anomaly-detection
+
+## Getting started
+
+### Setup
+* Python 3.8 
+* Create virtual environment `python3 -m venv /path/to/virtual/environment`
+* Activate virtual environment `source /path/to/virtual/environment/bin activate`
+* Install required packages `pip install -r requirements.txt`

anomaly-detection/build.py

@@ -0,0 +1,14 @@
+from pybuilder.core import use_plugin, init
+
+use_plugin("python.core")
+use_plugin("python.install_dependencies")
+use_plugin("python.distutils")
+
+default_task = "publish"
+name = "anomaly_detection"
+version = "3.6.0"
+license = "MIT License"
+
+@init
+def initialize(project):
+    project.depends_on_requirements("requirements.txt")

anomaly-detection/build.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+cd "$(dirname "$0")"
+
+rm -rf .local || true
+pybuilderInstalled=`pip3 freeze | grep 'pybuilder' | wc -l`
+
+if [ $pybuilderInstalled != 1 ]
+then
+   echo "Installing pybuilder"
+   pip3 install -U pybuilder --pre
+fi
+export PATH=$PATH:$(pwd)/.local/bin
+
+pyb
+
+if [ ! -d "bin" ]; then
+  mkdir 'bin'
+fi
+
+cp target/dist/anomaly_detection*/dist/* bin/
+mv bin/anomaly_detection-*.tar.gz bin/anomaly_detection.tar.gz
+mv bin/anomaly_detection-*.whl bin/anomaly_detection.whl
+
+rm -rf target

anomaly-detection/requirements.txt

@@ -0,0 +1,7 @@
+flask
+requests
+sqlparse
+ciso8601
+dataclasses-json
+jsons
+APScheduler==3.7.0