Merge pull request #656 from project-sunbird/release-2.2.0

Merge release-2.2.0 to Master

ansible/artifacts/sunbird/login/login.ftl

@@ -22,7 +22,7 @@
                     <div id="success-msg" class="ui text success hide">suceess</div>
                     <div id="error-msg" class="ui text error hide">error</div>
                 </div>
-                <form id="kc-form-login" class="ui form" method="POST" action="${url.loginAction}">
+                <form id="kc-form-login" onsubmit="login.disabled = true; return true;" class="ui form" method="POST" action="${url.loginAction}">
                     <div class="field">
                         <label id="usernameLabel" for="username" class="">
                             <#if !realm.loginWithEmailAllowed>${msg("username")}
@@ -59,7 +59,7 @@
                     <div class="field">
                         <button id="login" class="mt-36 ui fluid button">${msg("doSignIn")}</button>
                     </div>
-                    
+
                     <div id="selfSingUp" class="hide">
                         <p class="or mb-30 mt-30 textCenter">OR</p>
                         <div class="field">

ansible/cassandra-deploy-decrypt.yml

@@ -0,0 +1,11 @@
+- hosts: "{{groups['cassandra'][0]}}"
+  become: yes
+  environment:
+    sunbird_cassandra_host: "{{ groups['cassandra'][0] }}"
+    sunbird_cassandra_port: 9042
+    sunbird_cassandra_keyspace: sunbird
+    sunbird_encryption_key: "{{ core_vault_sunbird_encryption_key }}"
+  vars_files:
+    - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']
+  roles:
+    - cassandra-deploy-decrypt

ansible/es.yml

@@ -1,4 +1,5 @@
 - hosts: es-backup
+  gather_facts: no
   become: yes
   vars_files:
     - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']
@@ -11,18 +12,21 @@
   tags:
     - es_backup
 
-- hosts: es-backup
+- hosts: "{{remote}}"
+  gather_facts: no
   become: yes
   vars_files:
     - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']
   vars:
-    - es_restore_host: "{{app_es_restore_host}}"
+    - es_restore_host: "{{remote}}"
+    - snapshot_base_path: "{{app_snapshot_base_path}}"
   roles:
     - es-azure-restore
   tags:
     - es_restore
 
 - hosts: log-es-backup
+  gather_facts: no
   become: yes
   vars_files:
     - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']
@@ -35,18 +39,21 @@
   tags:
     - log_es_backup
 
-- hosts: log-es-backup
+- hosts: "{{remote}}"
+  gather_facts: no
   become: yes
   vars_files:
     - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']
   vars:
-    - es_restore_host: "{{log_es_restore_host}}"
+    - es_restore_host: "{{remote}}"
+    - snapshot_base_path: "{{log_snapshot_base_path}}"
   roles:
     - es-azure-restore
   tags:
     - log_es_restore
 
 - hosts: log-es
+  gather_facts: no
   become: yes
   vars_files:
     - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']

ansible/inventory/env/group_vars/all.yml

@@ -19,8 +19,8 @@ sunbird_app_name: "{{env}}"
 azure_account_key: "{{core_vault_sunbird_azure_storage_key}}"
 
 #artifact upload
-artifact_azure_account_name: "{{core_artifact_azure_account_name}}"
-artifact_azure_account_key: "{{core_vault_artifact_azure_account_key}}"
+artifact_azure_account_name: "{{azure_account_name}}"
+artifact_azure_account_key: "{{core_vault_sunbird_azure_storage_key}}"
 
 
 #plugins
@@ -84,6 +84,7 @@ kong_postgres_user: "{{core_vault_postgres_username}}"
 kong_postgres_password: "{{core_vault_postgres_password}}"
 enc_postgres_user: "{{core_vault_postgres_username}}"
 badger_postgres_user: "{{core_vault_postgres_username}}"
+user_org_service_postgres_user: "{{core_vault_postgres_username}}"
 sunbird_account_name: "{{azure_account_name}}"
 sunbird_account_key: "{{core_vault_sunbird_azure_storage_key}}"
 ansible_vault_password: "{{ core_vault_ansible_vault_password }}"
@@ -108,6 +109,9 @@ kong_host: api-manager_kong
 
 ## DB address
 application_postgres_host: "{{groups['postgres'][0]}}"  #Private IP of Postgres server
+enc_postgres_host: "{{groups['postgres'][0]}}"  #Private IP of Postgres server
+user_org_service_postgres_host: "{{groups['postgres'][0]}}"  #Private IP of Postgres server
+badger_postgres_host: "{{groups['postgres'][0]}}"  #Private IP of Postgres server
 keycloak_postgres_host: "{{groups['postgres'][0]}}"  #Private IP of Postgres server
 kong_postgres_host: "{{groups['postgres'][0]}}"  #Private IP of Postgres server
 sunbird_cassandra_host: "{{groups['cassandra']|join(',')}}"  #Private IP of Cassandra server
@@ -190,6 +194,7 @@ sunbird_msg_91_auth: "{{core_vault_msg_91_auth_key}}"
 sunbird_telemetry_pdata_id: "{{env}}.sunbird.learning.service"
 sunbird_encryption_key: "{{ core_vault_sunbird_encryption_key }}"
 #player_tenant_dir:
+user_org_service_postgres_database: userorg
 application_postgres_database: quartz
 keycloak_postgres_database: keycloak
 keyclaok_deployer_sudo_pass: "{{core_vault_keyclaok_deployer_sudo_pass}}"
@@ -201,7 +206,7 @@ sunbird_content_repo_api_key: "{{ core_vault_sunbird_ekstep_api_key }}"
 sunbird_search_service_api_key: "{{ core_vault_sunbird_ekstep_api_key }}"
 sunbird_dial_repo_api_base_url: "{{sunbird_ekstep_api_base_url}}"
 sunbird_dial_repo_api_key: "{{ core_vault_sunbird_ekstep_api_key }}"
-sunbird_plugin_repo_api_base_url: "{{sunbird_ekstep_api_base_url}}"
+sunbird_plugin_repo_api_base_url: "http://{{searchservice_ip}}:9000" 
 sunbird_data_service_api_base_url: "{{sunbird_ekstep_api_base_url}}"
 sunbird_data_service_api_key: "{{ core_vault_sunbird_ekstep_api_key }}"
 
@@ -290,7 +295,7 @@ badger_container: badgr
 sunbird_http_orgin: "{{proto}}://{{proxy_server_name}}/badging"
 
 #enc-service
-enc_postgres_database: encryption
+enc_postgres_database: "{{env}}-keys"
 
 #kafka vars
 zk_hosts: "127.0.0.1:2181"
@@ -347,14 +352,6 @@ ep_es_host: "{{ groups['telemetry-search-cluster'][0] }}"     #For kibana pipeli
 
 
 #encryption Service
-encryption_service_user: encryption
-encryption_db_name: encryption
-encryption_db_dialect: postgres
-encryption_reservation_memory: 750MB
-encryption_replicas: 1
-encryption_limit_memory: 750MB
-encryption_limit_cpu: 1
-encryption_reservation_cpu: 1
 sunbird_allowed_login:
 sunbird_course_batch_notification_enabled: 'true'
 sunbird_device_register_api: "{{proto}}://{{api_proxy_name}}/v3/device/register/"
@@ -384,34 +381,88 @@ grafana_dashboards_git_repo_url_with_credentails: "{{core_vault_grafana_dashboar
 monitor_alerts_slack_url: "{{core_vault_monitor_alerts_slack_url}}"
 kong__test_jwt: "{{core_vault_kong__test_jwt}}"
 
-# postgresql_users should only be created in master and replicated to slave
+############# Postgres users and databases ###############
 postgresql_users:
   - name: "{{kong_postgres_user}}"
+    login_host: "{{kong_postgres_host}}"
+    login_password: "{{kong_postgres_password}}"
     password: "{{postgres_password}}"
     db: "{{kong_postgres_database}}"
+    login_user: "{{kong_postgres_user}}"
     priv: "ALL"
   - name: "{{keycloak_postgres_user}}"
+    login_host: "{{keycloak_postgres_host}}"
+    login_password: "{{keycloak_postgres_password}}"
     password: "{{postgres_password}}"
     db: "{{keycloak_postgres_database}}"
+    login_user: "{{keycloak_postgres_user}}"
     priv: "ALL"
   - name: "{{application_postgres_user}}"
+    login_host: "{{application_postgres_host}}"
+    login_password: "{{application_postgres_password}}"
     password: "{{postgres_password}}"
     db: "{{application_postgres_database}}"
+    login_user: "{{application_postgres_user}}"
     priv: "ALL"
   - name: "{{badger_postgres_user}}"
+    login_host: "{{badger_postgres_host}}"
+    login_password: "{{badger_postgres_password}}"
     password: "{{postgres_password}}"
+    login_user: "{{badger_postgres_user}}"
     db: "{{badger_postgres_database}}"
     priv: "ALL"
+  - name: "{{user_org_service_postgres_user}}"
+    login_host: "{{user_org_service_postgres_host}}"
+    login_password: "{{user_org_service_postgres_password}}"
+    password: "{{user_org_service_postgres_password}}"
+    db: "{{user_org_service_postgres_database}}"
+    login_user: "{{user_org_service_postgres_user}}"
+    priv: "ALL"
+  - name: "{{enc_postgres_user}}"
+    login_host: "{{enc_postgres_host}}"
+    login_password: "{{enc_postgres_password}}"
+    password: "{{enc_postgres_password}}"
+    db: "{{enc_postgres_database}}"
+    login_user: "{{enc_postgres_user}}"
+    priv: "ALL"
 
 
 postgresql_databases:
   - name: "{{kong_postgres_database}}"
+    login_host: "{{kong_postgres_host}}"
+    login_password: "{{kong_postgres_password}}"
+    owner: "{{kong_postgres_user}}"
+    login_user: "{{kong_postgres_user}}"
   - name: "{{keycloak_postgres_database}}"
+    login_host: "{{keycloak_postgres_host}}"
+    login_password: "{{keycloak_postgres_password}}"
+    owner: "{{keycloak_postgres_user}}"
+    login_user: "{{keycloak_postgres_user}}"
   - name: "{{application_postgres_database}}"
+    login_host: "{{application_postgres_host}}"
+    login_password: "{{application_postgres_password}}"
+    owner: "{{application_postgres_user}}"
+    login_user: "{{application_postgres_user}}"
   - name: "{{badger_postgres_database}}"
+    login_host: "{{badger_postgres_host}}"
+    login_password: "{{badger_postgres_password}}"
+    owner: "{{badger_postgres_user}}"
+    login_user: "{{badger_postgres_user}}"
+  - name: "{{user_org_service_postgres_database}}"
+    login_host: "{{user_org_service_postgres_host}}"
+    login_password: "{{user_org_service_postgres_password}}"
+    owner: "{{user_org_service_postgres_user}}"
+    login_user: "{{user_org_service_postgres_user}}"
   - name: "{{enc_postgres_database}}"
+    login_host: "{{enc_postgres_host}}"
+    login_password: "{{enc_postgres_password}}"
+    owner: "{{enc_postgres_user}}"
+    login_user: "{{enc_postgres_user}}"
+
+##########################################################
+
+####### App ES ########
 
-#######App ES
 
 app_es_etc_cluster_name: "{{env}}"
 app_es_etc_discovery_zen_minimum_master_nodes: "{{groups['es']| length | int}}"
@@ -424,7 +475,6 @@ app_es_snapshot_base_path: application
 log_es_etc_cluster_name: "{{env}}-log"
 log_es_snapshot_host: "{{ groups['log-es'][0] }}"
 log_es_restore_host: "{{ groups['log-es'][0] }}"
-log_snapshot_base_path: logger
 log_es_host: "{{ groups['log-es'][0] }}"
 
 ####### Advanced Config ##########
@@ -538,11 +588,8 @@ trampoline_secret: "{{core_vault_trampoline_secret}}"
 es_api_host: "{{inventory_hostname}}"
 sunbird_linked_content_base_url: "{{proto}}://{{proxy_server_name}}/play/content/"
 enc_postgres_password: "{{core_vault_postgres_password}}"
-
-# Depricated value ??
+user_org_service_postgres_password: "{{core_vault_postgres_password}}"
 postgres_replication_user_password:  "{{core_vault_postgres_password}}"
-# Content service is calling ekstep ??
-# Config service variables, not required
 sunbird_config_service_url:
 config_refresh_interval: 10
 config_service_enabled: false
@@ -557,6 +604,8 @@ app_es_snapshot_host: "{{ groups['es'][0] }}"
 app_snapshot_base_path: applicationelasticsearch
 
 sunbird_es_host: "{{ groups['es'][0] }}"
+user_org_sunbird_es_host: "{{ sunbird_es_host }}"
+user_org_sunbird_es_port: "{{ sunbird_es_port }}"
 
 #log es backup
 log_es_snapshot_host: "{{ groups['log-es'][0] }}"
@@ -591,3 +640,11 @@ sunbird_portal_offline_app_release_date: ""
 sunbird_portal_offline_app_version: "1.0.0"
 sunbird_portal_offline_app_download_url: ""
 sunbird_portal_log_level: "debug"
+
+### Release 2.2.0 ###
+sunbird_google_android_keycloak_client_id: ''
+sunbird_google_android_keycloak_secret: ''
+sunbird_trampoline_android_keycloak_client_id: ''
+sunbird_trampoline_android_keycloak_secret: ''
+sunbird_android_keycloak_client_id: ''
+sunbird_user_org_api_base_url: http://learner-service:9000

ansible/keycloak.yml

@@ -1,5 +1,5 @@
 ---
-- hosts: keycloak
+- hosts: "{{host}}"
   become: true
   vars_files:
     - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']
@@ -8,7 +8,7 @@
   tags:
     - provision
 
-- hosts: keycloak
+- hosts: "{{host}}"
   become: true
   vars_files:
     - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']

ansible/logging.yml

@@ -15,7 +15,7 @@
     - stack-logger
   run_once: true
 
-- hosts: swarm-agent-dashboard
+- hosts: swarm-dashboard
   become: yes
   vars_files:
     - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']

ansible/monitoring-fed.yml

@@ -0,0 +1,6 @@
+- hosts: swarm-dashboard
+  become: yes
+  vars_files:
+    - ['{{ inventory_dir }}/secrets.yml']
+  roles:
+    - prometheus-fed

ansible/postgresql-data-update.yml

@@ -1,6 +1,7 @@
-- hosts: postgresql-master
+- hosts: localhost
+  gather_facts: no
   become: yes
   vars_files:
     - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']
   roles:
-    - postgresql-data-update
+    - postgresql-data-update

ansible/roles/artifacts-download-azure/tasks/main.yml

@@ -4,5 +4,5 @@
 
 - name: Download from azure blob storage
   command: az storage blob download -c {{ artifacts_container }} --name {{ artifact }} -f {{ artifact_path }}
-  async: 100
+  async: 3600
   poll: 10

ansible/roles/cassandra-deploy-decrypt/defaults/main.yml

@@ -0,0 +1,2 @@
+cassandra_jar_path: ../
+cassandra_deploy_path: /home/deployer

ansible/roles/cassandra-deploy-decrypt/tasks/main.yml

@@ -0,0 +1,10 @@
+- name: Copy the jar
+  become: yes
+  copy:
+    src: "{{cassandra_jar_path}}/decryptionUtil-1.0-SNAPSHOT-jar-with-dependencies.jar"
+    dest: "{{cassandra_deploy_path}}"
+  run_once: true
+
+- name: Run the application
+  shell: java -jar decryptionUtil-1.0-SNAPSHOT-jar-with-dependencies.jar
+  run_once: true

ansible/roles/elasticsearch_old/tasks/main.yml

@@ -46,6 +46,9 @@
   ignore_errors: yes
   notify: Restart elasticsearch
 
+- name: Install azure plugin for elasticsearch
+  shell: ES_PATH_CONF=/etc/elasticsearch /usr/share/elasticsearch/bin/plugin install cloud-azure
+
 - name: Install plugins
   command: bin/plugin install {{item.name}} {%if item.url is defined %} url {{item.url}}{% endif %}
   args:

ansible/roles/es-azure-restore/defaults/main.yml

@@ -1,2 +1,11 @@
 # Override these values
-es_restore_host: localhost
+snapshot_create_request_body: {
+    type: azure,
+    settings: {
+      container: "elasticsearch-snapshots",
+      base_path: "{{ snapshot_base_path }}"
+    }
+}
+
+es_restore_host: "{{groups['es'][0]}}"
+snapshot_base_path: application

ansible/roles/es-azure-restore/tasks/main.yml

@@ -1,4 +1,11 @@
 ---
+- name: Set azure snapshot for the first time
+  uri:
+    url: "http://{{ es_restore_host }}:9200/_snapshot/azurebackup"
+    method: PUT
+    body: "{{ snapshot_create_request_body | to_json }}"
+    headers:
+      Content-Type: "application/json"
 
 - name: Restore ES from Azure backup
   uri:
@@ -14,5 +21,5 @@
     body_format: json
   register: result
   until: result.json.snapshots[0].state == 'SUCCESS'
-  retries: 120
+  retries: 1200
   delay: 10

ansible/roles/es6/tasks/azure-plugin.yml

@@ -0,0 +1,3 @@
+- name: Install azure plugin
+  shell: ES_PATH_CONF=/etc/elasticsearch/"{{ es_instance_name }}" /usr/share/elasticsearch/bin/elasticsearch-plugin install repository-azure
+  notify: restart elasticsearch