Feature/813 postgres v2

.gitignore

@@ -22,6 +22,11 @@ dev-env
 venv
 scripts/dev-vm
 
+# PG migration
+pg_migrate_conf.yml
+pg_tests.log
+pg_db_password.tmp
+
 
 ## Deployment
 scripts/deployment/builders/osx/config/python_system.config

pg_alts.py

@@ -0,0 +1,227 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+""" Akvo RSR is covered by the GNU Affero General Public License.
+See more details in the license.txt file located at the root folder of the
+Akvo RSR module. For additional details on the GNU license please
+see < http://www.gnu.org/licenses/agpl.html >.
+
+This scripts alters the postgres database after errors have been found
+running the test suit.
+"""
+
+from __future__ import print_function
+import sys
+import os
+
+from django.db import connections
+
+
+def alter_paypal_ipn_ipaddress():
+    """
+    -> IPAddressField
+    Ipaddress fields should have pg inet type, varchar was used. Since
+    postgres didn't like to convert directly to inet we have to
+    dance by text"""
+    print('\tAbout to alter paypal_ipn ipaddress')
+    cursor = connections["default"].cursor()
+    sql = ('ALTER TABLE paypal_ipn ALTER COLUMN ipaddress TYPE text; '
+           'ALTER TABLE paypal_ipn ALTER COLUMN ipaddress '
+           'TYPE inet USING ipaddress::inet;')
+    cursor.execute(sql)
+    print('\tDid alter paypal_ipn ipaddress')
+
+
+def alter_invoice_amount():
+    """
+    -> PositiveIntegerField
+    """
+    print('\tAbout to alter rsr_invoice amount')
+    cursor = connections["default"].cursor()
+    sql = ('ALTER TABLE rsr_invoice ALTER COLUMN amount TYPE integer; '
+           'ALTER TABLE rsr_invoice '
+           'ADD CONSTRAINT amountchk CHECK (amount >= 0);')
+    cursor.execute(sql)
+    print('\tDid alter rsr_invoice amount')
+
+
+def alter_invoice_status():
+    """
+    -> PositiveSmallIntegerField
+    """
+    print('\tAbout to alter rsr_invoice status')
+    cursor = connections["default"].cursor()
+    sql = ('ALTER TABLE rsr_invoice ALTER COLUMN status TYPE smallint; '
+           'ALTER TABLE rsr_invoice '
+           'ADD CONSTRAINT statuschk CHECK (status >= 0);')
+    cursor.execute(sql)
+    print('\tDid alter rsr_invoice status')
+
+
+def alter_rsr_projectlocation_administrative_level():
+    """
+    -> PositiveSmallIntegerField
+    """
+    print('\tAbout to alter rsr_projectlocation administrative_level')
+    cursor = connections["default"].cursor()
+    sql = ('ALTER TABLE rsr_projectlocation '
+           'ALTER COLUMN administrative_level TYPE smallint; '
+           'ALTER TABLE rsr_projectlocation '
+           'ADD CONSTRAINT administrative_levelchk '
+           'CHECK (administrative_level >= 0);')
+    cursor.execute(sql)
+    print('\tDid alter rsr_projectlocation administrative_level')
+
+
+def alter_rsr_project_hierarchy():
+    """
+    -> PositiveIntegerField
+    """
+    print('\tAbout to alter rsr_project hierarchy')
+    cursor = connections["default"].cursor()
+    sql = ('ALTER TABLE rsr_project ALTER COLUMN hierarchy TYPE integer; '
+           'ALTER TABLE rsr_project '
+           'ADD CONSTRAINT hierarchychk CHECK (hierarchy >= 0);')
+    cursor.execute(sql)
+    print('\tDid alter rsr_project hierarchy')
+
+
+def alter_rsr_indicator_baseline_year():
+    """
+    -> PositiveIntegerField
+    """
+    print('\tAbout to alter rsr_indicator baseline_year')
+    cursor = connections["default"].cursor()
+    sql = ('ALTER TABLE rsr_indicator '
+           'ALTER COLUMN baseline_year TYPE integer; '
+           'ALTER TABLE rsr_indicator '
+           'ADD CONSTRAINT baseline_yearchk CHECK (baseline_year >= 0);')
+    cursor.execute(sql)
+    print('\tDid alter rsr_indicator baseline_year')
+
+
+def alter_django_counter_redircounter_count():
+    """
+    -> PositiveIntegerField
+    """
+    print('\tAbout to alter django_counter_redircounter count')
+    cursor = connections["default"].cursor()
+    sql = ('ALTER TABLE django_counter_redircounter '
+           'ALTER COLUMN count TYPE integer; '
+           'ALTER TABLE django_counter_redircounter '
+           'ADD CONSTRAINT countchk CHECK (count >= 0);')
+    cursor.execute(sql)
+    print('\tDid alter django_counter_redircounter count')
+
+
+def alter_rsr_partnersite_piwik_id():
+    """
+    -> PositiveIntegerField
+    """
+    print('\tAbout to alter rsr_partnersite piwik_id')
+    cursor = connections["default"].cursor()
+    sql = ('ALTER TABLE rsr_partnersite ALTER COLUMN piwik_id TYPE integer; '
+           'ALTER TABLE rsr_partnersite '
+           'ADD CONSTRAINT piwik_idchk CHECK (piwik_id >= 0);')
+    cursor.execute(sql)
+    print('\tDid alter rsr_partnersite piwik_id')
+
+
+def alter_django_counter_referer_count():
+    """
+    -> PositiveIntegerField
+    """
+    print('\tAbout to alter django_counter_referer count')
+    cursor = connections["default"].cursor()
+    sql = ('ALTER TABLE django_counter_referer '
+           'ALTER COLUMN count TYPE integer; '
+           'ALTER TABLE django_counter_referer '
+           'ADD CONSTRAINT countchk CHECK (count >= 0);')
+    cursor.execute(sql)
+    print('\tDid alter django_counter_referer count')
+
+
+def alter_django_admin_log_action_flag():
+    """
+    -> PositiveSmallIntegerField
+    """
+    print('\tAbout to alter django_admin_log status')
+    cursor = connections["default"].cursor()
+    sql = ('ALTER TABLE django_admin_log '
+           'ALTER COLUMN action_flag TYPE smallint; '
+           'ALTER TABLE django_admin_log '
+           'ADD CONSTRAINT action_flag_chk CHECK (action_flag >= 0);')
+    cursor.execute(sql)
+    print('\tDid alter django_admin_log status')
+
+
+def alter_django_counter_viewcounter_object_id():
+    """
+    -> PositiveIntegerField
+    """
+    print('\tAbout to alter django_counter_viewcounter object_id')
+    cursor = connections["default"].cursor()
+    sql = ('ALTER TABLE django_counter_viewcounter '
+           'ALTER COLUMN object_id TYPE integer; '
+           'ALTER TABLE django_counter_viewcounter '
+           'ADD CONSTRAINT object_idchk CHECK (object_id >= 0);')
+    cursor.execute(sql)
+    print('\tDid alter django_counter_viewcounter object_id')
+
+
+def alter_django_counter_viewcounter_count():
+    """
+    -> PositiveIntegerField
+    """
+    print('\tAbout to alter django_counter_viewcounter count')
+    cursor = connections["default"].cursor()
+    sql = ('ALTER TABLE django_counter_viewcounter '
+           'ALTER COLUMN count TYPE integer; '
+           'ALTER TABLE django_counter_viewcounter '
+           'ADD CONSTRAINT countchk CHECK (count >= 0);')
+    cursor.execute(sql)
+    print('\tDid alter django_counter_viewcounter count')
+
+
+def alter_tastypie_apiaccess_accessed():
+    """
+    -> PositiveIntegerField
+    """
+    print('\tAbout to alter tastypie_apiaccess count')
+    cursor = connections["default"].cursor()
+    sql = ('ALTER TABLE tastypie_apiaccess '
+           'ALTER COLUMN accessed TYPE integer; '
+           'ALTER TABLE tastypie_apiaccess '
+           'ADD CONSTRAINT accessedchk CHECK (accessed >= 0);')
+    cursor.execute(sql)
+    print('\tDid alter tastypie_apiaccess accessed')
+
+
+def alt():
+    alter_paypal_ipn_ipaddress()
+    alter_invoice_amount()
+    alter_invoice_status()
+    alter_rsr_projectlocation_administrative_level()
+    alter_rsr_project_hierarchy()
+    alter_rsr_indicator_baseline_year()
+    alter_django_counter_redircounter_count()
+    alter_rsr_partnersite_piwik_id()
+    alter_django_counter_referer_count()
+    alter_django_admin_log_action_flag()
+    alter_django_counter_viewcounter_object_id()
+    alter_django_counter_viewcounter_count()
+    alter_tastypie_apiaccess_accessed()
+
+if __name__ == '__main__':
+    # Setup the Django env
+    try:
+        sys.path.append('/var/akvo/rsr/code/akvo')
+        os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
+    except ImportError as error:
+        sys.stderr.write('Error: Can not find the \'settings\' module.')
+        sys.stderr.write('\nImportError: ' + str(error) + '\n')
+        sys.exit(1)
+
+    print('\nAbout to alter\n')
+    alt()
+    print('\nFinished\n')

pg_migrate.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+if [ -z "$1" ]; then
+    echo "No password argument supplied"
+    exit 2
+fi
+
+# Since we switch user we need to store password on disk
+# we're using the rsr disk location and not /tmp
+PASSWORD_FILE="/var/akvo/rsr/code/pg_db_password.tmp"
+echo $1 > $PASSWORD_FILE
+
+# Truncate Django session
+mysql --user=rsr --password=$1 rsr -e "Truncate table django_session;"
+
+# Create rsr user and database
+sudo su postgres <<'EOF'
+PASSWORD=`cat /var/akvo/rsr/code/pg_db_password.tmp`
+psql -c "CREATE USER rsr WITH PASSWORD '$PASSWORD';"
+psql -c "CREATE DATABASE rsr OWNER rsr TEMPLATE template0 ENCODING 'UTF8';"
+EOF
+
+# Remove password file
+rm -f $PASSWORD_FILE
+
+# Do migration
+sudo su rsr <<'EOF'
+cd /var/akvo/rsr
+. venv/bin/activate
+cd code
+if ! [ -f  /var/akvo/rsr/code/pg_migrate_conf.yml ]; then
+   echo "Could not find pg_migrate_conf.yml. Use pg_migrate_conf.yml.template."
+   exit 2
+fi
+py-mysql2pgsql -v -f pg_migrate_conf.yml
+python pg_alts.py > pg_tests.log
+python pg_tests.py > pg_tests.log
+EOF

pg_migrate_conf.yml.template

@@ -0,0 +1,63 @@
+# if a socket is specified we will use that
+# if tcp is chosen you can use compression
+mysql:
+ hostname: mysql-localdev.localdev.akvo.org
+ port: 3306
+ username: rsr
+ password: password
+ database: rsr
+ compress: false
+destination:
+ # if file is given, output goes to file, else postgres
+ file:
+ postgres:
+  hostname: 192.168.50.101
+  port: 5432
+  username: rsr
+  password: password
+  database: rsr
+
+# if tables is given, only the listed tables will be converted.  leave empty to convert all tables.
+#only_tables:
+#- table1
+#- table2
+# if exclude_tables is given, exclude the listed tables from the conversion.
+#exclude_tables:
+#- table3
+#- table4
+
+exclude_tables:
+- celery_taskmeta
+- celery_tasksetmeta
+- djcelery_crontabschedule
+- djcelery_intervalschedule
+- djcelery_periodictask
+- djcelery_periodictasks
+- djcelery_taskstate
+- djcelery_workerstate
+- djkombu_message
+- djkombu_queue
+- notification_notice
+- notification_noticequeuebatch
+- notification_noticesetting
+- notification_noticetype
+- notification_observeditem
+- workflows_state
+- workflows_state_transitions
+- workflows_stateinheritanceblock
+- workflows_stateobjectrelation
+- workflows_statepermissionrelation
+- workflows_transition
+- workflows_workflow
+- workflows_workflowmodelrelation
+- workflows_workflowobjectrelation
+- workflows_workflowpermissionrelation
+
+# if supress_data is true, only the schema definition will be exported/migrated, and not the data
+supress_data: false
+
+# if supress_ddl is true, only the data will be exported/imported, and not the schema
+supress_ddl: false
+
+# if force_truncate is true, forces a table truncate before table loading
+force_truncate: false