Merge pull request #165 from ZuluPro/dbconnector

Dbconnector system

.travis.yml

@@ -13,6 +13,8 @@ services:
   - mysql
   - postgresql
   - mongodb
+addons:
+  postgresql: "9.4"
 
 env:
   matrix:
@@ -43,9 +45,10 @@ matrix:
       install: pip install tox
       script: tox -e docs
     - python: "3.4"
-      env: ENV=functional
+      env: TOX_ENV=functional
       install:
         - pip install tox
+        - export PYTHON='coverage run -a'
       before_install: 
         - mysql -e 'CREATE DATABASE test;'
         - psql -c 'CREATE DATABASE test;' -U postgres
@@ -66,4 +69,3 @@ matrix:
       env: DJANGO=1.9
   allow_failures:
     - python: pypy3
-    - env: ENV=functional

dbbackup/db/base.py

@@ -0,0 +1,143 @@
+import shlex
+from tempfile import SpooledTemporaryFile
+from subprocess import Popen
+from importlib import import_module
+from dbbackup import settings, utils
+from . import exceptions
+
+CONNECTOR_MAPPING = {
+    'django.db.backends.sqlite3': 'dbbackup.db.sqlite.SqliteConnector',
+    'django.db.backends.mysql': 'dbbackup.db.mysql.MysqlDumpConnector',
+    'django.db.backends.postgresql': 'dbbackup.db.postgresql.PgDumpConnector',
+    'django.db.backends.postgresql_psycopg2': 'dbbackup.db.postgresql.PgDumpConnector',
+    'django.db.backends.oracle': None,
+    'django_mongodb_engine': 'dbbackup.db.mongo.MongoDumpConnector',
+    'django.contrib.gis.db.backends.postgis': 'dbbackup.db.postgresql.PgDumpGisConnector',
+    'django.contrib.gis.db.backends.mysql': 'dbbackup.db.mysql.MysqlDumpConnector',
+    'django.contrib.gis.db.backends.oracle': None,
+    'django.contrib.gis.db.backends.spatialite': 'dbbackup.db.sqlite.SqliteConnector',
+}
+
+
+def get_connector(database_name=None):
+    """
+    Get a connector from its database key in setttings.
+    """
+    from django.db import connections, DEFAULT_DB_ALIAS
+    database_name = database_name or DEFAULT_DB_ALIAS
+    connection = connections[database_name]
+    connector_settings = settings.CONNECTORS.get(database_name, {})
+    engine = connection.settings_dict['ENGINE']
+    connector_path = connector_settings.get('CONNECTOR', CONNECTOR_MAPPING[engine])
+    connector_module_path = '.'.join(connector_path.split('.')[:-1])
+    module = import_module(connector_module_path)
+    connector_name = connector_path.split('.')[-1]
+    connector = getattr(module, connector_name)
+    return connector(database_name, **connector_settings)
+
+
+class BaseDBConnector(object):
+    """
+    Base class for create database connector. This kind of object creates
+    interaction with database and allow backup and restore operations.
+    """
+    extension = 'dump'
+    exclude = []
+
+    def __init__(self, database_name=None, **kwargs):
+        from django.db import connections, DEFAULT_DB_ALIAS
+        self.database_name = database_name or DEFAULT_DB_ALIAS
+        self.connection = connections[self.database_name]
+        for attr, value in kwargs.items():
+            setattr(self, attr.lower(), value)
+
+    @property
+    def settings(self):
+        """Mix of database and connector settings."""
+        if not hasattr(self, '_settings'):
+            sett = self.connection.settings_dict.copy()
+            sett.update(settings.CONNECTORS.get(self.database_name, {}))
+            self._settings = sett
+        return self._settings
+
+    def generate_filename(self, server_name=None):
+        return utils.filename_generate(self.extension, self.settings['NAME'],
+                                       server_name)
+
+    def create_dump(self):
+        """
+        :return: File object
+        :rtype: file
+        """
+        dump = self._create_dump()
+        return dump
+
+    def _create_dump(self):
+        """
+        Override this method to define dump creation.
+        :return: File object
+        :rtype: file
+        """
+        raise NotImplementedError("_create_dump not implemented")
+
+    def restore_dump(self, dump):
+        """
+        :param dump: Dump file
+        :type dump: file
+        """
+        result = self._restore_dump(dump)
+        return result
+
+    def _restore_dump(self, dump):
+        """
+        Override this method to define dump creation.
+        :param dump: Dump file
+        :type dump: file
+        """
+        raise NotImplementedError("_restore_dump not implemented")
+
+
+class BaseCommandDBConnector(BaseDBConnector):
+    """
+    Base class for create database connector based on command line tools.
+    """
+    dump_prefix = ''
+    dump_suffix = ''
+    restore_prefix = ''
+    restore_suffix = ''
+    env = {}
+    dump_env = {}
+    restore_env = {}
+
+    def run_command(self, command, stdin=None, env=None):
+        """
+        Launch a shell command line.
+
+        :param command: Command line to launch
+        :type command: str
+        :param stdin: Standard input of command
+        :type stdin: file
+        :param env: Environment variable used in command
+        :type env: dict
+        :return: Standard output of command
+        :rtype: file
+        """
+        cmd = shlex.split(command)
+        stdout = SpooledTemporaryFile(max_size=10 * 1024 * 1024)
+        stderr = SpooledTemporaryFile(max_size=10 * 1024 * 1024)
+        full_env = self.env.copy()
+        full_env.update(env or {})
+        try:
+            process = Popen(cmd, stdin=stdin, stdout=stdout, stderr=stderr,
+                            env=full_env)
+            process.wait()
+            if process.poll():
+                stderr.seek(0)
+                raise exceptions.CommandConnectorError(
+                    "Error running: {}\n{}".format(command, stderr.read()))
+            stdout.seek(0)
+            stderr.seek(0)
+            return stdout, stderr
+        except OSError as err:
+            raise exceptions.CommandConnectorError(
+                "Error running: {}\n{}".format(command, str(err)))

dbbackup/db/exceptions.py

@@ -0,0 +1,14 @@
+class ConnectorError(Exception):
+    """Base connector error"""
+
+
+class DumpError(ConnectorError):
+    """Error on dump"""
+
+
+class RestoreError(ConnectorError):
+    """Error on restore"""
+
+
+class CommandConnectorError(ConnectorError):
+    """Failing command"""

dbbackup/db/mongodb.py

@@ -0,0 +1,41 @@
+from .base import BaseCommandDBConnector
+
+
+class MongoDumpConnector(BaseCommandDBConnector):
+    """
+    MongoDB connector, creates dump with ``mongodump`` and restore with
+    ``mongorestore``.
+    """
+    dump_cmd = 'mongodump'
+    restore_cmd = 'mongorestore'
+    object_check = True
+    drop = True
+
+    def _create_dump(self):
+        cmd = '{} --db {}'.format(self.dump_cmd, self.settings['NAME'])
+        cmd += ' --host {}:{}'.format(self.settings['HOST'], self.settings['PORT'])
+        if self.settings.get('USER'):
+            cmd += ' --username {}'.format(self.settings['USER'])
+        if self.settings.get('PASSWORD'):
+            cmd += ' --password {}'.format(self.settings['PASSWORD'])
+        for collection in self.exclude:
+            cmd += ' --excludeCollection {}'.format(collection)
+        cmd += ' --archive'
+        cmd = '{} {} {}'.format(self.dump_prefix, cmd, self.dump_suffix)
+        stdout, stderr = self.run_command(cmd, env=self.dump_env)
+        return stdout
+
+    def _restore_dump(self, dump):
+        cmd = self.restore_cmd
+        cmd += ' --host {}:{}'.format(self.settings['HOST'], self.settings['PORT'])
+        if self.settings.get('USER'):
+            cmd += ' --username {}'.format(self.settings['USER'])
+        if self.settings.get('PASSWORD'):
+            cmd += ' --password {}'.format(self.settings['PASSWORD'])
+        if self.object_check:
+            cmd += ' --objcheck'
+        if self.drop:
+            cmd += ' --drop'
+        cmd += ' --archive'
+        cmd = '{} {} {}'.format(self.restore_prefix, cmd, self.restore_suffix)
+        return self.run_command(cmd, stdin=dump, env=self.restore_env)

dbbackup/db/mysql.py

@@ -0,0 +1,40 @@
+from .base import BaseCommandDBConnector
+
+
+class MysqlDumpConnector(BaseCommandDBConnector):
+    """
+    MySQL connector, creates dump with ``mysqldump`` and restore with
+    ``mysql``.
+    """
+    dump_cmd = 'mysqldump'
+    restore_cmd = 'mysql'
+
+    def _create_dump(self):
+        cmd = '{} {} --quick'.format(self.dump_cmd, self.settings['NAME'])
+        if self.settings.get('HOST'):
+            cmd += ' --host={}'.format(self.settings['HOST'])
+        if self.settings.get('PORT'):
+            cmd += ' --port={}'.format(self.settings['PORT'])
+        if self.settings.get('USER'):
+            cmd += ' --user={}'.format(self.settings['USER'])
+        if self.settings.get('PASSWORD'):
+            cmd += ' --password={}'.format(self.settings['PASSWORD'])
+        for table in self.exclude:
+            cmd += ' --ignore-table={}.{}'.format(self.settings['NAME'], table)
+        cmd = '{} {} {}'.format(self.dump_prefix, cmd, self.dump_suffix)
+        stdout, stderr = self.run_command(cmd, env=self.dump_env)
+        return stdout
+
+    def _restore_dump(self, dump):
+        cmd = '{} {}'.format(self.restore_cmd, self.settings['NAME'])
+        if self.settings.get('HOST'):
+            cmd += ' --host={}'.format(self.settings['HOST'])
+        if self.settings.get('PORT'):
+            cmd += ' --port={}'.format(self.settings['PORT'])
+        if self.settings.get('USER'):
+            cmd += ' --user={}'.format(self.settings['USER'])
+        if self.settings.get('PASSWORD'):
+            cmd += ' --password={}'.format(self.settings['PASSWORD'])
+        cmd = '{} {} {}'.format(self.restore_prefix, cmd, self.restore_suffix)
+        stdout, stderr = self.run_command(cmd, stdin=dump, env=self.restore_env)
+        return stdout, stderr

dbbackup/db/postgresql.py

@@ -0,0 +1,78 @@
+from .base import BaseCommandDBConnector
+
+
+class PgDumpConnector(BaseCommandDBConnector):
+    """
+    PostgreSQL connector, it uses pg_dump`` to create an SQL text file
+    and ``psql`` for restore it.
+    """
+    extension = 'psql'
+    dump_cmd = 'pg_dump'
+    restore_cmd = 'psql'
+    single_transaction = True
+    drop = True
+
+    def _create_dump(self):
+        cmd = '{} {}'.format(self.dump_cmd, self.settings['NAME'])
+        if self.settings.get('HOST'):
+            cmd += ' --host={}'.format(self.settings['HOST'])
+        if self.settings.get('PORT'):
+            cmd += ' --port={}'.format(self.settings['PORT'])
+        if self.settings.get('USER'):
+            cmd += ' --user={}'.format(self.settings['USER'])
+        if self.settings.get('PASSWORD'):
+            cmd += ' --password={}'.format(self.settings['PASSWORD'])
+        else:
+            cmd += ' --no-password'
+        for table in self.exclude:
+            cmd += ' --exclude-table={}'.format(table)
+        if self.drop:
+            cmd += ' --clean'
+        cmd = '{} {} {}'.format(self.dump_prefix, cmd, self.dump_suffix)
+        stdout, stderr = self.run_command(cmd, env=self.dump_env)
+        return stdout
+
+    def _restore_dump(self, dump):
+        cmd = '{} {}'.format(self.restore_cmd, self.settings['NAME'])
+        if self.settings.get('HOST'):
+            cmd += ' --host={}'.format(self.settings['HOST'])
+        if self.settings.get('PORT'):
+            cmd += ' --port={}'.format(self.settings['PORT'])
+        if self.settings.get('USER'):
+            cmd += ' --user={}'.format(self.settings['USER'])
+        if self.settings.get('PASSWORD'):
+            cmd += ' --password={}'.format(self.settings['PASSWORD'])
+        else:
+            cmd += ' --no-password'
+        if self.single_transaction:
+            cmd += ' --single-transaction'
+        cmd = '{} {} {}'.format(self.restore_prefix, cmd, self.restore_suffix)
+        stdout, stderr = self.run_command(cmd, stdin=dump, env=self.restore_env)
+        return stdout, stderr
+
+
+class PgDumpGisConnector(PgDumpConnector):
+    """
+    PostgreGIS connector, same than :class:`PgDumpGisConnector` but enable
+    postgis if not made.
+    """
+    psql_cmd = 'psql'
+
+    def _enable_postgis(self):
+        cmd = '{} -c "CREATE EXTENSION IF NOT EXISTS postgis;"'.format(
+            self.psql_cmd)
+        cmd += ' --user={}'.format(self.settings['ADMIN_USER'])
+        if self.settings.get('ADMIN_PASSWORD'):
+            cmd += ' --password={}'.format(self.settings['ADMIN_PASSWORD'])
+        else:
+            cmd += ' --no-password'
+        if self.settings.get('HOST'):
+            cmd += ' --host={}'.format(self.settings['HOST'])
+        if self.settings.get('PORT'):
+            cmd += ' --port={}'.format(self.settings['PORT'])
+        return self.run_command(cmd)
+
+    def _restore_dump(self, dump):
+        if self.settings.get('ADMINUSER'):
+            self._enable_postgis()
+        return super(PgDumpGisConnector, self)._restore_dump(dump)