documents: import records from HEP BEJUNE

setup.py

@@ -49,15 +49,16 @@
             'sonar = invenio_app.cli:cli',
         ],
         'flask.commands': [
-            'fixtures = sonar.modules.cli:fixtures',
             'documents = sonar.modules.documents.cli.documents:documents',
             'ark = sonar.modules.ark.cli:ark',
             'oaiharvester = \
                 sonar.modules.documents.cli.oaiharvester:oaiharvester',
-            'utils = sonar.modules.cli:utils',
             'es = sonar.elasticsearch.cli:es',
             'heg = sonar.heg.cli:heg',
-            'resources = sonar.resources.cli:resources'
+            'resources = sonar.resources.cli:resources',
+            'imports = sonar.modules.cli.imports:imports',
+            'fixtures = sonar.modules.cli.fixtures:fixtures',
+            'utils = sonar.modules.cli.utils:utils'
         ],
         'invenio_base.apps': [
             'sonar = sonar.ext:Sonar',

sonar/modules/cli/__init__.py

@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+#
+# Swiss Open Access Repository
+# Copyright (C) 2021 RERO
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""SONAR CLI commands."""

sonar/modules/cli/fixtures.py

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+#
+# Swiss Open Access Repository
+# Copyright (C) 2021 RERO
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Fixtures commands."""
+
+import click
+
+from ..deposits.cli import deposits
+from ..organisations.cli.organisations import organisations
+from ..users.cli import users
+
+
+@click.group()
+def fixtures():
+    """Fixtures commands."""
+
+
+fixtures.add_command(users)
+fixtures.add_command(organisations)
+fixtures.add_command(deposits)

sonar/modules/cli/imports.py

@@ -0,0 +1,128 @@
+# -*- coding: utf-8 -*-
+#
+# Swiss Open Access Repository
+# Copyright (C) 2021 RERO
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""SONAR CLI commands."""
+
+import csv
+import os
+from datetime import datetime
+
+import click
+from flask.cli import with_appcontext
+
+from sonar.modules.documents.tasks import import_records
+from sonar.modules.utils import chunks
+
+DOCUMENT_TYPE_MAPPING = {
+    'master thesis': 'coar:c_bdcc',
+    'bachelor thesis': 'coar:c_7a1f',
+    'thesis': 'coar:c_46ec'
+}
+
+
+@click.group()
+def imports():
+    """Import commands."""
+
+
+@imports.command()
+@with_appcontext
+@click.argument('data_file', type=click.File('r'), required=True)
+@click.argument('pdf_directory', required=True)
+def hepbejune(data_file, pdf_directory):
+    """Import record from HEP BEJUNE.
+
+    :param data_file: CSV File.
+    :param pdf_directory: Directory containing the PDF files.
+    """
+    click.secho('Import records from HEPBEJUNE')
+
+    records = []
+
+    with open(data_file.name, 'r') as file:
+        reader = csv.reader(file, delimiter=',')
+
+        for row in reader:
+            date = datetime.strptime(row[9], '%d/%m/%Y')
+            if row[1] == 'SKIP':
+                continue
+
+            data = {
+                'title': [{
+                    'type': 'bf:Title',
+                    'mainTitle': [{
+                        'value': row[3],
+                        'language': 'fre'
+                    }]
+                }],
+                'identifiedBy': [{
+                    'type': 'bf:Local',
+                    'source': 'hepbejune',
+                    'value': row[0]
+                }],
+                'language': [{
+                    'type': 'bf:Language',
+                    'value': 'fre'
+                }],
+                'contribution': [{
+                    'agent': {
+                        'type': 'bf:Person',
+                        'preferred_name': row[8]
+                    },
+                    'role': ['cre']
+                }],
+                'dissertation': {
+                    'degree': row[12],
+                    'grantingInstitution': row[10],
+                    'date': date.strftime('%Y-%m-%d')
+                },
+                'provisionActivity': [{
+                    'type': 'bf:Publication',
+                    'startDate': row[11]
+                }],
+                'documentType':
+                DOCUMENT_TYPE_MAPPING.get(row[13], 'coar:c_1843'),
+                'usageAndAccessPolicy': {
+                    'license': 'CC BY-NC-ND'
+                },
+                'organisation': [{
+                    '$ref':
+                    'https://sonar.ch/api/organisations/hepbejune'
+                }],
+                'harvested':
+                True,
+                'masked':
+                True
+            }
+
+            file_path = os.path.join(pdf_directory, row[15])
+            if os.path.isfile(file_path):
+                data['files'] = [{
+                    'key': 'fulltext.pdf',
+                    'url': file_path,
+                    'label': 'Full-text',
+                    'type': 'file',
+                    'order': 0
+                }]
+
+            records.append(data)
+
+    # Chunk record list and send celery task
+    for chunk in list(chunks(records, 10)):
+        import_records.delay(chunk)
+
+    click.secho('Finished, records are imported in background', fg='green')

sonar/modules/cli.py → sonar/modules/cli/utils.py

@@ -15,7 +15,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-"""SONAR CLI commands."""
+"""Utils commands."""
 
 import json
 import pathlib
@@ -36,20 +36,6 @@
 
 from sonar.modules.api import SonarRecord
 
-from .deposits.cli import deposits
-from .organisations.cli.organisations import organisations
-from .users.cli import users
-
-
-@click.group()
-def fixtures():
-    """Fixtures management commands."""
-
-
-fixtures.add_command(users)
-fixtures.add_command(organisations)
-fixtures.add_command(deposits)
-
 
 @click.group()
 def utils():

sonar/modules/documents/tasks.py

@@ -66,7 +66,12 @@ def import_records(records_to_import):
                 key = file_data.pop('key')
 
                 try:
-                    record.add_file_from_url(url, key, **file_data)
+                    if url.startswith('http'):
+                        record.add_file_from_url(url, key, **file_data)
+                    else:
+                        with open(url, 'rb') as pdf_file:
+                            record.add_file(pdf_file.read(), key,
+                                            **file_data)
                 except Exception as exception:
                     current_app.logger.warning(
                         'Error during import of file {file} of record '

tests/ui/test_cli.py

@@ -23,7 +23,7 @@
 from click.testing import CliRunner
 from invenio_search.cli import destroy
 
-import sonar.modules.cli as Cli
+import sonar.modules.cli.utils as Cli
 
 
 def test_compile_json(app, script_info):