diff --git a/.dockerignore b/.dockerignore index 231f82c3..41ba9bae 100644 --- a/.dockerignore +++ b/.dockerignore @@ -13,3 +13,5 @@ docker-compose.yml docker-compose-dev.yml Procfile* + +celerybeat-schedule diff --git a/.gitignore b/.gitignore index 21bbcd04..062fb047 100644 --- a/.gitignore +++ b/.gitignore @@ -74,3 +74,6 @@ target/ # Generated JSON schema sonar/modules/documents/jsonschemas/documents/document-v1.0.0.json + +# Celery +celerybeat-schedule diff --git a/Pipfile b/Pipfile index e62dc8ec..45f643be 100644 --- a/Pipfile +++ b/Pipfile @@ -23,6 +23,7 @@ pycountry = "*" werkzeug = "==0.16.0" # TODO: check why we need this. flask-login = "<0.5" +invenio-oaiharvester = {editable = true,ref = "v1.0.0a4",git = "https://github.com/inveniosoftware/invenio-oaiharvester.git"} [dev-packages] Flask-Debugtoolbar = ">=0.10.1" diff --git a/Pipfile.lock b/Pipfile.lock index 1167bf5d..2474e94e 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "fbc43ec46db64015760d8bf93e3981cf0280e5f423900ddca338cb9454ef7862" + "sha256": "9d1b27bc22f1ee75ca13cfd3cfa13df25a1b985e35e084c4c070e28565eaab54" }, "pipfile-spec": 6, "requires": { @@ -655,6 +655,11 @@ ], "version": "==1.0.2" }, + "invenio-oaiharvester": { + "editable": true, + "git": "https://github.com/inveniosoftware/invenio-oaiharvester.git", + "ref": "fe55e095a8e78b36cfad875f752f2facc2908bae" + }, "invenio-oaiserver": { "hashes": [ "sha256:3f6f270464965d8e48eb548cf9eabd93b1985cc5034e9cb84f2f75e8f8b3109d", @@ -1335,6 +1340,13 @@ "index": "pypi", "version": "==0.14.1" }, + "sickle": { + "hashes": [ + "sha256:c0841b4df5edea33d2da01e893b3feadb1a8eacd2ca4dab9248e6047455ba14a", + "sha256:efdfa46c40cd34b3550f11b59c607ff0bd63adbc074efaa8b4dd662108269a2f" + ], + "version": "==0.6.5" + }, "simplejson": { "hashes": [ "sha256:0fe3994207485efb63d8f10a833ff31236ed27e3b23dadd0bf51c9900313f8f2", diff --git a/babel.ini b/babel.ini index 8167aad0..82f175c8 100644 --- a/babel.ini +++ b/babel.ini @@ -18,6 +18,8 @@ # Extraction from Python source files [python: **.py] encoding = utf-8 +[python: **/manual_translations.txt] +encoding = utf-8 # Extraction from Jinja2 templates [jinja2: **/templates/**.html] diff --git a/data/complete_document_sample.json b/data/complete_document_sample.json new file mode 100644 index 00000000..7e9eb956 --- /dev/null +++ b/data/complete_document_sample.json @@ -0,0 +1,162 @@ +{ + "identifiedBy": [ + { + "value": "urn:nbn:ch:rero-006-108713", + "type": "bf:Urn" + }, + { + "value": "oai:doc.rero.ch:20050302172954-WU", + "type": "bf:Identifier" + } + ], + "language": [ + { + "value": "eng", + "type": "bf:Language" + } + ], + "authors": [ + { + "type": "person", + "name": "Mancini, Loriano", + "date": "1975-03-23", + "qualifier": "Librarian" + }, + { + "type": "person", + "name": "Ronchetti, Elvezio" + }, + { + "type": "person", + "name": "Trojani, Fabio" + } + ], + "title": [ + { + "type": "bf:Title", + "mainTitle": [ + { + "language": "eng", + "value": "Title of the document" + } + ], + "subtitle": [ + { + "language": "eng", + "value": "Subtitle" + } + ] + } + ], + "extent": "103 p", + "abstracts": [ + { + "language": "eng", + "value": "Abstract of the document" + } + ], + "subjects": [ + { + "language": "eng", + "value": [ + "Time series models", + "GARCH models" + ] + } + ], + "provisionActivity": [ + { + "type": "bf:Manufacture", + "statement": [ + { + "label": [ + { + "value": "Bienne" + } + ], + "type": "bf:Place" + }, + { + "label": [ + { + "value": "Impr. Weber" + } + ], + "type": "bf:Agent" + }, + { + "label": [ + { + "value": "[2006]" + } + ], + "type": "Date" + }, + { + "label": [ + { + "value": "Lausanne" + } + ], + "type": "bf:Place" + }, + { + "label": [ + { + "value": "Rippone" + } + ], + "type": "bf:Place" + }, + { + "label": [ + { + "value": "Impr. Coustaud" + } + ], + "type": "bf:Agent" + } + ] + } + ], + "editionStatement": [ + { + "editionDesignation": [ + { + "value": "Di 3 ban" + }, + { + "value": "第3版", + "language": "chi-hani" + } + ], + "responsibility": [ + { + "value": "Zeng Lingliang zhu bian" + }, + { + "value": "曾令良主编", + "language": "chi-hani" + } + ] + } + ], + "is_part_of": "Is part of", + "copyrightDate": [ + "© 1971" + ], + "series": [ + { + "name": "Collection One", + "number": "5" + }, + { + "name": "Collection Two", + "number": "123" + } + ], + "notes": ["Note 1", "Note 2"], + "institution": { + "$ref": "https://sonar.ch/api/institutions/org" + } +} diff --git a/data/institutions.json b/data/institutions.json deleted file mode 100644 index 4a9731e1..00000000 --- a/data/institutions.json +++ /dev/null @@ -1,10 +0,0 @@ -[ - { - "pid": "usi", - "name": "Università della Svizzera italiana" - }, - { - "pid": "hevs", - "name": "HES-SO Valais/Wallis" - } -] \ No newline at end of file diff --git a/data/rerodoc_oai_sources.json b/data/rerodoc_oai_sources.json new file mode 100644 index 00000000..699cc91a --- /dev/null +++ b/data/rerodoc_oai_sources.json @@ -0,0 +1,130 @@ +[ + { + "key": "rerodoc-unisi", + "name": "rerodoc", + "url": "http://doc.rero.ch/oai2d", + "metadataprefix": "marcxml", + "comment": "", + "setspecs": "unisi" + }, + { + "key": "rerodoc-unifr", + "name": "rerodoc", + "url": "http://doc.rero.ch/oai2d", + "metadataprefix": "marcxml", + "comment": "", + "setspecs": "unifr" + }, + { + "key": "rerodoc-nl-epfl", + "name": "rerodoc", + "url": "http://doc.rero.ch/oai2d", + "metadataprefix": "marcxml", + "comment": "", + "setspecs": "nl-epfl" + }, + { + "key": "rerodoc-nl-ethz", + "name": "rerodoc", + "url": "http://doc.rero.ch/oai2d", + "metadataprefix": "marcxml", + "comment": "", + "setspecs": "nl-ethz" + }, + { + "key": "rerodoc-nl-fachhochschulen", + "name": "rerodoc", + "url": "http://doc.rero.ch/oai2d", + "metadataprefix": "marcxml", + "comment": "", + "setspecs": "nl-fachhochschulen" + }, + { + "key": "rerodoc-nl-lib4ri", + "name": "rerodoc", + "url": "http://doc.rero.ch/oai2d", + "metadataprefix": "marcxml", + "comment": "", + "setspecs": "nl-lib4ri" + }, + { + "key": "rerodoc-nl-unibas", + "name": "rerodoc", + "url": "http://doc.rero.ch/oai2d", + "metadataprefix": "marcxml", + "comment": "", + "setspecs": "nl-unibas" + }, + { + "key": "rerodoc-nl-unibe", + "name": "rerodoc", + "url": "http://doc.rero.ch/oai2d", + "metadataprefix": "marcxml", + "comment": "", + "setspecs": "nl-unibe" + }, + { + "key": "rerodoc-nl-unifr", + "name": "rerodoc", + "url": "http://doc.rero.ch/oai2d", + "metadataprefix": "marcxml", + "comment": "", + "setspecs": "nl-unifr" + }, + { + "key": "rerodoc-nl-unige", + "name": "rerodoc", + "url": "http://doc.rero.ch/oai2d", + "metadataprefix": "marcxml", + "comment": "", + "setspecs": "nl-unige" + }, + { + "key": "rerodoc-nl-unil", + "name": "rerodoc", + "url": "http://doc.rero.ch/oai2d", + "metadataprefix": "marcxml", + "comment": "", + "setspecs": "nl-unil" + }, + { + "key": "rerodoc-nl-unilu", + "name": "rerodoc", + "url": "http://doc.rero.ch/oai2d", + "metadataprefix": "marcxml", + "comment": "", + "setspecs": "nl-unilu" + }, + { + "key": "rerodoc-nl-unine", + "name": "rerodoc", + "url": "http://doc.rero.ch/oai2d", + "metadataprefix": "marcxml", + "comment": "", + "setspecs": "nl-unine" + }, + { + "key": "rerodoc-nl-unisg", + "name": "rerodoc", + "url": "http://doc.rero.ch/oai2d", + "metadataprefix": "marcxml", + "comment": "", + "setspecs": "nl-unisg" + }, + { + "key": "rerodoc-nl-usi", + "name": "rerodoc", + "url": "http://doc.rero.ch/oai2d", + "metadataprefix": "marcxml", + "comment": "", + "setspecs": "nl-usi" + }, + { + "key": "rerodoc-nl-uzh", + "name": "rerodoc", + "url": "http://doc.rero.ch/oai2d", + "metadataprefix": "marcxml", + "comment": "", + "setspecs": "nl-uzh" + } +] diff --git a/data/users.json b/data/users.json index bac270dd..2555027d 100644 --- a/data/users.json +++ b/data/users.json @@ -19,10 +19,7 @@ "street": "Brunnacherstrasse 112", "postal_code": "5330", "city": "Zurzach", - "phone": "+41625037543", - "institution": { - "$ref": "https://sonar.ch/api/institutions/usi" - } + "phone": "+41625037543" }, { "full_name": "Emanuele Fiorentini", @@ -33,10 +30,7 @@ "street": "Lichtmattstrasse 123", "postal_code": "5736", "city": "Burg", - "phone": "+41627676171", - "institution": { - "$ref": "https://sonar.ch/api/institutions/usi" - } + "phone": "+41627676171" }, { "full_name": "Jules Brochu", @@ -47,9 +41,6 @@ "street": "Rasenstrasse 2", "postal_code": "4577", "city": "Hessigkofen", - "phone": "+41323666556", - "institution": { - "$ref": "https://sonar.ch/api/institutions/usi" - } + "phone": "+41323666556" } ] diff --git a/scripts/server b/scripts/server index e42c95bc..64165cff 100755 --- a/scripts/server +++ b/scripts/server @@ -22,7 +22,8 @@ script_path=$(dirname "$0") export FLASK_ENV=development # Start Worker and Server -pipenv run celery worker -A invenio_app.celery -l INFO & pid_celery=$! +pipenv run celery worker -A invenio_app.celery -l INFO --beat --without-heartbeat & pid_celery=$! +# pipenv run celery worker -A invenio_app.celery -l INFO & pid_celery=$! pipenv run invenio run \ --cert "$script_path"/../docker/nginx/test.crt \ diff --git a/scripts/setup b/scripts/setup index 481580d6..1d704c39 100755 --- a/scripts/setup +++ b/scripts/setup @@ -18,6 +18,41 @@ set -e +# Size of data to create +size="minimal" +import=false +local=false +documents_api_url="https://localhost:5000/api/documents/" + +while test $# -gt 0; do + case "$1" in + --minimal) + size="minimal" + ;; + --small) + size="small" + ;; + --full) + size="full" + ;; + --local) + local=true + ;; + --import) + import=true + ;; + --documents-api-url) + # Shift to next value for getting option value + shift + documents_api_url=$1 + ;; + *) echo "Option $1 not recognized" ;; + esac + shift +done + +printf "\nSize of setup is set to \"$size\"\n\n" + # Clean redis pipenv run invenio shell --no-term-title -c "import redis; redis.StrictRedis.from_url(app.config['CACHE_REDIS_URL']).flushall(); print('Cache cleared')" pipenv run invenio db destroy --yes-i-know @@ -35,9 +70,25 @@ pipenv run invenio access allow superuser-access role superadmin pipenv run invenio access allow admin-access role admin pipenv run invenio access allow admin-access role moderator -# Import fixtures -pipenv run invenio fixtures institutions import -pipenv run invenio fixtures users import $(pipenv --where)/data/users.json -pipenv run invenio fixtures documents import hevs -pipenv run invenio fixtures documents import usi +# Create a default location for depositing files pipenv run invenio fixtures deposits create + +# Create configuration for RERODOC OAI harvesting +pipenv run invenio oaiharvester config create data/rerodoc_oai_sources.json + +# Create users +pipenv run invenio fixtures users import $(pipenv --where)/data/users.json + +# Create document sample +if $local; then + curl -L -H 'Content-Type:application/json' --data-binary "@./data/complete_document_sample.json" -XPOST $documents_api_url --insecure +fi + +# Import rerodoc data +if $import; then + case "$size" in + full) pipenv run invenio oaiharvester harvest-all ;; + small) pipenv run invenio oaiharvester harvest-all --max 100 ;; + *) pipenv run invenio oaiharvester harvest-all --max 10 ;; + esac +fi diff --git a/setup.py b/setup.py index 9bb6545e..82a3520e 100644 --- a/setup.py +++ b/setup.py @@ -50,10 +50,12 @@ ], 'flask.commands': [ 'fixtures = sonar.modules.cli:fixtures', + 'documents = sonar.modules.documents.cli:oaiharvester', 'utils = sonar.modules.cli:utils' ], 'invenio_base.apps': [ 'sonar = sonar.modules:Sonar', + 'documents = sonar.modules.documents:Documents', 'shibboleth_authenticator = \ sonar.modules.shibboleth_authenticator:ShibbolethAuthenticator', ], @@ -127,6 +129,9 @@ "invenio_records.jsonresolver": [ "institution = sonar.modules.institutions.jsonresolvers", "user = sonar.modules.users.jsonresolvers" + ], + 'invenio_celery.tasks' : [ + 'documents = sonar.modules.documents.tasks' ] }, classifiers=[ diff --git a/sonar/config.py b/sonar/config.py index 6c2498bd..feabc058 100644 --- a/sonar/config.py +++ b/sonar/config.py @@ -76,6 +76,8 @@ def _(x): # ======= LOGGING_SENTRY_LEVEL = "ERROR" LOGGING_SENTRY_CELERY = True +LOGGING_FS_LEVEL = "WARNING" +LOGGING_FS_LOGFILE = '{instance_path}/app.log' # Theme configuration # =================== @@ -238,134 +240,131 @@ def _(x): RECORDS_REST_ENDPOINTS = { 'doc': - dict( - pid_type='doc', - pid_minter='document_id', - pid_fetcher='document_id', - default_endpoint_prefix=True, - record_class=DocumentRecord, - search_class=DocumentSearch, - indexer_class=RecordIndexer, - search_index='documents', - search_type=None, - record_serializers={ - 'application/json': ('invenio_records_rest.serializers' - ':json_v1_response'), - }, - search_serializers={ - 'application/json': ('invenio_records_rest.serializers' - ':json_v1_search'), - }, - # record_loaders={ - # 'application/json': ('sonar.modules.documents.loaders' - # ':json_v1'), - # }, - list_route='/documents/', - item_route='/documents/', - default_media_type='application/json', - max_result_window=10000, - error_handlers=dict(), - create_permission_factory_imp=can_create_record_factory, - read_permission_factory_imp=can_read_record_factory, - update_permission_factory_imp=can_update_record_factory, - delete_permission_factory_imp=can_delete_record_factory, - list_permission_factory_imp=can_list_record_factory), + dict(pid_type='doc', + pid_minter='document_id', + pid_fetcher='document_id', + default_endpoint_prefix=True, + record_class=DocumentRecord, + search_class=DocumentSearch, + indexer_class=RecordIndexer, + search_index='documents', + search_type=None, + record_serializers={ + 'application/json': ('sonar.modules.documents.serializers' + ':json_v1_response'), + }, + search_serializers={ + 'application/json': ('sonar.modules.documents.serializers' + ':json_v1_search'), + }, + record_loaders={ + 'application/json': ('sonar.modules.documents.loaders' + ':json_v1'), + }, + list_route='/documents/', + item_route='/documents/', + default_media_type='application/json', + max_result_window=10000, + error_handlers=dict(), + create_permission_factory_imp=can_create_record_factory, + read_permission_factory_imp=can_read_record_factory, + update_permission_factory_imp=can_update_record_factory, + delete_permission_factory_imp=can_delete_record_factory, + list_permission_factory_imp=can_list_record_factory), 'inst': - dict( - pid_type='inst', - pid_minter='institution_id', - pid_fetcher='institution_id', - default_endpoint_prefix=True, - record_class=InstitutionRecord, - search_class=InstitutionSearch, - indexer_class=RecordIndexer, - search_index='institutions', - search_type=None, - record_serializers={ - 'application/json': ('invenio_records_rest.serializers' - ':json_v1_response'), - }, - search_serializers={ - 'application/json': ('invenio_records_rest.serializers' - ':json_v1_search'), - }, - # record_loaders={ - # 'application/json': ('sonar.modules.institutions.loaders' - # ':json_v1'), - # }, - list_route='/institutions/', - item_route='/institutions/', - default_media_type='application/json', - max_result_window=10000, - error_handlers=dict(), - create_permission_factory_imp=can_create_record_factory, - read_permission_factory_imp=can_read_record_factory, - update_permission_factory_imp=can_update_record_factory, - delete_permission_factory_imp=can_delete_record_factory, - list_permission_factory_imp=can_list_record_factory), + dict(pid_type='inst', + pid_minter='institution_id', + pid_fetcher='institution_id', + default_endpoint_prefix=True, + record_class=InstitutionRecord, + search_class=InstitutionSearch, + indexer_class=RecordIndexer, + search_index='institutions', + search_type=None, + record_serializers={ + 'application/json': ('sonar.modules.institutions.serializers' + ':json_v1_response'), + }, + search_serializers={ + 'application/json': ('sonar.modules.institutions.serializers' + ':json_v1_search'), + }, + record_loaders={ + 'application/json': ('sonar.modules.institutions.loaders' + ':json_v1'), + }, + list_route='/institutions/', + item_route='/institutions/', + default_media_type='application/json', + max_result_window=10000, + error_handlers=dict(), + create_permission_factory_imp=can_create_record_factory, + read_permission_factory_imp=can_read_record_factory, + update_permission_factory_imp=can_update_record_factory, + delete_permission_factory_imp=can_delete_record_factory, + list_permission_factory_imp=can_list_record_factory), 'user': - dict( - pid_type='user', - pid_minter='user_id', - pid_fetcher='user_id', - default_endpoint_prefix=True, - record_class=UserRecord, - search_class=UserSearch, - indexer_class=RecordIndexer, - search_index='users', - search_type=None, - record_serializers={ - 'application/json': ('invenio_records_rest.serializers' - ':json_v1_response'), - }, - search_serializers={ - 'application/json': ('invenio_records_rest.serializers' - ':json_v1_search'), - }, - # record_loaders={ - # 'application/json': ('sonar.modules.users.loaders' - # ':json_v1'), - # }, - list_route='/users/', - item_route='/users/', - default_media_type='application/json', - max_result_window=10000, - error_handlers=dict(), - create_permission_factory_imp=can_create_record_factory, - read_permission_factory_imp=can_read_record_factory, - update_permission_factory_imp=can_update_record_factory, - delete_permission_factory_imp=can_delete_record_factory, - list_permission_factory_imp=can_list_record_factory), + dict(pid_type='user', + pid_minter='user_id', + pid_fetcher='user_id', + default_endpoint_prefix=True, + record_class=UserRecord, + search_class=UserSearch, + indexer_class=RecordIndexer, + search_index='users', + search_type=None, + record_serializers={ + 'application/json': ('sonar.modules.users.serializers' + ':json_v1_response'), + }, + search_serializers={ + 'application/json': ('sonar.modules.users.serializers' + ':json_v1_search'), + }, + record_loaders={ + 'application/json': ('sonar.modules.users.loaders' + ':json_v1'), + }, + list_route='/users/', + item_route='/users/', + default_media_type='application/json', + max_result_window=10000, + error_handlers=dict(), + create_permission_factory_imp=can_create_record_factory, + read_permission_factory_imp=can_read_record_factory, + update_permission_factory_imp=can_update_record_factory, + delete_permission_factory_imp=can_delete_record_factory, + list_permission_factory_imp=can_list_record_factory), 'depo': - dict( - pid_type='depo', - pid_minter='deposit_id', - pid_fetcher='deposit_id', - default_endpoint_prefix=True, - record_class=DepositRecord, - search_class=DepositSearch, - indexer_class=RecordIndexer, - search_index='deposits', - search_type=None, - record_serializers={ - 'application/json': ('invenio_records_rest.serializers' - ':json_v1_response'), - }, - search_serializers={ - 'application/json': ('invenio_records_rest.serializers' - ':json_v1_search'), - }, - list_route='/deposits/', - item_route='/deposits/', - default_media_type='application/json', - max_result_window=10000, - error_handlers=dict(), - create_permission_factory_imp=can_create_record_factory, - read_permission_factory_imp=can_read_record_factory, - update_permission_factory_imp=can_update_record_factory, - delete_permission_factory_imp=can_delete_record_factory, - list_permission_factory_imp=can_list_record_factory) + dict(pid_type='depo', + pid_minter='deposit_id', + pid_fetcher='deposit_id', + default_endpoint_prefix=True, + record_class=DepositRecord, + search_class=DepositSearch, + indexer_class=RecordIndexer, + search_index='deposits', + search_type=None, + record_serializers={ + 'application/json': ('invenio_records_rest.serializers' + ':json_v1_response'), + }, + search_serializers={ + 'application/json': ('invenio_records_rest.serializers' + ':json_v1_search'), + }, + list_route='/deposits/', + item_route='/deposits/', + default_media_type='application/json', + max_result_window=10000, + error_handlers=dict(), + create_permission_factory_imp=can_create_record_factory, + read_permission_factory_imp=can_read_record_factory, + update_permission_factory_imp=can_update_record_factory, + delete_permission_factory_imp=can_delete_record_factory, + list_permission_factory_imp=can_list_record_factory) } """REST endpoints.""" @@ -422,7 +421,12 @@ def _(x): ), ) """Set default sorting options.""" -RECORDS_FILES_REST_ENDPOINTS = {'RECORDS_REST_ENDPOINTS': {'depo': '/files'}} +RECORDS_FILES_REST_ENDPOINTS = { + 'RECORDS_REST_ENDPOINTS': { + 'doc': '/files', + 'depo': '/files' + } +} SONAR_ENDPOINTS_ENABLED = True """Enable/disable automatic endpoint registration.""" @@ -484,3 +488,9 @@ def _(x): FILES_REST_PERMISSION_FACTORY = \ 'sonar.modules.permissions.files_permission_factory' + +# Database +# ========================= +DB_VERSIONING = False +# DB versioning is disabled globally, because of performances during documents +# importation. diff --git a/sonar/modules/api.py b/sonar/modules/api.py index 367a0191..f67c009a 100644 --- a/sonar/modules/api.py +++ b/sonar/modules/api.py @@ -18,10 +18,13 @@ """API for manipulating records.""" import re +from io import BytesIO from uuid import uuid4 +import requests from flask import current_app from invenio_db import db +from invenio_files_rest.helpers import compute_md5_checksum from invenio_indexer.api import RecordIndexer from invenio_jsonschemas import current_jsonschemas from invenio_pidstore.errors import PIDDoesNotExistError @@ -30,6 +33,9 @@ from invenio_search.api import RecordsSearch from sqlalchemy.orm.exc import NoResultFound +from sonar.modules.deposits.utils import change_filename_extension +from sonar.modules.pdf_extractor.utils import extract_text_from_content + class SonarRecord(Record, FilesMixin): """SONAR Record.""" @@ -65,7 +71,7 @@ def create(cls, id_=id_, **kwargs) - if (dbcommit): + if dbcommit: record.dbcommit() return record @@ -86,10 +92,12 @@ def get_record_by_pid(cls, pid, with_deleted=False): return None @classmethod - def get_ref_link(cls, type, id): + def get_ref_link(cls, record_type, record_id): """Get $ref link for the given type of record.""" return 'https://{host}/api/{type}/{id}'.format( - host=current_app.config.get('JSONSCHEMAS_HOST'), type=type, id=id) + host=current_app.config.get('JSONSCHEMAS_HOST'), + type=record_type, + id=record_id) @classmethod def get_pid_by_ref_link(cls, link): @@ -106,7 +114,8 @@ def get_record_by_ref_link(cls, link): """Get a record by its ref link.""" return cls.get_record_by_pid(cls.get_pid_by_ref_link(link)) - def dbcommit(self): + @staticmethod + def dbcommit(): """Commit changes to db.""" db.session.commit() @@ -114,6 +123,59 @@ def reindex(self): """Reindex record.""" RecordIndexer().index(self) + def add_file_from_url(self, url, key, **kwargs): + """Add file to record by getting data from given url. + + :param str url: External URL of the file + :param str key: File key + + for kwargs, see add_file method below. + """ + self.add_file(requests.get(url).content, key, **kwargs) + + def add_file(self, data, key, **kwargs): + """Create file and add it to record. + + :param data: Binary data of file + :param str key: File key + + kwargs may contain some additional data such as: file label, file type, + order. + """ + if not current_app.config.get('SONAR_DOCUMENTS_IMPORT_FILES'): + return + + # If file with the same key exists and checksum is the same as the + # registered file, we don't do anything + checksum = compute_md5_checksum(BytesIO(data)) + if key in self.files and checksum == self.files[key].file.checksum: + return + + # Create the file + self.files[key] = BytesIO(data) + self.files[key]['label'] = kwargs.get('label', key) + self.files[key]['type'] = kwargs.get('type', 'file') + self.files[key]['order'] = kwargs.get('order', 1) + + # Try to extract full text from file data, and generate a warning if + # it's not possible. For several cases, file is locked against fulltext + # copy. + if current_app.config.get( + 'SONAR_DOCUMENTS_EXTRACT_FULLTEXT_ON_IMPORT' + ) and self.files[key].mimetype == 'application/pdf': + try: + fulltext = extract_text_from_content(data) + + key = change_filename_extension(key, 'txt') + self.files[key] = BytesIO(fulltext.encode()) + self.files[key]['type'] = 'fulltext' + except Exception as exception: + current_app.logger.warning( + 'Error during fulltext extraction of {file} of record ' + '{record}: {error}'.format(file=key, + error=exception, + record=self['identifiedBy'])) + class SonarSearch(RecordsSearch): """Search Class SONAR.""" diff --git a/sonar/modules/cli.py b/sonar/modules/cli.py index b081d1e5..3abd0e7a 100644 --- a/sonar/modules/cli.py +++ b/sonar/modules/cli.py @@ -27,8 +27,6 @@ from invenio_search.proxies import current_search from .deposits.cli import deposits -from .documents.cli import documents -from .institutions.cli import institutions from .users.cli import users @@ -37,8 +35,6 @@ def fixtures(): """Fixtures management commands.""" -fixtures.add_command(documents) -fixtures.add_command(institutions) fixtures.add_command(users) fixtures.add_command(deposits) diff --git a/sonar/modules/config.py b/sonar/modules/config.py index 0c840c47..ab66d387 100644 --- a/sonar/modules/config.py +++ b/sonar/modules/config.py @@ -37,6 +37,9 @@ rus='ru' ) +SONAR_APP_PREFERRED_LANGUAGES = ['eng', 'fre', 'ger', 'ita'] +"""Order of preferred languages for displaying value in views.""" + SONAR_APP_ENABLE_CORS = True SONAR_APP_DISABLE_PERMISSION_CHECKS = False diff --git a/sonar/modules/documents/api.py b/sonar/modules/documents/api.py index ce7d9a1a..16f32ced 100644 --- a/sonar/modules/documents/api.py +++ b/sonar/modules/documents/api.py @@ -19,19 +19,13 @@ from functools import partial -from flask import current_app - from ..api import SonarRecord, SonarSearch from ..fetchers import id_fetcher from ..minters import id_minter from ..providers import Provider # provider -DocumentProvider = type( - 'DocumentProvider', - (Provider,), - dict(pid_type='doc') -) +DocumentProvider = type('DocumentProvider', (Provider, ), dict(pid_type='doc')) # minter document_pid_minter = partial(id_minter, provider=DocumentProvider) # fetcher @@ -55,3 +49,20 @@ class DocumentRecord(SonarRecord): fetcher = document_pid_fetcher provider = DocumentProvider schema = 'document' + + @classmethod + def get_record_by_identifier(cls, identifiers): + """Get a record by its identifier. + + :param list identifiers: List of identifiers + """ + for identifier in identifiers: + if identifier['type'] == 'bf:Identifier': + results = list(DocumentSearch().filter( + 'term', identifiedBy__value=identifier['value']).source( + includes=['pid'])) + + if results: + return cls.get_record_by_pid(results[0]['pid']) + + return None diff --git a/sonar/modules/documents/cli.py b/sonar/modules/documents/cli.py index 1006e77a..36650c3b 100644 --- a/sonar/modules/documents/cli.py +++ b/sonar/modules/documents/cli.py @@ -18,114 +18,113 @@ """Documents CLI commands.""" import json -from functools import partial import click -import requests from click.exceptions import ClickException -from dojson.contrib.marc21.utils import create_record, split_stream from flask import current_app from flask.cli import with_appcontext from invenio_db import db -from invenio_indexer.api import RecordIndexer -from invenio_records import Record +from invenio_oaiharvester.cli import harvest, oaiharvester +from invenio_oaiharvester.models import OAIHarvestConfig -from sonar.modules.documents.dojson.contrib.marc21tojson import marc21tojson -from sonar.modules.institutions.api import InstitutionRecord -from .api import DocumentRecord +@oaiharvester.group() +def config(): + """Configs commands for OAI harvesting.""" -@click.group() -def documents(): - """Documents CLI commands.""" - - -@documents.command('import') -@click.argument('institution') -@click.option('--pages', '-p', required=True, type=int, default=10) +@config.command('create') +@click.argument('config_file', type=click.File('r')) @with_appcontext -def import_documents(institution, pages): - """Import documents from RERO doc. +def oai_config_create(config_file): + """Creates configurations for OAI harvesting. - institution: String institution filter for retreiving documents - pages: Number of pages to import + :param config_file: File containing a list of sources to harvest. """ - url = current_app.config.get('SONAR_DOCUMENTS_RERO_DOC_URL') - click.secho( - 'Importing {pages} pages of records for "{institution}" ' - 'from {url}'.format(pages=pages, institution=institution, url=url)) - - # Get institution record from database - institution_record = InstitutionRecord.get_record_by_pid(institution) - - if not institution_record: - raise ClickException('Institution record not found in database') - - institution_ref_link = InstitutionRecord.get_ref_link( - 'institutions', institution_record['pid']) - - # mapping between institution key and RERO doc filter - institution_map = current_app.config.get( - 'SONAR_DOCUMENTS_INSTITUTIONS_MAP') - - if not institution_map: - raise ClickException('Institution map not found in configuration') - - if institution not in institution_map: - raise ClickException( - 'Institution map for "{institution}" not found in configuration, ' - 'keys available {keys}'.format( - institution=institution, - keys=institution_map.keys())) - - key = institution_map[institution] - current_page = 1 - - indexer = RecordIndexer() - - while(current_page <= pages): - click.echo('Importing records {start} to {end}... '.format( - start=(current_page*10-9), end=(current_page*10)), nl=False) - - # Read Marc21 data for current page - response = requests.get( - '{url}?of=xm&jrec={first_record}&c=NAVSITE.{institution}' - .format(url=url, - first_record=(current_page*10-9), - institution=key.upper()), stream=True) - - if response.status_code != 200: - raise ClickException('Request to "{url}" failed'.format(url=url)) - - response.raw.decode_content = True - - ids = [] - - for data in split_stream(response.raw): - # Convert from Marc XML to JSON - record = create_record(data) + '\nCreating configurations for OAI harvesting from file "{file}"...'. + format(file=config_file.name)) + + sources = json.load(config_file) + + if not sources or not isinstance(sources, list): + raise ClickException('Configurations file cannot be parsed') + + for source in sources: + try: + configuration = OAIHarvestConfig.query.filter_by( + name=source['key']).first() + + if configuration: + raise Exception( + 'Config already registered for "{name}"'.format( + name=source['key'])) + + configuration = OAIHarvestConfig( + name=source['key'], + baseurl=source['url'], + metadataprefix=source['metadataprefix'], + setspecs=source['setspecs'], + comment=source['comment']) + configuration.save() + db.session.commit() - # Transform JSON - record = marc21tojson.do(record) + click.secho('Created configuration for "{name}"'.format( + name=source['key']), + fg='green') + except Exception as exception: + click.secho(str(exception), fg='yellow') - # Add institution - record['institution'] = {'$ref': institution_ref_link} + click.secho('Done', fg='green') - # Register record to DB - db_record = DocumentRecord.create(record) - db.session.commit() - # Add ID for bulk index in elasticsearch - ids.append(str(db_record.id)) +@config.command('list') +@with_appcontext +def oai_config_info(): + """List infos for tasks.""" + oais = OAIHarvestConfig.query.all() + for oai in oais: + click.secho('\n' + oai.name, underline=True) + click.echo('\tlastrun : ', nl=False) + click.echo(oai.lastrun) + click.echo('\tbaseurl : ' + oai.baseurl) + click.echo('\tmetadataprefix: ' + oai.metadataprefix) + click.echo('\tcomment : ' + oai.comment) + click.echo('\tsetspecs : ' + oai.setspecs) + + +@oaiharvester.command('harvest-all') +@click.option('-m', + '--max', + type=int, + default=None, + help='Maximum of records to harvest (optional).') +@click.option('-k', + '--enqueue', + is_flag=True, + default=False, + help="Enqueue harvesting and return immediately.") +@with_appcontext +@click.pass_context +def oai_config_harvest_all(ctx, max, enqueue): + """Harvesting data for the set and the configuration. - # index and process queue in elasticsearch - indexer.bulk_index(ids) - indexer.process_bulk_queue() + :param ctx: App context. + :param max: Max records to harvest for each sources. + :param enqueue: Enqueue process or return immediately. + """ + with current_app.app_context(): + sources = OAIHarvestConfig.query.all() - current_page += 1 + arguments = [] - click.secho('Done', fg='green', nl=True) + if max: + arguments.append('max={max}'.format(max=max)) - click.secho('Finished', fg='green') + for source in sources: + name = source.name + ctx.invoke(harvest, + name=name, + arguments=arguments, + enqueue=enqueue, + quiet=True) diff --git a/sonar/modules/documents/config.py b/sonar/modules/documents/config.py index b7b2f292..a82623b6 100644 --- a/sonar/modules/documents/config.py +++ b/sonar/modules/documents/config.py @@ -17,9 +17,8 @@ """SONAR documents configuration.""" -SONAR_DOCUMENTS_RERO_DOC_URL = 'http://doc.rero.ch/search' +SONAR_DOCUMENTS_IMPORT_FILES = True +"""Import files associated with the document.""" -SONAR_DOCUMENTS_INSTITUTIONS_MAP = dict( - usi='ticino.unisi', - hevs='valais.hevs' -) +SONAR_DOCUMENTS_EXTRACT_FULLTEXT_ON_IMPORT = True +"""Automatically extract fulltext when a file is imported.""" diff --git a/sonar/modules/documents/dojson/contrib/marc21tojson/model.py b/sonar/modules/documents/dojson/contrib/marc21tojson/model.py index da76c88c..bef8105a 100644 --- a/sonar/modules/documents/dojson/contrib/marc21tojson/model.py +++ b/sonar/modules/documents/dojson/contrib/marc21tojson/model.py @@ -22,10 +22,12 @@ import requests from dojson import utils +from flask import current_app from sonar.modules.documents.dojson.utils import SonarMarc21Overdo, \ error_print, get_field_items, get_field_link_data, get_year_from_date, \ not_repetitive, remove_trailing_punctuation +from sonar.modules.institutions.api import InstitutionRecord marc21tojson = SonarMarc21Overdo() @@ -48,6 +50,7 @@ def get_language_script(script): 'kore': ('kor', ), 'zyyy': ('chi', ) } + if script in languages_scripts: languages = ([marc21tojson.lang_from_008] + marc21tojson.langs_from_041_a + @@ -90,51 +93,28 @@ def get_person_link(bibid, id, key, value): return mef_link -@marc21tojson.over('type', 'leader') -def marc21_to_type(self, key, value): - """ - Get document type. - - Books: LDR/6-7: am - Journals: LDR/6-7: as - Articles: LDR/6-7: aa + add field 773 (journal title) - Scores: LDR/6: c|d - Videos: LDR/6: g + 007/0: m|v - Sounds: LDR/6: i|j - E-books (imported from Cantook) - """ - type = 'other' - type_of_record = value[6] - bibliographic_level = value[7] - if type_of_record == 'a': - if bibliographic_level == 'm': - type = 'book' - elif bibliographic_level == 's': - type = 'journal' - elif bibliographic_level == 'a': - type = 'article' - elif type_of_record in ['c', 'd']: - type = 'score' - elif type_of_record in ['i', 'j']: - type = 'sound' - elif type_of_record == 'g': - type = 'video' - # Todo 007 - return type - - -@marc21tojson.over('pid', '^001') +@marc21tojson.over('type', '^980') @utils.ignore_value -def marc21_to_pid(self, key, value): - """Get pid. +def marc21_to_type_and_institution(self, key, value): + """Get document type and institution from 980 field.""" + institution = value.get('b') + document_type = value.get('a') - If 001 starts with 'REROILS:' save as pid. - """ - pid = None - value = value.strip().split(':') - if value[0] == 'REROILS': - pid = value[1] - return pid + if institution: + institution = institution.lower() + + if institution not in marc21tojson.registererd_organizations: + marc21tojson.create_institution(institution) + marc21tojson.registererd_organizations.append(institution) + + self['institution'] = { + '$ref': InstitutionRecord.get_ref_link('institutions', institution) + } + + if document_type: + self['type'] = document_type.lower() + + return None @marc21tojson.over('language', '^041') @@ -162,24 +142,55 @@ def marc21_to_language(self, key, value): # if not language: # error_print('ERROR LANGUAGE:', marc21tojson.bib_id, 'set to "und"') # language = [{'value': 'und', 'type': 'bf:Language'}] + return language or None @marc21tojson.over('title', '^245..') +@utils.for_each_value @utils.ignore_value -def marc21_to_title(self, key, value): - """Get title. +def marc21_to_title_245(self, key, value): + """Get title.""" + main_title = value.get('a') + language = value.get('9', 'eng') + subtitle = value.get('b') - title: 245$a - without the punctuaction. If there's a $b, then 245$a : $b without the " /" - """ - data = not_repetitive(marc21tojson.bib_id, key, value, 'a') - main_title = remove_trailing_punctuation(data) - sub_title = not_repetitive(marc21tojson.bib_id, key, value, 'b') - if sub_title: - main_title += ' : ' + ' : '.join( - utils.force_list(remove_trailing_punctuation(sub_title))) - return main_title + if not main_title: + return None + + title = { + 'type': 'bf:Title', + 'mainTitle': [{ + 'value': main_title, + 'language': language + }] + } + + if subtitle: + title['subtitle'] = [{'value': subtitle, 'language': language}] + + return title + + +@marc21tojson.over('title', '^246..') +@utils.for_each_value +@utils.ignore_value +def marc21_to_title_246(self, key, value): + """Get title.""" + main_title = value.get('a') + language = value.get('9', 'eng') + + if not main_title: + return None + + title = self.get('title', [{'type': 'bf:Title', 'mainTitle': []}]) + + # Add title 246 to last title in mainTitle propert + title[-1]['mainTitle'].append({'value': main_title, 'language': language}) + + self['title'] = title + + return None @marc21tojson.over('authors', '[17][01]0..') @@ -287,10 +298,10 @@ def build_edition_data(code, label, index, link): return edition_data or None -@marc21tojson.over('provisionActivity', '^264.[ 0-3]') +@marc21tojson.over('provisionActivity', '^(260..|264.[ 0-3])') @utils.for_each_value @utils.ignore_value -def marc21_to_provisionActivity(self, key, value): +def marc21_to_provision_activity(self, key, value): """Get publisher data. publisher.name: 264 [$b repetitive] (without the , but keep the ;) @@ -317,7 +328,7 @@ def build_agent_data(code, label, index, link): 'language': get_language_script(alt_gr['script']) }) - except Exception as err: + except Exception: pass return agent_data @@ -354,6 +365,8 @@ def build_place(): '2': 'bf:Distribution', '3': 'bf:Manufacture' } + if key[:3] == '260': + ind2 = '1' publication = { 'type': type_per_ind2[ind2], 'statement': [], @@ -389,7 +402,7 @@ def build_place(): 'language': get_language_script(alt_gr['script']) }) - except Exception as err: + except Exception: pass publication['statement'].append(date) @@ -445,15 +458,20 @@ def marc21_to_series(self, key, value): @marc21tojson.over('abstracts', '^520..') @utils.for_each_value @utils.ignore_value -def marc21_to_abstracts(self, key, value): - """Get abstracts. +def marc21_to_abstract(self, key, value): + """Get abstract.""" + abstract = value.get('a') + language = value.get('9', 'eng') - abstract: [520$a repetitive] - """ - abstracts = None - if value.get('a'): - abstracts = ', '.join(utils.force_list(value.get('a'))) - return abstracts + if not abstract: + return None + + abstracts_data = self.get('abstracts', []) + abstracts_data.append({'value': abstract, 'language': language}) + + self['abstracts'] = abstracts_data + + return None @marc21tojson.over('identifiedBy', '^020..') @@ -598,6 +616,7 @@ def populate_acquisitionTerms_note_qualifier(identifier): } } + identifiedBy = self.get('identifiedBy', []) identifier = {} subfield_a = not_repetitive(marc21tojson.bib_id, key, @@ -660,7 +679,6 @@ def populate_acquisitionTerms_note_qualifier(identifier): 'value': subfield_a, 'type': 'bf:Identifier' }) - identifiedBy = self.get('identifiedBy', []) if not identifier.get('type'): identifier['type'] = 'bf:Identifier' identifiedBy.append(identifier) @@ -773,7 +791,7 @@ def marc21_to_is_part_of(self, key, value): return not_repetitive(marc21tojson.bib_id, key, value, 't') -@marc21tojson.over('subjects', '^6....') +@marc21tojson.over('subjects', '^695..') @utils.for_each_value @utils.ignore_value def marc21_to_subjects(self, key, value): @@ -782,4 +800,51 @@ def marc21_to_subjects(self, key, value): subjects: 6xx [duplicates could exist between several vocabularies, if possible deduplicate] """ + if not value.get('9'): + return None + return dict(language=value.get('9'), value=value.get('a').split(' ; ')) + + +@marc21tojson.over('files', '^856..') +@utils.for_each_value +@utils.ignore_value +def marc21_to_files(self, key, value): + """Get files.""" + key = value.get('f') + url = value.get('u') + size = int(value.get('s', 0)) + mime_type = value.get('q', 'text/plain') + + if not key or not url: + return None + + # TODO: Check why this type of file exists. Real example with rerodoc ID + # 29085 + if mime_type == 'pdt/download': + current_app.logger.warning( + 'File {file} for record {record} has a strange pdt/download mime ' + 'type, skipping import of file...'.format( + file=key, record=self['identifiedBy'])) + return None + + url = url.strip() + + # Retreive file order + # If order not set we put a value to 99 for easily point theses files + order = 99 + if value.get('y'): + match = re.search(r'order:([0-9]+)$', value.get('y')) + if match: + order = int(match.group(1)) + + data = { + 'key': key, + 'url': url, + 'label': value.get('z', key), + 'type': 'file', + 'order': order, + 'size': size + } + + return data diff --git a/sonar/modules/documents/dojson/utils.py b/sonar/modules/documents/dojson/utils.py index 11f2adc7..5aae4792 100644 --- a/sonar/modules/documents/dojson/utils.py +++ b/sonar/modules/documents/dojson/utils.py @@ -24,6 +24,8 @@ import click from dojson import Overdo, utils +from sonar.modules.institutions.api import InstitutionRecord + def error_print(*args): """Error printing to sdterr.""" @@ -90,6 +92,9 @@ def remove_trailing_punctuation( punctuation characters needing one or more preceding space(s) in order to be removed. """ + punctuation = punctuation.replace('.', r'\.').replace('-', r'\-') + spaced_punctuation = \ + spaced_punctuation.replace('.', r'\.').replace('-', r'\-') return re.sub( r'([{0}]|\s+[{1}])$'.format(punctuation, spaced_punctuation), '', @@ -178,6 +183,7 @@ class SonarMarc21Overdo(SonarOverdo): langs_from_041_a = [] langs_from_041_h = [] unique_languages = [] + registererd_organizations = [] alternate_graphic = {} country = None cantons = [] @@ -240,6 +246,28 @@ def get_link_data(subfields_6_data): script_dir = '' return tag, link, script_code, script_dir + @staticmethod + def create_institution(institution_key): + """Create institution if not existing and return it. + + :param str institution_key: Key (PID) of the institution. + """ + if not institution_key: + raise Exception('Not key provided') + + # Get institution record from database + organization = InstitutionRecord.get_record_by_pid(institution_key) + + if not organization: + # Create organization record + organization = InstitutionRecord.create( + { + 'pid': institution_key, + 'name': institution_key + }, + dbcommit=True) + organization.reindex() + def init_country(self): """Initialization country (008 and 044).""" self.country = None diff --git a/sonar/modules/documents/ext.py b/sonar/modules/documents/ext.py index 40870063..959b4526 100644 --- a/sonar/modules/documents/ext.py +++ b/sonar/modules/documents/ext.py @@ -19,6 +19,12 @@ from __future__ import absolute_import, print_function +from invenio_indexer.signals import before_record_index +from invenio_oaiharvester.signals import oaiharvest_finished + +from sonar.modules.documents.receivers import populate_fulltext_field, \ + transform_harvested_records + from . import config @@ -35,6 +41,13 @@ def init_app(self, app): self.init_config(app) app.extensions['sonar_documents'] = self + # Connect to oaiharvester signal + oaiharvest_finished.connect(transform_harvested_records, weak=False) + + # Connect to record index signal, to modify record before indexing. + before_record_index.connect(populate_fulltext_field, + weak=False) + def init_config(self, app): """Initialize configuration.""" for k in dir(config): diff --git a/sonar/modules/documents/jsonschemas/documents/document-v1.0.0_src.json b/sonar/modules/documents/jsonschemas/documents/document-v1.0.0_src.json index b0d5a23e..19218aad 100644 --- a/sonar/modules/documents/jsonschemas/documents/document-v1.0.0_src.json +++ b/sonar/modules/documents/jsonschemas/documents/document-v1.0.0_src.json @@ -492,7 +492,8 @@ "zun", "zxx", "zza" - ] + ], + "minLength": 1 }, "language_script": { "enum": [ @@ -952,7 +953,89 @@ "title": "Schema", "description": "Schema to validate document against.", "type": "string", - "default": "https://ils.rero.ch/schema/documents/document-v0.0.1.json" + "default": "https://sonar.ch/schema/documents/document-v1.0.0.json", + "minLength": 1 + }, + "_bucket": { + "title": "Bucket UUID", + "description": "Bucket UUID used to store files", + "type": "string", + "minLength": 1 + }, + "_files": { + "title": "Files", + "description": "List of files attached to the record.", + "items": { + "title": "File item", + "description": "Describes the information of a single file in the record.", + "additionalProperties": false, + "properties": { + "bucket": { + "title": "Bucket UUID", + "description": "UUID of the bucket to which this file is assigned.", + "type": "string", + "minLength": 1 + }, + "file_id": { + "title": "File UUID", + "description": "UUID of the FileInstance object.", + "type": "string", + "minLength": 1 + }, + "version_id": { + "title": "Version UUID", + "description": "UUID of the ObjectVersion object.", + "type": "string", + "minLength": 1 + }, + "key": { + "title": "Key", + "description": "Key (filename) of the file.", + "type": "string", + "minLength": 1 + }, + "checksum": { + "title": "Checksum", + "description": "MD5 checksum of the file.", + "type": "string", + "minLength": 1 + }, + "size": { + "title": "Size", + "description": "Size of the file in bytes.", + "type": "integer" + }, + "label": { + "title": "Label", + "description": "Label of the file", + "type": "string", + "minLength": 1 + }, + "type": { + "title": "Type", + "description": "Type of the file", + "type": "string", + "enum": ["file", "fulltext", "thumbnail"], + "default": "file", + "minLength": 1 + }, + "order": { + "title": "Position", + "description": "Position of the file", + "type": "integer", + "default": 1, + "minimum": 1 + } + }, + "required": [ + "bucket", + "file_id", + "version_id", + "key" + ], + "type": "object" + }, + "type": "array" }, "pid": { "title": "Document ID", @@ -967,33 +1050,108 @@ "$ref": { "title": "Organization", "type": "string", - "pattern": "^https://sonar.ch/api/institutions/.*?$" + "pattern": "^https://sonar.ch/api/institutions/.*?$", + "minLength": 1 } }, - "required": ["$ref"] + "required": [ + "$ref" + ] }, "type": { "title": "Type", "description": "Required. Type of the document. Should be selected in the list below.", "type": "string", "enum": [ - "other", - "article", "book", - "ebook", "journal", - "score", - "sound", - "video" + "thesis", + "postprint", + "preprint", + "dissertation", + "report", + "newspaper", + "partition", + "newspapers", + "map", + "audio", + "print_media", + "issue" ], "minLength": 4 }, "title": { "title": "Title", - "description": "Required. Entire title without statement of responsibility.", - "validationMessage": "Required. Entire title without statement of responsibility.", - "type": "string", - "minLength": 1 + "type": "array", + "minItems": 1, + "items": { + "title": "Title item", + "type": "object", + "required": [ + "type", + "mainTitle" + ], + "additionalProperties": false, + "properties": { + "type": { + "title": "Type", + "description": "Type of the title.", + "type": "string", + "default": "bf:Title", + "enum": [ + "bf:Title" + ], + "minLength": 1 + }, + "mainTitle": { + "title": "Main title", + "description": "Main title.", + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": ["value", "language"], + "additionalProperties": false, + "properties": { + "value": { + "title": "Value of the title", + "type": "string", + "minLength": 1 + }, + "language": { + "title": "Language of the title", + "type": "string", + "$ref": "#/definitions/language", + "minLength": 1 + } + } + } + }, + "subtitle": { + "title": "Subtitle", + "description": "Subtitle.", + "type": "array", + "items": { + "type": "object", + "required": ["value", "language"], + "additionalProperties": false, + "properties": { + "value": { + "title": "Value of the title", + "type": "string", + "minLength": 1 + }, + "language": { + "title": "Language of the title", + "type": "string", + "$ref": "#/definitions/language", + "minLength": 1 + } + } + } + } + } + } }, "is_part_of": { "title": "Is part of", @@ -1021,12 +1179,14 @@ "default": "bf:Language", "enum": [ "bf:Language" - ] + ], + "minLength": 1 }, "value": { "title": "Language value", "type": "string", - "$ref": "#/definitions/language" + "$ref": "#/definitions/language", + "minLength": 1 } } } @@ -1046,7 +1206,8 @@ "$ref": { "title": "MEF person ref", "type": "string", - "pattern": "^https://mef.rero.ch/api/mef/.*?$" + "pattern": "^https://mef.rero.ch/api/mef/.*?$", + "minLength": 1 }, "pid": { "title": "pid", @@ -1057,7 +1218,8 @@ "name": { "title": "Name", "description": "Person's or organisation's name.", - "type": "string" + "type": "string", + "minLength": 1 }, "type": { "title": "Type", @@ -1067,17 +1229,20 @@ "person", "organisation" ], - "default": "person" + "default": "person", + "minLength": 1 }, "date": { "title": "Date", "description": "Information about the birth and the death of a person. Helpful to disambiguate people.", - "type": "string" + "type": "string", + "minLength": 1 }, "qualifier": { "title": "Qualifier", "description": "Information about the person, ie her profession. Helpful to disambiguate people.", - "type": "string" + "type": "string", + "minLength": 1 } } } @@ -1114,7 +1279,8 @@ "language": { "title": "Language", "type": "string", - "$ref": "#/definitions/language_script" + "$ref": "#/definitions/language_script", + "minLength": 1 } } } @@ -1132,7 +1298,8 @@ "language": { "title": "Language", "type": "string", - "$ref": "#/definitions/language_script" + "$ref": "#/definitions/language_script", + "minLength": 1 } } } @@ -1156,7 +1323,8 @@ "bf:Manufacture", "bf:Distribution", "bf:Production" - ] + ], + "minLength": 1 }, "place": { "title": "Country and/or canton of the provision activity", @@ -1174,12 +1342,14 @@ "country": { "title": "Country", "type": "string", - "$ref": "#/definitions/country" + "$ref": "#/definitions/country", + "minLength": 1 }, "canton": { "title": "Canton", "type": "string", - "$ref": "#/definitions/canton" + "$ref": "#/definitions/canton", + "minLength": 1 } } } @@ -1198,7 +1368,8 @@ "bf:Place", "bf:Agent", "Date" - ] + ], + "minLength": 1 }, "label": { "title": "Label", @@ -1215,7 +1386,8 @@ "language": { "title": "Language", "type": "string", - "$ref": "#/definitions/language_script" + "$ref": "#/definitions/language_script", + "minLength": 1 } } } @@ -1226,21 +1398,24 @@ "note": { "title": "Note", "description": "Note.", - "type": "string" + "type": "string", + "minLength": 1 }, "startDate": { "title": "Start date of publication", "description": "Start date of the publication. This must be an integer, ie 1989, 453, -50. Used to sort search results. Once this field is set, a free formed date of publication can be added in the next field.", "type": "string", "minimum": -9999, - "maximum": 2050 + "maximum": 2050, + "minLength": 1 }, "endDate": { "title": "End date of publication", "description": "End date of the publication. This must be an integer, ie 1989, 453, -50. Used to sort search results. Once this field is set, a free formed date of publication can be added in the next field.", "type": "string", "minimum": -9999, - "maximum": 2050 + "maximum": 2050, + "minLength": 1 } } } @@ -1288,12 +1463,14 @@ "name": { "title": "Title", "description": "Title of the series.", - "type": "string" + "type": "string", + "minLength": 1 }, "number": { "title": "Numbering", "description": "Numbering of the resource within the series.", - "type": "string" + "type": "string", + "minLength": 1 } } } @@ -1311,13 +1488,27 @@ }, "abstracts": { "title": "Abstracts", - "description": "Abstract of the resource.", + "description": "Abstracts.", "type": "array", "minItems": 1, "items": { - "title": "Abstract", - "type": "string", - "minLength": 2 + "title": "Abstract item", + "type": "object", + "required": ["value", "language"], + "additionalProperties": false, + "properties": { + "value": { + "title": "Value of the abstract", + "type": "string", + "minLength": 1 + }, + "language": { + "title": "Language of the abstract", + "type": "string", + "$ref": "#/definitions/language", + "minLength": 1 + } + } } }, "identifiedBy": { @@ -1357,7 +1548,8 @@ "bf:Urn", "bf:VideoRecordingNumber", "uri" - ] + ], + "minLength": 1 }, "value": { "title": "Identifier value", @@ -1398,7 +1590,8 @@ "invalid", "cancelled", "invalid or cancelled" - ] + ], + "minLength": 1 } } } @@ -1412,27 +1605,16 @@ "type": "object", "additionalProperties": false, "required": [ - "language", "value" + "language", + "value" ], "properties": { "language": { "title": "Subject language", "description": "Language of subject.", "type": "string", - "enum": [ - "fre", - "ger", - "eng", - "ita", - "spa", - "ara", - "chi", - "lat", - "heb", - "jpn", - "por", - "rus" - ] + "$ref": "#/definitions/language", + "minLength": 1 }, "value": { "title": "Subjects list", @@ -1441,7 +1623,8 @@ "minLength": 1, "items": { "type": "string", - "title": "Subject" + "title": "Subject", + "minLength": 1 } } } @@ -1451,7 +1634,8 @@ "title": "Cover art", "description": "Vendor cover art URL.", "type": "string", - "format": "uri" + "format": "uri", + "minLength": 1 }, "electronic_location": { "title": "Electronic Locations", @@ -1468,7 +1652,8 @@ "title": "Uniform Resource Identifier", "description": "Uniform Resource Identifier (URI), which provides standard syntax for locating an object using existing Internet protocols.", "type": "string", - "format": "uri" + "format": "uri", + "minLength": 1 } } } diff --git a/sonar/modules/documents/mappings/v6/documents/document-v1.0.0.json b/sonar/modules/documents/mappings/v6/documents/document-v1.0.0.json index 7e4161a3..33f65d1c 100644 --- a/sonar/modules/documents/mappings/v6/documents/document-v1.0.0.json +++ b/sonar/modules/documents/mappings/v6/documents/document-v1.0.0.json @@ -25,6 +25,9 @@ }, "mappings": { "document-v1.0.0": { + "_source": { + "excludes": ["fulltext"] + }, "date_detection": false, "numeric_detection": false, "properties": { @@ -35,6 +38,9 @@ "pid": { "type": "keyword" }, + "fulltext": { + "type": "text" + }, "institution": { "type": "object", "properties": { @@ -46,34 +52,6 @@ } } }, - "title": { - "type": "text", - "analyzer": "global_lowercase_asciifolding", - "copy_to": "autocomplete_title", - "fields": { - "eng": { - "type": "text", - "analyzer": "english" - }, - "fre": { - "type": "text", - "analyzer": "french" - }, - "ger": { - "type": "text", - "analyzer": "german" - }, - "ita": { - "type": "text", - "analyzer": "italian" - } - } - }, - "autocomplete_title": { - "type": "text", - "analyzer": "autocomplete", - "search_analyzer": "standard" - }, "type": { "type": "keyword" }, @@ -270,10 +248,6 @@ } } }, - "publisherStatement": { - "type": "keyword", - "index": false - }, "extent": { "type": "text", "analyzer": "global_lowercase_asciifolding" @@ -342,28 +316,6 @@ } } }, - "abstracts": { - "type": "text", - "analyzer": "global_lowercase_asciifolding", - "fields": { - "eng": { - "type": "text", - "analyzer": "english" - }, - "fre": { - "type": "text", - "analyzer": "french" - }, - "ger": { - "type": "text", - "analyzer": "german" - }, - "ita": { - "type": "text", - "analyzer": "italian" - } - } - }, "identifiedBy": { "type": "object", "properties": { @@ -402,76 +354,6 @@ "facet_subjects": { "type": "keyword" }, - "holdings": { - "type": "object", - "properties": { - "pid": { - "type": "keyword" - }, - "call_number": { - "type": "keyword" - }, - "location": { - "type": "object", - "properties": { - "pid": { - "type": "keyword" - }, - "name": { - "type": "text" - } - } - }, - "circulation_category": { - "type": "object", - "properties": { - "pid": { - "type": "keyword" - }, - "name": { - "type": "text" - } - } - }, - "organisation": { - "type": "object", - "properties": { - "organisation_pid": { - "type": "keyword" - }, - "library_pid": { - "type": "keyword" - }, - "organisation_library": { - "type": "keyword" - } - } - }, - "items": { - "type": "object", - "properties": { - "pid": { - "type": "keyword" - }, - "barcode": { - "type": "keyword" - }, - "call_number": { - "type": "keyword" - }, - "status": { - "type": "keyword" - }, - "available": { - "type": "boolean" - } - } - } - } - }, - "available": { - "type": "boolean" - }, "harvested": { "type": "boolean" }, diff --git a/sonar/modules/documents/marshmallow/json.py b/sonar/modules/documents/marshmallow/json.py index 630446d3..c5a3da70 100644 --- a/sonar/modules/documents/marshmallow/json.py +++ b/sonar/modules/documents/marshmallow/json.py @@ -19,29 +19,39 @@ from __future__ import absolute_import, print_function -from invenio_records_rest.schemas import Nested, StrictKeysMixin -from invenio_records_rest.schemas.fields import DateString, \ - PersistentIdentifier, SanitizedUnicode -from marshmallow import fields, missing, validate +from invenio_records_rest.schemas import StrictKeysMixin +from invenio_records_rest.schemas.fields import PersistentIdentifier, \ + SanitizedUnicode +from marshmallow import fields class DocumentMetadataSchemaV1(StrictKeysMixin): """Schema for the document metadata.""" pid = PersistentIdentifier() - title = SanitizedUnicode(required=True) + type = SanitizedUnicode() + title = fields.List(fields.Dict()) is_part_of = SanitizedUnicode() abstracts = fields.List(fields.Dict()) - authors = fields.Dict(dump_only=True) + authors = fields.List(fields.Dict()) institution = fields.Dict(dump_only=True) + _files = fields.Dict(dump_only=True) + language = fields.List(fields.Dict()) + copyrightDate = fields.List(fields.String()) + editionStatement = fields.List(fields.Dict()) + provisionActivity = fields.List(fields.Dict()) + extent = SanitizedUnicode() + otherMaterialCharacteristics = SanitizedUnicode() + formats = fields.Dict() + additionalMaterials = SanitizedUnicode() + series = fields.List(fields.Dict()) + notes = fields.List(fields.String()) + identifiedBy = fields.List(fields.Dict()) + subjects = fields.List(fields.Dict()) class DocumentSchemaV1(StrictKeysMixin): """Document schema.""" metadata = fields.Nested(DocumentMetadataSchemaV1) - # created = fields.Str(dump_only=True) - # revision = fields.Integer(dump_only=True) - # updated = fields.Str(dump_only=True) - # links = fields.Dict(dump_only=True) - id = PersistentIdentifier() + links = fields.Dict(dump_only=True) diff --git a/sonar/modules/documents/receivers.py b/sonar/modules/documents/receivers.py new file mode 100644 index 00000000..0f7f2c2f --- /dev/null +++ b/sonar/modules/documents/receivers.py @@ -0,0 +1,113 @@ +# -*- coding: utf-8 -*- +# +# Swiss Open Access Repository +# Copyright (C) 2019 RERO +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Signals connections for documents.""" + +import time + +from dojson.contrib.marc21.utils import create_record + +from .api import DocumentRecord +from .dojson.contrib.marc21tojson.model import marc21tojson +from .tasks import import_records + +CHUNK_SIZE = 100 + + +def transform_harvested_records(sender=None, records=None, **kwargs): + """Harvest records and transform them and send to the import queue. + + This function is called when the oaiharvester command is finished. + + :param sender: Sender of the signal. + :param list records: Liste of records to harvest. + """ + start_time = time.time() + + max_records = kwargs.get('max', None) + + if kwargs.get('name'): + print('Harvesting records from "{set}"'.format(set=kwargs.get('name'))) + + harvested_records = list(records) + + # Reduce array to max records + if max_records: + harvested_records = harvested_records[:int(max_records)] + + records = [] + + for harvested_record in harvested_records: + # Convert from Marc XML to JSON + data = create_record(harvested_record.xml) + + # Transform JSON + data = marc21tojson.do(data) + + # Add transformed data to list + records.append(data) + + # Chunk record list and send celery task + for chunk in list(chunks(records, CHUNK_SIZE)): + import_records.delay(chunk) + + print('{count} records harvested in {time} seconds'.format( + count=len(records), time=time.time() - start_time)) + + +def populate_fulltext_field(sender=None, + record=None, + json=None, + index=None, + **kwargs): + """Receive a signal before record is indexed, to add fulltext. + + This function is called just before a record is sent to index. + + :param sender: Sender of the signal. + :param Record record: Record to index. + :param dict json: JSON that will be indexed. + :param str index: Name of the index in which record will be sent. + """ + # Takes care only about documents indexing + if not index.startswith('documents'): + return + + # Transform record in DocumentRecord + if not isinstance(record, DocumentRecord): + record = DocumentRecord.get_record(record.id) + + # No files are present in record + if not record.files: + return + + # Store fulltext in array for indexing + json['fulltext'] = [] + for file in record.files: + if file['type'] == 'fulltext': + with file.file.storage().open() as pdf_file: + json['fulltext'].append(pdf_file.read().decode('utf-8')) + + +def chunks(records, size): + """Yield chunks from records. + + :param list records: Full records list. + :param int size: Size of chunks. + """ + for i in range(0, len(records), size): + yield records[i:i + size] diff --git a/sonar/modules/documents/tasks.py b/sonar/modules/documents/tasks.py new file mode 100644 index 00000000..d2e3195c --- /dev/null +++ b/sonar/modules/documents/tasks.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- +# +# Swiss Open Access Repository +# Copyright (C) 2019 RERO +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Tasks for document in celery.""" + +from celery import shared_task +from flask import current_app +from invenio_db import db +from invenio_indexer.api import RecordIndexer + +from sonar.modules.documents.api import DocumentRecord + + +@shared_task(ignore_result=True) +def import_records(records_to_import): + """Import records in database and index them. + + Used as celery task. "ignore_result" flag means that we don't want to + get the status and/or the result of the task, execution is faster. + + :param list records_to_import: List of records to import. + """ + indexer = RecordIndexer() + + ids = [] + + for data in records_to_import: + try: + files_data = data.pop('files', []) + + record = DocumentRecord.get_record_by_identifier( + data.get('identifiedBy', [])) + + if not record: + record = DocumentRecord.create(data, + dbcommit=False, + with_bucket=True) + + for file_data in files_data: + # Store url and key and remove it from dict to pass dict to + # kwargs in add_file_from_url method + url = file_data.pop('url') + key = file_data.pop('key') + + try: + record.add_file_from_url(url, key, **file_data) + except Exception as exception: + current_app.logger.error( + 'Error during import of file {file} of record ' + '{record}: {error}'.format( + file=key, + error=exception, + record=record['identifiedBy'])) + + # Merge record in database, at this time it's not saved into DB. + record.commit() + + # Pushing record to database, not yet persisted into DB + db.session.flush() + + # Add ID for bulk index in elasticsearch + ids.append(str(record.id)) + + current_app.logger.info( + 'Record with reference "{reference}" imported successfully'. + format(reference=record['identifiedBy'])) + + except Exception as exception: + current_app.logger.error( + 'Error during importation of record {record}: {exception}'. + format(record=data, exception=exception)) + + # Commit and index records + db.session.commit() + indexer.bulk_index(ids) + indexer.process_bulk_queue() diff --git a/sonar/modules/documents/templates/documents/record.html b/sonar/modules/documents/templates/documents/record.html index beaffef2..eccbaa3e 100644 --- a/sonar/modules/documents/templates/documents/record.html +++ b/sonar/modules/documents/templates/documents/record.html @@ -17,7 +17,7 @@ {%- extends config.RECORDS_UI_BASE_TEMPLATE %} -{% from 'sonar/macros/macro.html' import dl %} +{% from 'sonar/macros/macro.html' import dl, dl_dict, dl_list %} {%- macro record_content(data) %} {% for key, value in data.items() recursive %} @@ -48,13 +48,11 @@ {%- endmacro %} {%- block body %} - + --> {% set record = record.replace_refs() %} -

{{ record.title }}

+

{{ record.title[0] | title_format(current_i18n.language) }}

{% if record.institution %}
{{ record.institution.name }}
{% endif %} @@ -64,12 +62,12 @@
{{ record.institution.name }}
{% if record.authors %} - {{ dl(_('Author'), record.pid | authors_format(current_i18n.language, viewcode)) }} + {{ dl(_('Authors'), record.pid|authors_format(current_i18n.language, viewcode)) }} {% endif %} {% if record.editionStatement %} - {{ dl_list(_('Edition'), record.editionStatement | edition_format) }} + {{ dl_list(_('Edition'), record.editionStatement|edition_format) }} {% endif %} @@ -111,17 +109,28 @@
{{ record.institution.name }}
{% if record.subjects|length > 0 %} - {{ dl(_('Abstract'), record.subjects|subjects_format|nl2br) }} + {{ dl(_('Subjects'), record.subjects|subjects_format|nl2br) }} {% endif %} {% if record.notes|length > 0 %} - {{ dl(_('Notes'), record.notes|abstracts_format|nl2br) }} +
+ {{ _('Notes') }}: +
+
+
    + {% for note in record.notes %} +
  • + {{ note }} +
    • + {% endfor %} +
+
{% endif %} {% if record.identifiedBy|identifiedby_format|length > 0 %}
- {{ _('Identifier') }}: + {{ _('Identifiers') }}:
    @@ -129,24 +138,43 @@
    {{ record.institution.name }}
  • {{ identifier.value }} {{ identifier.type }} - {% endfor %}
    • + {% endfor %}
{% endif %} - {% if record.language|language_format(current_i18n.language)|length > 0 %} - {{ dl(_('Language'), record.language|language_format(current_i18n.language)) }} - {% endif %} - - {% if record.source %} - {{ dl(_('Source'), ''+ record.source +'') }} + {% if record.language %} +
+ {{ _('Languages') }}: +
+
+
    + {% for language in record.language %} +
  • + {{ _('lang-' + language.value) }} +
    • + {% endfor %} +
+
{% endif %} {% set link = url_for('invenio_records_ui.document', pid_value=record.pid, ir=g.ir|default('sonar'), _external=True) %} {{ dl(_('Permalink'), '' + link + '') }}
+ + {% if record._files %} +
+
{{ _('Files') }}
+
    + {% for file in record._files %} +
  • + {{ file.key }} +
    • + {% endfor %} +
+ {% endif %} {%- endblock %} diff --git a/sonar/modules/documents/utils.py b/sonar/modules/documents/utils.py index 8cd053d9..1b2a3d8b 100644 --- a/sonar/modules/documents/utils.py +++ b/sonar/modules/documents/utils.py @@ -40,8 +40,6 @@ def publication_statement_text(provision_activity): statement_with_language = {'default': ''} statement_type = None - print(provision_activity['statement']) - for statement in provision_activity['statement']: labels = statement['label'] @@ -52,7 +50,6 @@ def publication_statement_text(provision_activity): statement_with_language[language] = '' if statement_with_language[language]: - print(statement_type, statement['type']) if statement_type == statement['type']: statement_with_language[language] += punctuation[ statement_type diff --git a/sonar/modules/documents/views.py b/sonar/modules/documents/views.py index 0f482341..d2b29963 100644 --- a/sonar/modules/documents/views.py +++ b/sonar/modules/documents/views.py @@ -78,6 +78,41 @@ def nl2br(string): return string.replace('\n', '
') +@blueprint.app_template_filter() +def title_format(title, language): + """Format title for template. + + :param list title: List of titles. + :param str language: Language to retreive title from. + """ + language = get_bibliographic_code_from_language(language) + + preferred_languages = get_preferred_languages(language) + + def get_value(items): + """Return the value for the given language.""" + if not items: + return None + + for preferred_language in preferred_languages: + for item in items: + if item['language'] == preferred_language: + return item['value'] + + return items[0]['value'] + + output = [] + main_title = get_value(title.get('mainTitle', [])) + if main_title: + output.append(main_title) + + subtitle = get_value(title.get('subtitle', [])) + if subtitle: + output.append(subtitle) + + return " : ".join(output) + + @blueprint.app_template_filter() def authors_format(pid, language='en', viewcode='sonar'): """Format authors for template in given language.""" @@ -145,8 +180,8 @@ def abstracts_format(abstracts): """Format abstracts for template.""" output = [] for abstract in abstracts: - output.append(re.sub(r'\n+', '\n', abstract)) - return '\n'.join(str(x) for x in output) + output.append(re.sub(r'\n+', '\n', abstract['value'])) + return '\n\n'.join(str(x) for x in output) @blueprint.app_template_filter() @@ -178,23 +213,6 @@ def identifiedby_format(identifiedby): return output -@blueprint.app_template_filter() -def language_format(langs_list, language_interface): - """Converts language code to langauge name. - - langs_list: a code or a list of language codes - language_interface: the code of the language of the interface - Returns a comma separated list of language names. - """ - output = [] - if isinstance(langs_list, str): - langs_list = [{'type': 'bf:Language', 'value': langs_list}] - for lang in langs_list: - language_code = lang.get('value') - output.append(_(language_code)) - return ", ".join(output) - - def get_language_from_bibliographic_code(language_code): """Return language code from bibliographic language. @@ -228,3 +246,17 @@ def get_bibliographic_code_from_language(language_code): raise Exception('Language code not found for "{language_code}"'.format( language_code=language_code)) + + +def get_preferred_languages(force_language=None): + """Get the ordered list of preferred languages. + + :param forceLanguage: String, force a language to be the first. + """ + preferred_languages = current_app.config.get( + 'SONAR_APP_PREFERRED_LANGUAGES', []).copy() + + if force_language: + preferred_languages.insert(0, force_language) + + return list(dict.fromkeys(preferred_languages)) diff --git a/sonar/modules/institutions/cli.py b/sonar/modules/institutions/cli.py deleted file mode 100644 index 4e1ab7be..00000000 --- a/sonar/modules/institutions/cli.py +++ /dev/null @@ -1,69 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Swiss Open Access Repository -# Copyright (C) 2019 RERO -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, version 3 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -"""Documents CLI commands.""" - -import json -from functools import partial - -import click -import requests -from click.exceptions import ClickException -from flask import current_app -from flask.cli import with_appcontext -from invenio_db import db -from invenio_indexer.api import RecordIndexer - -from .api import InstitutionRecord - - -@click.group() -def institutions(): - """Institutions CLI commands.""" - - -@institutions.command('import') -@with_appcontext -def import_institutions(): - """Import institutions from JSON file.""" - institution_file = './data/institutions.json' - click.secho('Importing institution from {file}'.format( - file=institution_file)) - - indexer = RecordIndexer() - - with open(institution_file) as json_file: - records = json.load(json_file) - for record in records: - try: - # Check existence in DB - db_record = InstitutionRecord.get_record_by_pid(record['pid']) - - if db_record: - raise ClickException('Record already exists in DB') - - # Register record to DB - db_record = InstitutionRecord.create(record) - db.session.commit() - - indexer.index(db_record) - except Exception as error: - click.secho( - 'Institution {institution} could not be imported: {error}' - .format(institution=record, error=str(error)), fg='red') - - click.secho('Finished', fg='green') diff --git a/sonar/modules/institutions/jsonresolvers.py b/sonar/modules/institutions/jsonresolvers.py index 185e550c..ae5d9c00 100644 --- a/sonar/modules/institutions/jsonresolvers.py +++ b/sonar/modules/institutions/jsonresolvers.py @@ -32,5 +32,7 @@ def institution_resolver(pid): getter=Record.get_record) _, record = resolver.resolve(pid) - del record['$schema'] + if record.get('$schema'): + del record['$schema'] + return record diff --git a/sonar/modules/pdf_extractor/utils.py b/sonar/modules/pdf_extractor/utils.py index fd2382c3..7ff01ec3 100644 --- a/sonar/modules/pdf_extractor/utils.py +++ b/sonar/modules/pdf_extractor/utils.py @@ -39,7 +39,8 @@ def extract_text_from_file(file): """Extract full-text from file.""" # Process pdf text extraction text = subprocess.check_output( - 'pdftotext -enc UTF-8 {file} -'.format(file=file), shell=True) + 'pdftotext -enc UTF-8 {file} - 2> /dev/null'.format(file=file), + shell=True) text = text.decode() # Remove carriage returns diff --git a/sonar/theme/api_views.py b/sonar/theme/api_views.py index 25b3890b..051426f2 100644 --- a/sonar/theme/api_views.py +++ b/sonar/theme/api_views.py @@ -37,9 +37,14 @@ def prepare_jsonschema(schema): """Json schema prep.""" schema = copy.deepcopy(schema) - if schema.get('$schema'): - del schema['$schema'] - schema['required'].remove('pid') + + del schema['$schema'] + + if 'pid' in schema.get('required', []): + schema['required'].remove('pid') + + del schema['properties']['$schema'] + return schema diff --git a/sonar/theme/templates/sonar/macros/macro.html b/sonar/theme/templates/sonar/macros/macro.html index 42185710..b4930ea9 100644 --- a/sonar/theme/templates/sonar/macros/macro.html +++ b/sonar/theme/templates/sonar/macros/macro.html @@ -22,4 +22,42 @@
{{ content|safe }}
-{% endmacro %} \ No newline at end of file +{% endmacro %} + +{% macro dl_dict(name, dict_or_string) %} + {% if dict_or_string %} +
+ {{ name }}: +
+
+ {% if dict_or_string is string %} + {{ dict_or_string }} + {% else %} +
    + {% for element in dict_or_string %} +
  • {{ dict_or_string[element] }}
    • + {% endfor %} +
+ {% endif %} +
+ {% endif %} +{% endmacro %} + +{% macro dl_list(name, list_or_string) %} + {% if list_or_string %} +
+ {{ name }}: +
+
+ {% if list_or_string is string %} + {{ list_or_string }} + {% else %} +
    + {% for element in list_or_string %} +
  • {{ element }}
    • + {% endfor %} +
+ {% endif %} +
+ {% endif %} +{% endmacro %} diff --git a/sonar/translations/de/LC_MESSAGES/messages.po b/sonar/translations/de/LC_MESSAGES/messages.po index 344cdce8..e8f64a2f 100644 --- a/sonar/translations/de/LC_MESSAGES/messages.po +++ b/sonar/translations/de/LC_MESSAGES/messages.po @@ -2,33 +2,28 @@ # # Swiss Open Access Repository # Copyright (C) 2019 RERO -# # This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by +# it under the terms of the GNU Affero General Public License as published +# by # the Free Software Foundation, version 3 of the License. -# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. -# # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -# # Dockerfile that builds a fully functional image of your app. -# # Note: It is important to keep the commands in this file in sync with your # boostrap script located in ./scripts/bootstrap. -# -# In order to increase the build speed, we are extending this image from a base +# In order to increase the build speed, we are extending this image from a +# base # image (built with Dockerfile.base) which only includes your Python # dependencies. -# msgid "" msgstr "" "Project-Id-Version: sonar 0.0.1\n" "Report-Msgid-Bugs-To: software@rero.ch\n" -"POT-Creation-Date: 2019-07-22 16:12+0200\n" +"POT-Creation-Date: 2020-02-04 15:21+0100\n" "PO-Revision-Date: 2019-06-13 10:45+0200\n" "Last-Translator: FULL NAME \n" "Language: de\n" @@ -37,155 +32,170 @@ msgstr "" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" -"Generated-By: Babel 2.7.0\n" +"Generated-By: Babel 2.8.0\n" -#: sonar/config.py:49 +#: sonar/config.py:63 msgid "French" msgstr "Français" -#: sonar/config.py:50 +#: sonar/config.py:63 msgid "German" msgstr "Deutsch" -#: sonar/config.py:51 +#: sonar/config.py:64 msgid "Italian" msgstr "Italiano" -#: sonar/config.py:71 sonar/config.py:75 +#: sonar/config.py:83 sonar/config.py:87 msgid "Swiss Open Access Repository" msgstr "Swiss Open Access Repository" -#: sonar/config.py:98 +#: sonar/config.py:112 msgid "Welcome to Swiss Open Access Repository!" msgstr "Willkommen im Swiss Open Access Repository!" -#: sonar/config.py:286 +#: sonar/config.py:379 msgid "institution" msgstr "Institution" -#: sonar/config.py:287 +#: sonar/config.py:380 msgid "language" msgstr "Sprache" -#: sonar/config.py:288 -msgid "author" -msgstr "Autor" +#: sonar/config.py:381 +msgid "author__en" +msgstr "" -#: sonar/config.py:289 +#: sonar/config.py:382 +msgid "author__fr" +msgstr "" + +#: sonar/config.py:383 +msgid "author__de" +msgstr "" + +#: sonar/config.py:384 +msgid "author__it" +msgstr "" + +#: sonar/config.py:385 msgid "subject" msgstr "Stichwort" -#: sonar/config.py:298 +#: sonar/config.py:392 +msgid "pid" +msgstr "" + +#: sonar/config.py:393 +msgid "status" +msgstr "" + +#: sonar/config.py:394 +msgid "user" +msgstr "" + +#: sonar/config.py:395 +msgid "contributor" +msgstr "" + +#: sonar/config.py:402 msgid "Best match" msgstr "Beste Übereinstimmung" -#: sonar/config.py:304 +#: sonar/config.py:408 msgid "Most recent" msgstr "Neueste" -#: sonar/modules/documents/templates/documents/record.html:58 +#: sonar/modules/documents/templates/documents/record.html:65 msgid "Authors" -msgstr "Autoren" - -#: sonar/modules/documents/templates/documents/record.html:63 -msgid "Abstract" -msgstr "Zusammenfassung" +msgstr "" #: sonar/modules/documents/templates/documents/record.html:70 -msgid "Physical description" -msgstr "Physische Beschreibung" - -#: sonar/modules/documents/templates/documents/record.html:75 -msgid "Subject" -msgstr "Subjekt" +msgid "Edition" +msgstr "" #: sonar/modules/documents/templates/documents/record.html:80 -msgid "Language" -msgstr "Sprache" +msgid "Copyright date" +msgstr "" #: sonar/modules/documents/templates/documents/record.html:85 -msgid "Notes" -msgstr "Notizen" +msgid "Abstract" +msgstr "Zusammenfassung" -#: sonar/modules/documents/templates/documents/record.html:91 -msgid "ISBN" -msgstr "ISBN" +#: sonar/modules/documents/templates/documents/record.html:92 +msgid "Physical description" +msgstr "Physische Beschreibung" #: sonar/modules/documents/templates/documents/record.html:97 -msgid "Permalink" -msgstr "Permalink" - -#: sonar/theme/templates/sonar/401.html:13 -msgid "Unauthorized" -msgstr "Nicht autorisiert" +msgid "Additional Materials" +msgstr "" -#: sonar/theme/templates/sonar/401.html:14 -msgid "You need to be authenticated to view this page." -msgstr "Sie müssen authentifiziert sein, um diese Seite anzuzeigen." +#: sonar/modules/documents/templates/documents/record.html:102 +msgid "Series" +msgstr "" -#: sonar/theme/templates/sonar/403.html:13 -msgid "Permission required" -msgstr "Genehmigung erforderlich" +#: sonar/modules/documents/templates/documents/record.html:107 +msgid "Is part of" +msgstr "" -#: sonar/theme/templates/sonar/403.html:14 -msgid "You do not have sufficient permissions to view this page." -msgstr "Du hast nicht genügend Berechtigungen, um diese Seite anzuzeigen." +#: sonar/modules/documents/templates/documents/record.html:112 +msgid "Subjects" +msgstr "" -#: sonar/theme/templates/sonar/404.html:13 -msgid "Page not found" -msgstr "Seite nicht gefunden" +#: sonar/modules/documents/templates/documents/record.html:117 +msgid "Notes" +msgstr "Notizen" -#: sonar/theme/templates/sonar/404.html:14 -msgid "The page you are looking for could not be found." -msgstr "Die gesuchte Seite konnte nicht gefunden werden." +#: sonar/modules/documents/templates/documents/record.html:122 +msgid "Identifier" +msgstr "" -#: sonar/theme/templates/sonar/500.html:13 -msgid "Internal server error" -msgstr "Interner Serverfehler" +#: sonar/modules/documents/templates/documents/record.html:137 +msgid "Language" +msgstr "Sprache" -#: sonar/theme/templates/sonar/500.html:15 -msgid "Error identifier" -msgstr "Fehleridentifikationsnummer" +#: sonar/modules/documents/templates/documents/record.html:142 +msgid "Permalink" +msgstr "Permalink" -#: sonar/theme/templates/sonar/admin_header.html:72 -#: sonar/theme/templates/sonar/partial/dropdown_user.html:8 -msgid "Logout" -msgstr "Abmeldung" +#: sonar/modules/documents/templates/documents/record.html:147 +msgid "Files" +msgstr "" -#: sonar/theme/templates/sonar/footer.html:32 +#: sonar/theme/templates/sonar/footer.html:40 msgid "Help" msgstr "Hilfe" -#: sonar/theme/templates/sonar/footer.html:35 +#: sonar/theme/templates/sonar/footer.html:43 msgid "About" msgstr "Über uns" -#: sonar/theme/templates/sonar/footer.html:38 +#: sonar/theme/templates/sonar/footer.html:46 msgid "Contact" msgstr "Kontakt" -#: sonar/theme/templates/sonar/footer.html:43 +#: sonar/theme/templates/sonar/footer.html:51 #, python-format msgid "Powered by RERO" msgstr "Angetrieben von RERO" -#: sonar/theme/templates/sonar/frontpage.html:33 +#: sonar/theme/templates/sonar/frontpage.html:41 msgid "Search publications, authors, projects, ..." msgstr "Suche nach Publikationen, Autoren, Projekten, ..." -#: sonar/theme/templates/sonar/frontpage.html:35 +#: sonar/theme/templates/sonar/frontpage.html:43 msgid "Advanced search" msgstr "Erweiterte Suche" -#: sonar/theme/templates/sonar/frontpage.html:51 +#: sonar/theme/templates/sonar/frontpage.html:59 msgid "Software under development!" msgstr "Software in Entwicklung!" -#: sonar/theme/templates/sonar/frontpage.html:58 +#: sonar/theme/templates/sonar/frontpage.html:66 msgid "The project" msgstr "Das Projekt" -#: sonar/theme/templates/sonar/frontpage.html:59 +#: sonar/theme/templates/sonar/frontpage.html:67 msgid "" "The SONAR project aims to create a scholarly archive that collects, " "promotes and preserves the publications of authors affiliated with Swiss " @@ -196,53 +206,46 @@ msgstr "" "von mit der Schweiz verbundenen Autoren öffentliche " "Forschungseinrichtungen." -#: sonar/theme/templates/sonar/frontpage.html:60 +#: sonar/theme/templates/sonar/frontpage.html:68 msgid "Further info on the project page" msgstr "Weitere Informationen auf der Projektseite" -#: sonar/theme/templates/sonar/frontpage.html:63 +#: sonar/theme/templates/sonar/frontpage.html:71 msgid "Institution views (test)" msgstr "Institutsansichten (Test)" -#: sonar/theme/templates/sonar/frontpage.html:67 +#: sonar/theme/templates/sonar/frontpage.html:75 msgid "Follow us" msgstr "Folgen Sie uns" -#: sonar/theme/templates/sonar/frontpage.html:72 +#: sonar/theme/templates/sonar/frontpage.html:80 msgid "Project website on" msgstr "Projekt-Website auf" -#: sonar/theme/templates/sonar/frontpage.html:78 +#: sonar/theme/templates/sonar/frontpage.html:86 msgid "Source code on" msgstr "Sourcecode auf github" -#: sonar/theme/templates/sonar/page.html:34 -#: sonar/theme/templates/sonar/page_admin.html:34 +#: sonar/theme/templates/sonar/manage.html:4 +msgid "SONAR administration" +msgstr "" + +#: sonar/theme/templates/sonar/page.html:42 msgid "Invenio" msgstr "Invenio" -#: sonar/theme/templates/sonar/page_settings.html:19 +#: sonar/theme/templates/sonar/page_settings.html:27 msgid "Settings" msgstr "Einstellungen" -#: sonar/theme/templates/sonar/search.html:45 -msgid "Sort by" -msgstr "Sortiert nach" - -#: sonar/theme/templates/sonar/search.html:80 -msgid "Loading..." -msgstr "Ladung..." - -#: sonar/theme/templates/sonar/search.html:87 -msgid "Search failed." -msgstr "Suche fehlgeschlagen." - -#: sonar/theme/templates/sonar/accounts/forgot_password.html:11 -#: sonar/theme/templates/sonar/accounts/forgot_password.html:21 +#: sonar/theme/templates/sonar/accounts/forgot_password.html:28 +#: sonar/theme/templates/sonar/accounts/forgot_password.html:38 +#: sonar/theme/templates/sonar/accounts/reset_password.html:20 +#: sonar/theme/templates/sonar/accounts/reset_password.html:30 msgid "Reset Password" msgstr "Passwort zurücksetzen" -#: sonar/theme/templates/sonar/accounts/forgot_password.html:17 +#: sonar/theme/templates/sonar/accounts/forgot_password.html:34 msgid "" "Enter your email address below and we will send you a link to reset your " "password." @@ -250,46 +253,48 @@ msgstr "" "Geben Sie unten Ihre E-Mail-Adresse ein und wir senden Ihnen einen Link, " "um Ihr Passwort zurückzusetzen." -#: sonar/theme/templates/sonar/accounts/forgot_password.html:28 -#: sonar/theme/templates/sonar/accounts/login.html:16 -#: sonar/theme/templates/sonar/accounts/signup.html:49 +#: sonar/theme/templates/sonar/accounts/forgot_password.html:45 +#: sonar/theme/templates/sonar/accounts/login.html:33 +#: sonar/theme/templates/sonar/accounts/reset_password.html:37 +#: sonar/theme/templates/sonar/accounts/signup.html:65 msgid "Log In" msgstr "Einloggen" -#: sonar/theme/templates/sonar/accounts/forgot_password.html:30 -#: sonar/theme/templates/sonar/accounts/login.html:39 -#: sonar/theme/templates/sonar/accounts/signup.html:28 -#: sonar/theme/templates/sonar/oauth/signup.html:38 +#: sonar/theme/templates/sonar/accounts/forgot_password.html:47 +#: sonar/theme/templates/sonar/accounts/login.html:56 +#: sonar/theme/templates/sonar/accounts/reset_password.html:39 +#: sonar/theme/templates/sonar/accounts/signup.html:44 +#: sonar/theme/templates/sonar/oauth/signup.html:46 msgid "Sign Up" msgstr "Registrieren" -#: sonar/theme/templates/sonar/accounts/login.html:9 +#: sonar/theme/templates/sonar/accounts/login.html:26 msgid "Log in to account" msgstr "Melden Sie sich bei Ihrem Konto an" -#: sonar/theme/templates/sonar/accounts/login.html:24 -#: sonar/theme/templates/sonar/accounts/login.html:30 +#: sonar/theme/templates/sonar/accounts/login.html:41 +#: sonar/theme/templates/sonar/accounts/login.html:47 #, python-format msgid "Sign-in with %(type)s" msgstr "Anmelden mit %(type)s" -#: sonar/theme/templates/sonar/accounts/login.html:37 +#: sonar/theme/templates/sonar/accounts/login.html:54 msgid "Forgot password?" msgstr "Passwort vergessen?" -#: sonar/theme/templates/sonar/accounts/signup.html:11 +#: sonar/theme/templates/sonar/accounts/signup.html:27 #, python-format msgid "Sign up for a %(sitename)s account" msgstr "Melde dich für ein %(sitename)s-Konto an!" -#: sonar/theme/templates/sonar/accounts/signup.html:36 -#: sonar/theme/templates/sonar/accounts/signup.html:42 -#: sonar/theme/templates/sonar/oauth/signup.html:25 +#: sonar/theme/templates/sonar/accounts/signup.html:52 +#: sonar/theme/templates/sonar/accounts/signup.html:58 +#: sonar/theme/templates/sonar/oauth/signup.html:33 #, python-format msgid "Sign-up with %(type)s" msgstr "Anmelden mit %(type)s" -#: sonar/theme/templates/sonar/oauth/signup.html:27 +#: sonar/theme/templates/sonar/oauth/signup.html:35 msgid "" "Fill in your details to complete your registration. You only have to do " "this once." @@ -297,23 +302,115 @@ msgstr "" "Füllen Sie Ihre Daten aus, um Ihre Registrierung abzuschließen. Du musst " "das nur einmal machen." -#: sonar/theme/templates/sonar/partial/dropdown_user.html:5 +#: sonar/theme/templates/sonar/partial/dropdown_user.html:22 msgid "Profile" msgstr "Profil" -#: sonar/theme/templates/sonar/partial/navbar.html:26 +#: sonar/theme/templates/sonar/partial/dropdown_user.html:27 +msgid "Super administration" +msgstr "" + +#: sonar/theme/templates/sonar/partial/dropdown_user.html:32 +#: sonar/theme/templates/sonar/partial/navbar.html:50 +msgid "Administration" +msgstr "" + +#: sonar/theme/templates/sonar/partial/dropdown_user.html:36 +msgid "Logout" +msgstr "Abmeldung" + +#: sonar/theme/templates/sonar/partial/navbar.html:44 msgid "Search" msgstr "Suche" -#: sonar/theme/templates/sonar/partial/navbar.html:36 +#: sonar/theme/templates/sonar/partial/navbar.html:57 msgid "Back to SONAR" msgstr "Zurück zu SONAR" -#: sonar/theme/templates/sonar/partial/navbar.html:44 +#: sonar/theme/templates/sonar/partial/navbar.html:65 msgid "Log in" msgstr "Einloggen" -#: sonar/theme/templates/sonar/partial/navbar.html:50 +#: sonar/theme/templates/sonar/partial/navbar.html:71 msgid "Sign up" msgstr "Registrieren" +#: sonar/translations/manual_translations.txt:17 +msgid "bf:Publication" +msgstr "" + +#: sonar/translations/manual_translations.txt:18 +msgid "bf:Manufacture" +msgstr "" + +#: sonar/translations/manual_translations.txt:19 +msgid "bf:Distribution" +msgstr "" + +#: sonar/translations/manual_translations.txt:20 +msgid "bf:Production" +msgstr "" + +#: sonar/translations/manual_translations.txt:21 +msgid "bf:Place" +msgstr "" + +#: sonar/translations/manual_translations.txt:22 +msgid "Date" +msgstr "" + +#: sonar/translations/manual_translations.txt:24 +msgid "lang-eng" +msgstr "" + +#: sonar/translations/manual_translations.txt:25 +msgid "lang-fre" +msgstr "" + +#: sonar/translations/manual_translations.txt:26 +msgid "lang-ger" +msgstr "" + +#: sonar/translations/manual_translations.txt:27 +msgid "lang-ita" +msgstr "" + +#~ msgid "Authors" +#~ msgstr "Autoren" + +#~ msgid "Subject" +#~ msgstr "Subjekt" + +#~ msgid "ISBN" +#~ msgstr "ISBN" + +#~ msgid "Unauthorized" +#~ msgstr "Nicht autorisiert" + +#~ msgid "You need to be authenticated to view this page." +#~ msgstr "Sie müssen authentifiziert sein, um diese Seite anzuzeigen." + +#~ msgid "Permission required" +#~ msgstr "Genehmigung erforderlich" + +#~ msgid "You do not have sufficient permissions to view this page." +#~ msgstr "Du hast nicht genügend Berechtigungen, um diese Seite anzuzeigen." + +#~ msgid "Page not found" +#~ msgstr "Seite nicht gefunden" + +#~ msgid "The page you are looking for could not be found." +#~ msgstr "Die gesuchte Seite konnte nicht gefunden werden." + +#~ msgid "Internal server error" +#~ msgstr "Interner Serverfehler" + +#~ msgid "Sort by" +#~ msgstr "Sortiert nach" + +#~ msgid "Loading..." +#~ msgstr "Ladung..." + +#~ msgid "Search failed." +#~ msgstr "Suche fehlgeschlagen." + diff --git a/sonar/translations/fr/LC_MESSAGES/messages.po b/sonar/translations/fr/LC_MESSAGES/messages.po index d4995798..c12091de 100644 --- a/sonar/translations/fr/LC_MESSAGES/messages.po +++ b/sonar/translations/fr/LC_MESSAGES/messages.po @@ -2,33 +2,28 @@ # # Swiss Open Access Repository # Copyright (C) 2019 RERO -# # This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by +# it under the terms of the GNU Affero General Public License as published +# by # the Free Software Foundation, version 3 of the License. -# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. -# # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -# # Dockerfile that builds a fully functional image of your app. -# # Note: It is important to keep the commands in this file in sync with your # boostrap script located in ./scripts/bootstrap. -# -# In order to increase the build speed, we are extending this image from a base +# In order to increase the build speed, we are extending this image from a +# base # image (built with Dockerfile.base) which only includes your Python # dependencies. -# msgid "" msgstr "" "Project-Id-Version: sonar 0.0.1\n" "Report-Msgid-Bugs-To: software@rero.ch\n" -"POT-Creation-Date: 2019-07-22 16:12+0200\n" +"POT-Creation-Date: 2020-02-04 15:21+0100\n" "PO-Revision-Date: 2019-06-06 11:09+0200\n" "Last-Translator: FULL NAME \n" "Language: fr\n" @@ -37,155 +32,170 @@ msgstr "" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" -"Generated-By: Babel 2.7.0\n" +"Generated-By: Babel 2.8.0\n" -#: sonar/config.py:49 +#: sonar/config.py:63 msgid "French" msgstr "Français" -#: sonar/config.py:50 +#: sonar/config.py:63 msgid "German" msgstr "Deutsch" -#: sonar/config.py:51 +#: sonar/config.py:64 msgid "Italian" msgstr "Italiano" -#: sonar/config.py:71 sonar/config.py:75 +#: sonar/config.py:83 sonar/config.py:87 msgid "Swiss Open Access Repository" msgstr "Swiss Open Access Repository" -#: sonar/config.py:98 +#: sonar/config.py:112 msgid "Welcome to Swiss Open Access Repository!" msgstr "Bienvenue sur Swiss Open Access Repository" -#: sonar/config.py:286 +#: sonar/config.py:379 msgid "institution" msgstr "Institution" -#: sonar/config.py:287 +#: sonar/config.py:380 msgid "language" msgstr "Langue" -#: sonar/config.py:288 -msgid "author" +#: sonar/config.py:381 +msgid "author__en" +msgstr "Auteur" + +#: sonar/config.py:382 +msgid "author__fr" +msgstr "Auteur" + +#: sonar/config.py:383 +msgid "author__de" msgstr "Auteur" -#: sonar/config.py:289 +#: sonar/config.py:384 +msgid "author__it" +msgstr "Auteur" + +#: sonar/config.py:385 msgid "subject" msgstr "Sujet" -#: sonar/config.py:298 +#: sonar/config.py:392 +msgid "pid" +msgstr "pid" + +#: sonar/config.py:393 +msgid "status" +msgstr "statut" + +#: sonar/config.py:394 +msgid "user" +msgstr "utilisateur" + +#: sonar/config.py:395 +msgid "contributor" +msgstr "contributeur" + +#: sonar/config.py:402 msgid "Best match" msgstr "Meilleure correspondance" -#: sonar/config.py:304 +#: sonar/config.py:408 msgid "Most recent" msgstr "Les plus récents" -#: sonar/modules/documents/templates/documents/record.html:58 +#: sonar/modules/documents/templates/documents/record.html:65 msgid "Authors" msgstr "Auteurs" -#: sonar/modules/documents/templates/documents/record.html:63 -msgid "Abstract" -msgstr "Résumé" - #: sonar/modules/documents/templates/documents/record.html:70 -msgid "Physical description" -msgstr "Description" - -#: sonar/modules/documents/templates/documents/record.html:75 -msgid "Subject" -msgstr "Sujet" +msgid "Edition" +msgstr "Edition" #: sonar/modules/documents/templates/documents/record.html:80 -msgid "Language" -msgstr "Langue" +msgid "Copyright date" +msgstr "Date du droit d'auteur" #: sonar/modules/documents/templates/documents/record.html:85 -msgid "Notes" -msgstr "Notes" +msgid "Abstract" +msgstr "Résumé" -#: sonar/modules/documents/templates/documents/record.html:91 -msgid "ISBN" -msgstr "ISBN" +#: sonar/modules/documents/templates/documents/record.html:92 +msgid "Physical description" +msgstr "Description matérielle" #: sonar/modules/documents/templates/documents/record.html:97 -msgid "Permalink" -msgstr "Lien permanent" - -#: sonar/theme/templates/sonar/401.html:13 -msgid "Unauthorized" -msgstr "Non autorisé" +msgid "Additional Materials" +msgstr "Matériel d'accompagnement" -#: sonar/theme/templates/sonar/401.html:14 -msgid "You need to be authenticated to view this page." -msgstr "Vous devez être authentifié pour voir cette page." +#: sonar/modules/documents/templates/documents/record.html:102 +msgid "Series" +msgstr "Collections" -#: sonar/theme/templates/sonar/403.html:13 -msgid "Permission required" -msgstr "Permission requise" +#: sonar/modules/documents/templates/documents/record.html:107 +msgid "Is part of" +msgstr "Fait partie de" -#: sonar/theme/templates/sonar/403.html:14 -msgid "You do not have sufficient permissions to view this page." -msgstr "Vous n'avez pas suffisamment de permissions pour voir cette page." +#: sonar/modules/documents/templates/documents/record.html:112 +msgid "Subjects" +msgstr "Sujets" -#: sonar/theme/templates/sonar/404.html:13 -msgid "Page not found" -msgstr "Page non trouvée" +#: sonar/modules/documents/templates/documents/record.html:117 +msgid "Notes" +msgstr "Notes" -#: sonar/theme/templates/sonar/404.html:14 -msgid "The page you are looking for could not be found." -msgstr "La page que vous recherchez n'a pas pu être trouvée." +#: sonar/modules/documents/templates/documents/record.html:122 +msgid "Identifier" +msgstr "Identifiant" -#: sonar/theme/templates/sonar/500.html:13 -msgid "Internal server error" -msgstr "Erreur interne du serveur" +#: sonar/modules/documents/templates/documents/record.html:137 +msgid "Language" +msgstr "Langue" -#: sonar/theme/templates/sonar/500.html:15 -msgid "Error identifier" -msgstr "Identifiant d'erreur" +#: sonar/modules/documents/templates/documents/record.html:142 +msgid "Permalink" +msgstr "Lien permanent" -#: sonar/theme/templates/sonar/admin_header.html:72 -#: sonar/theme/templates/sonar/partial/dropdown_user.html:8 -msgid "Logout" -msgstr "Déconnexion" +#: sonar/modules/documents/templates/documents/record.html:147 +msgid "Files" +msgstr "Fichiers" -#: sonar/theme/templates/sonar/footer.html:32 +#: sonar/theme/templates/sonar/footer.html:40 msgid "Help" msgstr "Aide" -#: sonar/theme/templates/sonar/footer.html:35 +#: sonar/theme/templates/sonar/footer.html:43 msgid "About" msgstr "A propos" -#: sonar/theme/templates/sonar/footer.html:38 +#: sonar/theme/templates/sonar/footer.html:46 msgid "Contact" msgstr "Contact" -#: sonar/theme/templates/sonar/footer.html:43 +#: sonar/theme/templates/sonar/footer.html:51 #, python-format msgid "Powered by RERO" msgstr "Réalisé par RERO" -#: sonar/theme/templates/sonar/frontpage.html:33 +#: sonar/theme/templates/sonar/frontpage.html:41 msgid "Search publications, authors, projects, ..." msgstr "Recherche de publications, auteurs, projets, ..." -#: sonar/theme/templates/sonar/frontpage.html:35 +#: sonar/theme/templates/sonar/frontpage.html:43 msgid "Advanced search" msgstr "Recherche avancée" -#: sonar/theme/templates/sonar/frontpage.html:51 +#: sonar/theme/templates/sonar/frontpage.html:59 msgid "Software under development!" msgstr "Logiciel en cours de développement !" -#: sonar/theme/templates/sonar/frontpage.html:58 +#: sonar/theme/templates/sonar/frontpage.html:66 msgid "The project" msgstr "Le projet" -#: sonar/theme/templates/sonar/frontpage.html:59 +#: sonar/theme/templates/sonar/frontpage.html:67 msgid "" "The SONAR project aims to create a scholarly archive that collects, " "promotes and preserves the publications of authors affiliated with Swiss " @@ -195,53 +205,46 @@ msgstr "" "recueille, promeut et préserve les publications des auteurs affiliés aux " "institutions publiques suisses de recherche." -#: sonar/theme/templates/sonar/frontpage.html:60 +#: sonar/theme/templates/sonar/frontpage.html:68 msgid "Further info on the project page" msgstr "Plus d'informations sur la page du projet" -#: sonar/theme/templates/sonar/frontpage.html:63 +#: sonar/theme/templates/sonar/frontpage.html:71 msgid "Institution views (test)" msgstr "Vues institutionelles (test)" -#: sonar/theme/templates/sonar/frontpage.html:67 +#: sonar/theme/templates/sonar/frontpage.html:75 msgid "Follow us" msgstr "Suivez-nous" -#: sonar/theme/templates/sonar/frontpage.html:72 +#: sonar/theme/templates/sonar/frontpage.html:80 msgid "Project website on" msgstr "Site web du projet sur" -#: sonar/theme/templates/sonar/frontpage.html:78 +#: sonar/theme/templates/sonar/frontpage.html:86 msgid "Source code on" msgstr "Code source sur" -#: sonar/theme/templates/sonar/page.html:34 -#: sonar/theme/templates/sonar/page_admin.html:34 +#: sonar/theme/templates/sonar/manage.html:4 +msgid "SONAR administration" +msgstr "SONAR administration" + +#: sonar/theme/templates/sonar/page.html:42 msgid "Invenio" msgstr "Invenio" -#: sonar/theme/templates/sonar/page_settings.html:19 +#: sonar/theme/templates/sonar/page_settings.html:27 msgid "Settings" msgstr "Réglages" -#: sonar/theme/templates/sonar/search.html:45 -msgid "Sort by" -msgstr "Trier par" - -#: sonar/theme/templates/sonar/search.html:80 -msgid "Loading..." -msgstr "Chargement..." - -#: sonar/theme/templates/sonar/search.html:87 -msgid "Search failed." -msgstr "Echec de la recherche." - -#: sonar/theme/templates/sonar/accounts/forgot_password.html:11 -#: sonar/theme/templates/sonar/accounts/forgot_password.html:21 +#: sonar/theme/templates/sonar/accounts/forgot_password.html:28 +#: sonar/theme/templates/sonar/accounts/forgot_password.html:38 +#: sonar/theme/templates/sonar/accounts/reset_password.html:20 +#: sonar/theme/templates/sonar/accounts/reset_password.html:30 msgid "Reset Password" msgstr "Réinitialiser le mot de passe" -#: sonar/theme/templates/sonar/accounts/forgot_password.html:17 +#: sonar/theme/templates/sonar/accounts/forgot_password.html:34 msgid "" "Enter your email address below and we will send you a link to reset your " "password." @@ -249,46 +252,48 @@ msgstr "" "Entrez votre adresse email ci-dessous et nous vous enverrons un lien pour" " réinitialiser votre mot de passe." -#: sonar/theme/templates/sonar/accounts/forgot_password.html:28 -#: sonar/theme/templates/sonar/accounts/login.html:16 -#: sonar/theme/templates/sonar/accounts/signup.html:49 +#: sonar/theme/templates/sonar/accounts/forgot_password.html:45 +#: sonar/theme/templates/sonar/accounts/login.html:33 +#: sonar/theme/templates/sonar/accounts/reset_password.html:37 +#: sonar/theme/templates/sonar/accounts/signup.html:65 msgid "Log In" msgstr "Identification" -#: sonar/theme/templates/sonar/accounts/forgot_password.html:30 -#: sonar/theme/templates/sonar/accounts/login.html:39 -#: sonar/theme/templates/sonar/accounts/signup.html:28 -#: sonar/theme/templates/sonar/oauth/signup.html:38 +#: sonar/theme/templates/sonar/accounts/forgot_password.html:47 +#: sonar/theme/templates/sonar/accounts/login.html:56 +#: sonar/theme/templates/sonar/accounts/reset_password.html:39 +#: sonar/theme/templates/sonar/accounts/signup.html:44 +#: sonar/theme/templates/sonar/oauth/signup.html:46 msgid "Sign Up" msgstr "S'inscrire" -#: sonar/theme/templates/sonar/accounts/login.html:9 +#: sonar/theme/templates/sonar/accounts/login.html:26 msgid "Log in to account" msgstr "Connectez-vous à votre compte" -#: sonar/theme/templates/sonar/accounts/login.html:24 -#: sonar/theme/templates/sonar/accounts/login.html:30 +#: sonar/theme/templates/sonar/accounts/login.html:41 +#: sonar/theme/templates/sonar/accounts/login.html:47 #, python-format msgid "Sign-in with %(type)s" msgstr "Se connecter avec %(type)s" -#: sonar/theme/templates/sonar/accounts/login.html:37 +#: sonar/theme/templates/sonar/accounts/login.html:54 msgid "Forgot password?" msgstr "Mot de passe oublié?" -#: sonar/theme/templates/sonar/accounts/signup.html:11 +#: sonar/theme/templates/sonar/accounts/signup.html:27 #, python-format msgid "Sign up for a %(sitename)s account" msgstr "Inscrivez-vous sur %(sitename)s" -#: sonar/theme/templates/sonar/accounts/signup.html:36 -#: sonar/theme/templates/sonar/accounts/signup.html:42 -#: sonar/theme/templates/sonar/oauth/signup.html:25 +#: sonar/theme/templates/sonar/accounts/signup.html:52 +#: sonar/theme/templates/sonar/accounts/signup.html:58 +#: sonar/theme/templates/sonar/oauth/signup.html:33 #, python-format msgid "Sign-up with %(type)s" msgstr "S'inscrire avec %(type)s" -#: sonar/theme/templates/sonar/oauth/signup.html:27 +#: sonar/theme/templates/sonar/oauth/signup.html:35 msgid "" "Fill in your details to complete your registration. You only have to do " "this once." @@ -296,23 +301,115 @@ msgstr "" "Remplissez vos coordonnées pour compléter votre inscription. Vous n'avez " "qu'à le faire une seule fois." -#: sonar/theme/templates/sonar/partial/dropdown_user.html:5 +#: sonar/theme/templates/sonar/partial/dropdown_user.html:22 msgid "Profile" msgstr "Profile" -#: sonar/theme/templates/sonar/partial/navbar.html:26 +#: sonar/theme/templates/sonar/partial/dropdown_user.html:27 +msgid "Super administration" +msgstr "Super administration" + +#: sonar/theme/templates/sonar/partial/dropdown_user.html:32 +#: sonar/theme/templates/sonar/partial/navbar.html:50 +msgid "Administration" +msgstr "Administration" + +#: sonar/theme/templates/sonar/partial/dropdown_user.html:36 +msgid "Logout" +msgstr "Déconnexion" + +#: sonar/theme/templates/sonar/partial/navbar.html:44 msgid "Search" msgstr "Recherche" -#: sonar/theme/templates/sonar/partial/navbar.html:36 +#: sonar/theme/templates/sonar/partial/navbar.html:57 msgid "Back to SONAR" msgstr "Retour à SONAR" -#: sonar/theme/templates/sonar/partial/navbar.html:44 +#: sonar/theme/templates/sonar/partial/navbar.html:65 msgid "Log in" msgstr "Identification" -#: sonar/theme/templates/sonar/partial/navbar.html:50 +#: sonar/theme/templates/sonar/partial/navbar.html:71 msgid "Sign up" msgstr "S'inscrire" +#: sonar/translations/manual_translations.txt:17 +msgid "bf:Publication" +msgstr "Publication" + +#: sonar/translations/manual_translations.txt:18 +msgid "bf:Manufacture" +msgstr "Fabrication" + +#: sonar/translations/manual_translations.txt:19 +msgid "bf:Distribution" +msgstr "Distribution" + +#: sonar/translations/manual_translations.txt:20 +msgid "bf:Production" +msgstr "Production" + +#: sonar/translations/manual_translations.txt:21 +msgid "bf:Place" +msgstr "Lieu" + +#: sonar/translations/manual_translations.txt:22 +msgid "Date" +msgstr "Date" + +#: sonar/translations/manual_translations.txt:24 +msgid "lang-eng" +msgstr "Anglais" + +#: sonar/translations/manual_translations.txt:25 +msgid "lang-fre" +msgstr "Français" + +#: sonar/translations/manual_translations.txt:26 +msgid "lang-ger" +msgstr "Allemand" + +#: sonar/translations/manual_translations.txt:27 +msgid "lang-ita" +msgstr "Italien" + +#~ msgid "Authors" +#~ msgstr "Auteurs" + +#~ msgid "Subject" +#~ msgstr "Sujet" + +#~ msgid "ISBN" +#~ msgstr "ISBN" + +#~ msgid "Unauthorized" +#~ msgstr "Non autorisé" + +#~ msgid "You need to be authenticated to view this page." +#~ msgstr "Vous devez être authentifié pour voir cette page." + +#~ msgid "Permission required" +#~ msgstr "Permission requise" + +#~ msgid "You do not have sufficient permissions to view this page." +#~ msgstr "Vous n'avez pas suffisamment de permissions pour voir cette page." + +#~ msgid "Page not found" +#~ msgstr "Page non trouvée" + +#~ msgid "The page you are looking for could not be found." +#~ msgstr "La page que vous recherchez n'a pas pu être trouvée." + +#~ msgid "Internal server error" +#~ msgstr "Erreur interne du serveur" + +#~ msgid "Sort by" +#~ msgstr "Trier par" + +#~ msgid "Loading..." +#~ msgstr "Chargement..." + +#~ msgid "Search failed." +#~ msgstr "Echec de la recherche." + diff --git a/sonar/translations/it/LC_MESSAGES/messages.po b/sonar/translations/it/LC_MESSAGES/messages.po index 7a1cae2f..f0cf96f4 100644 --- a/sonar/translations/it/LC_MESSAGES/messages.po +++ b/sonar/translations/it/LC_MESSAGES/messages.po @@ -2,33 +2,28 @@ # # Swiss Open Access Repository # Copyright (C) 2019 RERO -# # This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by +# it under the terms of the GNU Affero General Public License as published +# by # the Free Software Foundation, version 3 of the License. -# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. -# # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -# # Dockerfile that builds a fully functional image of your app. -# # Note: It is important to keep the commands in this file in sync with your # boostrap script located in ./scripts/bootstrap. -# -# In order to increase the build speed, we are extending this image from a base +# In order to increase the build speed, we are extending this image from a +# base # image (built with Dockerfile.base) which only includes your Python # dependencies. -# msgid "" msgstr "" "Project-Id-Version: sonar 0.0.1\n" "Report-Msgid-Bugs-To: software@rero.ch\n" -"POT-Creation-Date: 2019-07-22 16:12+0200\n" +"POT-Creation-Date: 2020-02-04 15:21+0100\n" "PO-Revision-Date: 2019-07-15 17:09+0200\n" "Last-Translator: FULL NAME \n" "Language: it\n" @@ -37,155 +32,170 @@ msgstr "" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" -"Generated-By: Babel 2.7.0\n" +"Generated-By: Babel 2.8.0\n" -#: sonar/config.py:49 +#: sonar/config.py:63 msgid "French" msgstr "Français" -#: sonar/config.py:50 +#: sonar/config.py:63 msgid "German" msgstr "Deutsch" -#: sonar/config.py:51 +#: sonar/config.py:64 msgid "Italian" msgstr "Italiano" -#: sonar/config.py:71 sonar/config.py:75 +#: sonar/config.py:83 sonar/config.py:87 msgid "Swiss Open Access Repository" msgstr "Swiss Open Access Repository" -#: sonar/config.py:98 +#: sonar/config.py:112 msgid "Welcome to Swiss Open Access Repository!" msgstr "Benvenuti nel Swiss Open Access Repository!" -#: sonar/config.py:286 +#: sonar/config.py:379 msgid "institution" msgstr "istituzione" -#: sonar/config.py:287 +#: sonar/config.py:380 msgid "language" msgstr "linguaggio" -#: sonar/config.py:288 -msgid "author" +#: sonar/config.py:381 +msgid "author__en" msgstr "autore" -#: sonar/config.py:289 +#: sonar/config.py:382 +msgid "author__fr" +msgstr "autore" + +#: sonar/config.py:383 +msgid "author__de" +msgstr "autore" + +#: sonar/config.py:384 +msgid "author__it" +msgstr "autore" + +#: sonar/config.py:385 msgid "subject" msgstr "tema" -#: sonar/config.py:298 +#: sonar/config.py:392 +msgid "pid" +msgstr "" + +#: sonar/config.py:393 +msgid "status" +msgstr "" + +#: sonar/config.py:394 +msgid "user" +msgstr "" + +#: sonar/config.py:395 +msgid "contributor" +msgstr "" + +#: sonar/config.py:402 msgid "Best match" msgstr "Migliori risultati" -#: sonar/config.py:304 +#: sonar/config.py:408 msgid "Most recent" msgstr "Più recenti" -#: sonar/modules/documents/templates/documents/record.html:58 +#: sonar/modules/documents/templates/documents/record.html:65 msgid "Authors" -msgstr "Autori" - -#: sonar/modules/documents/templates/documents/record.html:63 -msgid "Abstract" -msgstr "Astratto" +msgstr "autore" #: sonar/modules/documents/templates/documents/record.html:70 -msgid "Physical description" -msgstr "Descrizione fisica" - -#: sonar/modules/documents/templates/documents/record.html:75 -msgid "Subject" -msgstr "Soggetto" +msgid "Edition" +msgstr "" #: sonar/modules/documents/templates/documents/record.html:80 -msgid "Language" -msgstr "Lingua" +msgid "Copyright date" +msgstr "" #: sonar/modules/documents/templates/documents/record.html:85 -msgid "Notes" -msgstr "Note" +msgid "Abstract" +msgstr "Astratto" -#: sonar/modules/documents/templates/documents/record.html:91 -msgid "ISBN" -msgstr "ISBN" +#: sonar/modules/documents/templates/documents/record.html:92 +msgid "Physical description" +msgstr "Descrizione fisica" #: sonar/modules/documents/templates/documents/record.html:97 -msgid "Permalink" -msgstr "Collegamento permanente" - -#: sonar/theme/templates/sonar/401.html:13 -msgid "Unauthorized" -msgstr "Non autorizzato" +msgid "Additional Materials" +msgstr "" -#: sonar/theme/templates/sonar/401.html:14 -msgid "You need to be authenticated to view this page." -msgstr "Devi essere autenticato per visualizzare questa pagina." +#: sonar/modules/documents/templates/documents/record.html:102 +msgid "Series" +msgstr "" -#: sonar/theme/templates/sonar/403.html:13 -msgid "Permission required" -msgstr "Permesso richiesto" +#: sonar/modules/documents/templates/documents/record.html:107 +msgid "Is part of" +msgstr "" -#: sonar/theme/templates/sonar/403.html:14 -msgid "You do not have sufficient permissions to view this page." -msgstr "Non hai permessi sufficienti per visualizzare questa pagina." +#: sonar/modules/documents/templates/documents/record.html:112 +msgid "Subjects" +msgstr "Tema" -#: sonar/theme/templates/sonar/404.html:13 -msgid "Page not found" -msgstr "Pagina non trovata" +#: sonar/modules/documents/templates/documents/record.html:117 +msgid "Notes" +msgstr "Note" -#: sonar/theme/templates/sonar/404.html:14 -msgid "The page you are looking for could not be found." -msgstr "La pagina che stai cercando non è stata trovata." +#: sonar/modules/documents/templates/documents/record.html:122 +msgid "Identifier" +msgstr "" -#: sonar/theme/templates/sonar/500.html:13 -msgid "Internal server error" -msgstr "Errore interno del server" +#: sonar/modules/documents/templates/documents/record.html:137 +msgid "Language" +msgstr "Lingua" -#: sonar/theme/templates/sonar/500.html:15 -msgid "Error identifier" -msgstr "Identificazione dell'errore" +#: sonar/modules/documents/templates/documents/record.html:142 +msgid "Permalink" +msgstr "Collegamento permanente" -#: sonar/theme/templates/sonar/admin_header.html:72 -#: sonar/theme/templates/sonar/partial/dropdown_user.html:8 -msgid "Logout" -msgstr "Disconnessione" +#: sonar/modules/documents/templates/documents/record.html:147 +msgid "Files" +msgstr "" -#: sonar/theme/templates/sonar/footer.html:32 +#: sonar/theme/templates/sonar/footer.html:40 msgid "Help" msgstr "Assistenza" -#: sonar/theme/templates/sonar/footer.html:35 +#: sonar/theme/templates/sonar/footer.html:43 msgid "About" msgstr "A proposito" -#: sonar/theme/templates/sonar/footer.html:38 +#: sonar/theme/templates/sonar/footer.html:46 msgid "Contact" msgstr "Contatto" -#: sonar/theme/templates/sonar/footer.html:43 +#: sonar/theme/templates/sonar/footer.html:51 #, python-format msgid "Powered by RERO" msgstr "Diretto da RERO" -#: sonar/theme/templates/sonar/frontpage.html:33 +#: sonar/theme/templates/sonar/frontpage.html:41 msgid "Search publications, authors, projects, ..." msgstr "Ricerca di pubblicazioni, autori, progetti, ...." -#: sonar/theme/templates/sonar/frontpage.html:35 +#: sonar/theme/templates/sonar/frontpage.html:43 msgid "Advanced search" msgstr "Ricerca avanzata" -#: sonar/theme/templates/sonar/frontpage.html:51 +#: sonar/theme/templates/sonar/frontpage.html:59 msgid "Software under development!" msgstr "Software in fase di sviluppo!" -#: sonar/theme/templates/sonar/frontpage.html:58 +#: sonar/theme/templates/sonar/frontpage.html:66 msgid "The project" msgstr "Il progetto" -#: sonar/theme/templates/sonar/frontpage.html:59 +#: sonar/theme/templates/sonar/frontpage.html:67 msgid "" "The SONAR project aims to create a scholarly archive that collects, " "promotes and preserves the publications of authors affiliated with Swiss " @@ -195,53 +205,46 @@ msgstr "" "promuove e conserva le pubblicazioni di autori affiliati a istituzioni " "pubbliche di ricerca svizzere." -#: sonar/theme/templates/sonar/frontpage.html:60 +#: sonar/theme/templates/sonar/frontpage.html:68 msgid "Further info on the project page" msgstr "Ulteriori informazioni sulla pagina del progetto" -#: sonar/theme/templates/sonar/frontpage.html:63 +#: sonar/theme/templates/sonar/frontpage.html:71 msgid "Institution views (test)" msgstr "Viste dell'istituzione (test)" -#: sonar/theme/templates/sonar/frontpage.html:67 +#: sonar/theme/templates/sonar/frontpage.html:75 msgid "Follow us" msgstr "Seguici" -#: sonar/theme/templates/sonar/frontpage.html:72 +#: sonar/theme/templates/sonar/frontpage.html:80 msgid "Project website on" msgstr "Sito web del progetto" -#: sonar/theme/templates/sonar/frontpage.html:78 +#: sonar/theme/templates/sonar/frontpage.html:86 msgid "Source code on" msgstr "Codice sorgente su" -#: sonar/theme/templates/sonar/page.html:34 -#: sonar/theme/templates/sonar/page_admin.html:34 +#: sonar/theme/templates/sonar/manage.html:4 +msgid "SONAR administration" +msgstr "" + +#: sonar/theme/templates/sonar/page.html:42 msgid "Invenio" msgstr "Invenio" -#: sonar/theme/templates/sonar/page_settings.html:19 +#: sonar/theme/templates/sonar/page_settings.html:27 msgid "Settings" msgstr "Impostazioni" -#: sonar/theme/templates/sonar/search.html:45 -msgid "Sort by" -msgstr "Ordina per" - -#: sonar/theme/templates/sonar/search.html:80 -msgid "Loading..." -msgstr "Caricamento..." - -#: sonar/theme/templates/sonar/search.html:87 -msgid "Search failed." -msgstr "La ricerca è fallita." - -#: sonar/theme/templates/sonar/accounts/forgot_password.html:11 -#: sonar/theme/templates/sonar/accounts/forgot_password.html:21 +#: sonar/theme/templates/sonar/accounts/forgot_password.html:28 +#: sonar/theme/templates/sonar/accounts/forgot_password.html:38 +#: sonar/theme/templates/sonar/accounts/reset_password.html:20 +#: sonar/theme/templates/sonar/accounts/reset_password.html:30 msgid "Reset Password" msgstr "Reimpostare la password" -#: sonar/theme/templates/sonar/accounts/forgot_password.html:17 +#: sonar/theme/templates/sonar/accounts/forgot_password.html:34 msgid "" "Enter your email address below and we will send you a link to reset your " "password." @@ -249,68 +252,162 @@ msgstr "" "Inserisci il tuo indirizzo email qui sotto e ti invieremo un link per " "reimpostare la tua password." -#: sonar/theme/templates/sonar/accounts/forgot_password.html:28 -#: sonar/theme/templates/sonar/accounts/login.html:16 -#: sonar/theme/templates/sonar/accounts/signup.html:49 +#: sonar/theme/templates/sonar/accounts/forgot_password.html:45 +#: sonar/theme/templates/sonar/accounts/login.html:33 +#: sonar/theme/templates/sonar/accounts/reset_password.html:37 +#: sonar/theme/templates/sonar/accounts/signup.html:65 msgid "Log In" msgstr "Accedi" -#: sonar/theme/templates/sonar/accounts/forgot_password.html:30 -#: sonar/theme/templates/sonar/accounts/login.html:39 -#: sonar/theme/templates/sonar/accounts/signup.html:28 -#: sonar/theme/templates/sonar/oauth/signup.html:38 +#: sonar/theme/templates/sonar/accounts/forgot_password.html:47 +#: sonar/theme/templates/sonar/accounts/login.html:56 +#: sonar/theme/templates/sonar/accounts/reset_password.html:39 +#: sonar/theme/templates/sonar/accounts/signup.html:44 +#: sonar/theme/templates/sonar/oauth/signup.html:46 msgid "Sign Up" msgstr "Iscriviti" -#: sonar/theme/templates/sonar/accounts/login.html:9 +#: sonar/theme/templates/sonar/accounts/login.html:26 msgid "Log in to account" msgstr "Accedi al tuo account" -#: sonar/theme/templates/sonar/accounts/login.html:24 -#: sonar/theme/templates/sonar/accounts/login.html:30 +#: sonar/theme/templates/sonar/accounts/login.html:41 +#: sonar/theme/templates/sonar/accounts/login.html:47 #, python-format msgid "Sign-in with %(type)s" msgstr "" -#: sonar/theme/templates/sonar/accounts/login.html:37 +#: sonar/theme/templates/sonar/accounts/login.html:54 msgid "Forgot password?" msgstr "Hai dimenticato la password?" -#: sonar/theme/templates/sonar/accounts/signup.html:11 +#: sonar/theme/templates/sonar/accounts/signup.html:27 #, python-format msgid "Sign up for a %(sitename)s account" msgstr "Iscriviti per avere un account %(sitename)s!" -#: sonar/theme/templates/sonar/accounts/signup.html:36 -#: sonar/theme/templates/sonar/accounts/signup.html:42 -#: sonar/theme/templates/sonar/oauth/signup.html:25 +#: sonar/theme/templates/sonar/accounts/signup.html:52 +#: sonar/theme/templates/sonar/accounts/signup.html:58 +#: sonar/theme/templates/sonar/oauth/signup.html:33 #, python-format msgid "Sign-up with %(type)s" msgstr "" -#: sonar/theme/templates/sonar/oauth/signup.html:27 +#: sonar/theme/templates/sonar/oauth/signup.html:35 msgid "" "Fill in your details to complete your registration. You only have to do " "this once." msgstr "" -#: sonar/theme/templates/sonar/partial/dropdown_user.html:5 +#: sonar/theme/templates/sonar/partial/dropdown_user.html:22 msgid "Profile" msgstr "Profilo" -#: sonar/theme/templates/sonar/partial/navbar.html:26 +#: sonar/theme/templates/sonar/partial/dropdown_user.html:27 +msgid "Super administration" +msgstr "" + +#: sonar/theme/templates/sonar/partial/dropdown_user.html:32 +#: sonar/theme/templates/sonar/partial/navbar.html:50 +msgid "Administration" +msgstr "" + +#: sonar/theme/templates/sonar/partial/dropdown_user.html:36 +msgid "Logout" +msgstr "Disconnessione" + +#: sonar/theme/templates/sonar/partial/navbar.html:44 msgid "Search" msgstr "Ricerca" -#: sonar/theme/templates/sonar/partial/navbar.html:36 +#: sonar/theme/templates/sonar/partial/navbar.html:57 msgid "Back to SONAR" msgstr "Torna a SONAR" -#: sonar/theme/templates/sonar/partial/navbar.html:44 +#: sonar/theme/templates/sonar/partial/navbar.html:65 msgid "Log in" msgstr "Accedi" -#: sonar/theme/templates/sonar/partial/navbar.html:50 +#: sonar/theme/templates/sonar/partial/navbar.html:71 msgid "Sign up" msgstr "Iscriviti" +#: sonar/translations/manual_translations.txt:17 +msgid "bf:Publication" +msgstr "" + +#: sonar/translations/manual_translations.txt:18 +msgid "bf:Manufacture" +msgstr "" + +#: sonar/translations/manual_translations.txt:19 +msgid "bf:Distribution" +msgstr "" + +#: sonar/translations/manual_translations.txt:20 +msgid "bf:Production" +msgstr "" + +#: sonar/translations/manual_translations.txt:21 +msgid "bf:Place" +msgstr "" + +#: sonar/translations/manual_translations.txt:22 +msgid "Date" +msgstr "" + +#: sonar/translations/manual_translations.txt:24 +msgid "lang-eng" +msgstr "" + +#: sonar/translations/manual_translations.txt:25 +msgid "lang-fre" +msgstr "" + +#: sonar/translations/manual_translations.txt:26 +msgid "lang-ger" +msgstr "" + +#: sonar/translations/manual_translations.txt:27 +msgid "lang-ita" +msgstr "" + +#~ msgid "Authors" +#~ msgstr "Autori" + +#~ msgid "Subject" +#~ msgstr "Soggetto" + +#~ msgid "ISBN" +#~ msgstr "ISBN" + +#~ msgid "Unauthorized" +#~ msgstr "Non autorizzato" + +#~ msgid "You need to be authenticated to view this page." +#~ msgstr "Devi essere autenticato per visualizzare questa pagina." + +#~ msgid "Permission required" +#~ msgstr "Permesso richiesto" + +#~ msgid "You do not have sufficient permissions to view this page." +#~ msgstr "Non hai permessi sufficienti per visualizzare questa pagina." + +#~ msgid "Page not found" +#~ msgstr "Pagina non trovata" + +#~ msgid "The page you are looking for could not be found." +#~ msgstr "La pagina che stai cercando non è stata trovata." + +#~ msgid "Internal server error" +#~ msgstr "Errore interno del server" + +#~ msgid "Sort by" +#~ msgstr "Ordina per" + +#~ msgid "Loading..." +#~ msgstr "Caricamento..." + +#~ msgid "Search failed." +#~ msgstr "La ricerca è fallita." + diff --git a/sonar/translations/manual_translations.txt b/sonar/translations/manual_translations.txt new file mode 100644 index 00000000..8159e9d7 --- /dev/null +++ b/sonar/translations/manual_translations.txt @@ -0,0 +1,27 @@ +# Swiss Open Access Repository +# Copyright (C) 2019 RERO +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +# provisionActivity +_('bf:Publication') +_('bf:Manufacture') +_('bf:Distribution') +_('bf:Production') +_('bf:Place') +_('Date') + +_('lang-eng') +_('lang-fre') +_('lang-ger') +_('lang-ita') diff --git a/sonar/translations/messages.pot b/sonar/translations/messages.pot index 39f857ea..6a9f89ec 100644 --- a/sonar/translations/messages.pot +++ b/sonar/translations/messages.pot @@ -1,310 +1,352 @@ -# -*- coding: utf-8 -*- -# -# Swiss Open Access Repository -# Copyright (C) 2019 RERO -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, version 3 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -# Dockerfile that builds a fully functional image of your app. -# -# Note: It is important to keep the commands in this file in sync with your -# boostrap script located in ./scripts/bootstrap. -# -# In order to increase the build speed, we are extending this image from a base -# image (built with Dockerfile.base) which only includes your Python -# dependencies. +# Translations template for sonar. +# Copyright (C) 2020 RERO +# This file is distributed under the same license as the sonar project. +# FIRST AUTHOR , 2020. # #, fuzzy msgid "" msgstr "" -"Project-Id-Version: sonar 0.0.1\n" +"Project-Id-Version: sonar 0.2.2\n" "Report-Msgid-Bugs-To: software@rero.ch\n" -"POT-Creation-Date: 2019-07-22 16:12+0200\n" +"POT-Creation-Date: 2020-02-04 15:21+0100\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" -"Generated-By: Babel 2.7.0\n" +"Generated-By: Babel 2.8.0\n" -#: sonar/config.py:49 +#: sonar/config.py:63 msgid "French" msgstr "" -#: sonar/config.py:50 +#: sonar/config.py:63 msgid "German" msgstr "" -#: sonar/config.py:51 +#: sonar/config.py:64 msgid "Italian" msgstr "" -#: sonar/config.py:71 sonar/config.py:75 +#: sonar/config.py:83 sonar/config.py:87 msgid "Swiss Open Access Repository" msgstr "" -#: sonar/config.py:98 +#: sonar/config.py:112 msgid "Welcome to Swiss Open Access Repository!" msgstr "" -#: sonar/config.py:286 +#: sonar/config.py:379 msgid "institution" msgstr "" -#: sonar/config.py:287 +#: sonar/config.py:380 msgid "language" msgstr "" -#: sonar/config.py:288 -msgid "author" +#: sonar/config.py:381 +msgid "author__en" +msgstr "" + +#: sonar/config.py:382 +msgid "author__fr" +msgstr "" + +#: sonar/config.py:383 +msgid "author__de" +msgstr "" + +#: sonar/config.py:384 +msgid "author__it" msgstr "" -#: sonar/config.py:289 +#: sonar/config.py:385 msgid "subject" msgstr "" -#: sonar/config.py:298 -msgid "Best match" +#: sonar/config.py:392 +msgid "pid" msgstr "" -#: sonar/config.py:304 -msgid "Most recent" +#: sonar/config.py:393 +msgid "status" msgstr "" -#: sonar/modules/documents/templates/documents/record.html:58 -msgid "Authors" +#: sonar/config.py:394 +msgid "user" msgstr "" -#: sonar/modules/documents/templates/documents/record.html:63 -msgid "Abstract" +#: sonar/config.py:395 +msgid "contributor" msgstr "" -#: sonar/modules/documents/templates/documents/record.html:70 -msgid "Physical description" +#: sonar/config.py:402 +msgid "Best match" msgstr "" -#: sonar/modules/documents/templates/documents/record.html:75 -msgid "Subject" +#: sonar/config.py:408 +msgid "Most recent" +msgstr "" + +#: sonar/modules/documents/templates/documents/record.html:65 +msgid "Authors" +msgstr "" + +#: sonar/modules/documents/templates/documents/record.html:70 +msgid "Edition" msgstr "" #: sonar/modules/documents/templates/documents/record.html:80 -msgid "Language" +msgid "Copyright date" msgstr "" #: sonar/modules/documents/templates/documents/record.html:85 -msgid "Notes" +msgid "Abstract" msgstr "" -#: sonar/modules/documents/templates/documents/record.html:91 -msgid "ISBN" +#: sonar/modules/documents/templates/documents/record.html:92 +msgid "Physical description" msgstr "" #: sonar/modules/documents/templates/documents/record.html:97 -msgid "Permalink" +msgid "Additional Materials" msgstr "" -#: sonar/theme/templates/sonar/401.html:13 -msgid "Unauthorized" +#: sonar/modules/documents/templates/documents/record.html:102 +msgid "Series" msgstr "" -#: sonar/theme/templates/sonar/401.html:14 -msgid "You need to be authenticated to view this page." +#: sonar/modules/documents/templates/documents/record.html:107 +msgid "Is part of" msgstr "" -#: sonar/theme/templates/sonar/403.html:13 -msgid "Permission required" +#: sonar/modules/documents/templates/documents/record.html:112 +msgid "Subjects" msgstr "" -#: sonar/theme/templates/sonar/403.html:14 -msgid "You do not have sufficient permissions to view this page." -msgstr "" - -#: sonar/theme/templates/sonar/404.html:13 -msgid "Page not found" +#: sonar/modules/documents/templates/documents/record.html:117 +msgid "Notes" msgstr "" -#: sonar/theme/templates/sonar/404.html:14 -msgid "The page you are looking for could not be found." +#: sonar/modules/documents/templates/documents/record.html:122 +msgid "Identifier" msgstr "" -#: sonar/theme/templates/sonar/500.html:13 -msgid "Internal server error" +#: sonar/modules/documents/templates/documents/record.html:137 +msgid "Language" msgstr "" -#: sonar/theme/templates/sonar/500.html:15 -msgid "Error identifier" +#: sonar/modules/documents/templates/documents/record.html:142 +msgid "Permalink" msgstr "" -#: sonar/theme/templates/sonar/admin_header.html:72 -#: sonar/theme/templates/sonar/partial/dropdown_user.html:8 -msgid "Logout" +#: sonar/modules/documents/templates/documents/record.html:147 +msgid "Files" msgstr "" -#: sonar/theme/templates/sonar/footer.html:32 +#: sonar/theme/templates/sonar/footer.html:40 msgid "Help" msgstr "" -#: sonar/theme/templates/sonar/footer.html:35 +#: sonar/theme/templates/sonar/footer.html:43 msgid "About" msgstr "" -#: sonar/theme/templates/sonar/footer.html:38 +#: sonar/theme/templates/sonar/footer.html:46 msgid "Contact" msgstr "" -#: sonar/theme/templates/sonar/footer.html:43 +#: sonar/theme/templates/sonar/footer.html:51 #, python-format msgid "Powered by RERO" msgstr "" -#: sonar/theme/templates/sonar/frontpage.html:33 +#: sonar/theme/templates/sonar/frontpage.html:41 msgid "Search publications, authors, projects, ..." msgstr "" -#: sonar/theme/templates/sonar/frontpage.html:35 +#: sonar/theme/templates/sonar/frontpage.html:43 msgid "Advanced search" msgstr "" -#: sonar/theme/templates/sonar/frontpage.html:51 +#: sonar/theme/templates/sonar/frontpage.html:59 msgid "Software under development!" msgstr "" -#: sonar/theme/templates/sonar/frontpage.html:58 +#: sonar/theme/templates/sonar/frontpage.html:66 msgid "The project" msgstr "" -#: sonar/theme/templates/sonar/frontpage.html:59 +#: sonar/theme/templates/sonar/frontpage.html:67 msgid "" "The SONAR project aims to create a scholarly archive that collects, " "promotes and preserves the publications of authors affiliated with Swiss " "public research institutions." msgstr "" -#: sonar/theme/templates/sonar/frontpage.html:60 +#: sonar/theme/templates/sonar/frontpage.html:68 msgid "Further info on the project page" msgstr "" -#: sonar/theme/templates/sonar/frontpage.html:63 +#: sonar/theme/templates/sonar/frontpage.html:71 msgid "Institution views (test)" msgstr "" -#: sonar/theme/templates/sonar/frontpage.html:67 +#: sonar/theme/templates/sonar/frontpage.html:75 msgid "Follow us" msgstr "" -#: sonar/theme/templates/sonar/frontpage.html:72 +#: sonar/theme/templates/sonar/frontpage.html:80 msgid "Project website on" msgstr "" -#: sonar/theme/templates/sonar/frontpage.html:78 +#: sonar/theme/templates/sonar/frontpage.html:86 msgid "Source code on" msgstr "" -#: sonar/theme/templates/sonar/page.html:34 -#: sonar/theme/templates/sonar/page_admin.html:34 -msgid "Invenio" +#: sonar/theme/templates/sonar/manage.html:4 +msgid "SONAR administration" msgstr "" -#: sonar/theme/templates/sonar/page_settings.html:19 -msgid "Settings" -msgstr "" - -#: sonar/theme/templates/sonar/search.html:45 -msgid "Sort by" -msgstr "" - -#: sonar/theme/templates/sonar/search.html:80 -msgid "Loading..." +#: sonar/theme/templates/sonar/page.html:42 +msgid "Invenio" msgstr "" -#: sonar/theme/templates/sonar/search.html:87 -msgid "Search failed." +#: sonar/theme/templates/sonar/page_settings.html:27 +msgid "Settings" msgstr "" -#: sonar/theme/templates/sonar/accounts/forgot_password.html:11 -#: sonar/theme/templates/sonar/accounts/forgot_password.html:21 +#: sonar/theme/templates/sonar/accounts/forgot_password.html:28 +#: sonar/theme/templates/sonar/accounts/forgot_password.html:38 +#: sonar/theme/templates/sonar/accounts/reset_password.html:20 +#: sonar/theme/templates/sonar/accounts/reset_password.html:30 msgid "Reset Password" msgstr "" -#: sonar/theme/templates/sonar/accounts/forgot_password.html:17 +#: sonar/theme/templates/sonar/accounts/forgot_password.html:34 msgid "" "Enter your email address below and we will send you a link to reset your " "password." msgstr "" -#: sonar/theme/templates/sonar/accounts/forgot_password.html:28 -#: sonar/theme/templates/sonar/accounts/login.html:16 -#: sonar/theme/templates/sonar/accounts/signup.html:49 +#: sonar/theme/templates/sonar/accounts/forgot_password.html:45 +#: sonar/theme/templates/sonar/accounts/login.html:33 +#: sonar/theme/templates/sonar/accounts/reset_password.html:37 +#: sonar/theme/templates/sonar/accounts/signup.html:65 msgid "Log In" msgstr "" -#: sonar/theme/templates/sonar/accounts/forgot_password.html:30 -#: sonar/theme/templates/sonar/accounts/login.html:39 -#: sonar/theme/templates/sonar/accounts/signup.html:28 -#: sonar/theme/templates/sonar/oauth/signup.html:38 +#: sonar/theme/templates/sonar/accounts/forgot_password.html:47 +#: sonar/theme/templates/sonar/accounts/login.html:56 +#: sonar/theme/templates/sonar/accounts/reset_password.html:39 +#: sonar/theme/templates/sonar/accounts/signup.html:44 +#: sonar/theme/templates/sonar/oauth/signup.html:46 msgid "Sign Up" msgstr "" -#: sonar/theme/templates/sonar/accounts/login.html:9 +#: sonar/theme/templates/sonar/accounts/login.html:26 msgid "Log in to account" msgstr "" -#: sonar/theme/templates/sonar/accounts/login.html:24 -#: sonar/theme/templates/sonar/accounts/login.html:30 +#: sonar/theme/templates/sonar/accounts/login.html:41 +#: sonar/theme/templates/sonar/accounts/login.html:47 #, python-format msgid "Sign-in with %(type)s" msgstr "" -#: sonar/theme/templates/sonar/accounts/login.html:37 +#: sonar/theme/templates/sonar/accounts/login.html:54 msgid "Forgot password?" msgstr "" -#: sonar/theme/templates/sonar/accounts/signup.html:11 +#: sonar/theme/templates/sonar/accounts/signup.html:27 #, python-format msgid "Sign up for a %(sitename)s account" msgstr "" -#: sonar/theme/templates/sonar/accounts/signup.html:36 -#: sonar/theme/templates/sonar/accounts/signup.html:42 -#: sonar/theme/templates/sonar/oauth/signup.html:25 +#: sonar/theme/templates/sonar/accounts/signup.html:52 +#: sonar/theme/templates/sonar/accounts/signup.html:58 +#: sonar/theme/templates/sonar/oauth/signup.html:33 #, python-format msgid "Sign-up with %(type)s" msgstr "" -#: sonar/theme/templates/sonar/oauth/signup.html:27 +#: sonar/theme/templates/sonar/oauth/signup.html:35 msgid "" "Fill in your details to complete your registration. You only have to do " "this once." msgstr "" -#: sonar/theme/templates/sonar/partial/dropdown_user.html:5 +#: sonar/theme/templates/sonar/partial/dropdown_user.html:22 msgid "Profile" msgstr "" -#: sonar/theme/templates/sonar/partial/navbar.html:26 +#: sonar/theme/templates/sonar/partial/dropdown_user.html:27 +msgid "Super administration" +msgstr "" + +#: sonar/theme/templates/sonar/partial/dropdown_user.html:32 +#: sonar/theme/templates/sonar/partial/navbar.html:50 +msgid "Administration" +msgstr "" + +#: sonar/theme/templates/sonar/partial/dropdown_user.html:36 +msgid "Logout" +msgstr "" + +#: sonar/theme/templates/sonar/partial/navbar.html:44 msgid "Search" msgstr "" -#: sonar/theme/templates/sonar/partial/navbar.html:36 +#: sonar/theme/templates/sonar/partial/navbar.html:57 msgid "Back to SONAR" msgstr "" -#: sonar/theme/templates/sonar/partial/navbar.html:44 +#: sonar/theme/templates/sonar/partial/navbar.html:65 msgid "Log in" msgstr "" -#: sonar/theme/templates/sonar/partial/navbar.html:50 +#: sonar/theme/templates/sonar/partial/navbar.html:71 msgid "Sign up" msgstr "" +#: sonar/translations/manual_translations.txt:17 +msgid "bf:Publication" +msgstr "Publication" + +#: sonar/translations/manual_translations.txt:18 +msgid "bf:Manufacture" +msgstr "Manufacture" + +#: sonar/translations/manual_translations.txt:19 +msgid "bf:Distribution" +msgstr "Distribution" + +#: sonar/translations/manual_translations.txt:20 +msgid "bf:Production" +msgstr "Production" + +#: sonar/translations/manual_translations.txt:21 +msgid "bf:Place" +msgstr "Place" + +#: sonar/translations/manual_translations.txt:22 +msgid "Date" +msgstr "Date" + +#: sonar/translations/manual_translations.txt:24 +msgid "lang-eng" +msgstr "English" + +#: sonar/translations/manual_translations.txt:25 +msgid "lang-fre" +msgstr "French" + +#: sonar/translations/manual_translations.txt:26 +msgid "German" +msgstr "" + +#: sonar/translations/manual_translations.txt:27 +msgid "lang-ita" +msgstr "Italian" + diff --git a/tests/api/test_api_simple_flow.py b/tests/api/test_api_simple_flow.py index 3f58ed9d..010a0e7f 100644 --- a/tests/api/test_api_simple_flow.py +++ b/tests/api/test_api_simple_flow.py @@ -26,19 +26,19 @@ @mock.patch('invenio_records_rest.views.verify_record_permission', mock.MagicMock(return_value=VerifyRecordPermissionPatch)) -def test_simple_flow(client): +def test_simple_flow(client, document_json_fixture): """Test simple flow using REST API.""" headers = [('Content-Type', 'application/json')] - data = { - 'title': 'The title of the record' - } - url = 'https://localhost:5000/documents/' # create a record - response = client.post(url, data=json.dumps(data), headers=headers) + response = client.post('https://localhost:5000/documents/', + data=json.dumps(document_json_fixture), + headers=headers) assert response.status_code == 201 current_search.flush_and_refresh('documents') # retrieve record res = client.get('https://localhost:5000/documents/1') assert res.status_code == 200 + assert response.json['metadata']['title'][0]['mainTitle'][0][ + 'value'] == 'Title of the document' diff --git a/tests/conftest.py b/tests/conftest.py index 1daf2a22..486c8972 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -17,9 +17,14 @@ """Common pytest fixtures and plugins.""" +import os +import tempfile + import pytest from flask import url_for from flask_security.utils import encrypt_password +from invenio_files_rest.models import Location +from invenio_search import current_search from sonar.modules.documents.api import DocumentRecord from sonar.modules.institutions.api import InstitutionRecord @@ -113,8 +118,8 @@ def organization_fixture(app, db): @pytest.fixture() -def document_fixture(app, db, organization_fixture): - """Create a document.""" +def document_json_fixture(app, db, organization_fixture): + """JSON document fixture.""" data = { "pid": "10000", @@ -141,11 +146,20 @@ def document_fixture(app, db, organization_fixture): "type": "person", "name": "Trojani, Fabio" }], - "title": - "Title of the document", + 'title': [{ + 'type': + 'bf:Title', + 'mainTitle': [{ + 'language': 'eng', + 'value': 'Title of the document' + }] + }], "extent": "103 p", - "abstracts": ["Abstract of the document"], + "abstracts": [{ + "language": "eng", + "value": "Abstract of the document" + }], "subjects": [{ "language": "eng", "value": ["Time series models", "GARCH models"] @@ -204,7 +218,32 @@ def document_fixture(app, db, organization_fixture): } } - document = DocumentRecord.create(data, dbcommit=True) - document.reindex() + return data + + +@pytest.fixture() +def document_fixture(app, db, document_json_fixture, bucket_location_fixture): + """Create a document.""" + document = DocumentRecord.create(document_json_fixture, + dbcommit=True, + with_bucket=True) db.session.commit() + document.reindex() + + current_search.flush_and_refresh('documents') + return document + + +@pytest.fixture() +def bucket_location_fixture(app, db): + """Create a default location for managing files.""" + tmppath = tempfile.mkdtemp() + db.session.add(Location(name='default', uri=tmppath, default=True)) + db.session.commit() + + +@pytest.fixture() +def pdf_file(): + """Return test PDF file path.""" + return os.path.dirname(os.path.abspath(__file__)) + '/data/test.pdf' diff --git a/tests/data/test.pdf b/tests/data/test.pdf new file mode 100755 index 00000000..38572d1a Binary files /dev/null and b/tests/data/test.pdf differ diff --git a/tests/ui/data/json_to_compile.json b/tests/ui/data/json_to_compile.json index 489a35a0..c23879c5 100644 --- a/tests/ui/data/json_to_compile.json +++ b/tests/ui/data/json_to_compile.json @@ -19,6 +19,12 @@ ], "additionalProperties": false, "properties": { + "$schema": { + "title": "Schema", + "description": "Schema to validate document against.", + "type": "string", + "default": "https://sonar.ch/schema/documents/document-v1.0.0.json" + }, "pid": { "title": "PID", "type": "string" diff --git a/tests/ui/documents/data/oai_sources.json b/tests/ui/documents/data/oai_sources.json new file mode 100644 index 00000000..98fb9011 --- /dev/null +++ b/tests/ui/documents/data/oai_sources.json @@ -0,0 +1,10 @@ +[ + { + "key": "fake", + "name": "domain", + "url": "http://some.domain.com/oai2d", + "metadataprefix": "marcxml", + "comment": "", + "setspecs": "fake-set" + } +] diff --git a/tests/ui/documents/data/oai_sources_error.json b/tests/ui/documents/data/oai_sources_error.json new file mode 100644 index 00000000..369ca3f1 --- /dev/null +++ b/tests/ui/documents/data/oai_sources_error.json @@ -0,0 +1,8 @@ +{ + "key": "fake", + "name": "domain", + "url": "http://some.domain.com/oai2d", + "metadataprefix": "marcxml", + "comment": "", + "setspecs": "fake-set" +} diff --git a/tests/ui/institutions/test_institutions_cli.py b/tests/ui/documents/test_documents_api.py similarity index 54% rename from tests/ui/institutions/test_institutions_cli.py rename to tests/ui/documents/test_documents_api.py index 3df78f3d..9b6430f8 100644 --- a/tests/ui/institutions/test_institutions_cli.py +++ b/tests/ui/documents/test_documents_api.py @@ -15,24 +15,23 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -"""Test CLI for importing documents.""" +"""Test documents API.""" -import click -from click.testing import CliRunner -from pytest_invenio.fixtures import script_info +from sonar.modules.documents.api import DocumentRecord -import sonar.modules.institutions.cli as Cli -from sonar.modules.institutions.api import InstitutionRecord +def test_get_record_by_identifier(app, document_fixture): + """Test getting record by its identifier.""" + # Record found + record = DocumentRecord.get_record_by_identifier([{ + 'value': 'oai:doc.rero.ch:20050302172954-WU', + 'type': 'bf:Identifier' + }]) + assert record['pid'] == '10000' -def test_import_institutions(app, script_info): - """Test import institutions.""" - InstitutionRecord.create({ - "pid": "usi", - "name": "Università della Svizzera italiana" - }) - - runner = CliRunner() - - result = runner.invoke(Cli.import_institutions, obj=script_info) - assert result.exit_code == 0 + # Record not found + record = DocumentRecord.get_record_by_identifier([{ + 'value': 'oai:unknown', + 'type': 'bf:Identifier' + }]) + assert not record diff --git a/tests/ui/documents/test_documents_cli.py b/tests/ui/documents/test_documents_cli.py index 05d37041..d397a010 100644 --- a/tests/ui/documents/test_documents_cli.py +++ b/tests/ui/documents/test_documents_cli.py @@ -15,66 +15,65 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -"""Test CLI for importing documents.""" +"""Test documents CLI commands.""" -import requests +import pytest from click.testing import CliRunner +from invenio_oaiharvester.models import OAIHarvestConfig import sonar.modules.documents.cli as Cli -from sonar.modules.institutions.api import InstitutionRecord -def test_import_documents(app, script_info, monkeypatch): - """Test import documents.""" +def test_oai_config_create(app, script_info): + """Test create configuration for harvesting.""" runner = CliRunner() - result = runner.invoke(Cli.import_documents, ['test'], obj=script_info) - assert result.output.find( - 'Institution record not found in database') != -1 + # Test create configuration + result = runner.invoke(Cli.oai_config_create, + ['./tests/ui/documents/data/oai_sources.json'], + obj=script_info) + assert result.output.find('Created configuration for "fake"') != -1 - InstitutionRecord.create({ - "pid": "test", - "name": "Test" - }, dbcommit=True) + # Test already created configurations + result = runner.invoke(Cli.oai_config_create, + ['./tests/ui/documents/data/oai_sources.json'], + obj=script_info) + assert result.output.find('Config already registered for "fake"') != -1 - app.config.update(SONAR_DOCUMENTS_INSTITUTIONS_MAP=None) - - result = runner.invoke(Cli.import_documents, ['test'], obj=script_info) - assert result.output.find( - 'Institution map not found in configuration') != -1 - - app.config.update(SONAR_DOCUMENTS_INSTITUTIONS_MAP=dict( - usi='ticino.unisi', - hevs='valais.hevs' - )) - - result = runner.invoke(Cli.import_documents, ['test'], obj=script_info) - assert result.output.find( - 'Institution map for "test" not found in configuration') != -1 + # Test error on configuration JSON file + result = runner.invoke( + Cli.oai_config_create, + ['./tests/ui/documents/data/oai_sources_error.json'], + obj=script_info) + assert result.output.find('Configurations file cannot be parsed') != -1 - result = runner.invoke(Cli.import_documents, ['usi'], obj=script_info) - assert result.exit_code == 1 - InstitutionRecord.create({ - "pid": "usi", - "name": "Università della Svizzera italiana" - }, dbcommit=True) +def test_oai_config_info(app, script_info): + """Test list configurations.""" + runner = CliRunner() - result = runner.invoke( - Cli.import_documents, ['usi', '--pages=1'], obj=script_info) - assert result.exit_code == 0 + # Create configurations + runner.invoke(Cli.oai_config_create, + ['./tests/ui/documents/data/oai_sources.json'], + obj=script_info) - class MockResponse(): - """Mock response.""" - def __init__(self): - self.status_code = 500 + # List configurations + result = runner.invoke(Cli.oai_config_info, obj=script_info) + assert result.output.startswith('\nfake') - def mock_get(*args, **kwargs): - return MockResponse() - monkeypatch.setattr(requests, 'get', mock_get) +def test_oai_config_harvest_all(app, db, script_info): + """Test harvest all.""" + runner = CliRunner() - result = runner.invoke( - Cli.import_documents, ['usi', '--pages=1'], obj=script_info) - assert result.exit_code == 1 - assert result.output.find('failed') != -1 + # Create configurations + configuration = OAIHarvestConfig(name='fake', + baseurl='http://some.domain.com/oai2d', + metadataprefix='marcxml', + setspecs='fake-set') + configuration.save() + db.session.commit() + + result = runner.invoke(Cli.oai_config_harvest_all, ['--max', '1'], + obj=script_info) + assert result.output == '' diff --git a/tests/ui/documents/test_documents_receivers.py b/tests/ui/documents/test_documents_receivers.py new file mode 100644 index 00000000..23fa42f4 --- /dev/null +++ b/tests/ui/documents/test_documents_receivers.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 -*- +# +# Swiss Open Access Repository +# Copyright (C) 2019 RERO +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Test documents recievers.""" + +from invenio_oaiharvester.tasks import get_records + +from sonar.modules.documents.receivers import chunks, \ + populate_fulltext_field, transform_harvested_records + + +def test_transform_harvested_records(app, bucket_location_fixture): + """Test harvested record transformation.""" + request, records = get_records( + ['oai:doc.rero.ch:20120503160026-MV'], + metadata_prefix="marcxml", + url='http://doc.rero.ch/oai2d', + ) + + transform_harvested_records(None, records, **{'name': 'test', 'max': 1}) + + +def test_populate_fulltext_field(app, db, document_fixture, pdf_file): + """Test add full text to document.""" + with open(pdf_file, 'rb') as file: + content = file.read() + + # Successful file add + document_fixture.add_file(content, 'test1.pdf', type='file') + assert document_fixture.files['test1.pdf'] + assert document_fixture.files['test1.txt'] + + db.session.commit() + + json = {} + populate_fulltext_field(record=document_fixture, + index='documents', + json=json) + + assert len(json['fulltext']) == 1 + assert json['fulltext'][0].startswith('PHYSICAL REVIEW B 99') + + +def test_chunks(): + """Test chunks.""" + records = chunks([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 3) + records = list(records) + assert len(records) == 4 + assert records[0] == [1, 2, 3] + assert records[-1] == [10] diff --git a/tests/ui/documents/test_documents_tasks.py b/tests/ui/documents/test_documents_tasks.py new file mode 100644 index 00000000..f2b5c6cf --- /dev/null +++ b/tests/ui/documents/test_documents_tasks.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +# +# Swiss Open Access Repository +# Copyright (C) 2019 RERO +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Test documents tasks.""" + +import mock + +from sonar.modules.documents.api import DocumentRecord +from sonar.modules.documents.tasks import import_records + + +@mock.patch( + 'sonar.modules.documents.tasks.DocumentRecord.get_record_by_identifier') +def test_import_records(mock_record_by_identifier, app, document_json_fixture, + bucket_location_fixture): + """Test import records.""" + # Successful importing record + mock_record_by_identifier.return_value = None + document_json_fixture['files'] = [{ + 'key': 'test.pdf', + 'url': 'http://some.url/file.pdf' + }] + import_records([document_json_fixture]) + assert DocumentRecord.get_record_by_pid('10000') + + # Error during importation of record + def exception_side_effect(data): + raise Exception("No record found for identifier") + + mock_record_by_identifier.side_effect = exception_side_effect + document_json_fixture['pid'] = '10001' + document_json_fixture['files'] = [{ + 'key': 'test.pdf', + 'url': 'http://some.url/file.pdf' + }] + + import_records([document_json_fixture]) + + assert not DocumentRecord.get_record_by_pid('10001') diff --git a/tests/ui/documents/test_documents_views.py b/tests/ui/documents/test_documents_views.py index e7a553b7..8a9b11e6 100644 --- a/tests/ui/documents/test_documents_views.py +++ b/tests/ui/documents/test_documents_views.py @@ -42,13 +42,50 @@ def test_search(app, client): '/organization/sonar/search/documents').status_code == 200 -def test_detail(app, client): +def test_detail(app, client, document_fixture): """Test document detail page.""" - record = DocumentRecord.create({"title": "The title of the record"}, - dbcommit=True) - - # assert isinstance(views.detail('1', record, ir='sonar'), str) - assert client.get('/organization/sonar/documents/1').status_code == 200 + assert client.get('/organization/sonar/documents/10000').status_code == 200 + + +def test_title_format(document_fixture): + """Test title format for display it in template.""" + # No title + assert views.title_format({'mainTitle': [], 'subtitle': []}, 'en') == '' + + # Take the first one as fallback + assert views.title_format( + {'mainTitle': [{ + 'language': 'spa', + 'value': 'Title ES' + }]}, 'fr') == 'Title ES' + + title = { + 'mainTitle': [{ + 'language': 'ger', + 'value': 'Title DE' + }, { + 'language': 'eng', + 'value': 'Title EN' + }, { + 'language': 'fre', + 'value': 'Title FR' + }], + 'subtitle': [{ + 'language': 'ita', + 'value': 'Subtitle IT' + }, { + 'language': 'fre', + 'value': 'Subtitle FR' + }, { + 'language': 'eng', + 'value': 'Subtitle EN' + }] + } + + assert views.title_format(title, 'en') == 'Title EN : Subtitle EN' + assert views.title_format(title, 'fr') == 'Title FR : Subtitle FR' + assert views.title_format(title, 'de') == 'Title DE : Subtitle EN' + assert views.title_format(title, 'it') == 'Title EN : Subtitle IT' def test_authors_format(document_fixture): @@ -83,8 +120,16 @@ def test_series_format(): def test_abstracts_format(): """Test series format.""" - result = 'line1\nline2\nline3' - assert result == views.abstracts_format(['line1\n\n\nline2', 'line3']) + abstracts = [{ + 'language': 'eng', + 'value': 'Abstract' + }, { + 'language': 'fre', + 'value': 'Résumé' + }] + + result = 'Abstract\n\nRésumé' + assert result == views.abstracts_format(abstracts) def test_subjects_format(document_fixture): @@ -120,23 +165,6 @@ def test_identifiedby_format(): assert results == views.identifiedby_format(identifiedby) -def test_language_format(app): - """Test language format.""" - language = [{ - 'type': 'bf:Language', - 'value': 'ger' - }, { - 'type': 'bf:Language', - 'value': 'fre' - }] - results = 'ger, fre' - assert results == views.language_format(language, 'en') - - language = 'fre' - results = 'fre' - assert results == views.language_format(language, 'en') - - def test_create_publication_statement(document_fixture): """Test create publication statement.""" publication_statement = views.create_publication_statement( @@ -176,3 +204,9 @@ def test_get_bibliographic_code_from_language(app): assert str(e.value) == 'Language code not found for "zz"' assert views.get_bibliographic_code_from_language('de') == 'ger' + + +def test_get_preferred_languages(app): + """Test getting the list of prefererred languages.""" + assert views.get_preferred_languages() == ['eng', 'fre', 'ger', 'ita'] + assert views.get_preferred_languages('fre') == ['fre', 'eng', 'ger', 'ita'] diff --git a/tests/ui/documents/test_marc21tojson.py b/tests/ui/documents/test_marc21tojson.py index 9a694800..0d496349 100644 --- a/tests/ui/documents/test_marc21tojson.py +++ b/tests/ui/documents/test_marc21tojson.py @@ -39,8 +39,7 @@ def test_not_repetetive(capsys): assert data == 'first' out, err = capsys.readouterr() assert err == 'WARNING NOT REPETITIVE:\tpid1\tkey\tsub\t{data}\t\n'.format( - data=str(data_dict) - ) + data=str(data_dict)) data = {'sub': 'only'} data = not_repetitive('pid1', 'key', data, 'sub', '') assert data == 'only' @@ -48,157 +47,323 @@ def test_not_repetetive(capsys): assert err == "" -def test_marc21_to_type(): - """ - Test dojson marc21_to_type. +def test_marc21_to_type_and_institution(app): + """Test type and institution.""" - Books: LDR/6-7: am - Journals: LDR/6-7: as - Articles: LDR/6-7: aa + add field 773 (journal title) - Scores: LDR/6: c|d - Videos: LDR/6: g + 007/0: m|v - Sounds: LDR/6: i|j - E-books (imported from Cantook) + # Type and institution + marc21xml = """ + + + BOOK + BAAGE + + """ + marc21json = create_record(marc21xml) + data = marc21tojson.do(marc21json) + assert data.get('type') == 'book' + assert data.get('institution') == { + '$ref': 'https://sonar.ch/api/institutions/baage' + } + # Type only marc21xml = """ - 00501nam a2200133 a 4500 + + BOOK + """ marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) assert data.get('type') == 'book' + assert not data.get('institution') + # Institution only marc21xml = """ - 00501nas a2200133 a 4500 + + BAAGE + """ marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) - assert data.get('type') == 'journal' + assert not data.get('type') + assert data.get('institution') == { + '$ref': 'https://sonar.ch/api/institutions/baage' + } + +def test_marc21_to_title_245(): + """Test dojson marc21_to_title.""" + + # One title with subtitle marc21xml = """ - 00501naa a2200133 a 4500 + + Main title + eng + Subtitle + """ marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) - assert data.get('type') == 'article' + assert data.get('title') == [{ + 'type': + 'bf:Title', + 'mainTitle': [{ + 'value': 'Main title', + 'language': 'eng' + }], + 'subtitle': [{ + 'value': 'Subtitle', + 'language': 'eng' + }] + }] + # Multiple titles with subtitles marc21xml = """ - 00501nca a2200133 a 4500 + + Main title + eng + Subtitle + + + Titre principal + fre + Sous-titre + """ marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) - assert data.get('type') == 'score' + assert data.get('title') == [{ + 'type': + 'bf:Title', + 'mainTitle': [{ + 'value': 'Main title', + 'language': 'eng' + }], + 'subtitle': [{ + 'value': 'Subtitle', + 'language': 'eng' + }] + }, { + 'type': + 'bf:Title', + 'mainTitle': [{ + 'value': 'Titre principal', + 'language': 'fre' + }], + 'subtitle': [{ + 'value': 'Sous-titre', + 'language': 'fre' + }] + }] + + # One title without subtitle marc21xml = """ - 00501nda a2200133 a 4500 + + Main title + eng + """ marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) - assert data.get('type') == 'score' + assert data.get('title') == [{ + 'type': + 'bf:Title', + 'mainTitle': [{ + 'value': 'Main title', + 'language': 'eng' + }] + }] + # No title marc21xml = """ - 00501nia a2200133 a 4500 + + eng + Subtitle + """ marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) - assert data.get('type') == 'sound' + assert not data.get('title') + + # No language marc21xml = """ - 00501nja a2200133 a 4500 + + Main title + """ marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) - assert data.get('type') == 'sound' + assert data.get('title') == [{ + 'type': + 'bf:Title', + 'mainTitle': [{ + 'value': 'Main title', + 'language': 'eng' + }] + }] + # Multiple title with one without title marc21xml = """ - 00501nga a2200133 a 4500 + + Main title + eng + Subtitle + + + fre + Sous-titre + """ marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) - assert data.get('type') == 'video' + assert data.get('title') == [{ + 'type': + 'bf:Title', + 'mainTitle': [{ + 'value': 'Main title', + 'language': 'eng' + }], + 'subtitle': [{ + 'value': 'Subtitle', + 'language': 'eng' + }] + }] -# pid: 001 -def test_marc21_to_pid(): - """Test dojson marc21languages.""" +def test_marc21_to_title_246(): + """Test dojson marc21_to_title.""" + # One title 246 without 245 marc21xml = """ - - REROILS:123456789 - + + Main title + eng + """ marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) - assert data.get('pid') == '123456789' + assert data.get('title') == [{ + 'type': + 'bf:Title', + 'mainTitle': [{ + 'value': 'Main title', + 'language': 'eng' + }] + }] + + # One title 246 without $a and without 245 marc21xml = """ - - 123456789 - + + eng + """ marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) - assert data.get('pid') is None + assert not data.get('title') - -# title: 245$a -# without the punctuaction. If there's a $b, then 245$a : $b without the " /" -def test_marc21_to_title(): - """Test dojson marc21_to_title.""" - - # subfields $a $b $c + # One title 246 without language and without 245 marc21xml = """ - - main title : - subtitle / - responsibility - + + Main title + """ marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) - assert data.get('title') == 'main title : subtitle' - # subfields $a $c + assert data.get('title') == [{ + 'type': + 'bf:Title', + 'mainTitle': [{ + 'value': 'Main title', + 'language': 'eng' + }] + }] + + # One title 246 with one 245 title marc21xml = """ - - main title - responsibility - + + Main title 245 + eng + + + Main title 246 + fre + """ marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) - assert data.get('title') == 'main title' - # subfield $a + assert data.get('title') == [{ + 'type': + 'bf:Title', + 'mainTitle': [{ + 'value': 'Main title 245', + 'language': 'eng' + }, { + 'value': 'Main title 246', + 'language': 'fre' + }] + }] + + # One title 246 with multiple 245 title marc21xml = """ - - main title - + + Main title 245 1 + eng + + + Main title 245 2 + eng + + + Main title 246 + fre + """ marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) - assert data.get('title') == 'main title' + assert data.get('title') == [{ + 'type': + 'bf:Title', + 'mainTitle': [{ + 'value': 'Main title 245 1', + 'language': 'eng' + }] + }, { + 'type': + 'bf:Title', + 'mainTitle': [{ + 'value': 'Main title 245 2', + 'language': 'eng' + }, { + 'value': 'Main title 246', + 'language': 'fre' + }] + }] # languages: 008 and 041 [$a, repetitive] @@ -213,22 +378,17 @@ def test_marc21_to_language(): eng - """.format( - field_008='881005s1984 xxu|||||| ||||00|| |ara d' - ) + """.format(field_008='881005s1984 xxu|||||| ||||00|| |ara d') marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) - assert data.get('language') == [ - { - 'type': 'bf:Language', - 'value': 'ara' - }, - { - 'type': 'bf:Language', - 'value': 'eng' - } - ] + assert data.get('language') == [{ + 'type': 'bf:Language', + 'value': 'ara' + }, { + 'type': 'bf:Language', + 'value': 'eng' + }] marc21xml = """ @@ -240,25 +400,19 @@ def test_marc21_to_language(): fre - """.format( - field_008='881005s1984 xxu|||||| ||||00|| |ara d' - ) - marc21json = create_record(marc21xml) - data = marc21tojson.do(marc21json) - assert data.get('language') == [ - { - 'type': 'bf:Language', - 'value': 'ara' - }, - { - 'type': 'bf:Language', - 'value': 'eng' - }, - { - 'type': 'bf:Language', - 'value': 'fre' - } - ] + """.format(field_008='881005s1984 xxu|||||| ||||00|| |ara d') + marc21json = create_record(marc21xml) + data = marc21tojson.do(marc21json) + assert data.get('language') == [{ + 'type': 'bf:Language', + 'value': 'ara' + }, { + 'type': 'bf:Language', + 'value': 'eng' + }, { + 'type': 'bf:Language', + 'value': 'fre' + }] marc21xml = """ @@ -267,22 +421,17 @@ def test_marc21_to_language(): {field_008} - """.format( - field_008='881005s1984 xxu|||||| ||||00|| |ara d' - ) + """.format(field_008='881005s1984 xxu|||||| ||||00|| |ara d') marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) - assert data.get('language') == [ - { - 'type': 'bf:Language', - 'value': 'ara' - }, - { - 'type': 'bf:Language', - 'value': 'eng' - } - ] + assert data.get('language') == [{ + 'type': 'bf:Language', + 'value': 'ara' + }, { + 'type': 'bf:Language', + 'value': 'eng' + }] marc21xml = """ @@ -292,33 +441,25 @@ def test_marc21_to_language(): rus - """.format( - field_008='881005s1984 xxu|||||| ||||00|| |ara d' - ) - marc21json = create_record(marc21xml) - data = marc21tojson.do(marc21json) - assert data.get('language') == [ - { - 'type': 'bf:Language', - 'value': 'ara' - }, - { - 'type': 'bf:Language', - 'value': 'eng' - }, - { - 'type': 'bf:Language', - 'value': 'rus' - } - ] + """.format(field_008='881005s1984 xxu|||||| ||||00|| |ara d') + marc21json = create_record(marc21xml) + data = marc21tojson.do(marc21json) + assert data.get('language') == [{ + 'type': 'bf:Language', + 'value': 'ara' + }, { + 'type': 'bf:Language', + 'value': 'eng' + }, { + 'type': 'bf:Language', + 'value': 'rus' + }] marc21xml = """ {field_008} - """.format( - field_008='881005s1984 xxu|||||| ||||00|| |ara d' - ) + """.format(field_008='881005s1984 xxu|||||| ||||00|| |ara d') marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) assert not data.get('language') @@ -356,24 +497,20 @@ def test_marc21_to_authors(mock_get): marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) authors = data.get('authors') - assert authors == [ - { - 'name': 'Jean-Paul II', - 'type': 'person', - 'date': '1954-', - 'qualifier': 'Pape' - }, - { - 'name': 'Dumont, Jean', - 'type': 'person', - 'date': '1921-2014', - 'qualifier': 'Historien' - }, - { - 'name': 'RERO', - 'type': 'organisation' - } - ] + assert authors == [{ + 'name': 'Jean-Paul II', + 'type': 'person', + 'date': '1954-', + 'qualifier': 'Pape' + }, { + 'name': 'Dumont, Jean', + 'type': 'person', + 'date': '1921-2014', + 'qualifier': 'Historien' + }, { + 'name': 'RERO', + 'type': 'organisation' + }] marc21xml = """ @@ -396,18 +533,15 @@ def test_marc21_to_authors(mock_get): marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) authors = data.get('authors') - assert authors == [ - { - 'name': 'Jean-Paul II', - 'type': 'person', - 'date': '1954-', - 'qualifier': 'Pape' - }, - { - 'name': 'RERO', - 'type': 'organisation' - } - ] + assert authors == [{ + 'name': 'Jean-Paul II', + 'type': 'person', + 'date': '1954-', + 'qualifier': 'Pape' + }, { + 'name': 'RERO', + 'type': 'organisation' + }] marc21xml = """ @@ -416,13 +550,16 @@ def test_marc21_to_authors(mock_get): """ - mock_get.return_value = mock_response(json_data={ - 'hits': { - 'hits': [{ - 'links': {'self': 'https://mef.rero.ch/api/rero/XXXXXXXX'} - }] - } - }) + mock_get.return_value = mock_response( + json_data={ + 'hits': { + 'hits': [{ + 'links': { + 'self': 'https://mef.rero.ch/api/rero/XXXXXXXX' + } + }] + } + }) marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) authors = data.get('authors') @@ -472,12 +609,12 @@ def test_marc21_to_provision_activity_manufacture_date(): fre ger - + Bienne : Impr. Weber [2006] - + © 2006 @@ -485,21 +622,49 @@ def test_marc21_to_provision_activity_manufacture_date(): marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) assert data.get('provisionActivity') == [{ - 'type': 'bf:Manufacture', - 'statement': [ - { - 'label': [{'value': 'Bienne'}], - 'type': 'bf:Place' - }, - { - 'label': [{'value': 'Impr. Weber'}], - 'type': 'bf:Agent' - }, - { - 'label': [{'value': '[2006]'}], - 'type': 'Date' - } - ] + 'type': + 'bf:Publication', + 'statement': [{ + 'type': 'bf:Place', + 'label': [{ + 'value': 'Bienne' + }] + }, { + 'type': 'bf:Agent', + 'label': [{ + 'value': 'Impr. Weber' + }] + }, { + 'label': [{ + 'value': '[2006]' + }], + 'type': 'Date' + }], + 'startDate': + '2006', + 'endDate': + '2010', + 'place': [{ + 'country': 'sz', + 'type': 'bf:Place' + }] + }, { + 'type': + 'bf:Publication', + 'statement': [{ + 'label': [{ + 'value': '© 2006' + }], + 'type': 'Date' + }], + 'startDate': + '2006', + 'endDate': + '2010', + 'place': [{ + 'country': 'sz', + 'type': 'bf:Place' + }] }] @@ -521,7 +686,7 @@ def test_marc21_to_provision_activity_canton(): sz ch-be - + Biel/Bienne : Centre PasquArt ; Nürnberg : @@ -530,72 +695,108 @@ def test_marc21_to_provision_activity_canton(): distrib. in the United Kingdom [etc.], [2006-2010] - + Bienne : Impr. Weber - + © 2006 """ marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) + assert data.get('provisionActivity') == [{ - 'type': 'bf:Publication', + 'type': + 'bf:Publication', + 'statement': [{ + 'type': 'bf:Place', + 'label': [{ + 'value': 'Biel/Bienne' + }] + }, { + 'type': 'bf:Agent', + 'label': [{ + 'value': 'Centre PasquArt' + }] + }, { + 'type': 'bf:Place', + 'label': [{ + 'value': 'Nürnberg' + }] + }, { + 'type': 'bf:Agent', + 'label': [{ + 'value': 'Verlag für Moderne Kunst' + }] + }, { + 'type': 'bf:Place', + 'label': [{ + 'value': 'Manchester' + }] + }, { + 'type': + 'bf:Agent', + 'label': [{ + 'value': 'distrib. in the United Kingdom [etc.]' + }] + }, { + 'label': [{ + 'value': '[2006-2010]' + }], + 'type': 'Date' + }], + 'startDate': + '2006', + 'endDate': + '2010', 'place': [{ 'canton': 'be', 'country': 'sz', 'type': 'bf:Place' + }] + }, { + 'type': + 'bf:Publication', + 'statement': [{ + 'type': 'bf:Place', + 'label': [{ + 'value': 'Bienne' + }] + }, { + 'type': 'bf:Agent', + 'label': [{ + 'value': 'Impr. Weber' + }] }], - 'statement': [ - { - 'label': [{'value': 'Biel/Bienne'}], - 'type': 'bf:Place' - }, - { - 'label': [{'value': 'Centre PasquArt'}], - 'type': 'bf:Agent' - }, - { - 'label': [{'value': 'Nürnberg'}], - 'type': 'bf:Place' - }, - { - 'label': [{'value': 'Verlag für Moderne Kunst'}], - 'type': 'bf:Agent' - }, - { - 'label': [{'value': 'Manchester'}], - 'type': 'bf:Place' - }, - { - 'label': [{'value': 'distrib. in the United Kingdom [etc.]'}], - 'type': 'bf:Agent' - }, - { - 'label': [{'value': '[2006-2010]'}], - 'type': 'Date' - } - ], - 'startDate': '2006', - 'endDate': '2010', + 'startDate': + '2006', + 'endDate': + '2010', + 'place': [{ + 'canton': 'be', + 'country': 'sz', + 'type': 'bf:Place' + }] }, { - 'type': 'bf:Manufacture', - 'statement': [ - { - 'label': [ - {'value': 'Bienne'} - ], - 'type': 'bf:Place' - }, - { - 'label': [ - {'value': 'Impr. Weber'} - ], - 'type': 'bf:Agent' - } - ] + 'type': + 'bf:Publication', + 'statement': [{ + 'label': [{ + 'value': '© 2006' + }], + 'type': 'Date' + }], + 'startDate': + '2006', + 'endDate': + '2010', + 'place': [{ + 'canton': 'be', + 'country': 'sz', + 'type': 'bf:Place' + }] }] @@ -608,7 +809,7 @@ def test_marc21_to_provision_activity_1_place_2_agents(): 940202m19699999fr |||||| ||||00|| |fre d - + [Paris] : Desclée de Brouwer [puis] Etudes augustiniennes, @@ -619,30 +820,35 @@ def test_marc21_to_provision_activity_1_place_2_agents(): marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) assert data.get('provisionActivity') == [{ - 'type': 'bf:Publication', + 'type': + 'bf:Publication', 'place': [{ 'country': 'fr', 'type': 'bf:Place' }], - 'statement': [ - { - 'label': [{'value': '[Paris]'}], - 'type': 'bf:Place' - }, - { - 'label': [{'value': 'Desclée de Brouwer [puis]'}], - 'type': 'bf:Agent' - }, - { - 'label': [{'value': 'Etudes augustiniennes'}], - 'type': 'bf:Agent' - }, - { - 'label': [{'value': '1969-'}], - 'type': 'Date' - } - ], - 'startDate': '1969' + 'statement': [{ + 'label': [{ + 'value': '[Paris]' + }], + 'type': 'bf:Place' + }, { + 'label': [{ + 'value': 'Desclée de Brouwer [puis]' + }], + 'type': 'bf:Agent' + }, { + 'label': [{ + 'value': 'Etudes augustiniennes' + }], + 'type': 'bf:Agent' + }, { + 'label': [{ + 'value': '1969-' + }], + 'type': 'Date' + }], + 'startDate': + '1969' }] @@ -654,7 +860,7 @@ def test_marc21_to_provision_activity_unknown_place_2_agents(): 960525s1968 be |||||| ||||00|| |fre d - + [Lieu de publication non identifié] : Labor : Nathan, @@ -665,30 +871,36 @@ def test_marc21_to_provision_activity_unknown_place_2_agents(): marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) assert data.get('provisionActivity') == [{ - 'type': 'bf:Publication', + 'type': + 'bf:Publication', 'place': [{ 'country': 'be', 'type': 'bf:Place' }], - 'statement': [ - { - 'label': [{'value': '[Lieu de publication non identifié]'}], - 'type': 'bf:Place' - }, - { - 'label': [{'value': 'Labor'}], - 'type': 'bf:Agent' - }, - { - 'label': [{'value': 'Nathan'}], - 'type': 'bf:Agent' - }, - { - 'label': [{'value': '1968'}], - 'type': 'Date' - } - ], - 'startDate': '1968' + 'statement': [{ + 'label': [{ + 'value': '[Lieu de publication non identifié]' + }], + 'type': + 'bf:Place' + }, { + 'label': [{ + 'value': 'Labor' + }], + 'type': 'bf:Agent' + }, { + 'label': [{ + 'value': 'Nathan' + }], + 'type': 'bf:Agent' + }, { + 'label': [{ + 'value': '1968' + }], + 'type': 'Date' + }], + 'startDate': + '1968' }] assert create_publication_statement(data.get('provisionActivity')[0]) == { 'default': '[Lieu de publication non identifié] : Labor, Nathan, 1968' @@ -704,7 +916,7 @@ def test_marc21_to_provision_activity_3_places_dann_2_agents(): 000927m19759999gw |||||| ||||00| |ger d - + Hamm (Westf.) ; [dann] Herzberg ; [dann] Nordhausen : @@ -716,37 +928,44 @@ def test_marc21_to_provision_activity_3_places_dann_2_agents(): marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) assert data.get('provisionActivity') == [{ - 'type': 'bf:Publication', + 'type': + 'bf:Publication', 'place': [{ 'country': 'gw', 'type': 'bf:Place' }], - 'statement': [ - { - 'label': [{'value': 'Hamm (Westf.)'}], - 'type': 'bf:Place' - }, - { - 'label': [{'value': '[dann] Herzberg'}], - 'type': 'bf:Place' - }, - { - 'label': [{'value': '[dann] Nordhausen'}], - 'type': 'bf:Place' - }, - { - 'label': [{'value': 'T. Bautz'}], - 'type': 'bf:Agent' - }, - { - 'label': [{'value': '1975-'}], - 'type': 'Date' - } - ], - 'startDate': '1975' + 'statement': [{ + 'label': [{ + 'value': 'Hamm (Westf.)' + }], + 'type': 'bf:Place' + }, { + 'label': [{ + 'value': '[dann] Herzberg' + }], + 'type': 'bf:Place' + }, { + 'label': [{ + 'value': '[dann] Nordhausen' + }], + 'type': 'bf:Place' + }, { + 'label': [{ + 'value': 'T. Bautz' + }], + 'type': 'bf:Agent' + }, { + 'label': [{ + 'value': '1975-' + }], + 'type': 'Date' + }], + 'startDate': + '1975' }] assert create_publication_statement(data.get('provisionActivity')[0]) == { - 'default': 'Hamm (Westf.) ; [dann] Herzberg ; [dann] Nordhausen : ' + + 'default': + 'Hamm (Westf.) ; [dann] Herzberg ; [dann] Nordhausen : ' + 'T. Bautz, 1975-' } @@ -760,7 +979,7 @@ def test_marc21_to_provision_activity_2_places_1_agent(): 960525s1966 sz |||||| ||||00|| |fre d - + [Louvain] ; [Paris] : [éditeur non identifié], @@ -771,30 +990,35 @@ def test_marc21_to_provision_activity_2_places_1_agent(): marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) assert data.get('provisionActivity') == [{ - 'type': 'bf:Publication', + 'type': + 'bf:Publication', 'place': [{ 'country': 'sz', 'type': 'bf:Place' }], - 'statement': [ - { - 'label': [{'value': '[Louvain]'}], - 'type': 'bf:Place' - }, - { - 'label': [{'value': '[Paris]'}], - 'type': 'bf:Place' - }, - { - 'label': [{'value': '[éditeur non identifié]'}], - 'type': 'bf:Agent' - }, - { - 'label': [{'value': '[1966]'}], - 'type': 'Date' - } - ], - 'startDate': '1966' + 'statement': [{ + 'label': [{ + 'value': '[Louvain]' + }], + 'type': 'bf:Place' + }, { + 'label': [{ + 'value': '[Paris]' + }], + 'type': 'bf:Place' + }, { + 'label': [{ + 'value': '[éditeur non identifié]' + }], + 'type': 'bf:Agent' + }, { + 'label': [{ + 'value': '[1966]' + }], + 'type': 'Date' + }], + 'startDate': + '1966' }] assert create_publication_statement(data.get('provisionActivity')[0]) == { 'default': '[Louvain] ; [Paris] : [éditeur non identifié], [1966]' @@ -814,7 +1038,7 @@ def test_marc21_to_provision_activity_1_place_1_agent_reprint_date(): eng fre - + Washington : Carnegie Institution of Washington, 1916 @@ -824,27 +1048,33 @@ def test_marc21_to_provision_activity_1_place_1_agent_reprint_date(): marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) assert data.get('provisionActivity') == [{ - 'type': 'bf:Publication', + 'type': + 'bf:Publication', 'place': [{ 'country': 'xxu', 'type': 'bf:Place' }], - 'statement': [ - { - 'label': [{'value': 'Washington'}], - 'type': 'bf:Place' - }, - { - 'label': [{'value': 'Carnegie Institution of Washington'}], - 'type': 'bf:Agent' - }, - { - 'label': [{'value': '1916'}], - 'type': 'Date' - } - ], - 'startDate': '1758', - 'endDate': '1916' + 'statement': [{ + 'label': [{ + 'value': 'Washington' + }], + 'type': 'bf:Place' + }, { + 'label': [{ + 'value': 'Carnegie Institution of Washington' + }], + 'type': + 'bf:Agent' + }, { + 'label': [{ + 'value': '1916' + }], + 'type': 'Date' + }], + 'startDate': + '1758', + 'endDate': + '1916' }] @@ -857,7 +1087,7 @@ def test_marc21_to_provision_activity_1_place_1_agent_uncertain_date(): 160126q1941 fr ||| | ||||00| |fre d - + Aurillac : Impr. moderne, [1941?] @@ -867,30 +1097,36 @@ def test_marc21_to_provision_activity_1_place_1_agent_uncertain_date(): marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) assert data.get('provisionActivity') == [{ - 'type': 'bf:Publication', + 'type': + 'bf:Publication', 'place': [{ 'country': 'fr', 'type': 'bf:Place' }], - 'statement': [ - { - 'label': [{'value': 'Aurillac'}], - 'type': 'bf:Place' - }, - { - 'label': [{'value': 'Impr. moderne'}], - 'type': 'bf:Agent' - }, - { - 'label': [{'value': '[1941?]'}], - 'type': 'Date' - } - ], - 'note': 'Date(s) incertaine(s) ou inconnue(s)', - 'startDate': '1941' + 'statement': [{ + 'label': [{ + 'value': 'Aurillac' + }], + 'type': 'bf:Place' + }, { + 'label': [{ + 'value': 'Impr. moderne' + }], + 'type': 'bf:Agent' + }, { + 'label': [{ + 'value': '[1941?]' + }], + 'type': 'Date' + }], + 'note': + 'Date(s) incertaine(s) ou inconnue(s)', + 'startDate': + '1941' }] assert create_publication_statement(data.get('provisionActivity')[0]) == { - 'default': 'Aurillac : Impr. moderne, [1941?]'} + 'default': 'Aurillac : Impr. moderne, [1941?]' + } def test_marc21_to_provision_activity_1_place_1_agent_chi_hani(): @@ -902,7 +1138,7 @@ def test_marc21_to_provision_activity_1_place_1_agent_chi_hani(): 180323s2017 cc ||| | ||||00| |chi d - + 880-04 Beijing : Beijing da xue chu ban she, @@ -928,35 +1164,42 @@ def test_marc21_to_provision_activity_1_place_1_agent_chi_hani(): marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) assert data.get('provisionActivity') == [{ - 'type': 'bf:Publication', + 'type': + 'bf:Publication', 'place': [{ 'country': 'cc', 'type': 'bf:Place' }], - 'statement': [ - { - 'label': [ - {'value': 'Beijing'}, - {'value': '北京', 'language': 'chi-hani'} - ], - 'type': 'bf:Place' - }, - { - 'label': [ - {'value': 'Beijing da xue chu ban she'}, - {'value': '北京大学出版社', 'language': 'chi-hani'} - ], - 'type': 'bf:Agent' - }, - { - 'label': [ - {'value': '2017'}, - {'language': 'chi-hani', 'value': '2017'} - ], - 'type': 'Date' - } - ], - 'startDate': '2017' + 'statement': [{ + 'label': [{ + 'value': 'Beijing' + }, { + 'value': '北京', + 'language': 'chi-hani' + }], + 'type': + 'bf:Place' + }, { + 'label': [{ + 'value': 'Beijing da xue chu ban she' + }, { + 'value': '北京大学出版社', + 'language': 'chi-hani' + }], + 'type': + 'bf:Agent' + }, { + 'label': [{ + 'value': '2017' + }, { + 'language': 'chi-hani', + 'value': '2017' + }], + 'type': + 'Date' + }], + 'startDate': + '2017' }] assert create_publication_statement(data.get('provisionActivity')[0]) == { 'chi-hani': '北京 : 北京大学出版社, 2017', @@ -966,7 +1209,7 @@ def test_marc21_to_provision_activity_1_place_1_agent_chi_hani(): 180323s2017 cc ||| | ||||00| |eng d - + 880-04 Beijing : Beijing da xue chu ban she, @@ -992,36 +1235,42 @@ def test_marc21_to_provision_activity_1_place_1_agent_chi_hani(): marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) assert data.get('provisionActivity') == [{ - 'type': 'bf:Publication', + 'type': + 'bf:Publication', 'place': [{ 'country': 'cc', 'type': 'bf:Place' }], - 'statement': [ - { - 'label': [ - {'value': 'Beijing'}, - {'value': '北京', 'language': 'und-hani'} - ], - 'type': 'bf:Place' - }, - { - 'label': [ - {'value': 'Beijing da xue chu ban she'}, - {'value': '北京大学出版社', - 'language': 'und-hani'} - ], - 'type': 'bf:Agent' - }, - { - 'label': [ - {'value': '2017'}, - {'language': 'und-hani', 'value': '2017'} - ], - 'type': 'Date' - } - ], - 'startDate': '2017' + 'statement': [{ + 'label': [{ + 'value': 'Beijing' + }, { + 'value': '北京', + 'language': 'und-hani' + }], + 'type': + 'bf:Place' + }, { + 'label': [{ + 'value': 'Beijing da xue chu ban she' + }, { + 'value': '北京大学出版社', + 'language': 'und-hani' + }], + 'type': + 'bf:Agent' + }, { + 'label': [{ + 'value': '2017' + }, { + 'language': 'und-hani', + 'value': '2017' + }], + 'type': + 'Date' + }], + 'startDate': + '2017' }] assert create_publication_statement(data.get('provisionActivity')[0]) == { 'und-hani': '北京 : 北京大学出版社, 2017', @@ -1053,24 +1302,18 @@ def test_marc21_to_edition_statement_one_field_250(): marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) assert data.get('editionStatement') == [{ - 'editionDesignation': [ - { - 'value': 'Di 3 ban' - }, - { - 'value': '第3版', - 'language': 'chi-hani' - } - ], - 'responsibility': [ - { - 'value': 'Zeng Lingliang zhu bian' - }, - { - 'value': '曾令良主编', - 'language': 'chi-hani' - } - ] + 'editionDesignation': [{ + 'value': 'Di 3 ban' + }, { + 'value': '第3版', + 'language': 'chi-hani' + }], + 'responsibility': [{ + 'value': 'Zeng Lingliang zhu bian' + }, { + 'value': '曾令良主编', + 'language': 'chi-hani' + }] }] @@ -1102,35 +1345,25 @@ def test_marc21_to_edition_statement_two_fields_250(): marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) assert data.get('editionStatement') == [{ - 'editionDesignation': [ - { - 'value': 'Di 3 ban' - }, - { - 'value': '第3版', - 'language': 'chi-hani' - } - ], - 'responsibility': [ - { - 'value': 'Zeng Lingliang zhu bian' - }, - { - 'value': '曾令良主编', - 'language': 'chi-hani' - } - ] + 'editionDesignation': [{ + 'value': 'Di 3 ban' + }, { + 'value': '第3版', + 'language': 'chi-hani' + }], + 'responsibility': [{ + 'value': 'Zeng Lingliang zhu bian' + }, { + 'value': '曾令良主编', + 'language': 'chi-hani' + }] }, { - 'editionDesignation': [ - { - 'value': 'Edition' - } - ], - 'responsibility': [ - { - 'value': 'Responsibility' - } - ] + 'editionDesignation': [{ + 'value': 'Edition' + }], + 'responsibility': [{ + 'value': 'Responsibility' + }] }] @@ -1161,24 +1394,18 @@ def test_marc21_to_edition_statement_with_two_subfield_a(): data = marc21tojson.do(marc21json) assert data.get('editionStatement') == [{ - 'editionDesignation': [ - { - 'value': 'Di 3 ban' - }, - { - 'value': '第3版', - 'language': 'chi-hani' - } - ], - 'responsibility': [ - { - 'value': 'Zeng Lingliang zhu bian' - }, - { - 'value': '曾令良主编', - 'language': 'chi-hani' - } - ] + 'editionDesignation': [{ + 'value': 'Di 3 ban' + }, { + 'value': '第3版', + 'language': 'chi-hani' + }], + 'responsibility': [{ + 'value': 'Zeng Lingliang zhu bian' + }, { + 'value': '曾令良主编', + 'language': 'chi-hani' + }] }] @@ -1215,30 +1442,22 @@ def test_marc21_to_edition_statement_with_one_bad_field_250(): marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) assert data.get('editionStatement') == [{ - 'editionDesignation': [ - { - 'value': 'Di 3 ban' - }, - { - 'value': '第3版', - 'language': 'chi-hani' - } - ], - 'responsibility': [ - { - 'value': 'Zeng Lingliang zhu bian' - }, - { - 'value': '曾令良主编', - 'language': 'chi-hani' - } - ] + 'editionDesignation': [{ + 'value': 'Di 3 ban' + }, { + 'value': '第3版', + 'language': 'chi-hani' + }], + 'responsibility': [{ + 'value': 'Zeng Lingliang zhu bian' + }, { + 'value': '曾令良主编', + 'language': 'chi-hani' + }] }, { - 'editionDesignation': [ - { - 'value': 'Edition' - } - ] + 'editionDesignation': [{ + 'value': 'Edition' + }] }] @@ -1268,42 +1487,48 @@ def test_marc21_to_provision_activity_1_place_1_agent_ara_arab(): marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) assert data.get('provisionActivity') == [{ - 'type': 'bf:Publication', + 'type': + 'bf:Publication', 'place': [{ 'country': 'ua', 'type': 'bf:Place' }], - 'statement': [ - { - 'label': [ - {'value': 'al-Qāhirah'}, - {'value': 'القاهرة', - 'language': 'ara-arab'} - ], - 'type': 'bf:Place' - }, - { - 'label': [ - {'value': 'Al-Hayʾat al-ʿāmmah li quṣūr al-thaqāfah'}, - {'value': 'الهيئة العامة لقصور الثقافة', - 'language': 'ara-arab'} - ], - 'type': 'bf:Agent' - }, - { - 'label': [ - {'value': '2014'}, - {'value': '2014', 'language': 'ara-arab'} - ], - 'type': 'Date' - } - ], - 'startDate': '2014' + 'statement': [{ + 'label': [{ + 'value': 'al-Qāhirah' + }, { + 'value': 'القاهرة', + 'language': 'ara-arab' + }], + 'type': + 'bf:Place' + }, { + 'label': [{ + 'value': 'Al-Hayʾat al-ʿāmmah li quṣūr al-thaqāfah' + }, { + 'value': 'الهيئة العامة لقصور الثقافة', + 'language': 'ara-arab' + }], + 'type': + 'bf:Agent' + }, { + 'label': [{ + 'value': '2014' + }, { + 'value': '2014', + 'language': 'ara-arab' + }], + 'type': + 'Date' + }], + 'startDate': + '2014' }] assert create_publication_statement(data.get('provisionActivity')[0]) == { - 'ara-arab': 'القاهرة : الهيئة العامة لقصور الثقافة, 2014', - 'default': 'al-Qāhirah : Al-Hayʾat al-ʿāmmah li quṣūr al-thaqāfah,' + - ' 2014' + 'ara-arab': + 'القاهرة : الهيئة العامة لقصور الثقافة, 2014', + 'default': + 'al-Qāhirah : Al-Hayʾat al-ʿāmmah li quṣūr al-thaqāfah,' + ' 2014' } @@ -1351,53 +1576,60 @@ def test_marc21_to_provision_activity_2_places_2_agents_rus_cyrl(): marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) assert data.get('provisionActivity') == [{ - 'type': 'bf:Publication', + 'type': + 'bf:Publication', 'place': [{ 'country': 'ru', 'type': 'bf:Place' }], - 'statement': [ - { - 'label': [ - {'value': 'Ierusalim'}, - {'value': 'Иерусалим', - 'language': 'rus-cyrl'} - ], - 'type': 'bf:Place' - }, - { - 'label': [ - {'value': 'Gesharim'}, - {'value': 'Гешарим', - 'language': 'rus-cyrl'} - ], - 'type': 'bf:Agent' - }, - { - 'label': [ - {'value': 'Moskva'}, - {'value': 'Москва', - 'language': 'rus-cyrl'} - ], - 'type': 'bf:Place' - }, - { - 'label': [ - {'value': 'Mosty Kulʹtury'}, - {'value': 'Мосты Культуры', - 'language': 'rus-cyrl'} - ], - 'type': 'bf:Agent' - }, - { - 'label': [ - {'value': '2017'}, - {'language': 'rus-cyrl', 'value': '2017'} - ], - 'type': 'Date' - } - ], - 'startDate': '2017' + 'statement': [{ + 'label': [{ + 'value': 'Ierusalim' + }, { + 'value': 'Иерусалим', + 'language': 'rus-cyrl' + }], + 'type': + 'bf:Place' + }, { + 'label': [{ + 'value': 'Gesharim' + }, { + 'value': 'Гешарим', + 'language': 'rus-cyrl' + }], + 'type': + 'bf:Agent' + }, { + 'label': [{ + 'value': 'Moskva' + }, { + 'value': 'Москва', + 'language': 'rus-cyrl' + }], + 'type': + 'bf:Place' + }, { + 'label': [{ + 'value': 'Mosty Kulʹtury' + }, { + 'value': 'Мосты Культуры', + 'language': 'rus-cyrl' + }], + 'type': + 'bf:Agent' + }, { + 'label': [{ + 'value': '2017' + }, { + 'language': 'rus-cyrl', + 'value': '2017' + }], + 'type': + 'Date' + }], + 'startDate': + '2017' }] assert create_publication_statement(data.get('provisionActivity')[0]) == { 'default': 'Ierusalim : Gesharim ; Moskva : Mosty Kulʹtury, 2017', @@ -1427,22 +1659,26 @@ def test_marc21_to_provision_activity_exceptions(capsys): data = marc21tojson.do(marc21json) out, err = capsys.readouterr() assert data.get('provisionActivity') == [{ - 'type': 'bf:Publication', + 'type': + 'bf:Publication', 'place': [{ 'country': 'ru', 'type': 'bf:Place' }], 'statement': [ { - 'label': [ - {'value': 'Ierusalim'}, - {'value': 'Иерусалим', - 'language': 'und-zyyy'} - ], - 'type': 'bf:Place' + 'label': [{ + 'value': 'Ierusalim' + }, { + 'value': 'Иерусалим', + 'language': 'und-zyyy' + }], + 'type': + 'bf:Place' }, ], - 'startDate': '2017' + 'startDate': + '2017' }] assert err.strip() == ('WARNING LANGUAGE SCRIPTS:\t???\tzyyy\t008:' '\t\t041$a:\t[]\t041$h:\t[]') @@ -1535,32 +1771,77 @@ def test_marc21_to_series(): """ marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) - assert data.get('series') == [ - { - 'name': 'Collection One', - 'number': '5' - }, - { - 'name': 'Collection Two', - 'number': '123' - } - ] + assert data.get('series') == [{ + 'name': 'Collection One', + 'number': '5' + }, { + 'name': 'Collection Two', + 'number': '123' + }] -# abstract: [520$a repetitive] def test_marc21_to_abstract(): """Test dojson abstract.""" + # One abstract without language + marc21xml = """ + + + Abstract + + + """ + marc21json = create_record(marc21xml) + data = marc21tojson.do(marc21json) + assert data.get('abstracts') == [{'value': 'Abstract', 'language': 'eng'}] + + # One abstract with language + marc21xml = """ + + + Résumé + fre + + + """ + marc21json = create_record(marc21xml) + data = marc21tojson.do(marc21json) + assert data.get('abstracts') == [{'value': 'Résumé', 'language': 'fre'}] + + # Multiple abstracts + marc21xml = """ + + + Abstract + eng + + + Résumé + fre + + + """ + marc21json = create_record(marc21xml) + data = marc21tojson.do(marc21json) + assert data.get('abstracts') == [{ + 'value': 'Abstract', + 'language': 'eng' + }, { + 'value': 'Résumé', + 'language': 'fre' + }] + + # Without abstract marc21xml = """ - This book is about + eng """ marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) - assert data.get('abstracts') == ["This book is about"] + assert not data.get('abstracts') # notes: [500$a repetitive] @@ -1618,10 +1899,13 @@ def test_marc21_to_subjects(): """ marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) - assert data.get('subjects') == [ - {'language': 'eng', 'value': ['subject 1', 'subject 2']}, - {'language': 'fre', 'value': ['sujet 1', 'sujet 2']} - ] + assert data.get('subjects') == [{ + 'language': 'eng', + 'value': ['subject 1', 'subject 2'] + }, { + 'language': 'fre', + 'value': ['sujet 1', 'sujet 2'] + }] def test_marc21_to_identifiedby_from_020(): @@ -1644,24 +1928,20 @@ def test_marc21_to_identifiedby_from_020(): """ marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) - assert data.get('identifiedBy') == [ - { - 'type': 'bf:Isbn', - 'status': 'invalid or cancelled', - 'value': '8124605254' - }, - { - 'type': 'bf:Isbn', - 'qualifier': 'broché', - 'value': '9788124605257' - }, - { - 'type': 'bf:Isbn', - 'qualifier': 'hbk.', - 'acquisitionTerms': '£125.00', - 'value': '9788189997212' - } - ] + assert data.get('identifiedBy') == [{ + 'type': 'bf:Isbn', + 'status': 'invalid or cancelled', + 'value': '8124605254' + }, { + 'type': 'bf:Isbn', + 'qualifier': 'broché', + 'value': '9788124605257' + }, { + 'type': 'bf:Isbn', + 'qualifier': 'hbk.', + 'acquisitionTerms': '£125.00', + 'value': '9788189997212' + }] def test_marc21_to_identifiedby_from_022(): @@ -1684,30 +1964,24 @@ def test_marc21_to_identifiedby_from_022(): """ marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) - assert data.get('identifiedBy') == [ - { - 'type': 'bf:Issn', - 'value': '0264-2875' - }, - { - 'type': 'bf:IssnL', - 'value': '0264-2875' - }, - { - 'type': 'bf:Issn', - 'value': '0264-2875' - }, - { - 'type': 'bf:Issn', - 'status': 'invalid', - 'value': '0080-4649' - }, - { - 'type': 'bf:IssnL', - 'status': 'cancelled', - 'value': '0080-4650' - } - ] + assert data.get('identifiedBy') == [{ + 'type': 'bf:Issn', + 'value': '0264-2875' + }, { + 'type': 'bf:IssnL', + 'value': '0264-2875' + }, { + 'type': 'bf:Issn', + 'value': '0264-2875' + }, { + 'type': 'bf:Issn', + 'status': 'invalid', + 'value': '0080-4649' + }, { + 'type': 'bf:IssnL', + 'status': 'cancelled', + 'value': '0080-4650' + }] def test_marc21_to_identifiedby_from_024_snl_bnf(): @@ -1726,18 +2000,21 @@ def test_marc21_to_identifiedby_from_024_snl_bnf(): """ marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) - assert data.get('identifiedBy') == [ - { - 'type': 'uri', - 'source': 'SNL', - 'value': 'http://permalink.snl.ch/bib/chccsa86779' - }, - { - 'type': 'uri', - 'source': 'BNF', - 'value': 'http://catalogue.bnf.fr/ark:/12148/cb312v' - } - ] + assert data.get('identifiedBy') == [{ + 'type': + 'uri', + 'source': + 'SNL', + 'value': + 'http://permalink.snl.ch/bib/chccsa86779' + }, { + 'type': + 'uri', + 'source': + 'BNF', + 'value': + 'http://catalogue.bnf.fr/ark:/12148/cb312v' + }] def test_marc21_to_identifiedby_from_024_with_subfield_2(): @@ -1775,37 +2052,32 @@ def test_marc21_to_identifiedby_from_024_with_subfield_2(): """ marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) - assert data.get('identifiedBy') == [ - { - 'type': 'bf:Doi', - 'value': '10.1007/978-3-540-37973-7', - 'acquisitionTerms': '£125.00', - 'note': 'note' - }, - { - 'type': 'bf:Urn', - 'value': 'urn:nbn:de:101:1-201609052530' - }, - { - 'type': 'bf:Local', - 'source': 'NIPO', - 'value': 'NIPO 035-16-060-7' - }, - { - 'type': 'bf:Local', - 'source': 'danacode', - 'value': '7290105422026' - }, - { - 'type': 'bf:Local', - 'source': 'vd18', - 'value': 'VD18 10153438' - }, - { - 'type': 'bf:Gtin14Number', - 'value': '00028947969525' - } - ] + assert data.get('identifiedBy') == [{ + 'type': 'bf:Doi', + 'value': '10.1007/978-3-540-37973-7', + 'acquisitionTerms': '£125.00', + 'note': 'note' + }, { + 'type': + 'bf:Urn', + 'value': + 'urn:nbn:de:101:1-201609052530' + }, { + 'type': 'bf:Local', + 'source': 'NIPO', + 'value': 'NIPO 035-16-060-7' + }, { + 'type': 'bf:Local', + 'source': 'danacode', + 'value': '7290105422026' + }, { + 'type': 'bf:Local', + 'source': 'vd18', + 'value': 'VD18 10153438' + }, { + 'type': 'bf:Gtin14Number', + 'value': '00028947969525' + }] def test_marc21_to_identifiedby_from_024_without_subfield_2(): @@ -1864,71 +2136,58 @@ def test_marc21_to_identifiedby_from_024_without_subfield_2(): """ marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) - assert data.get('identifiedBy') == [ - { - 'type': 'bf:Identifier', - 'value': '9782100745463' - }, - { - 'type': 'bf:Isrc', - 'qualifier': 'vol. 2', - 'value': '702391010582' - }, - { - 'type': 'bf:Isrc', - 'value': 'Erato ECD 88030' - }, - { - 'type': 'bf:Upc', - 'qualifier': 'vol. 5', - 'value': '604907014223' - }, - { - 'type': 'bf:Upc', - 'value': 'EMI Classics 5 55585 2' - }, - { - 'type': 'bf:Ismn', - 'qualifier': 'kritischer B., kartoniert, vol. 1', - 'value': 'M006546565' - }, - { - 'type': 'bf:Ismn', - 'qualifier': 'Kritischer Bericht', - 'value': '9790201858135' - }, - { - 'type': 'bf:Identifier', - 'qualifier': 'Bd. 1', - 'value': '4018262101065' - }, - { - 'type': 'bf:Identifier', - 'qualifier': 'CD audio classe', - 'value': '309-5-56-196162-1' - }, - { - 'type': 'bf:Ean', - 'qualifier': 'Bd 1, pbk.', - 'value': '9783737407427' - }, - { - 'type': 'bf:Identifier', - 'value': 'EP 2305' - }, - { - 'type': 'bf:Ean', - 'value': '97 EP 1234' - }, - { - 'type': 'bf:Identifier', - 'value': 'ELC1283925' - }, - { - 'type': 'bf:Isan', - 'value': '0000-0002-A3B1-0000-0-0000-0000-2' - } - ] + assert data.get('identifiedBy') == [{ + 'type': 'bf:Identifier', + 'value': '9782100745463' + }, { + 'type': 'bf:Isrc', + 'qualifier': 'vol. 2', + 'value': '702391010582' + }, { + 'type': 'bf:Isrc', + 'value': 'Erato ECD 88030' + }, { + 'type': 'bf:Upc', + 'qualifier': 'vol. 5', + 'value': '604907014223' + }, { + 'type': 'bf:Upc', + 'value': 'EMI Classics 5 55585 2' + }, { + 'type': 'bf:Ismn', + 'qualifier': 'kritischer B., kartoniert, vol. 1', + 'value': 'M006546565' + }, { + 'type': 'bf:Ismn', + 'qualifier': 'Kritischer Bericht', + 'value': '9790201858135' + }, { + 'type': 'bf:Identifier', + 'qualifier': 'Bd. 1', + 'value': '4018262101065' + }, { + 'type': 'bf:Identifier', + 'qualifier': 'CD audio classe', + 'value': '309-5-56-196162-1' + }, { + 'type': 'bf:Ean', + 'qualifier': 'Bd 1, pbk.', + 'value': '9783737407427' + }, { + 'type': 'bf:Identifier', + 'value': 'EP 2305' + }, { + 'type': 'bf:Ean', + 'value': '97 EP 1234' + }, { + 'type': 'bf:Identifier', + 'value': 'ELC1283925' + }, { + 'type': + 'bf:Isan', + 'value': + '0000-0002-A3B1-0000-0-0000-0000-2' + }] def test_marc21_to_identifiedby_from_028(): @@ -1946,14 +2205,12 @@ def test_marc21_to_identifiedby_from_028(): """ marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) - assert data.get('identifiedBy') == [ - { - 'type': 'bf:MusicPublisherNumber', - 'source': 'SRC', - 'qualifier': 'Qualif1, Qualif2', - 'value': '1234' - } - ] + assert data.get('identifiedBy') == [{ + 'type': 'bf:MusicPublisherNumber', + 'source': 'SRC', + 'qualifier': 'Qualif1, Qualif2', + 'value': '1234' + }] marc21xml = """ @@ -1967,14 +2224,12 @@ def test_marc21_to_identifiedby_from_028(): """ marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) - assert data.get('identifiedBy') == [ - { - 'type': 'bf:Identifier', - 'source': 'SRC', - 'qualifier': 'Qualif1, Qualif2', - 'value': '1234' - } - ] + assert data.get('identifiedBy') == [{ + 'type': 'bf:Identifier', + 'source': 'SRC', + 'qualifier': 'Qualif1, Qualif2', + 'value': '1234' + }] def test_marc21_to_identifiedby_from_035(): @@ -1989,13 +2244,11 @@ def test_marc21_to_identifiedby_from_035(): """ marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) - assert data.get('identifiedBy') == [ - { - 'type': 'bf:Local', - 'source': 'RERO', - 'value': 'R008945501' - } - ] + assert data.get('identifiedBy') == [{ + 'type': 'bf:Local', + 'source': 'RERO', + 'value': 'R008945501' + }] def test_marc21_to_identifiedby_from_930(): @@ -2011,13 +2264,11 @@ def test_marc21_to_identifiedby_from_930(): """ marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) - assert data.get('identifiedBy') == [ - { - 'type': 'bf:Local', - 'source': 'OCoLC', - 'value': 'ocm11113722' - } - ] + assert data.get('identifiedBy') == [{ + 'type': 'bf:Local', + 'source': 'OCoLC', + 'value': 'ocm11113722' + }] # identifier without source in parenthesis marc21xml = """ @@ -2028,59 +2279,78 @@ def test_marc21_to_identifiedby_from_930(): """ marc21json = create_record(marc21xml) data = marc21tojson.do(marc21json) - assert data.get('identifiedBy') == [ - { - 'type': 'bf:Local', - 'value': 'ocm11113722' - } - ] + assert data.get('identifiedBy') == [{ + 'type': 'bf:Local', + 'value': 'ocm11113722' + }] @mock.patch('requests.get') def test_get_person_link(mock_get, capsys): """Test get mef person link""" - mock_get.return_value = mock_response(json_data={ - 'hits': { - 'hits': [{'links': {'self': 'mocked_url'}}] - } - }) - mef_url = get_person_link( - bibid='1', - id='(RERO)A003945843', - key='100..', - value={'0': '(RERO)A003945843'} - ) + mock_get.return_value = mock_response( + json_data={'hits': { + 'hits': [{ + 'links': { + 'self': 'mocked_url' + } + }] + }}) + mef_url = get_person_link(bibid='1', + id='(RERO)A003945843', + key='100..', + value={'0': '(RERO)A003945843'}) assert mef_url == 'mocked_url' os.environ['RERO_ILS_MEF_URL'] = 'https://mefdev.test.rero.ch/api/mef' - mef_url = get_person_link( - bibid='1', - id='(RERO)A003945843', - key='100..', - value={'0': '(RERO)A003945843'} - ) + mef_url = get_person_link(bibid='1', + id='(RERO)A003945843', + key='100..', + value={'0': '(RERO)A003945843'}) assert mef_url == 'mocked_url' mock_get.return_value = mock_response(status=400) - mef_url = get_person_link( - bibid='1', - id='(RERO)A123456789', - key='100..', - value={'0': '(RERO)A123456789'} - ) + mef_url = get_person_link(bibid='1', + id='(RERO)A123456789', + key='100..', + value={'0': '(RERO)A123456789'}) assert not mef_url out, err = capsys.readouterr() assert err == "ERROR MEF REQUEST:\t1\t" + \ 'https://mefdev.test.rero.ch/api/mef/?q=rero.pid:A123456789\t400\t\n' mock_get.return_value = mock_response(status=400) - mef_url = get_person_link( - bibid='1', - id='X123456789', - key='100..', - value={'0': 'X123456789'} - ) + mef_url = get_person_link(bibid='1', + id='X123456789', + key='100..', + value={'0': 'X123456789'}) assert not mef_url out, err = capsys.readouterr() assert err == 'WARNING NOT MEF REF:\t1\tX123456789\t100..\t' + \ "{'0': 'X123456789'}\t\n" + + +def test_marc21_to_files(): + """Test getting files from field 856.""" + # Only one file + marc21xml = """ + + + file.pdf + application/pdf + 1467377 + + http://some.url/file.pdf + + order:1 + Dépliant de l'exposition + + + """ + marc21json = create_record(marc21xml) + data = marc21tojson.do(marc21json) + assert len(data.get('files')) == 1 + assert data.get('files')[0]['key'] == 'file.pdf' + assert data.get('files')[0]['url'] == 'http://some.url/file.pdf' + assert data.get('files')[0]['label'] == 'Dépliant de l\'exposition' + assert data.get('files')[0]['order'] == 1 diff --git a/tests/ui/institutions/test_jsonresolvers.py b/tests/ui/institutions/test_jsonresolvers.py index 54736d51..feac4f4f 100644 --- a/tests/ui/institutions/test_jsonresolvers.py +++ b/tests/ui/institutions/test_jsonresolvers.py @@ -17,24 +17,12 @@ """Test institutions jsonresolvers.""" -import pytest -from flask import url_for -from sonar.modules.documents.api import DocumentRecord -from sonar.modules.institutions.api import InstitutionRecord - - -def test_institution_resolver(client): +def test_institution_resolver(document_fixture): """Test institution resolver.""" - InstitutionRecord.create({ - "pid": "usi", - "name": "Università della Svizzera italiana" - }) - - record = DocumentRecord.create({ - "title": "The title of the record", - "institution": {"$ref": "https://sonar.ch/api/institutions/usi"} - }) + assert document_fixture['institution'].get('$ref') + assert document_fixture['institution'][ + '$ref'] == 'https://sonar.ch/api/institutions/org' - assert record.replace_refs().get('institution')['name'] == 'Università ' \ - 'della Svizzera italiana' + assert document_fixture.replace_refs().get( + 'institution')['name'] == 'Fake organization' diff --git a/tests/ui/test_api.py b/tests/ui/test_api.py index 34b14d36..2a9be104 100644 --- a/tests/ui/test_api.py +++ b/tests/ui/test_api.py @@ -17,6 +17,7 @@ """Test SONAR api.""" +import mock import pytest from flask import url_for from invenio_app.factory import create_api @@ -30,15 +31,12 @@ create_app = create_api -def test_create(app): +def test_create(app, document_json_fixture): """Test creating a record.""" - DocumentRecord.create({"pid": "1", "title": "The title of the record"}) - - DocumentRecord.create({ - "pid": "2", - "title": "The title of the record" - }, - dbcommit=True) + DocumentRecord.create(document_json_fixture) + assert DocumentRecord.get_record_by_pid('10000')['pid'] == '10000' + DocumentRecord.create(document_json_fixture, dbcommit=True) + assert DocumentRecord.get_record_by_pid('10000')['pid'] == '10000' def test_get_ref_link(app): @@ -47,37 +45,30 @@ def test_get_ref_link(app): '/api/document/1' -def test_get_record_by_pid(app): +def test_get_record_by_pid(app, document_json_fixture): """Test get record by PID.""" - assert DocumentRecord.get_record_by_pid('ABCD') is None + assert DocumentRecord.get_record_by_pid('10000') is None - record = DocumentRecord.create({ - "pid": "ABCD", - "title": "The title of the record" - }) + record = DocumentRecord.create(document_json_fixture) - assert DocumentRecord.get_record_by_pid('ABCD')['pid'] == 'ABCD' + assert DocumentRecord.get_record_by_pid('10000')['pid'] == '10000' record.delete() - assert DocumentRecord.get_record_by_pid('ABCD') is None + assert DocumentRecord.get_record_by_pid('10000') is None -def test_dbcommit(app): +def test_dbcommit(app, document_json_fixture): """Test record commit to db.""" - record = DocumentRecord.create({"title": "The title of the record"}) - + record = DocumentRecord.create(document_json_fixture) record.dbcommit() - assert DocumentRecord.get_record_by_pid( - '1')['title'] == 'The title of the record' + assert DocumentRecord.get_record_by_pid('10000')['pid'] == '10000' -def test_reindex(app, db, client): + +def test_reindex(app, db, client, document_json_fixture): """Test record reindex.""" - record = DocumentRecord.create({ - "pid": "100", - "title": "The title of the record" - }) + record = DocumentRecord.create(document_json_fixture) db.session.commit() indexer = RecordIndexer() @@ -88,13 +79,13 @@ def test_reindex(app, db, client): headers = [('Content-Type', 'application/json')] - url = url_for('invenio_records_rest.doc_item', pid_value='100') + url = url_for('invenio_records_rest.doc_item', pid_value='10000') response = client.get(url, headers=headers) data = response.json assert response.status_code == 200 - assert data['metadata']['title'] == 'The title of the record' + assert data['metadata']['pid'] == '10000' def test_get_pid_by_ref_link(app): @@ -104,19 +95,63 @@ def test_get_pid_by_ref_link(app): assert str(e.value) == 'falsy-link is not a valid ref link' pid = SonarRecord.get_pid_by_ref_link( - 'https://sonar.ch/api/institutions/usi') - assert pid == 'usi' + 'https://sonar.ch/api/documents/10000') + assert pid == '10000' -def test_get_record_by_ref_link(app): +def test_get_record_by_ref_link(app, document_fixture): """Test getting a record by a reference link.""" - DocumentRecord.create({ - "pid": "1", - "title": "The title of the record" - }, - dbcommit=True) record = DocumentRecord.get_record_by_ref_link( - 'https://sonar.ch/api/documents/1') - assert record['pid'] == '1' - assert record['title'] == 'The title of the record' + 'https://sonar.ch/api/documents/10000') + assert record['pid'] == '10000' + + +def test_add_file_from_url(app, document_fixture): + """Test add file to document by giving its URL.""" + document_fixture.add_file_from_url( + 'http://doc.rero.ch/record/328028/files/nor_irc.pdf', 'test.pdf') + + assert len(document_fixture.files) == 2 + assert document_fixture.files['test.pdf']['label'] == 'test.pdf' + + +@mock.patch('sonar.modules.api.extract_text_from_content') +def test_add_file(mock_extract, app, pdf_file, document_fixture): + """Test add file to document.""" + with open(pdf_file, 'rb') as file: + content = file.read() + + mock_extract.return_value = 'Fulltext content' + + # Successful file add + document_fixture.add_file(content, 'test1.pdf') + assert document_fixture.files['test1.pdf'] + assert document_fixture.files['test1.txt'] + + # Test already existing files + document_fixture.add_file(content, 'test1.pdf', size=4242167) + assert len(document_fixture.files) == 2 + + # Importing files is disabled + app.config['SONAR_DOCUMENTS_IMPORT_FILES'] = False + document_fixture.add_file(b'Hello, World', 'test3.pdf') + assert 'test3.pdf' not in document_fixture.files + + # Extracting fulltext is disabled + app.config['SONAR_DOCUMENTS_IMPORT_FILES'] = True + app.config['SONAR_DOCUMENTS_EXTRACT_FULLTEXT_ON_IMPORT'] = False + document_fixture.add_file(b'Hello, World', 'test4.pdf') + assert document_fixture.files['test4.pdf'] + assert 'test4.txt' not in document_fixture.files + + # Test exception when extracting fulltext + app.config['SONAR_DOCUMENTS_EXTRACT_FULLTEXT_ON_IMPORT'] = True + + def exception_side_effect(data): + raise Exception("Fulltext extraction error") + + mock_extract.side_effect = exception_side_effect + document_fixture.add_file(content, 'test5.pdf') + assert document_fixture.files['test5.pdf'] + assert 'test5.txt' not in document_fixture.files diff --git a/tests/ui/test_permissions.py b/tests/ui/test_permissions.py index 9c5e3467..e728727c 100644 --- a/tests/ui/test_permissions.py +++ b/tests/ui/test_permissions.py @@ -27,8 +27,7 @@ has_super_admin_access, has_user_access -def test_has_user_access(app, client, user_without_role_fixture, - user_fixture): +def test_has_user_access(app, client, user_without_role_fixture, user_fixture): """Test if user has an admin access.""" app.config.update(SONAR_APP_DISABLE_PERMISSION_CHECKS=True) @@ -41,9 +40,7 @@ def test_has_user_access(app, client, user_without_role_fixture, assert not has_user_access() - login_user_via_view(client, - email=user_fixture.email, - password='123456') + login_user_via_view(client, email=user_fixture.email, password='123456') assert not has_user_access() @@ -89,7 +86,8 @@ def test_has_super_admin_access(app, client, user_without_role_fixture, assert not has_super_admin_access() -def test_permissions_factories(app, client, admin_user_fixture): +def test_permissions_factories(app, client, admin_user_fixture, + document_fixture): """Test is user can list record.""" app.config.update(SONAR_APP_DISABLE_PERMISSION_CHECKS=True) @@ -108,14 +106,7 @@ def test_permissions_factories(app, client, admin_user_fixture): assert can_create_record_factory() assert can_update_record_factory() assert can_delete_record_factory() - - record = DocumentRecord.create( - { - "pid": "2", - "title": "The title of the record" - }, dbcommit=True) - - assert can_read_record_factory(record) + assert can_read_record_factory(document_fixture) def test_files_permission_factory(app, client, admin_user_fixture):