diff --git a/.env b/.env index e3ececc2e54..d5cffcec0aa 100644 --- a/.env +++ b/.env @@ -1,4 +1,5 @@ APP_IMAGE=gdcc/dataverse:unstable -POSTGRES_VERSION=13 +POSTGRES_VERSION=16 DATAVERSE_DB_USER=dataverse SOLR_VERSION=9.3.0 +SKIP_DEPLOY=0 \ No newline at end of file diff --git a/.github/workflows/maven_cache_management.yml b/.github/workflows/maven_cache_management.yml new file mode 100644 index 00000000000..fedf63b7c54 --- /dev/null +++ b/.github/workflows/maven_cache_management.yml @@ -0,0 +1,101 @@ +name: Maven Cache Management + +on: + # Every push to develop should trigger cache rejuvenation (dependencies might have changed) + push: + branches: + - develop + # According to https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#usage-limits-and-eviction-policy + # all caches are deleted after 7 days of no access. Make sure we rejuvenate every 7 days to keep it available. + schedule: + - cron: '23 2 * * 0' # Run for 'develop' every Sunday at 02:23 UTC (3:23 CET, 21:23 ET) + # Enable manual cache management + workflow_dispatch: + # Delete branch caches once a PR is merged + pull_request: + types: + - closed + +env: + COMMON_CACHE_KEY: "dataverse-maven-cache" + COMMON_CACHE_PATH: "~/.m2/repository" + +jobs: + seed: + name: Drop and Re-Seed Local Repository + runs-on: ubuntu-latest + if: ${{ github.event_name != 'pull_request' }} + permissions: + # Write permission needed to delete caches + # See also: https://docs.github.com/en/rest/actions/cache?apiVersion=2022-11-28#delete-a-github-actions-cache-for-a-repository-using-a-cache-id + actions: write + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Determine Java version from Parent POM + run: echo "JAVA_VERSION=$(grep '' modules/dataverse-parent/pom.xml | cut -f2 -d'>' | cut -f1 -d'<')" >> ${GITHUB_ENV} + - name: Set up JDK ${{ env.JAVA_VERSION }} + uses: actions/setup-java@v4 + with: + java-version: ${{ env.JAVA_VERSION }} + distribution: temurin + - name: Seed common cache + run: | + mvn -B -f modules/dataverse-parent dependency:go-offline dependency:resolve-plugins + # This non-obvious order is due to the fact that the download via Maven above will take a very long time (7-8 min). + # Jobs should not be left without a cache. Deleting and saving in one go leaves only a small chance for a cache miss. + - name: Drop common cache + run: | + gh extension install actions/gh-actions-cache + echo "🛒 Fetching list of cache keys" + cacheKeys=$(gh actions-cache list -R ${{ github.repository }} -B develop | cut -f 1 ) + + ## Setting this to not fail the workflow while deleting cache keys. + set +e + echo "🗑️ Deleting caches..." + for cacheKey in $cacheKeys + do + gh actions-cache delete $cacheKey -R ${{ github.repository }} -B develop --confirm + done + echo "✅ Done" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Save the common cache + uses: actions/cache@v4 + with: + path: ${{ env.COMMON_CACHE_PATH }} + key: ${{ env.COMMON_CACHE_KEY }} + enableCrossOsArchive: true + + # Let's delete feature branch caches once their PR is merged - we only have 10 GB of space before eviction kicks in + deplete: + name: Deplete feature branch caches + runs-on: ubuntu-latest + if: ${{ github.event_name == 'pull_request' }} + permissions: + # `actions:write` permission is required to delete caches + # See also: https://docs.github.com/en/rest/actions/cache?apiVersion=2022-11-28#delete-a-github-actions-cache-for-a-repository-using-a-cache-id + actions: write + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Cleanup caches + run: | + gh extension install actions/gh-actions-cache + + BRANCH=refs/pull/${{ github.event.pull_request.number }}/merge + echo "🛒 Fetching list of cache keys" + cacheKeysForPR=$(gh actions-cache list -R ${{ github.repository }} -B $BRANCH | cut -f 1 ) + + ## Setting this to not fail the workflow while deleting cache keys. + set +e + echo "🗑️ Deleting caches..." + for cacheKey in $cacheKeysForPR + do + gh actions-cache delete $cacheKey -R ${{ github.repository }} -B $BRANCH --confirm + done + echo "✅ Done" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/maven_unit_test.yml b/.github/workflows/maven_unit_test.yml index a70c55fc31d..4ad4798bc64 100644 --- a/.github/workflows/maven_unit_test.yml +++ b/.github/workflows/maven_unit_test.yml @@ -4,6 +4,7 @@ on: push: paths: - "**.java" + - "**.sql" - "pom.xml" - "modules/**/pom.xml" - "!modules/container-base/**" @@ -11,6 +12,7 @@ on: pull_request: paths: - "**.java" + - "**.sql" - "pom.xml" - "modules/**/pom.xml" - "!modules/container-base/**" diff --git a/.gitignore b/.gitignore index 7f0d3a2b466..514f82116de 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,7 @@ oauth-credentials.md /src/main/webapp/oauth2/newAccount.html scripts/api/setup-all.sh* scripts/api/setup-all.*.log +src/main/resources/edu/harvard/iq/dataverse/openapi/ # ctags generated tag file tags @@ -61,3 +62,4 @@ src/main/webapp/resources/images/dataverseproject.png.thumb140 # Docker development volumes /docker-dev-volumes +/.vs diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index b0864a0c55f..00000000000 --- a/Dockerfile +++ /dev/null @@ -1 +0,0 @@ -# See http://guides.dataverse.org/en/latest/developers/containers.html diff --git a/README.md b/README.md index 831dbfed5ff..77720453d5f 100644 --- a/README.md +++ b/README.md @@ -3,11 +3,11 @@ Dataverse® Dataverse is an [open source][] software platform for sharing, finding, citing, and preserving research data (developed by the [Dataverse team](https://dataverse.org/about) at the [Institute for Quantitative Social Science](https://iq.harvard.edu/) and the [Dataverse community][]). -[dataverse.org][] is our home on the web and shows a map of Dataverse installations around the world, a list of [features][], [integrations][] that have been made possible through [REST APIs][], our development [roadmap][], and more. +[dataverse.org][] is our home on the web and shows a map of Dataverse installations around the world, a list of [features][], [integrations][] that have been made possible through [REST APIs][], our [project board][], our development [roadmap][], and more. We maintain a demo site at [demo.dataverse.org][] which you are welcome to use for testing and evaluating Dataverse. -To install Dataverse, please see our [Installation Guide][] which will prompt you to download our [latest release][]. +To install Dataverse, please see our [Installation Guide][] which will prompt you to download our [latest release][]. Docker users should consult the [Container Guide][]. To discuss Dataverse with the community, please join our [mailing list][], participate in a [community call][], chat with us at [chat.dataverse.org][], or attend our annual [Dataverse Community Meeting][]. @@ -28,7 +28,9 @@ Dataverse is a trademark of President and Fellows of Harvard College and is regi [Dataverse community]: https://dataverse.org/developers [Installation Guide]: https://guides.dataverse.org/en/latest/installation/index.html [latest release]: https://github.com/IQSS/dataverse/releases +[Container Guide]: https://guides.dataverse.org/en/latest/container/index.html [features]: https://dataverse.org/software-features +[project board]: https://github.com/orgs/IQSS/projects/34 [roadmap]: https://www.iq.harvard.edu/roadmap-dataverse-project [integrations]: https://dataverse.org/integrations [REST APIs]: https://guides.dataverse.org/en/latest/api/index.html diff --git a/conf/proxy/Caddyfile b/conf/proxy/Caddyfile new file mode 100644 index 00000000000..70e6904d26e --- /dev/null +++ b/conf/proxy/Caddyfile @@ -0,0 +1,12 @@ +# This configuration is intended to be used with Caddy, a very small high perf proxy. +# It will serve the application containers Payara Admin GUI via HTTP instead of HTTPS, +# avoiding the trouble of self signed certificates for local development. + +:4848 { + reverse_proxy https://dataverse:4848 { + transport http { + tls_insecure_skip_verify + } + header_down Location "^https://" "http://" + } +} diff --git a/conf/solr/9.3.0/schema.xml b/conf/solr/schema.xml similarity index 99% rename from conf/solr/9.3.0/schema.xml rename to conf/solr/schema.xml index 3711ffeddba..5dde750573d 100644 --- a/conf/solr/9.3.0/schema.xml +++ b/conf/solr/schema.xml @@ -157,7 +157,8 @@ - + + @@ -229,6 +230,8 @@ + + - ${solr.autoCommit.maxTime:15000} + ${solr.autoCommit.maxTime:30000} false @@ -301,7 +301,7 @@ --> - ${solr.autoSoftCommit.maxTime:-1} + ${solr.autoSoftCommit.maxTime:1000} - conf/solr/9.3.0 + conf/solr solr diff --git a/modules/container-configbaker/scripts/bootstrap/demo/init.sh b/modules/container-configbaker/scripts/bootstrap/demo/init.sh new file mode 100644 index 00000000000..e8d1d07dd2d --- /dev/null +++ b/modules/container-configbaker/scripts/bootstrap/demo/init.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +set -euo pipefail + +# Set some defaults +DATAVERSE_URL=${DATAVERSE_URL:-"http://dataverse:8080"} +export DATAVERSE_URL + +BLOCKED_API_KEY=${BLOCKED_API_KEY:-"unblockme"} +export BLOCKED_API_KEY + +# --insecure is used so we can configure a few things but +# later in this script we'll apply the changes as if we had +# run the script without --insecure. +echo "Running base setup-all.sh..." +"${BOOTSTRAP_DIR}"/base/setup-all.sh --insecure -p=admin1 | tee /tmp/setup-all.sh.out + +echo "" +echo "Setting DOI provider to \"FAKE\"..." +curl -sS -X PUT -d FAKE "${DATAVERSE_URL}/api/admin/settings/:DoiProvider" + +echo "" +echo "Revoke the key that allows for creation of builtin users..." +curl -sS -X DELETE "${DATAVERSE_URL}/api/admin/settings/BuiltinUsers.KEY" + +echo "" +echo "Set key for accessing blocked API endpoints..." +curl -sS -X PUT -d "$BLOCKED_API_KEY" "${DATAVERSE_URL}/api/admin/settings/:BlockedApiKey" + +echo "" +echo "Set policy to only allow access to admin APIs with with a key..." +curl -sS -X PUT -d unblock-key "${DATAVERSE_URL}/api/admin/settings/:BlockedApiPolicy" + +echo "" +echo "Block admin and other sensitive API endpoints..." +curl -sS -X PUT -d 'admin,builtin-users' "${DATAVERSE_URL}/api/admin/settings/:BlockedApiEndpoints" + +echo "" +echo "Done, your instance has been configured for demo or eval. Have a nice day!" diff --git a/modules/container-configbaker/scripts/bootstrap/dev/init.sh b/modules/container-configbaker/scripts/bootstrap/dev/init.sh index efdaee3d0c3..f8770436652 100644 --- a/modules/container-configbaker/scripts/bootstrap/dev/init.sh +++ b/modules/container-configbaker/scripts/bootstrap/dev/init.sh @@ -9,9 +9,6 @@ export DATAVERSE_URL echo "Running base setup-all.sh (INSECURE MODE)..." "${BOOTSTRAP_DIR}"/base/setup-all.sh --insecure -p=admin1 | tee /tmp/setup-all.sh.out -echo "Setting system mail address..." -curl -X PUT -d "dataverse@localhost" "${DATAVERSE_URL}/api/admin/settings/:SystemEmail" - echo "Setting DOI provider to \"FAKE\"..." curl "${DATAVERSE_URL}/api/admin/settings/:DoiProvider" -X PUT -d FAKE diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml index 386d4934cb1..02f601ba0f6 100644 --- a/modules/dataverse-parent/pom.xml +++ b/modules/dataverse-parent/pom.xml @@ -131,7 +131,7 @@ - 6.1 + 6.2 17 UTF-8 @@ -149,18 +149,18 @@ 6.2023.8 - 42.6.0 - 9.3.0 + 42.7.2 + 9.4.1 1.12.290 26.30.0 8.0.0 1.7.35 - 2.11.0 + 2.15.1 1.2 3.12.0 - 1.21 + 1.26.0 4.5.13 4.4.14 @@ -168,11 +168,11 @@ 5.2.0 - 1.19.0 - 2.10.1 - 5.10.0 - 5.4.0 - 0.8.10 + 1.19.7 + 3.7.1 + 5.10.2 + 5.11.0 + 0.8.11 9.3 @@ -182,8 +182,8 @@ 3.3.2 3.5.0 3.1.1 - 3.1.0 - 3.1.0 + 3.2.5 + 3.2.5 3.6.0 3.3.1 3.0.0-M7 diff --git a/modules/nginx/Dockerfile b/modules/nginx/Dockerfile deleted file mode 100644 index 3900076599f..00000000000 --- a/modules/nginx/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM nginx:latest - -# Remove the default NGINX configuration file -RUN rm /etc/nginx/conf.d/default.conf - -# Copy the contents of the local default.conf to the container -COPY default.conf /etc/nginx/conf.d/ - -EXPOSE 4849 \ No newline at end of file diff --git a/modules/nginx/README.md b/modules/nginx/README.md deleted file mode 100644 index 9d2ff785577..00000000000 --- a/modules/nginx/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# nginx proxy - -nginx can be used to proxy various services at other ports/protocols from docker. - -Currently, this is used to work around a problem with the IntelliJ Payara plugin, which doesn't allow remote redeployment in case the Payara admin is served via HTTPS using a self-signed certificate, which is the case of the default dataverse container installation. This configuration provides an HTTP endpoint at port 4849, and proxies requests to the Payara admin console's HTTPS 4848 endpoint. From the IntelliJ Payara plugin one has to specify the localhost 4849 port (without SSL). - -![img.png](img.png) diff --git a/modules/nginx/default.conf b/modules/nginx/default.conf deleted file mode 100644 index 8381a66c19a..00000000000 --- a/modules/nginx/default.conf +++ /dev/null @@ -1,12 +0,0 @@ -server { - listen 4849; - - # Make it big, so that .war files can be submitted - client_max_body_size 300M; - - location / { - proxy_pass https://dataverse:4848; - proxy_ssl_verify off; - proxy_ssl_server_name on; - } -} diff --git a/pom.xml b/pom.xml index f45e8fd9033..8fa1d993f6e 100644 --- a/pom.xml +++ b/pom.xml @@ -32,6 +32,12 @@ 5.2.1 2.4.1 5.5.3 + + Dataverse API + ${project.version} + Open source research data repository software. + + ${project.build.outputDirectory}/META-INF - + + + jakarta.json.bind + jakarta.json.bind-api + + + + org.eclipse + yasson + test + + org.glassfish @@ -283,7 +322,7 @@ org.apache.solr solr-solrj - 9.3.0 + 9.4.1 colt @@ -532,6 +571,12 @@ java-json-canonicalization 1.1 + + + io.gdcc + sitemapgen4j + 2.1.2 + edu.ucar cdm-core @@ -542,6 +587,10 @@ dataverse-spi 2.0.0 + + javax.cache + cache-api + org.junit.jupiter @@ -653,6 +702,11 @@ 3.9.0 test + + com.hazelcast + hazelcast + test + @@ -682,6 +736,7 @@ **/firstNames/*.* **/*.xsl **/services/* + **/*.map @@ -721,6 +776,7 @@ true false + ${project.build.directory}/${project.artifactId} true @@ -885,6 +941,30 @@ true + + io.smallrye + smallrye-open-api-maven-plugin + 3.10.0 + + + + generate-schema + + + process-classes + + ${openapi.outputDirectory} + openapi + ${openapi.infoTitle} + ${openapi.infoVersion} + ${openapi.infoDescription} + CLASS_METHOD + edu.harvard.iq.dataverse + true + + + + @@ -915,6 +995,7 @@ gdcc/dataverse:${app.image.tag} unstable + false gdcc/base:${base.image.tag} unstable gdcc/configbaker:${conf.image.tag} @@ -927,6 +1008,7 @@ ${postgresql.server.version} ${solr.version} dataverse + ${app.skipDeploy} @@ -1012,14 +1094,6 @@ true - - - gdcc/dev_nginx:unstable - - ${project.basedir}/modules/nginx - - - true @@ -1048,4 +1122,4 @@ - + \ No newline at end of file diff --git a/scripts/api/data/dataset-create-new-all-default-fields.json b/scripts/api/data/dataset-create-new-all-default-fields.json index 1118ed98a03..cc856c6372f 100644 --- a/scripts/api/data/dataset-create-new-all-default-fields.json +++ b/scripts/api/data/dataset-create-new-all-default-fields.json @@ -231,6 +231,12 @@ "typeClass": "primitive", "value": "KeywordTerm1" }, + "keywordTermURI": { + "typeName": "keywordTermURI", + "multiple": false, + "typeClass": "primitive", + "value": "http://keywordTermURI1.org" + }, "keywordVocabulary": { "typeName": "keywordVocabulary", "multiple": false, @@ -251,6 +257,12 @@ "typeClass": "primitive", "value": "KeywordTerm2" }, + "keywordTermURI": { + "typeName": "keywordTermURI", + "multiple": false, + "typeClass": "primitive", + "value": "http://keywordTermURI2.org" + }, "keywordVocabulary": { "typeName": "keywordVocabulary", "multiple": false, @@ -907,14 +919,14 @@ "typeClass": "primitive", "value": "-70" }, - "northLongitude": { - "typeName": "northLongitude", + "northLatitude": { + "typeName": "northLatitude", "multiple": false, "typeClass": "primitive", "value": "43" }, - "southLongitude": { - "typeName": "southLongitude", + "southLatitude": { + "typeName": "southLatitude", "multiple": false, "typeClass": "primitive", "value": "42" @@ -933,14 +945,14 @@ "typeClass": "primitive", "value": "-13" }, - "northLongitude": { - "typeName": "northLongitude", + "northLatitude": { + "typeName": "northLatitude", "multiple": false, "typeClass": "primitive", "value": "29" }, - "southLongitude": { - "typeName": "southLongitude", + "southLatitude": { + "typeName": "southLatitude", "multiple": false, "typeClass": "primitive", "value": "28" diff --git a/scripts/api/data/licenses/licenseApache-2.0.json b/scripts/api/data/licenses/licenseApache-2.0.json new file mode 100644 index 00000000000..5b7c3cf5c95 --- /dev/null +++ b/scripts/api/data/licenses/licenseApache-2.0.json @@ -0,0 +1,8 @@ +{ + "name": "Apache-2.0", + "uri": "http://www.apache.org/licenses/LICENSE-2.0", + "shortDescription": "Apache License 2.0", + "active": true, + "sortOrder": 9 + } + \ No newline at end of file diff --git a/scripts/api/data/licenses/licenseMIT.json b/scripts/api/data/licenses/licenseMIT.json new file mode 100644 index 00000000000..a879e8a5595 --- /dev/null +++ b/scripts/api/data/licenses/licenseMIT.json @@ -0,0 +1,7 @@ +{ + "name": "MIT", + "uri": "https://opensource.org/licenses/MIT", + "shortDescription": "MIT License", + "active": true, + "sortOrder": 8 +} diff --git a/scripts/api/data/metadatablocks/astrophysics.tsv b/scripts/api/data/metadatablocks/astrophysics.tsv index 4039d32cb75..92792d404c9 100644 --- a/scripts/api/data/metadatablocks/astrophysics.tsv +++ b/scripts/api/data/metadatablocks/astrophysics.tsv @@ -2,13 +2,13 @@ astrophysics Astronomy and Astrophysics Metadata #datasetField name title description watermark fieldType displayOrder displayFormat advancedSearchField allowControlledVocabulary allowmultiples facetable displayoncreate required parent metadatablock_id astroType Type The nature or genre of the content of the files in the dataset. text 0 TRUE TRUE TRUE TRUE FALSE FALSE astrophysics - astroFacility Facility The observatory or facility where the data was obtained. text 1 TRUE TRUE TRUE TRUE FALSE FALSE astrophysics - astroInstrument Instrument The instrument used to collect the data. text 2 TRUE TRUE TRUE TRUE FALSE FALSE astrophysics + astroFacility Facility The observatory or facility where the data was obtained. text 1 TRUE FALSE TRUE TRUE FALSE FALSE astrophysics + astroInstrument Instrument The instrument used to collect the data. text 2 TRUE FALSE TRUE TRUE FALSE FALSE astrophysics astroObject Object Astronomical Objects represented in the data (Given as SIMBAD recognizable names preferred). text 3 TRUE FALSE TRUE TRUE FALSE FALSE astrophysics resolution.Spatial Spatial Resolution The spatial (angular) resolution that is typical of the observations, in decimal degrees. text 4 TRUE FALSE FALSE TRUE FALSE FALSE astrophysics resolution.Spectral Spectral Resolution The spectral resolution that is typical of the observations, given as the ratio \u03bb/\u0394\u03bb. text 5 TRUE FALSE FALSE TRUE FALSE FALSE astrophysics resolution.Temporal Time Resolution The temporal resolution that is typical of the observations, given in seconds. text 6 FALSE FALSE FALSE FALSE FALSE FALSE astrophysics - coverage.Spectral.Bandpass Bandpass Conventional bandpass name text 7 TRUE TRUE TRUE TRUE FALSE FALSE astrophysics + coverage.Spectral.Bandpass Bandpass Conventional bandpass name text 7 TRUE FALSE TRUE TRUE FALSE FALSE astrophysics coverage.Spectral.CentralWavelength Central Wavelength (m) The central wavelength of the spectral bandpass, in meters. Enter a floating-point number. float 8 TRUE FALSE TRUE TRUE FALSE FALSE astrophysics coverage.Spectral.Wavelength Wavelength Range The minimum and maximum wavelength of the spectral bandpass. Enter a floating-point number. none 9 FALSE FALSE TRUE FALSE FALSE FALSE astrophysics coverage.Spectral.MinimumWavelength Minimum (m) The minimum wavelength of the spectral bandpass, in meters. Enter a floating-point number. float 10 TRUE FALSE FALSE TRUE FALSE FALSE coverage.Spectral.Wavelength astrophysics diff --git a/scripts/api/data/metadatablocks/biomedical.tsv b/scripts/api/data/metadatablocks/biomedical.tsv index 28d59130c34..06f1ebec1b4 100644 --- a/scripts/api/data/metadatablocks/biomedical.tsv +++ b/scripts/api/data/metadatablocks/biomedical.tsv @@ -13,7 +13,7 @@ studyAssayOtherTechnologyType Other Technology Type If Other was selected in Technology Type, list any other technology types that were used in this Dataset. text 9 TRUE FALSE TRUE TRUE FALSE FALSE biomedical studyAssayPlatform Technology Platform The manufacturer and name of the technology platform used in the assay (e.g. Bruker AVANCE). text 10 TRUE TRUE TRUE TRUE FALSE FALSE biomedical studyAssayOtherPlatform Other Technology Platform If Other was selected in Technology Platform, list any other technology platforms that were used in this Dataset. text 11 TRUE FALSE TRUE TRUE FALSE FALSE biomedical - studyAssayCellType Cell Type The name of the cell line from which the source or sample derives. text 12 TRUE TRUE TRUE TRUE FALSE FALSE biomedical + studyAssayCellType Cell Type The name of the cell line from which the source or sample derives. text 12 TRUE FALSE TRUE TRUE FALSE FALSE biomedical #controlledVocabulary DatasetField Value identifier displayOrder studyDesignType Case Control EFO_0001427 0 studyDesignType Cross Sectional EFO_0001428 1 @@ -45,6 +45,7 @@ studyFactorType Treatment Compound EFO_0000369 17 studyFactorType Treatment Type EFO_0000727 18 studyFactorType Other OTHER_FACTOR 19 + studyAssayMeasurementType cell counting ERO_0001899 0 studyAssayMeasurementType cell sorting CHMO_0001085 1 studyAssayMeasurementType clinical chemistry analysis OBI_0000520 2 studyAssayMeasurementType copy number variation profiling OBI_0000537 3 diff --git a/scripts/api/data/metadatablocks/citation.tsv b/scripts/api/data/metadatablocks/citation.tsv index b21b6bcce57..18354f2b1f7 100644 --- a/scripts/api/data/metadatablocks/citation.tsv +++ b/scripts/api/data/metadatablocks/citation.tsv @@ -23,62 +23,63 @@ subject Subject The area of study relevant to the Dataset text 19 TRUE TRUE TRUE TRUE TRUE TRUE citation http://purl.org/dc/terms/subject keyword Keyword A key term that describes an important aspect of the Dataset and information about any controlled vocabulary used none 20 FALSE FALSE TRUE FALSE TRUE FALSE citation keywordValue Term A key term that describes important aspects of the Dataset text 21 #VALUE TRUE FALSE FALSE TRUE TRUE FALSE keyword citation - keywordVocabulary Controlled Vocabulary Name The controlled vocabulary used for the keyword term (e.g. LCSH, MeSH) text 22 (#VALUE) FALSE FALSE FALSE FALSE TRUE FALSE keyword citation - keywordVocabularyURI Controlled Vocabulary URL The URL where one can access information about the term's controlled vocabulary https:// url 23 #VALUE FALSE FALSE FALSE FALSE TRUE FALSE keyword citation - topicClassification Topic Classification Indicates a broad, important topic or subject that the Dataset covers and information about any controlled vocabulary used none 24 FALSE FALSE TRUE FALSE FALSE FALSE citation - topicClassValue Term A topic or subject term text 25 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE topicClassification citation - topicClassVocab Controlled Vocabulary Name The controlled vocabulary used for the keyword term (e.g. LCSH, MeSH) text 26 (#VALUE) FALSE FALSE FALSE FALSE FALSE FALSE topicClassification citation - topicClassVocabURI Controlled Vocabulary URL The URL where one can access information about the term's controlled vocabulary https:// url 27 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE topicClassification citation - publication Related Publication The article or report that uses the data in the Dataset. The full list of related publications will be displayed on the metadata tab none 28 FALSE FALSE TRUE FALSE TRUE FALSE citation http://purl.org/dc/terms/isReferencedBy - publicationCitation Citation The full bibliographic citation for the related publication textbox 29 #VALUE TRUE FALSE FALSE FALSE TRUE FALSE publication citation http://purl.org/dc/terms/bibliographicCitation - publicationIDType Identifier Type The type of identifier that uniquely identifies a related publication text 30 #VALUE: TRUE TRUE FALSE FALSE TRUE FALSE publication citation http://purl.org/spar/datacite/ResourceIdentifierScheme - publicationIDNumber Identifier The identifier for a related publication text 31 #VALUE TRUE FALSE FALSE FALSE TRUE FALSE publication citation http://purl.org/spar/datacite/ResourceIdentifier - publicationURL URL The URL form of the identifier entered in the Identifier field, e.g. the DOI URL if a DOI was entered in the Identifier field. Used to display what was entered in the ID Type and ID Number fields as a link. If what was entered in the Identifier field has no URL form, the URL of the publication webpage is used, e.g. a journal article webpage https:// url 32 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE publication citation https://schema.org/distribution - notesText Notes Additional information about the Dataset textbox 33 FALSE FALSE FALSE FALSE TRUE FALSE citation - language Language A language that the Dataset's files is written in text 34 TRUE TRUE TRUE TRUE FALSE FALSE citation http://purl.org/dc/terms/language - producer Producer The entity, such a person or organization, managing the finances or other administrative processes involved in the creation of the Dataset none 35 FALSE FALSE TRUE FALSE FALSE FALSE citation - producerName Name The name of the entity, e.g. the person's name or the name of an organization 1) FamilyName, GivenName or 2) Organization text 36 #VALUE TRUE FALSE FALSE TRUE FALSE TRUE producer citation - producerAffiliation Affiliation The name of the entity affiliated with the producer, e.g. an organization's name Organization XYZ text 37 (#VALUE) FALSE FALSE FALSE FALSE FALSE FALSE producer citation - producerAbbreviation Abbreviated Name The producer's abbreviated name (e.g. IQSS, ICPSR) text 38 (#VALUE) FALSE FALSE FALSE FALSE FALSE FALSE producer citation - producerURL URL The URL of the producer's website https:// url 39 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE producer citation - producerLogoURL Logo URL The URL of the producer's logo https:// url 40
FALSE FALSE FALSE FALSE FALSE FALSE producer citation - productionDate Production Date The date when the data were produced (not distributed, published, or archived) YYYY-MM-DD date 41 TRUE FALSE FALSE TRUE FALSE FALSE citation - productionPlace Production Location The location where the data and any related materials were produced or collected text 42 TRUE FALSE TRUE TRUE FALSE FALSE citation - contributor Contributor The entity, such as a person or organization, responsible for collecting, managing, or otherwise contributing to the development of the Dataset none 43 : FALSE FALSE TRUE FALSE FALSE FALSE citation http://purl.org/dc/terms/contributor - contributorType Type Indicates the type of contribution made to the dataset text 44 #VALUE TRUE TRUE FALSE TRUE FALSE FALSE contributor citation - contributorName Name The name of the contributor, e.g. the person's name or the name of an organization 1) FamilyName, GivenName or 2) Organization text 45 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE contributor citation - grantNumber Funding Information Information about the Dataset's financial support none 46 : FALSE FALSE TRUE FALSE FALSE FALSE citation https://schema.org/sponsor - grantNumberAgency Agency The agency that provided financial support for the Dataset Organization XYZ text 47 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE grantNumber citation - grantNumberValue Identifier The grant identifier or contract identifier of the agency that provided financial support for the Dataset text 48 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE grantNumber citation - distributor Distributor The entity, such as a person or organization, designated to generate copies of the Dataset, including any editions or revisions none 49 FALSE FALSE TRUE FALSE FALSE FALSE citation - distributorName Name The name of the entity, e.g. the person's name or the name of an organization 1) FamilyName, GivenName or 2) Organization text 50 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE distributor citation - distributorAffiliation Affiliation The name of the entity affiliated with the distributor, e.g. an organization's name Organization XYZ text 51 (#VALUE) FALSE FALSE FALSE FALSE FALSE FALSE distributor citation - distributorAbbreviation Abbreviated Name The distributor's abbreviated name (e.g. IQSS, ICPSR) text 52 (#VALUE) FALSE FALSE FALSE FALSE FALSE FALSE distributor citation - distributorURL URL The URL of the distributor's webpage https:// url 53 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE distributor citation - distributorLogoURL Logo URL The URL of the distributor's logo image, used to show the image on the Dataset's page https:// url 54
FALSE FALSE FALSE FALSE FALSE FALSE distributor citation - distributionDate Distribution Date The date when the Dataset was made available for distribution/presentation YYYY-MM-DD date 55 TRUE FALSE FALSE TRUE FALSE FALSE citation - depositor Depositor The entity, such as a person or organization, that deposited the Dataset in the repository 1) FamilyName, GivenName or 2) Organization text 56 FALSE FALSE FALSE FALSE FALSE FALSE citation - dateOfDeposit Deposit Date The date when the Dataset was deposited into the repository YYYY-MM-DD date 57 FALSE FALSE FALSE TRUE FALSE FALSE citation http://purl.org/dc/terms/dateSubmitted - timePeriodCovered Time Period The time period that the data refer to. Also known as span. This is the time period covered by the data, not the dates of coding, collecting data, or making documents machine-readable none 58 ; FALSE FALSE TRUE FALSE FALSE FALSE citation https://schema.org/temporalCoverage - timePeriodCoveredStart Start Date The start date of the time period that the data refer to YYYY-MM-DD date 59 #NAME: #VALUE TRUE FALSE FALSE TRUE FALSE FALSE timePeriodCovered citation - timePeriodCoveredEnd End Date The end date of the time period that the data refer to YYYY-MM-DD date 60 #NAME: #VALUE TRUE FALSE FALSE TRUE FALSE FALSE timePeriodCovered citation - dateOfCollection Date of Collection The dates when the data were collected or generated none 61 ; FALSE FALSE TRUE FALSE FALSE FALSE citation - dateOfCollectionStart Start Date The date when the data collection started YYYY-MM-DD date 62 #NAME: #VALUE FALSE FALSE FALSE FALSE FALSE FALSE dateOfCollection citation - dateOfCollectionEnd End Date The date when the data collection ended YYYY-MM-DD date 63 #NAME: #VALUE FALSE FALSE FALSE FALSE FALSE FALSE dateOfCollection citation - kindOfData Data Type The type of data included in the files (e.g. survey data, clinical data, or machine-readable text) text 64 TRUE FALSE TRUE TRUE FALSE FALSE citation http://rdf-vocabulary.ddialliance.org/discovery#kindOfData - series Series Information about the dataset series to which the Dataset belong none 65 : FALSE FALSE TRUE FALSE FALSE FALSE citation - seriesName Name The name of the dataset series text 66 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE series citation - seriesInformation Information Can include 1) a history of the series and 2) a summary of features that apply to the series textbox 67 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE series citation - software Software Information about the software used to generate the Dataset none 68 , FALSE FALSE TRUE FALSE FALSE FALSE citation https://www.w3.org/TR/prov-o/#wasGeneratedBy - softwareName Name The name of software used to generate the Dataset text 69 #VALUE FALSE TRUE FALSE FALSE FALSE FALSE software citation - softwareVersion Version The version of the software used to generate the Dataset, e.g. 4.11 text 70 #NAME: #VALUE FALSE FALSE FALSE FALSE FALSE FALSE software citation - relatedMaterial Related Material Information, such as a persistent ID or citation, about the material related to the Dataset, such as appendices or sampling information available outside of the Dataset textbox 71 FALSE FALSE TRUE FALSE FALSE FALSE citation - relatedDatasets Related Dataset Information, such as a persistent ID or citation, about a related dataset, such as previous research on the Dataset's subject textbox 72 FALSE FALSE TRUE FALSE FALSE FALSE citation http://purl.org/dc/terms/relation - otherReferences Other Reference Information, such as a persistent ID or citation, about another type of resource that provides background or supporting material to the Dataset text 73 FALSE FALSE TRUE FALSE FALSE FALSE citation http://purl.org/dc/terms/references - dataSources Data Source Information, such as a persistent ID or citation, about sources of the Dataset (e.g. a book, article, serial, or machine-readable data file) textbox 74 FALSE FALSE TRUE FALSE FALSE FALSE citation https://www.w3.org/TR/prov-o/#wasDerivedFrom - originOfSources Origin of Historical Sources For historical sources, the origin and any rules followed in establishing them as sources textbox 75 FALSE FALSE FALSE FALSE FALSE FALSE citation - characteristicOfSources Characteristic of Sources Characteristics not already noted elsewhere textbox 76 FALSE FALSE FALSE FALSE FALSE FALSE citation - accessToSources Documentation and Access to Sources 1) Methods or procedures for accessing data sources and 2) any special permissions needed for access textbox 77 FALSE FALSE FALSE FALSE FALSE FALSE citation + keywordTermURI Term URI A URI that points to the web presence of the Keyword Term https:// url 22 #VALUE FALSE FALSE FALSE FALSE TRUE FALSE keyword citation + keywordVocabulary Controlled Vocabulary Name The controlled vocabulary used for the keyword term (e.g. LCSH, MeSH) text 23 (#VALUE) FALSE FALSE FALSE FALSE TRUE FALSE keyword citation + keywordVocabularyURI Controlled Vocabulary URL The URL where one can access information about the term's controlled vocabulary https:// url 24 #VALUE FALSE FALSE FALSE FALSE TRUE FALSE keyword citation + topicClassification Topic Classification Indicates a broad, important topic or subject that the Dataset covers and information about any controlled vocabulary used none 25 FALSE FALSE TRUE FALSE FALSE FALSE citation + topicClassValue Term A topic or subject term text 26 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE topicClassification citation + topicClassVocab Controlled Vocabulary Name The controlled vocabulary used for the keyword term (e.g. LCSH, MeSH) text 27 (#VALUE) FALSE FALSE FALSE FALSE FALSE FALSE topicClassification citation + topicClassVocabURI Controlled Vocabulary URL The URL where one can access information about the term's controlled vocabulary https:// url 28 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE topicClassification citation + publication Related Publication The article or report that uses the data in the Dataset. The full list of related publications will be displayed on the metadata tab none 29 FALSE FALSE TRUE FALSE TRUE FALSE citation http://purl.org/dc/terms/isReferencedBy + publicationCitation Citation The full bibliographic citation for the related publication textbox 30 #VALUE TRUE FALSE FALSE FALSE TRUE FALSE publication citation http://purl.org/dc/terms/bibliographicCitation + publicationIDType Identifier Type The type of identifier that uniquely identifies a related publication text 31 #VALUE: TRUE TRUE FALSE FALSE TRUE FALSE publication citation http://purl.org/spar/datacite/ResourceIdentifierScheme + publicationIDNumber Identifier The identifier for a related publication text 32 #VALUE TRUE FALSE FALSE FALSE TRUE FALSE publication citation http://purl.org/spar/datacite/ResourceIdentifier + publicationURL URL The URL form of the identifier entered in the Identifier field, e.g. the DOI URL if a DOI was entered in the Identifier field. Used to display what was entered in the ID Type and ID Number fields as a link. If what was entered in the Identifier field has no URL form, the URL of the publication webpage is used, e.g. a journal article webpage https:// url 33 #VALUE FALSE FALSE FALSE FALSE TRUE FALSE publication citation https://schema.org/distribution + notesText Notes Additional information about the Dataset textbox 34 FALSE FALSE FALSE FALSE TRUE FALSE citation + language Language A language that the Dataset's files is written in text 35 TRUE TRUE TRUE TRUE FALSE FALSE citation http://purl.org/dc/terms/language + producer Producer The entity, such a person or organization, managing the finances or other administrative processes involved in the creation of the Dataset none 36 FALSE FALSE TRUE FALSE FALSE FALSE citation + producerName Name The name of the entity, e.g. the person's name or the name of an organization 1) FamilyName, GivenName or 2) Organization text 37 #VALUE TRUE FALSE FALSE TRUE FALSE TRUE producer citation + producerAffiliation Affiliation The name of the entity affiliated with the producer, e.g. an organization's name Organization XYZ text 38 (#VALUE) FALSE FALSE FALSE FALSE FALSE FALSE producer citation + producerAbbreviation Abbreviated Name The producer's abbreviated name (e.g. IQSS, ICPSR) text 39 (#VALUE) FALSE FALSE FALSE FALSE FALSE FALSE producer citation + producerURL URL The URL of the producer's website https:// url 40 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE producer citation + producerLogoURL Logo URL The URL of the producer's logo https:// url 41
FALSE FALSE FALSE FALSE FALSE FALSE producer citation + productionDate Production Date The date when the data were produced (not distributed, published, or archived) YYYY-MM-DD date 42 TRUE FALSE FALSE TRUE FALSE FALSE citation + productionPlace Production Location The location where the data and any related materials were produced or collected text 43 TRUE FALSE TRUE TRUE FALSE FALSE citation + contributor Contributor The entity, such as a person or organization, responsible for collecting, managing, or otherwise contributing to the development of the Dataset none 44 : FALSE FALSE TRUE FALSE FALSE FALSE citation http://purl.org/dc/terms/contributor + contributorType Type Indicates the type of contribution made to the dataset text 45 #VALUE TRUE TRUE FALSE TRUE FALSE FALSE contributor citation + contributorName Name The name of the contributor, e.g. the person's name or the name of an organization 1) FamilyName, GivenName or 2) Organization text 46 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE contributor citation + grantNumber Funding Information Information about the Dataset's financial support none 47 : FALSE FALSE TRUE FALSE FALSE FALSE citation https://schema.org/sponsor + grantNumberAgency Agency The agency that provided financial support for the Dataset Organization XYZ text 48 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE grantNumber citation + grantNumberValue Identifier The grant identifier or contract identifier of the agency that provided financial support for the Dataset text 49 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE grantNumber citation + distributor Distributor The entity, such as a person or organization, designated to generate copies of the Dataset, including any editions or revisions none 50 FALSE FALSE TRUE FALSE FALSE FALSE citation + distributorName Name The name of the entity, e.g. the person's name or the name of an organization 1) FamilyName, GivenName or 2) Organization text 51 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE distributor citation + distributorAffiliation Affiliation The name of the entity affiliated with the distributor, e.g. an organization's name Organization XYZ text 52 (#VALUE) FALSE FALSE FALSE FALSE FALSE FALSE distributor citation + distributorAbbreviation Abbreviated Name The distributor's abbreviated name (e.g. IQSS, ICPSR) text 53 (#VALUE) FALSE FALSE FALSE FALSE FALSE FALSE distributor citation + distributorURL URL The URL of the distributor's webpage https:// url 54 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE distributor citation + distributorLogoURL Logo URL The URL of the distributor's logo image, used to show the image on the Dataset's page https:// url 55
FALSE FALSE FALSE FALSE FALSE FALSE distributor citation + distributionDate Distribution Date The date when the Dataset was made available for distribution/presentation YYYY-MM-DD date 56 TRUE FALSE FALSE TRUE FALSE FALSE citation + depositor Depositor The entity, such as a person or organization, that deposited the Dataset in the repository 1) FamilyName, GivenName or 2) Organization text 57 FALSE FALSE FALSE FALSE FALSE FALSE citation + dateOfDeposit Deposit Date The date when the Dataset was deposited into the repository YYYY-MM-DD date 58 FALSE FALSE FALSE TRUE FALSE FALSE citation http://purl.org/dc/terms/dateSubmitted + timePeriodCovered Time Period The time period that the data refer to. Also known as span. This is the time period covered by the data, not the dates of coding, collecting data, or making documents machine-readable none 59 ; FALSE FALSE TRUE FALSE FALSE FALSE citation https://schema.org/temporalCoverage + timePeriodCoveredStart Start Date The start date of the time period that the data refer to YYYY-MM-DD date 60 #NAME: #VALUE TRUE FALSE FALSE TRUE FALSE FALSE timePeriodCovered citation + timePeriodCoveredEnd End Date The end date of the time period that the data refer to YYYY-MM-DD date 61 #NAME: #VALUE TRUE FALSE FALSE TRUE FALSE FALSE timePeriodCovered citation + dateOfCollection Date of Collection The dates when the data were collected or generated none 62 ; FALSE FALSE TRUE FALSE FALSE FALSE citation + dateOfCollectionStart Start Date The date when the data collection started YYYY-MM-DD date 63 #NAME: #VALUE FALSE FALSE FALSE FALSE FALSE FALSE dateOfCollection citation + dateOfCollectionEnd End Date The date when the data collection ended YYYY-MM-DD date 64 #NAME: #VALUE FALSE FALSE FALSE FALSE FALSE FALSE dateOfCollection citation + kindOfData Data Type The type of data included in the files (e.g. survey data, clinical data, or machine-readable text) text 65 TRUE FALSE TRUE TRUE FALSE FALSE citation http://rdf-vocabulary.ddialliance.org/discovery#kindOfData + series Series Information about the dataset series to which the Dataset belong none 66 : FALSE FALSE TRUE FALSE FALSE FALSE citation + seriesName Name The name of the dataset series text 67 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE series citation + seriesInformation Information Can include 1) a history of the series and 2) a summary of features that apply to the series textbox 68 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE series citation + software Software Information about the software used to generate the Dataset none 69 , FALSE FALSE TRUE FALSE FALSE FALSE citation https://www.w3.org/TR/prov-o/#wasGeneratedBy + softwareName Name The name of software used to generate the Dataset text 70 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE software citation + softwareVersion Version The version of the software used to generate the Dataset, e.g. 4.11 text 71 #NAME: #VALUE FALSE FALSE FALSE FALSE FALSE FALSE software citation + relatedMaterial Related Material Information, such as a persistent ID or citation, about the material related to the Dataset, such as appendices or sampling information available outside of the Dataset textbox 72 FALSE FALSE TRUE FALSE FALSE FALSE citation + relatedDatasets Related Dataset Information, such as a persistent ID or citation, about a related dataset, such as previous research on the Dataset's subject textbox 73 FALSE FALSE TRUE FALSE FALSE FALSE citation http://purl.org/dc/terms/relation + otherReferences Other Reference Information, such as a persistent ID or citation, about another type of resource that provides background or supporting material to the Dataset text 74 FALSE FALSE TRUE FALSE FALSE FALSE citation http://purl.org/dc/terms/references + dataSources Data Source Information, such as a persistent ID or citation, about sources of the Dataset (e.g. a book, article, serial, or machine-readable data file) textbox 75 FALSE FALSE TRUE FALSE FALSE FALSE citation https://www.w3.org/TR/prov-o/#wasDerivedFrom + originOfSources Origin of Historical Sources For historical sources, the origin and any rules followed in establishing them as sources textbox 76 FALSE FALSE FALSE FALSE FALSE FALSE citation + characteristicOfSources Characteristic of Sources Characteristics not already noted elsewhere textbox 77 FALSE FALSE FALSE FALSE FALSE FALSE citation + accessToSources Documentation and Access to Sources 1) Methods or procedures for accessing data sources and 2) any special permissions needed for access textbox 78 FALSE FALSE FALSE FALSE FALSE FALSE citation #controlledVocabulary DatasetField Value identifier displayOrder subject Agricultural Sciences D01 0 subject Arts and Humanities D0 1 @@ -138,189 +139,189 @@ authorIdentifierScheme DAI 5 authorIdentifierScheme ResearcherID 6 authorIdentifierScheme ScopusID 7 - language Abkhaz 0 - language Afar 1 aar aa - language Afrikaans 2 afr af - language Akan 3 aka ak - language Albanian 4 sqi alb sq - language Amharic 5 amh am - language Arabic 6 ara ar - language Aragonese 7 arg an - language Armenian 8 hye arm hy - language Assamese 9 asm as - language Avaric 10 ava av - language Avestan 11 ave ae - language Aymara 12 aym ay - language Azerbaijani 13 aze az - language Bambara 14 bam bm - language Bashkir 15 bak ba - language Basque 16 eus baq eu - language Belarusian 17 bel be - language Bengali, Bangla 18 ben bn - language Bihari 19 bih bh - language Bislama 20 bis bi - language Bosnian 21 bos bs - language Breton 22 bre br - language Bulgarian 23 bul bg - language Burmese 24 mya bur my - language Catalan,Valencian 25 cat ca - language Chamorro 26 cha ch - language Chechen 27 che ce - language Chichewa, Chewa, Nyanja 28 nya ny - language Chinese 29 zho chi zh - language Chuvash 30 chv cv - language Cornish 31 cor kw - language Corsican 32 cos co - language Cree 33 cre cr - language Croatian 34 hrv src hr - language Czech 35 ces cze cs - language Danish 36 dan da - language Divehi, Dhivehi, Maldivian 37 div dv - language Dutch 38 nld dut nl - language Dzongkha 39 dzo dz - language English 40 eng en - language Esperanto 41 epo eo - language Estonian 42 est et - language Ewe 43 ewe ee - language Faroese 44 fao fo - language Fijian 45 fij fj - language Finnish 46 fin fi - language French 47 fra fre fr - language Fula, Fulah, Pulaar, Pular 48 ful ff - language Galician 49 glg gl - language Georgian 50 kat geo ka - language German 51 deu ger de - language Greek (modern) 52 gre ell el - language Guaraní 53 grn gn - language Gujarati 54 guj gu - language Haitian, Haitian Creole 55 hat ht - language Hausa 56 hau ha - language Hebrew (modern) 57 heb he - language Herero 58 her hz - language Hindi 59 hin hi - language Hiri Motu 60 hmo ho - language Hungarian 61 hun hu - language Interlingua 62 ina ia - language Indonesian 63 ind id - language Interlingue 64 ile ie - language Irish 65 gle ga - language Igbo 66 ibo ig - language Inupiaq 67 ipk ik - language Ido 68 ido io - language Icelandic 69 isl ice is - language Italian 70 ita it - language Inuktitut 71 iku iu - language Japanese 72 jpn ja - language Javanese 73 jav jv - language Kalaallisut, Greenlandic 74 kal kl - language Kannada 75 kan kn - language Kanuri 76 kau kr - language Kashmiri 77 kas ks - language Kazakh 78 kaz kk - language Khmer 79 khm km - language Kikuyu, Gikuyu 80 kik ki - language Kinyarwanda 81 kin rw - language Kyrgyz 82 - language Komi 83 kom kv - language Kongo 84 kon kg - language Korean 85 kor ko - language Kurdish 86 kur ku - language Kwanyama, Kuanyama 87 kua kj - language Latin 88 lat la - language Luxembourgish, Letzeburgesch 89 ltz lb - language Ganda 90 lug lg - language Limburgish, Limburgan, Limburger 91 lim li - language Lingala 92 lin ln - language Lao 93 lao lo - language Lithuanian 94 lit lt - language Luba-Katanga 95 lub lu - language Latvian 96 lav lv - language Manx 97 glv gv - language Macedonian 98 mkd mac mk - language Malagasy 99 mlg mg - language Malay 100 may msa ms - language Malayalam 101 mal ml - language Maltese 102 mlt mt - language Māori 103 mao mri mi - language Marathi (Marāṭhī) 104 mar mr - language Marshallese 105 mah mh - language Mixtepec Mixtec 106 mix - language Mongolian 107 mon mn - language Nauru 108 nau na - language Navajo, Navaho 109 nav nv - language Northern Ndebele 110 nde nd - language Nepali 111 nep ne - language Ndonga 112 ndo ng - language Norwegian Bokmål 113 nob nb - language Norwegian Nynorsk 114 nno nn - language Norwegian 115 nor no - language Nuosu 116 - language Southern Ndebele 117 nbl nr - language Occitan 118 oci oc - language Ojibwe, Ojibwa 119 oji oj - language Old Church Slavonic,Church Slavonic,Old Bulgarian 120 chu cu - language Oromo 121 orm om - language Oriya 122 ori or - language Ossetian, Ossetic 123 oss os - language Panjabi, Punjabi 124 pan pa - language Pāli 125 pli pi - language Persian (Farsi) 126 per fas fa - language Polish 127 pol pl - language Pashto, Pushto 128 pus ps - language Portuguese 129 por pt - language Quechua 130 que qu - language Romansh 131 roh rm - language Kirundi 132 run rn - language Romanian 133 ron rum ro - language Russian 134 rus ru - language Sanskrit (Saṁskṛta) 135 san sa - language Sardinian 136 srd sc - language Sindhi 137 snd sd - language Northern Sami 138 sme se - language Samoan 139 smo sm - language Sango 140 sag sg - language Serbian 141 srp scc sr - language Scottish Gaelic, Gaelic 142 gla gd - language Shona 143 sna sn - language Sinhala, Sinhalese 144 sin si - language Slovak 145 slk slo sk - language Slovene 146 slv sl - language Somali 147 som so - language Southern Sotho 148 sot st - language Spanish, Castilian 149 spa es - language Sundanese 150 sun su - language Swahili 151 swa sw - language Swati 152 ssw ss - language Swedish 153 swe sv - language Tamil 154 tam ta - language Telugu 155 tel te - language Tajik 156 tgk tg - language Thai 157 tha th - language Tigrinya 158 tir ti - language Tibetan Standard, Tibetan, Central 159 tib bod bo - language Turkmen 160 tuk tk - language Tagalog 161 tgl tl - language Tswana 162 tsn tn - language Tonga (Tonga Islands) 163 ton to - language Turkish 164 tur tr - language Tsonga 165 tso ts - language Tatar 166 tat tt - language Twi 167 twi tw - language Tahitian 168 tah ty - language Uyghur, Uighur 169 uig ug - language Ukrainian 170 ukr uk - language Urdu 171 urd ur - language Uzbek 172 uzb uz - language Venda 173 ven ve - language Vietnamese 174 vie vi - language Volapük 175 vol vo - language Walloon 176 wln wa - language Welsh 177 cym wel cy - language Wolof 178 wol wo - language Western Frisian 179 fry fy - language Xhosa 180 xho xh - language Yiddish 181 yid yi - language Yoruba 182 yor yo - language Zhuang, Chuang 183 zha za - language Zulu 184 zul zu + language Abkhaz abk 0 abk ab + language Afar aar 1 aar aa + language Afrikaans afr 2 afr af + language Akan aka 3 aka ak + language Albanian sqi 4 sqi alb sq + language Amharic amh 5 amh am + language Arabic ara 6 ara ar + language Aragonese arg 7 arg an + language Armenian hye 8 hye arm hy + language Assamese asm 9 asm as + language Avaric ava 10 ava av + language Avestan ave 11 ave ae + language Aymara aym 12 aym ay + language Azerbaijani aze 13 aze az + language Bambara bam 14 bam bm + language Bashkir bak 15 bak ba + language Basque eus 16 eus baq eu + language Belarusian bel 17 bel be + language Bengali, Bangla ben 18 ben bn Bengali Bangla + language Bihari bih 19 bih bh + language Bislama bis 20 bis bi + language Bosnian bos 21 bos bs + language Breton bre 22 bre br + language Bulgarian bul 23 bul bg + language Burmese mya 24 mya bur my + language Catalan,Valencian cat 25 cat ca Catalan Valencian + language Chamorro cha 26 cha ch + language Chechen che 27 che ce + language Chichewa, Chewa, Nyanja nya 28 nya ny Chichewa Chewa Nyanja + language Chinese zho 29 zho chi zh + language Chuvash chv 30 chv cv + language Cornish cor 31 cor kw + language Corsican cos 32 cos co + language Cree cre 33 cre cr + language Croatian hrv 34 hrv src hr + language Czech ces 35 ces cze cs + language Danish dan 36 dan da + language Divehi, Dhivehi, Maldivian div 37 div dv Divehi Dhivehi Maldivian + language Dutch nld 38 nld dut nl + language Dzongkha dzo 39 dzo dz + language English eng 40 eng en + language Esperanto epo 41 epo eo + language Estonian est 42 est et + language Ewe ewe 43 ewe ee + language Faroese fao 44 fao fo + language Fijian fij 45 fij fj + language Finnish fin 46 fin fi + language French fra 47 fra fre fr + language Fula, Fulah, Pulaar, Pular ful 48 ful ff Fula Fulah Pulaar Pular + language Galician glg 49 glg gl + language Georgian kat 50 kat geo ka + language German deu 51 deu ger de + language Greek (modern) ell 52 ell gre el Greek + language Guaraní grn 53 grn gn + language Gujarati guj 54 guj gu + language Haitian, Haitian Creole hat 55 hat ht Haitian Haitian Creole + language Hausa hau 56 hau ha + language Hebrew (modern) heb 57 heb he + language Herero her 58 her hz + language Hindi hin 59 hin hi + language Hiri Motu hmo 60 hmo ho + language Hungarian hun 61 hun hu + language Interlingua ina 62 ina ia + language Indonesian ind 63 ind id + language Interlingue ile 64 ile ie + language Irish gle 65 gle ga + language Igbo ibo 66 ibo ig + language Inupiaq ipk 67 ipk ik + language Ido ido 68 ido io + language Icelandic isl 69 isl ice is + language Italian ita 70 ita it + language Inuktitut iku 71 iku iu + language Japanese jpn 72 jpn ja + language Javanese jav 73 jav jv + language Kalaallisut, Greenlandic kal 74 kal kl Kalaallisut Greenlandic + language Kannada kan 75 kan kn + language Kanuri kau 76 kau kr + language Kashmiri kas 77 kas ks + language Kazakh kaz 78 kaz kk + language Khmer khm 79 khm km + language Kikuyu, Gikuyu kik 80 kik ki Kikuyu Gikuyu + language Kinyarwanda kin 81 kin rw + language Kyrgyz kir 82 kir ky Kirghiz + language Komi kom 83 kom kv + language Kongo kon 84 kon kg + language Korean kor 85 kor ko + language Kurdish kur 86 kur ku + language Kwanyama, Kuanyama kua 87 kua kj Kwanyama Kuanyama + language Latin lat 88 lat la + language Luxembourgish, Letzeburgesch ltz 89 ltz lb Luxembourgish Letzeburgesch + language Ganda lug 90 lug lg + language Limburgish, Limburgan, Limburger lim 91 lim li Limburgish Limburgan Limburger + language Lingala lin 92 lin ln + language Lao lao 93 lao lo + language Lithuanian lit 94 lit lt + language Luba-Katanga lub 95 lub lu + language Latvian lav 96 lav lv + language Manx glv 97 glv gv + language Macedonian mkd 98 mkd mac mk + language Malagasy mlg 99 mlg mg + language Malay msa 100 msa may ms + language Malayalam mal 101 mal ml + language Maltese mlt 102 mlt mt + language Māori mri 103 mri mao mi Maori + language Marathi (Marāṭhī) mar 104 mar mr + language Marshallese mah 105 mah mh + language Mixtepec Mixtec mix 106 mix + language Mongolian mon 107 mon mn + language Nauru nau 108 nau na + language Navajo, Navaho nav 109 nav nv Navajo Navaho + language Northern Ndebele nde 110 nde nd + language Nepali nep 111 nep ne + language Ndonga ndo 112 ndo ng + language Norwegian Bokmål nob 113 nob nb + language Norwegian Nynorsk nno 114 nno nn + language Norwegian nor 115 nor no + language Nuosu iii 116 iii ii Sichuan Yi + language Southern Ndebele nbl 117 nbl nr + language Occitan oci 118 oci oc + language Ojibwe, Ojibwa oji 119 oji oj Ojibwe Ojibwa + language Old Church Slavonic,Church Slavonic,Old Bulgarian chu 120 chu cu + language Oromo orm 121 orm om + language Oriya ori 122 ori or + language Ossetian, Ossetic oss 123 oss os Ossetian Ossetic + language Panjabi, Punjabi pan 124 pan pa Panjabi Punjabi + language Pāli pli 125 pli pi + language Persian (Farsi) fas 126 fas per fa + language Polish pol 127 pol pl + language Pashto, Pushto pus 128 pus ps Pashto Pushto + language Portuguese por 129 por pt + language Quechua que 130 que qu + language Romansh roh 131 roh rm + language Kirundi run 132 run rn + language Romanian ron 133 ron rum ro + language Russian rus 134 rus ru + language Sanskrit (Saṁskṛta) san 135 san sa + language Sardinian srd 136 srd sc + language Sindhi snd 137 snd sd + language Northern Sami sme 138 sme se + language Samoan smo 139 smo sm + language Sango sag 140 sag sg + language Serbian srp 141 srp scc sr + language Scottish Gaelic, Gaelic gla 142 gla gd Scottish Gaelic Gaelic + language Shona sna 143 sna sn + language Sinhala, Sinhalese sin 144 sin si Sinhala Sinhalese + language Slovak slk 145 slk slo sk + language Slovene slv 146 slv sl Slovenian + language Somali som 147 som so + language Southern Sotho sot 148 sot st + language Spanish, Castilian spa 149 spa es Spanish Castilian + language Sundanese sun 150 sun su + language Swahili swa 151 swa sw + language Swati ssw 152 ssw ss + language Swedish swe 153 swe sv + language Tamil tam 154 tam ta + language Telugu tel 155 tel te + language Tajik tgk 156 tgk tg + language Thai tha 157 tha th + language Tigrinya tir 158 tir ti + language Tibetan Standard, Tibetan, Central bod 159 bod tib bo Tibetan Standard Tibetan Central + language Turkmen tuk 160 tuk tk + language Tagalog tgl 161 tgl tl + language Tswana tsn 162 tsn tn + language Tonga (Tonga Islands) ton 163 ton to Tonga + language Turkish tur 164 tur tr + language Tsonga tso 165 tso ts + language Tatar tat 166 tat tt + language Twi twi 167 twi tw + language Tahitian tah 168 tah ty + language Uyghur, Uighur uig 169 uig ug Uyghur Uighur + language Ukrainian ukr 170 ukr uk + language Urdu urd 171 urd ur + language Uzbek uzb 172 uzb uz + language Venda ven 173 ven ve + language Vietnamese vie 174 vie vi + language Volapük vol 175 vol vo + language Walloon wln 176 wln wa + language Welsh cym 177 cym wel cy + language Wolof wol 178 wol wo + language Western Frisian fry 179 fry fy + language Xhosa xho 180 xho xh + language Yiddish yid 181 yid yi + language Yoruba yor 182 yor yo + language Zhuang, Chuang zha 183 zha za Zhuang Chuang + language Zulu zul 184 zul zu language Not applicable 185 diff --git a/scripts/api/data/metadatablocks/computational_workflow.tsv b/scripts/api/data/metadatablocks/computational_workflow.tsv index 51b69cfdb80..3cd0c26a464 100644 --- a/scripts/api/data/metadatablocks/computational_workflow.tsv +++ b/scripts/api/data/metadatablocks/computational_workflow.tsv @@ -2,7 +2,7 @@ computationalworkflow Computational Workflow Metadata #datasetField name title description watermark fieldType displayOrder displayFormat advancedSearchField allowControlledVocabulary allowmultiples facetable displayoncreate required parent metadatablock_id termURI workflowType Computational Workflow Type The kind of Computational Workflow, which is designed to compose and execute a series of computational or data manipulation steps in a scientific application text 0 TRUE TRUE TRUE TRUE TRUE FALSE computationalworkflow - workflowCodeRepository External Code Repository URL A link to the repository where the un-compiled, human readable code and related code is located (e.g. GitHub, GitLab, SVN) https://... url 1 FALSE FALSE TRUE FALSE TRUE FALSE computationalworkflow + workflowCodeRepository External Code Repository URL A link to the repository where the un-compiled, human readable code and related code is located (e.g. GitHub, GitLab, SVN) https://... url 1 #VALUE FALSE FALSE TRUE FALSE TRUE FALSE computationalworkflow workflowDocumentation Documentation A link (URL) to the documentation or text describing the Computational Workflow and its use textbox 2 FALSE FALSE TRUE FALSE TRUE FALSE computationalworkflow #controlledVocabulary DatasetField Value identifier displayOrder workflowType Common Workflow Language (CWL) workflowtype_cwl 1 diff --git a/scripts/api/data/metadatablocks/geospatial.tsv b/scripts/api/data/metadatablocks/geospatial.tsv index ce481c1bf84..11408317410 100644 --- a/scripts/api/data/metadatablocks/geospatial.tsv +++ b/scripts/api/data/metadatablocks/geospatial.tsv @@ -8,10 +8,10 @@ otherGeographicCoverage Other Other information on the geographic coverage of the data. text 4 #VALUE, FALSE FALSE FALSE TRUE FALSE FALSE geographicCoverage geospatial geographicUnit Geographic Unit Lowest level of geographic aggregation covered by the Dataset, e.g., village, county, region. text 5 TRUE FALSE TRUE TRUE FALSE FALSE geospatial geographicBoundingBox Geographic Bounding Box The fundamental geometric description for any Dataset that models geography is the geographic bounding box. It describes the minimum box, defined by west and east longitudes and north and south latitudes, which includes the largest geographic extent of the Dataset's geographic coverage. This element is used in the first pass of a coordinate-based search. Inclusion of this element in the codebook is recommended, but is required if the bound polygon box is included. none 6 FALSE FALSE TRUE FALSE FALSE FALSE geospatial - westLongitude Westernmost (Left) Longitude Westernmost coordinate delimiting the geographic extent of the Dataset. A valid range of values, expressed in decimal degrees, is -180,0 <= West Bounding Longitude Value <= 180,0. text 7 FALSE FALSE FALSE FALSE FALSE FALSE geographicBoundingBox geospatial - eastLongitude Easternmost (Right) Longitude Easternmost coordinate delimiting the geographic extent of the Dataset. A valid range of values, expressed in decimal degrees, is -180,0 <= East Bounding Longitude Value <= 180,0. text 8 FALSE FALSE FALSE FALSE FALSE FALSE geographicBoundingBox geospatial - northLongitude Northernmost (Top) Latitude Northernmost coordinate delimiting the geographic extent of the Dataset. A valid range of values, expressed in decimal degrees, is -90,0 <= North Bounding Latitude Value <= 90,0. text 9 FALSE FALSE FALSE FALSE FALSE FALSE geographicBoundingBox geospatial - southLongitude Southernmost (Bottom) Latitude Southernmost coordinate delimiting the geographic extent of the Dataset. A valid range of values, expressed in decimal degrees, is -90,0 <= South Bounding Latitude Value <= 90,0. text 10 FALSE FALSE FALSE FALSE FALSE FALSE geographicBoundingBox geospatial + westLongitude Westernmost (Left) Longitude Westernmost coordinate delimiting the geographic extent of the Dataset. A valid range of values, expressed in decimal degrees, is -180.0 <= West Bounding Longitude Value <= 180.0. text 7 FALSE FALSE FALSE FALSE FALSE FALSE geographicBoundingBox geospatial + eastLongitude Easternmost (Right) Longitude Easternmost coordinate delimiting the geographic extent of the Dataset. A valid range of values, expressed in decimal degrees, is -180.0 <= East Bounding Longitude Value <= 180.0. text 8 FALSE FALSE FALSE FALSE FALSE FALSE geographicBoundingBox geospatial + northLatitude Northernmost (Top) Latitude Northernmost coordinate delimiting the geographic extent of the Dataset. A valid range of values, expressed in decimal degrees, is -90.0 <= North Bounding Latitude Value <= 90.0. text 9 FALSE FALSE FALSE FALSE FALSE FALSE geographicBoundingBox geospatial + southLatitude Southernmost (Bottom) Latitude Southernmost coordinate delimiting the geographic extent of the Dataset. A valid range of values, expressed in decimal degrees, is -90.0 <= South Bounding Latitude Value <= 90.0. text 10 FALSE FALSE FALSE FALSE FALSE FALSE geographicBoundingBox geospatial #controlledVocabulary DatasetField Value identifier displayOrder country Afghanistan 0 country Albania 1 diff --git a/scripts/api/data/storageSites/add-storage-site.json b/scripts/api/data/storageSites/add-storage-site.json deleted file mode 100644 index d13ec2f165d..00000000000 --- a/scripts/api/data/storageSites/add-storage-site.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "hostname": "dataverse.librascholar.edu", - "name": "LibraScholar, USA", - "primaryStorage": true, - "transferProtocols": "rsync,posix,globus" -} diff --git a/scripts/api/setup-all.sh b/scripts/api/setup-all.sh index e247caa72b5..b7f962209e4 100755 --- a/scripts/api/setup-all.sh +++ b/scripts/api/setup-all.sh @@ -57,10 +57,6 @@ echo "- Allow internal signup" curl -X PUT -d yes "${DATAVERSE_URL}/api/admin/settings/:AllowSignUp" curl -X PUT -d "/dataverseuser.xhtml?editMode=CREATE" "${DATAVERSE_URL}/api/admin/settings/:SignUpUrl" -curl -X PUT -d doi "${DATAVERSE_URL}/api/admin/settings/:Protocol" -curl -X PUT -d 10.5072 "${DATAVERSE_URL}/api/admin/settings/:Authority" -curl -X PUT -d "FK2/" "${DATAVERSE_URL}/api/admin/settings/:Shoulder" -curl -X PUT -d DataCite "${DATAVERSE_URL}/api/admin/settings/:DoiProvider" curl -X PUT -d burrito "${DATAVERSE_URL}/api/admin/settings/BuiltinUsers.KEY" curl -X PUT -d localhost-only "${DATAVERSE_URL}/api/admin/settings/:BlockedApiPolicy" curl -X PUT -d 'native/http' "${DATAVERSE_URL}/api/admin/settings/:UploadMethods" @@ -69,7 +65,7 @@ echo echo "Setting up the admin user (and as superuser)" adminResp=$(curl -s -H "Content-type:application/json" -X POST -d @"$SCRIPT_PATH"/data/user-admin.json "${DATAVERSE_URL}/api/builtin-users?password=$DV_SU_PASSWORD&key=burrito") echo "$adminResp" -curl -X POST "${DATAVERSE_URL}/api/admin/superuser/dataverseAdmin" +curl -X PUT "${DATAVERSE_URL}/api/admin/superuser/dataverseAdmin" -d "true" echo echo "Setting up the root dataverse" diff --git a/scripts/api/setup-optional-harvard.sh b/scripts/api/setup-optional-harvard.sh index fcbcc08a8e6..1311464e8ff 100755 --- a/scripts/api/setup-optional-harvard.sh +++ b/scripts/api/setup-optional-harvard.sh @@ -3,6 +3,7 @@ SERVER=http://localhost:8080/api echo "Setting up Harvard-specific settings" # :Authority and :Shoulder are commented out so this script can be used on test servers +# Should now use the new multipid JVM options instead of these settings #curl -X PUT -d 10.7910 "$SERVER/admin/settings/:Authority" #curl -X PUT -d "DVN/" "$SERVER/admin/settings/:Shoulder" echo "- Application Status header" diff --git a/scripts/deploy/phoenix.dataverse.org/post b/scripts/deploy/phoenix.dataverse.org/post index e4c8817844b..9d37c183a1a 100755 --- a/scripts/deploy/phoenix.dataverse.org/post +++ b/scripts/deploy/phoenix.dataverse.org/post @@ -4,7 +4,6 @@ cd scripts/api cd ../.. psql -U dvnapp dvndb -f scripts/database/reference_data.sql psql -U dvnapp dvndb -f doc/sphinx-guides/source/_static/util/createsequence.sql -curl http://localhost:8080/api/admin/settings/:DoiProvider -X PUT -d FAKE scripts/search/tests/publish-dataverse-root git checkout scripts/api/data/dv-root.json scripts/search/tests/grant-authusers-add-on-root diff --git a/scripts/dev/dev-rebuild.sh b/scripts/dev/dev-rebuild.sh index 9eae195b135..898212b4664 100755 --- a/scripts/dev/dev-rebuild.sh +++ b/scripts/dev/dev-rebuild.sh @@ -56,9 +56,6 @@ cd ../.. echo "Creating SQL sequence..." psql -h localhost -U $DB_USER $DB_NAME -f doc/sphinx-guides/source/_static/util/createsequence.sql -echo "Setting DOI provider to \"FAKE\"..." -curl http://localhost:8080/api/admin/settings/:DoiProvider -X PUT -d FAKE - echo "Allowing GUI edits to be visible without redeploy..." $PAYARA_DIR/glassfish/bin/asadmin create-system-properties "dataverse.jsf.refresh-period=1" diff --git a/scripts/dev/docker-final-setup.sh b/scripts/dev/docker-final-setup.sh index d2453619ec2..e20ce7ad6b6 100755 --- a/scripts/dev/docker-final-setup.sh +++ b/scripts/dev/docker-final-setup.sh @@ -10,9 +10,6 @@ cd ../.. echo "Setting system mail address..." curl -X PUT -d "dataverse@localhost" "http://localhost:8080/api/admin/settings/:SystemEmail" -echo "Setting DOI provider to \"FAKE\"..." -curl "http://localhost:8080/api/admin/settings/:DoiProvider" -X PUT -d FAKE - API_TOKEN=$(grep apiToken "/tmp/setup-all.sh.out" | jq ".data.apiToken" | tr -d \") export API_TOKEN diff --git a/scripts/installer/Makefile b/scripts/installer/Makefile index 399bc65168a..8ea95534986 100644 --- a/scripts/installer/Makefile +++ b/scripts/installer/Makefile @@ -55,13 +55,13 @@ ${JHOVE_SCHEMA}: ../../conf/jhove/jhoveConfig.xsd ${INSTALLER_ZIP_DIR} @echo copying jhove schema file /bin/cp ../../conf/jhove/jhoveConfig.xsd ${INSTALLER_ZIP_DIR} -${SOLR_SCHEMA}: ../../conf/solr/9.3.0/schema.xml ../../conf/solr/9.3.0/update-fields.sh ${INSTALLER_ZIP_DIR} +${SOLR_SCHEMA}: ../../conf/solr/schema.xml ../../conf/solr/update-fields.sh ${INSTALLER_ZIP_DIR} @echo copying Solr schema file - /bin/cp ../../conf/solr/9.3.0/schema.xml ../../conf/solr/9.3.0/update-fields.sh ${INSTALLER_ZIP_DIR} + /bin/cp ../../conf/solr/schema.xml ../../conf/solr/update-fields.sh ${INSTALLER_ZIP_DIR} -${SOLR_CONFIG}: ../../conf/solr/9.3.0/solrconfig.xml ${INSTALLER_ZIP_DIR} +${SOLR_CONFIG}: ../../conf/solr/solrconfig.xml ${INSTALLER_ZIP_DIR} @echo copying Solr config file - /bin/cp ../../conf/solr/9.3.0/solrconfig.xml ${INSTALLER_ZIP_DIR} + /bin/cp ../../conf/solr/solrconfig.xml ${INSTALLER_ZIP_DIR} ${PYTHON_FILES}: README_python.txt install.py installConfig.py installAppServer.py installUtils.py requirements.txt default.config interactive.config ${INSTALLER_ZIP_DIR} @echo copying Python installer files diff --git a/scripts/installer/as-setup.sh b/scripts/installer/as-setup.sh index fc5b378cff5..34deddf51a3 100755 --- a/scripts/installer/as-setup.sh +++ b/scripts/installer/as-setup.sh @@ -102,17 +102,18 @@ function preliminary_setup() # password reset token timeout in minutes ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.auth.password-reset-timeout-in-minutes=60" - # DataCite DOI Settings + # Fake DOI Settings # (we can no longer offer EZID with their shared test account) # jvm-options use colons as separators, escape as literal DOI_BASEURL_ESC=`echo $DOI_BASEURL | sed -e 's/:/\\\:/'` - ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.pid.datacite.username=${DOI_USERNAME}" - ./asadmin $ASADMIN_OPTS create-jvm-options '\-Ddataverse.pid.datacite.password=${ALIAS=doi_password_alias}' - ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.pid.datacite.mds-api-url=$DOI_BASEURL_ESC" - + ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.pid.providers=fake" + ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.pid.fake.type=FAKE" + ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.pid.fake.label=Fake DOI Provider" + ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.pid.fake.authority=10.5072" + ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.pid.fake.shoulder=FK2/" # jvm-options use colons as separators, escape as literal - DOI_DATACITERESTAPIURL_ESC=`echo $DOI_DATACITERESTAPIURL | sed -e 's/:/\\\:/'` - ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.pid.datacite.rest-api-url=$DOI_DATACITERESTAPIURL_ESC" + #DOI_DATACITERESTAPIURL_ESC=`echo $DOI_DATACITERESTAPIURL | sed -e 's/:/\\\:/'` + #./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.pid.testDC.datacite.rest-api-url=$DOI_DATACITERESTAPIURL_ESC" ./asadmin $ASADMIN_OPTS create-jvm-options "-Ddataverse.timerServer=true" @@ -146,12 +147,10 @@ function final_setup(){ # delete any existing mail/notifyMailSession; configure port, if provided: ./asadmin delete-javamail-resource mail/notifyMailSession - - if [ $SMTP_SERVER_PORT"x" != "x" ] - then - ./asadmin $ASADMIN_OPTS create-javamail-resource --mailhost "$SMTP_SERVER" --mailuser "dataversenotify" --fromaddress "do-not-reply@${HOST_ADDRESS}" --property mail.smtp.port="${SMTP_SERVER_PORT}" mail/notifyMailSession - else - ./asadmin $ASADMIN_OPTS create-javamail-resource --mailhost "$SMTP_SERVER" --mailuser "dataversenotify" --fromaddress "do-not-reply@${HOST_ADDRESS}" mail/notifyMailSession + ./asadmin $ASADMIN_OPTS create-system-properties "dataverse.mail.system-email='${ADMIN_EMAIL}'" + ./asadmin $ASADMIN_OPTS create-system-properties "dataverse.mail.mta.host='${SMTP_SERVER}'" + if [ "x${SMTP_SERVER_PORT}" != "x" ]; then + ./asadmin $ASADMIN_OPTS create-system-properties "dataverse.mail.mta.port='${SMTP_SERVER_PORT}'" fi } @@ -279,6 +278,12 @@ if [ ! -d "$DOMAIN_DIR" ] exit 2 fi +if [ -z "$ADMIN_EMAIL" ] + then + echo "You must specify the system admin email address (ADMIN_EMAIL)." + exit 1 +fi + echo "Setting up your app. server (Payara) to support Dataverse" echo "Payara directory: "$GLASSFISH_ROOT echo "Domain directory: "$DOMAIN_DIR diff --git a/scripts/installer/install.py b/scripts/installer/install.py index 18995695638..005fbad46e0 100644 --- a/scripts/installer/install.py +++ b/scripts/installer/install.py @@ -568,14 +568,6 @@ except: sys.exit("Failure to execute setup-all.sh! aborting.") -# 7b. configure admin email in the application settings -print("configuring system email address...") -returnCode = subprocess.call(["curl", "-X", "PUT", "-d", adminEmail, apiUrl+"/admin/settings/:SystemEmail"]) -if returnCode != 0: - print("\nWARNING: failed to configure the admin email in the Dataverse settings!") -else: - print("\ndone.") - # 8c. configure remote Solr location, if specified if solrLocation != "LOCAL": print("configuring remote Solr location... ("+solrLocation+")") @@ -591,15 +583,14 @@ print("\n\nYou should now have a running Dataverse instance at") print(" http://" + hostName + ":8080\n\n") -# DataCite instructions: +# PID instructions: -print("\nYour Dataverse has been configured to use DataCite, to register DOI global identifiers in the ") +print("\nYour Dataverse has been configured to use a Fake DOI Provider, registering (non-resolvable) DOI global identifiers in the ") print("test name space \"10.5072\" with the \"shoulder\" \"FK2\"") -print("However, you have to contact DataCite (support\@datacite.org) and request a test account, before you ") -print("can publish datasets. Once you receive the account name and password, add them to your domain.xml,") -print("as the following two JVM options:") -print("\t-Ddataverse.pid.datacite.username=...") -print("\t-Ddataverse.pid.datacite.password=...") +print("You can reconfigure to use additional/alternative providers.") +print("If you intend to use DOIs, you should contact DataCite (support\@datacite.org) or GDCC (see https://www.gdcc.io/about.html) and request a test account.") +print("Once you receive the account information (name, password, authority, shoulder), add them to your configuration ") +print("as described in the Dataverse Guides (see https://guides.dataverse.org/en/latest/installation/config.html#persistent-identifiers-and-publishing-datasets),") print("and restart payara") print("If this is a production Dataverse and you are planning to register datasets as ") print("\"real\", non-test DOIs or Handles, consult the \"Persistent Identifiers and Publishing Datasets\"") diff --git a/scripts/installer/installAppServer.py b/scripts/installer/installAppServer.py index 698f5ba9a58..7636490c583 100644 --- a/scripts/installer/installAppServer.py +++ b/scripts/installer/installAppServer.py @@ -6,8 +6,9 @@ def runAsadminScript(config): # commands to set up all the app. server (payara6) components for the application. # All the parameters must be passed to that script as environmental # variables: - os.environ['GLASSFISH_DOMAIN'] = "domain1"; - os.environ['ASADMIN_OPTS'] = ""; + os.environ['GLASSFISH_DOMAIN'] = "domain1" + os.environ['ASADMIN_OPTS'] = "" + os.environ['ADMIN_EMAIL'] = config.get('system','ADMIN_EMAIL') os.environ['HOST_ADDRESS'] = config.get('glassfish','HOST_DNS_ADDRESS') os.environ['GLASSFISH_ROOT'] = config.get('glassfish','GLASSFISH_DIRECTORY') diff --git a/scripts/intellij/cpwebapp.sh b/scripts/intellij/cpwebapp.sh deleted file mode 100755 index 6ecad367048..00000000000 --- a/scripts/intellij/cpwebapp.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash -# -# cpwebapp -# -# Usage: -# -# Add a File watcher by importing watchers.xml into IntelliJ IDEA, and let it do the copying whenever you save a -# file under webapp. -# -# https://www.jetbrains.com/help/idea/settings-tools-file-watchers.html -# -# Alternatively, you can add an External tool and trigger via menu or shortcut to do the copying manually: -# -# https://www.jetbrains.com/help/idea/configuring-third-party-tools.html -# - -PROJECT_DIR=$1 -FILE_TO_COPY=$2 -RELATIVE_PATH="${FILE_TO_COPY#$PROJECT_DIR/}" - -# Check if RELATIVE_PATH starts with 'src/main/webapp', otherwise ignore -if [[ $RELATIVE_PATH == src/main/webapp* ]]; then - # Get current version. Any other way to do this? A simple VERSION file would help. - VERSION=`perl -ne 'print $1 if /(.*?)<\/revision>/' ./modules/dataverse-parent/pom.xml` - RELATIVE_PATH_WITHOUT_WEBAPP="${RELATIVE_PATH#src/main/webapp/}" - TARGET_DIR=./docker-dev-volumes/glassfish/applications/dataverse-$VERSION - TARGET_PATH="${TARGET_DIR}/${RELATIVE_PATH_WITHOUT_WEBAPP}" - - mkdir -p "$(dirname "$TARGET_PATH")" - cp "$FILE_TO_COPY" "$TARGET_PATH" - - echo "File $FILE_TO_COPY copied to $TARGET_PATH" -fi diff --git a/src/main/docker/assembly.xml b/src/main/docker/assembly.xml index 9f9b39617a3..62cd910ef9b 100644 --- a/src/main/docker/assembly.xml +++ b/src/main/docker/assembly.xml @@ -3,7 +3,7 @@ - target/${project.artifactId}-${project.version} + target/${project.artifactId} app WEB-INF/lib/**/* @@ -11,7 +11,7 @@ - target/${project.artifactId}-${project.version}/WEB-INF/lib + target/${project.artifactId}/WEB-INF/lib deps diff --git a/src/main/docker/scripts/init_2_configure.sh b/src/main/docker/scripts/init_2_configure.sh index a98f08088c1..b31cfac37b7 100755 --- a/src/main/docker/scripts/init_2_configure.sh +++ b/src/main/docker/scripts/init_2_configure.sh @@ -31,10 +31,6 @@ echo "# Dataverse postboot configuration for Payara" > "${DV_POSTBOOT}" # EE 8 code annotations or at least glassfish-resources.xml # NOTE: postboot commands is not multi-line capable, thus spaghetti needed. -# JavaMail -echo "INFO: Defining JavaMail." -echo "create-javamail-resource --mailhost=${DATAVERSE_MAIL_HOST:-smtp} --mailuser=${DATAVERSE_MAIL_USER:-dataversenotify} --fromaddress=${DATAVERSE_MAIL_FROM:-dataverse@localhost} mail/notifyMailSession" >> "${DV_POSTBOOT}" - # 3. Domain based configuration options # Set Dataverse environment variables echo "INFO: Defining system properties for Dataverse configuration options." diff --git a/src/main/java/edu/harvard/iq/dataverse/AbstractGlobalIdServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/AbstractGlobalIdServiceBean.java deleted file mode 100644 index f1bfc3e290b..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/AbstractGlobalIdServiceBean.java +++ /dev/null @@ -1,700 +0,0 @@ -package edu.harvard.iq.dataverse; - -import edu.harvard.iq.dataverse.settings.SettingsServiceBean; -import edu.harvard.iq.dataverse.util.SystemConfig; -import java.io.InputStream; -import jakarta.ejb.EJB; -import jakarta.inject.Inject; -import java.util.*; -import java.util.logging.Level; -import java.util.logging.Logger; - -import org.apache.commons.lang3.RandomStringUtils; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; - -public abstract class AbstractGlobalIdServiceBean implements GlobalIdServiceBean { - - private static final Logger logger = Logger.getLogger(AbstractGlobalIdServiceBean.class.getCanonicalName()); - - @Inject - DataverseServiceBean dataverseService; - @EJB - protected - SettingsServiceBean settingsService; - @Inject - protected - DvObjectServiceBean dvObjectService; - @Inject - SystemConfig systemConfig; - - protected Boolean configured = null; - - public static String UNAVAILABLE = ":unav"; - - @Override - public Map getMetadataForCreateIndicator(DvObject dvObjectIn) { - logger.log(Level.FINE,"getMetadataForCreateIndicator(DvObject)"); - Map metadata = new HashMap<>(); - metadata = addBasicMetadata(dvObjectIn, metadata); - metadata.put("datacite.publicationyear", generateYear(dvObjectIn)); - metadata.put("_target", getTargetUrl(dvObjectIn)); - return metadata; - } - - protected Map getUpdateMetadata(DvObject dvObjectIn) { - logger.log(Level.FINE,"getUpdateMetadataFromDataset"); - Map metadata = new HashMap<>(); - metadata = addBasicMetadata(dvObjectIn, metadata); - return metadata; - } - - protected Map addBasicMetadata(DvObject dvObjectIn, Map metadata) { - - String authorString = dvObjectIn.getAuthorString(); - if (authorString.isEmpty() || authorString.contains(DatasetField.NA_VALUE)) { - authorString = UNAVAILABLE; - } - - String producerString = dataverseService.getRootDataverseName(); - - if (producerString.isEmpty() || producerString.equals(DatasetField.NA_VALUE)) { - producerString = UNAVAILABLE; - } - - String titleString = dvObjectIn.getCurrentName(); - - if (titleString.isEmpty() || titleString.equals(DatasetField.NA_VALUE)) { - titleString = UNAVAILABLE; - } - - metadata.put("datacite.creator", authorString); - metadata.put("datacite.title", titleString); - metadata.put("datacite.publisher", producerString); - metadata.put("datacite.publicationyear", generateYear(dvObjectIn)); - return metadata; - } - - protected Map addDOIMetadataForDestroyedDataset(DvObject dvObjectIn) { - Map metadata = new HashMap<>(); - String authorString = UNAVAILABLE; - String producerString = UNAVAILABLE; - String titleString = "This item has been removed from publication"; - - metadata.put("datacite.creator", authorString); - metadata.put("datacite.title", titleString); - metadata.put("datacite.publisher", producerString); - metadata.put("datacite.publicationyear", "9999"); - return metadata; - } - - protected String getTargetUrl(DvObject dvObjectIn) { - logger.log(Level.FINE,"getTargetUrl"); - return systemConfig.getDataverseSiteUrl() + dvObjectIn.getTargetUrl() + dvObjectIn.getGlobalId().asString(); - } - - @Override - public String getIdentifier(DvObject dvObject) { - GlobalId gid = dvObject.getGlobalId(); - return gid != null ? gid.asString() : null; - } - - protected String generateYear (DvObject dvObjectIn){ - return dvObjectIn.getYearPublishedCreated(); - } - - public Map getMetadataForTargetURL(DvObject dvObject) { - logger.log(Level.FINE,"getMetadataForTargetURL"); - HashMap metadata = new HashMap<>(); - metadata.put("_target", getTargetUrl(dvObject)); - return metadata; - } - - @Override - public boolean alreadyRegistered(DvObject dvo) throws Exception { - if(dvo==null) { - logger.severe("Null DvObject sent to alreadyRegistered()."); - return false; - } - GlobalId globalId = dvo.getGlobalId(); - if(globalId == null) { - return false; - } - return alreadyRegistered(globalId, false); - } - - public abstract boolean alreadyRegistered(GlobalId globalId, boolean noProviderDefault) throws Exception; - - /* - * ToDo: the DvObject being sent in provides partial support for the case where - * it has a different authority/protocol than what is configured (i.e. a legacy - * Pid that can actually be updated by the Pid account being used.) Removing - * this now would potentially break/make it harder to handle that case prior to - * support for configuring multiple Pid providers. Once that exists, it would be - * cleaner to always find the PidProvider associated with the - * protocol/authority/shoulder of the current dataset and then not pass the - * DvObject as a param. (This would also remove calls to get the settings since - * that would be done at construction.) - */ - @Override - public DvObject generateIdentifier(DvObject dvObject) { - - String protocol = dvObject.getProtocol() == null ? settingsService.getValueForKey(SettingsServiceBean.Key.Protocol) : dvObject.getProtocol(); - String authority = dvObject.getAuthority() == null ? settingsService.getValueForKey(SettingsServiceBean.Key.Authority) : dvObject.getAuthority(); - if (dvObject.isInstanceofDataset()) { - dvObject.setIdentifier(generateDatasetIdentifier((Dataset) dvObject)); - } else { - dvObject.setIdentifier(generateDataFileIdentifier((DataFile) dvObject)); - } - if (dvObject.getProtocol() == null) { - dvObject.setProtocol(protocol); - } - if (dvObject.getAuthority() == null) { - dvObject.setAuthority(authority); - } - return dvObject; - } - - //ToDo just send the DvObject.DType - public String generateDatasetIdentifier(Dataset dataset) { - //ToDo - track these in the bean - String identifierType = settingsService.getValueForKey(SettingsServiceBean.Key.IdentifierGenerationStyle, "randomString"); - String shoulder = settingsService.getValueForKey(SettingsServiceBean.Key.Shoulder, ""); - - switch (identifierType) { - case "randomString": - return generateIdentifierAsRandomString(dataset, shoulder); - case "storedProcGenerated": - return generateIdentifierFromStoredProcedureIndependent(dataset, shoulder); - default: - /* Should we throw an exception instead?? -- L.A. 4.6.2 */ - return generateIdentifierAsRandomString(dataset, shoulder); - } - } - - - /** - * Check that a identifier entered by the user is unique (not currently used - * for any other study in this Dataverse Network) also check for duplicate - * in EZID if needed - * @param userIdentifier - * @param dataset - * @return {@code true} if the identifier is unique, {@code false} otherwise. - */ - public boolean isGlobalIdUnique(GlobalId globalId) { - if ( ! dvObjectService.isGlobalIdLocallyUnique(globalId) ) { - return false; // duplication found in local database - } - - // not in local DB, look in the persistent identifier service - try { - return ! alreadyRegistered(globalId, false); - } catch (Exception e){ - //we can live with failure - means identifier not found remotely - } - - return true; - } - - /** - * Parse a Persistent Id and set the protocol, authority, and identifier - * - * Example 1: doi:10.5072/FK2/BYM3IW - * protocol: doi - * authority: 10.5072 - * identifier: FK2/BYM3IW - * - * Example 2: hdl:1902.1/111012 - * protocol: hdl - * authority: 1902.1 - * identifier: 111012 - * - * @param identifierString - * @param separator the string that separates the authority from the identifier. - * @param destination the global id that will contain the parsed data. - * @return {@code destination}, after its fields have been updated, or - * {@code null} if parsing failed. - */ - @Override - public GlobalId parsePersistentId(String fullIdentifierString) { - if(!isConfigured()) { - return null; - } - // Occasionally, the protocol separator character ':' comes in still - // URL-encoded as %3A (usually as a result of the URL having been - // encoded twice): - fullIdentifierString = fullIdentifierString.replace("%3A", ":"); - - int index1 = fullIdentifierString.indexOf(':'); - if (index1 > 0) { // ':' found with one or more characters before it - String protocol = fullIdentifierString.substring(0, index1); - GlobalId globalId = parsePersistentId(protocol, fullIdentifierString.substring(index1+1)); - return globalId; - } - logger.log(Level.INFO, "Error parsing identifier: {0}: '':'' not found in string", fullIdentifierString); - return null; - } - - protected GlobalId parsePersistentId(String protocol, String identifierString) { - if(!isConfigured()) { - return null; - } - String authority; - String identifier; - if (identifierString == null) { - return null; - } - int index = identifierString.indexOf('/'); - if (index > 0 && (index + 1) < identifierString.length()) { - // '/' found with one or more characters - // before and after it - // Strip any whitespace, ; and ' from authority (should finding them cause a - // failure instead?) - authority = GlobalIdServiceBean.formatIdentifierString(identifierString.substring(0, index)); - if (GlobalIdServiceBean.testforNullTerminator(authority)) { - return null; - } - identifier = GlobalIdServiceBean.formatIdentifierString(identifierString.substring(index + 1)); - if (GlobalIdServiceBean.testforNullTerminator(identifier)) { - return null; - } - } else { - logger.log(Level.INFO, "Error parsing identifier: {0}: '':/'' not found in string", - identifierString); - return null; - } - return parsePersistentId(protocol, authority, identifier); - } - - public GlobalId parsePersistentId(String protocol, String authority, String identifier) { - if(!isConfigured()) { - return null; - } - logger.fine("Parsing: " + protocol + ":" + authority + getSeparator() + identifier + " in " + getProviderInformation().get(0)); - if(!GlobalIdServiceBean.isValidGlobalId(protocol, authority, identifier)) { - return null; - } - return new GlobalId(protocol, authority, identifier, getSeparator(), getUrlPrefix(), - getProviderInformation().get(0)); - } - - - public String getSeparator() { - //The standard default - return "/"; - } - - @Override - public String generateDataFileIdentifier(DataFile datafile) { - String doiIdentifierType = settingsService.getValueForKey(SettingsServiceBean.Key.IdentifierGenerationStyle, "randomString"); - String doiDataFileFormat = settingsService.getValueForKey(SettingsServiceBean.Key.DataFilePIDFormat, SystemConfig.DataFilePIDFormat.DEPENDENT.toString()); - - String prepend = ""; - if (doiDataFileFormat.equals(SystemConfig.DataFilePIDFormat.DEPENDENT.toString())){ - //If format is dependent then pre-pend the dataset identifier - prepend = datafile.getOwner().getIdentifier() + "/"; - datafile.setProtocol(datafile.getOwner().getProtocol()); - datafile.setAuthority(datafile.getOwner().getAuthority()); - } else { - //If there's a shoulder prepend independent identifiers with it - prepend = settingsService.getValueForKey(SettingsServiceBean.Key.Shoulder, ""); - datafile.setProtocol(settingsService.getValueForKey(SettingsServiceBean.Key.Protocol)); - datafile.setAuthority(settingsService.getValueForKey(SettingsServiceBean.Key.Authority)); - } - - switch (doiIdentifierType) { - case "randomString": - return generateIdentifierAsRandomString(datafile, prepend); - case "storedProcGenerated": - if (doiDataFileFormat.equals(SystemConfig.DataFilePIDFormat.INDEPENDENT.toString())){ - return generateIdentifierFromStoredProcedureIndependent(datafile, prepend); - } else { - return generateIdentifierFromStoredProcedureDependent(datafile, prepend); - } - default: - /* Should we throw an exception instead?? -- L.A. 4.6.2 */ - return generateIdentifierAsRandomString(datafile, prepend); - } - } - - - /* - * This method checks locally for a DvObject with the same PID and if that is OK, checks with the PID service. - * @param dvo - the object to check (ToDo - get protocol/authority from this PidProvider object) - * @param prepend - for Datasets, this is always the shoulder, for DataFiles, it could be the shoulder or the parent Dataset identifier - */ - private String generateIdentifierAsRandomString(DvObject dvo, String prepend) { - String identifier = null; - do { - identifier = prepend + RandomStringUtils.randomAlphanumeric(6).toUpperCase(); - } while (!isGlobalIdUnique(new GlobalId(dvo.getProtocol(), dvo.getAuthority(), identifier, this.getSeparator(), this.getUrlPrefix(), this.getProviderInformation().get(0)))); - - return identifier; - } - - /* - * This method checks locally for a DvObject with the same PID and if that is OK, checks with the PID service. - * @param dvo - the object to check (ToDo - get protocol/authority from this PidProvider object) - * @param prepend - for Datasets, this is always the shoulder, for DataFiles, it could be the shoulder or the parent Dataset identifier - */ - - private String generateIdentifierFromStoredProcedureIndependent(DvObject dvo, String prepend) { - String identifier; - do { - String identifierFromStoredProcedure = dvObjectService.generateNewIdentifierByStoredProcedure(); - // some diagnostics here maybe - is it possible to determine that it's failing - // because the stored procedure hasn't been created in the database? - if (identifierFromStoredProcedure == null) { - return null; - } - identifier = prepend + identifierFromStoredProcedure; - } while (!isGlobalIdUnique(new GlobalId(dvo.getProtocol(), dvo.getAuthority(), identifier, this.getSeparator(), this.getUrlPrefix(), this.getProviderInformation().get(0)))); - - return identifier; - } - - /*This method is only used for DataFiles with DEPENDENT Pids. It is not for Datasets - * - */ - private String generateIdentifierFromStoredProcedureDependent(DataFile datafile, String prepend) { - String identifier; - Long retVal; - retVal = Long.valueOf(0L); - //ToDo - replace loops with one lookup for largest entry? (the do loop runs ~n**2/2 calls). The check for existingIdentifiers means this is mostly a local loop now, versus involving db or PidProvider calls, but still...) - - // This will catch identifiers already assigned in the current transaction (e.g. - // in FinalizeDatasetPublicationCommand) that haven't been committed to the db - // without having to make a call to the PIDProvider - Set existingIdentifiers = new HashSet(); - List files = datafile.getOwner().getFiles(); - for(DataFile f:files) { - existingIdentifiers.add(f.getIdentifier()); - } - - do { - retVal++; - identifier = prepend + retVal.toString(); - - } while (existingIdentifiers.contains(identifier) || !isGlobalIdUnique(new GlobalId(datafile.getProtocol(), datafile.getAuthority(), identifier, this.getSeparator(), this.getUrlPrefix(), this.getProviderInformation().get(0)))); - - return identifier; - } - - - class GlobalIdMetadataTemplate { - - - private String template; - - public GlobalIdMetadataTemplate(){ - try (InputStream in = GlobalIdMetadataTemplate.class.getResourceAsStream("datacite_metadata_template.xml")) { - template = Util.readAndClose(in, "utf-8"); - } catch (Exception e) { - logger.log(Level.SEVERE, "datacite metadata template load error"); - logger.log(Level.SEVERE, "String " + e.toString()); - logger.log(Level.SEVERE, "localized message " + e.getLocalizedMessage()); - logger.log(Level.SEVERE, "cause " + e.getCause()); - logger.log(Level.SEVERE, "message " + e.getMessage()); - } - } - - private String xmlMetadata; - private String identifier; - private List datafileIdentifiers; - private List creators; - private String title; - private String publisher; - private String publisherYear; - private List authors; - private String description; - private List contacts; - private List producers; - - public List getProducers() { - return producers; - } - - public void setProducers(List producers) { - this.producers = producers; - } - - public List getContacts() { - return contacts; - } - - public void setContacts(List contacts) { - this.contacts = contacts; - } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - - public List getAuthors() { - return authors; - } - - public void setAuthors(List authors) { - this.authors = authors; - } - - - public List getDatafileIdentifiers() { - return datafileIdentifiers; - } - - public void setDatafileIdentifiers(List datafileIdentifiers) { - this.datafileIdentifiers = datafileIdentifiers; - } - - public GlobalIdMetadataTemplate(String xmlMetaData) { - this.xmlMetadata = xmlMetaData; - Document doc = Jsoup.parseBodyFragment(xmlMetaData); - Elements identifierElements = doc.select("identifier"); - if (identifierElements.size() > 0) { - identifier = identifierElements.get(0).html(); - } - Elements creatorElements = doc.select("creatorName"); - creators = new ArrayList<>(); - for (Element creatorElement : creatorElements) { - creators.add(creatorElement.html()); - } - Elements titleElements = doc.select("title"); - if (titleElements.size() > 0) { - title = titleElements.get(0).html(); - } - Elements publisherElements = doc.select("publisher"); - if (publisherElements.size() > 0) { - publisher = publisherElements.get(0).html(); - } - Elements publisherYearElements = doc.select("publicationYear"); - if (publisherYearElements.size() > 0) { - publisherYear = publisherYearElements.get(0).html(); - } - } - - public String generateXML(DvObject dvObject) { - // Can't use "UNKNOWN" here because DataCite will respond with "[facet 'pattern'] the value 'unknown' is not accepted by the pattern '[\d]{4}'" - String publisherYearFinal = "9999"; - // FIXME: Investigate why this.publisherYear is sometimes null now that pull request #4606 has been merged. - if (this.publisherYear != null) { - // Added to prevent a NullPointerException when trying to destroy datasets when using DataCite rather than EZID. - publisherYearFinal = this.publisherYear; - } - xmlMetadata = template.replace("${identifier}", getIdentifier().trim()) - .replace("${title}", this.title) - .replace("${publisher}", this.publisher) - .replace("${publisherYear}", publisherYearFinal) - .replace("${description}", this.description); - StringBuilder creatorsElement = new StringBuilder(); - for (DatasetAuthor author : authors) { - creatorsElement.append(""); - creatorsElement.append(author.getName().getDisplayValue()); - creatorsElement.append(""); - - if (author.getIdType() != null && author.getIdValue() != null && !author.getIdType().isEmpty() && !author.getIdValue().isEmpty() && author.getAffiliation() != null && !author.getAffiliation().getDisplayValue().isEmpty()) { - - if (author.getIdType().equals("ORCID")) { - creatorsElement.append("" + author.getIdValue() + ""); - } - if (author.getIdType().equals("ISNI")) { - creatorsElement.append("" + author.getIdValue() + ""); - } - if (author.getIdType().equals("LCNA")) { - creatorsElement.append("" + author.getIdValue() + ""); - } - } - if (author.getAffiliation() != null && !author.getAffiliation().getDisplayValue().isEmpty()) { - creatorsElement.append("" + author.getAffiliation().getDisplayValue() + ""); - } - creatorsElement.append(""); - } - xmlMetadata = xmlMetadata.replace("${creators}", creatorsElement.toString()); - - StringBuilder contributorsElement = new StringBuilder(); - for (String[] contact : this.getContacts()) { - if (!contact[0].isEmpty()) { - contributorsElement.append("" + contact[0] + ""); - if (!contact[1].isEmpty()) { - contributorsElement.append("" + contact[1] + ""); - } - contributorsElement.append(""); - } - } - for (String[] producer : this.getProducers()) { - contributorsElement.append("" + producer[0] + ""); - if (!producer[1].isEmpty()) { - contributorsElement.append("" + producer[1] + ""); - } - contributorsElement.append(""); - } - - String relIdentifiers = generateRelatedIdentifiers(dvObject); - - xmlMetadata = xmlMetadata.replace("${relatedIdentifiers}", relIdentifiers); - - xmlMetadata = xmlMetadata.replace("{$contributors}", contributorsElement.toString()); - return xmlMetadata; - } - - private String generateRelatedIdentifiers(DvObject dvObject) { - - StringBuilder sb = new StringBuilder(); - if (dvObject.isInstanceofDataset()) { - Dataset dataset = (Dataset) dvObject; - if (!dataset.getFiles().isEmpty() && !(dataset.getFiles().get(0).getIdentifier() == null)) { - - datafileIdentifiers = new ArrayList<>(); - for (DataFile dataFile : dataset.getFiles()) { - if (!dataFile.getGlobalId().asString().isEmpty()) { - if (sb.toString().isEmpty()) { - sb.append(""); - } - sb.append("" + dataFile.getGlobalId() + ""); - } - } - - if (!sb.toString().isEmpty()) { - sb.append(""); - } - } - } else if (dvObject.isInstanceofDataFile()) { - DataFile df = (DataFile) dvObject; - sb.append(""); - sb.append("" + df.getOwner().getGlobalId() + ""); - sb.append(""); - } - return sb.toString(); - } - - public void generateFileIdentifiers(DvObject dvObject) { - - if (dvObject.isInstanceofDataset()) { - Dataset dataset = (Dataset) dvObject; - - if (!dataset.getFiles().isEmpty() && !(dataset.getFiles().get(0).getIdentifier() == null)) { - - datafileIdentifiers = new ArrayList<>(); - for (DataFile dataFile : dataset.getFiles()) { - datafileIdentifiers.add(dataFile.getIdentifier()); - int x = xmlMetadata.indexOf("") - 1; - xmlMetadata = xmlMetadata.replace("{relatedIdentifier}", dataFile.getIdentifier()); - xmlMetadata = xmlMetadata.substring(0, x) + "${relatedIdentifier}" + template.substring(x, template.length() - 1); - - } - - } else { - xmlMetadata = xmlMetadata.replace("${relatedIdentifier}", ""); - } - } - } - - public String getTemplate() { - return template; - } - - public void setTemplate(String templateIn) { - template = templateIn; - } - - public String getIdentifier() { - return identifier; - } - - public void setIdentifier(String identifier) { - this.identifier = identifier; - } - - public List getCreators() { - return creators; - } - - public void setCreators(List creators) { - this.creators = creators; - } - - public String getTitle() { - return title; - } - - public void setTitle(String title) { - this.title = title; - } - - public String getPublisher() { - return publisher; - } - - public void setPublisher(String publisher) { - this.publisher = publisher; - } - - public String getPublisherYear() { - return publisherYear; - } - - public void setPublisherYear(String publisherYear) { - this.publisherYear = publisherYear; - } -} - public String getMetadataFromDvObject(String identifier, Map metadata, DvObject dvObject) { - - Dataset dataset = null; - - if (dvObject instanceof Dataset) { - dataset = (Dataset) dvObject; - } else { - dataset = (Dataset) dvObject.getOwner(); - } - - GlobalIdMetadataTemplate metadataTemplate = new GlobalIdMetadataTemplate(); - metadataTemplate.setIdentifier(identifier.substring(identifier.indexOf(':') + 1)); - metadataTemplate.setCreators(Util.getListFromStr(metadata.get("datacite.creator"))); - metadataTemplate.setAuthors(dataset.getLatestVersion().getDatasetAuthors()); - if (dvObject.isInstanceofDataset()) { - metadataTemplate.setDescription(dataset.getLatestVersion().getDescriptionPlainText()); - } - if (dvObject.isInstanceofDataFile()) { - DataFile df = (DataFile) dvObject; - String fileDescription = df.getDescription(); - metadataTemplate.setDescription(fileDescription == null ? "" : fileDescription); - } - - metadataTemplate.setContacts(dataset.getLatestVersion().getDatasetContacts()); - metadataTemplate.setProducers(dataset.getLatestVersion().getDatasetProducers()); - metadataTemplate.setTitle(dvObject.getCurrentName()); - String producerString = dataverseService.getRootDataverseName(); - if (producerString.isEmpty() || producerString.equals(DatasetField.NA_VALUE) ) { - producerString = UNAVAILABLE; - } - metadataTemplate.setPublisher(producerString); - metadataTemplate.setPublisherYear(metadata.get("datacite.publicationyear")); - - String xmlMetadata = metadataTemplate.generateXML(dvObject); - logger.log(Level.FINE, "XML to send to DataCite: {0}", xmlMetadata); - return xmlMetadata; - } - - @Override - public boolean canManagePID() { - //The default expectation is that PID providers are configured to manage some set (i.e. based on protocol/authority/shoulder) of PIDs - return true; - } - - @Override - public boolean isConfigured() { - if(configured==null) { - return false; - } else { - return configured.booleanValue(); - } - } -} diff --git a/src/main/java/edu/harvard/iq/dataverse/BannerMessage.java b/src/main/java/edu/harvard/iq/dataverse/BannerMessage.java index 214e26965fa..003d1057972 100644 --- a/src/main/java/edu/harvard/iq/dataverse/BannerMessage.java +++ b/src/main/java/edu/harvard/iq/dataverse/BannerMessage.java @@ -46,7 +46,7 @@ public void setBannerMessageTexts(Collection bannerMessageTex public String getDisplayValue(){ - String retVal = ""; + String retVal = null; for (BannerMessageText msgTxt : this.getBannerMessageTexts()) { if (msgTxt.getLang().equals(BundleUtil.getCurrentLocale().getLanguage())) { retVal = msgTxt.getMessage(); diff --git a/src/main/java/edu/harvard/iq/dataverse/BannerMessageServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/BannerMessageServiceBean.java index 0e757998d58..3961bd064db 100644 --- a/src/main/java/edu/harvard/iq/dataverse/BannerMessageServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/BannerMessageServiceBean.java @@ -46,8 +46,10 @@ public List findAllBannerMessages() { public void save( BannerMessage message ) { em.persist(message); + em.flush(); } + public void deleteBannerMessage(Object pk) { BannerMessage message = em.find(BannerMessage.class, pk); diff --git a/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterCache.java b/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterCache.java deleted file mode 100644 index 7c75b1a4da6..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterCache.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ -package edu.harvard.iq.dataverse; - - -import java.io.Serializable; -import jakarta.persistence.Column; -import jakarta.persistence.Entity; -import jakarta.persistence.GeneratedValue; -import jakarta.persistence.GenerationType; -import jakarta.persistence.Id; -import jakarta.persistence.Lob; -import jakarta.persistence.NamedQueries; -import jakarta.persistence.NamedQuery; -import org.hibernate.validator.constraints.NotBlank; - -/** - * - * @author luopc - */ -@NamedQueries( - @NamedQuery( name="DOIDataCiteRegisterCache.findByDoi", - query="SELECT d FROM DOIDataCiteRegisterCache d WHERE d.doi=:doi") -) -@Entity -public class DOIDataCiteRegisterCache implements Serializable{ - - private static final long serialVersionUID = 8030143094734315681L; - - @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) - private Long id; - - @NotBlank - @Column(unique=true) - private String doi; - - @NotBlank - private String url; - - @NotBlank - private String status; - - @NotBlank - @Lob - private String xml; - - public Long getId() { - return id; - } - - public void setId(Long id) { - this.id = id; - } - - public String getDoi() { - return doi; - } - - public void setDoi(String doi) { - this.doi = doi; - } - - public String getStatus() { - return status; - } - - public void setStatus(String status) { - this.status = status; - } - - public String getXml() { - return xml; - } - - public void setXml(String xml) { - this.xml = xml; - } - - public String getUrl() { - return url; - } - - public void setUrl(String url) { - this.url = url; - } -} \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java b/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java deleted file mode 100644 index 9ecc4a3ecc9..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java +++ /dev/null @@ -1,707 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ -package edu.harvard.iq.dataverse; - -import edu.harvard.iq.dataverse.branding.BrandingUtil; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.UnsupportedEncodingException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.logging.Level; -import java.util.logging.Logger; -import jakarta.ejb.EJB; -import jakarta.ejb.Stateless; -import jakarta.persistence.EntityManager; -import jakarta.persistence.PersistenceContext; -import jakarta.persistence.TypedQuery; - -import edu.harvard.iq.dataverse.settings.JvmSettings; -import org.apache.commons.text.StringEscapeUtils; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; - -/** - * - * @author luopc - */ -@Stateless -public class DOIDataCiteRegisterService { - - private static final Logger logger = Logger.getLogger(DOIDataCiteRegisterService.class.getCanonicalName()); - - @PersistenceContext(unitName = "VDCNet-ejbPU") - private EntityManager em; - - @EJB - DataverseServiceBean dataverseService; - - @EJB - DOIDataCiteServiceBean doiDataCiteServiceBean; - - - //A singleton since it, and the httpClient in it can be reused. - private DataCiteRESTfullClient client=null; - - private DataCiteRESTfullClient getClient() throws IOException { - if (client == null) { - client = new DataCiteRESTfullClient( - JvmSettings.DATACITE_MDS_API_URL.lookup(), - JvmSettings.DATACITE_USERNAME.lookup(), - JvmSettings.DATACITE_PASSWORD.lookup() - ); - } - return client; - } - - /** - * This method is deprecated and unused. We switched away from this method - * when adjusting the code to reserve DOIs from DataCite on dataset create. - * - * Note that the DOIDataCiteRegisterCache entity/table used in this method - * might be a candidate for deprecation as well. Removing it would require - * some refactoring as it is used throughout the DataCite code. - */ - @Deprecated - public String createIdentifierLocal(String identifier, Map metadata, DvObject dvObject) { - - String xmlMetadata = getMetadataFromDvObject(identifier, metadata, dvObject); - String status = metadata.get("_status").trim(); - String target = metadata.get("_target"); - String retString = ""; - DOIDataCiteRegisterCache rc = findByDOI(identifier); - if (rc == null) { - rc = new DOIDataCiteRegisterCache(); - rc.setDoi(identifier); - rc.setXml(xmlMetadata); - rc.setStatus("reserved"); - rc.setUrl(target); - em.persist(rc); - } else { - rc.setDoi(identifier); - rc.setXml(xmlMetadata); - rc.setStatus("reserved"); - rc.setUrl(target); - } - retString = "success to reserved " + identifier; - - return retString; - } - - /** - * This "reserveIdentifier" method is heavily based on the - * "registerIdentifier" method below but doesn't, this one doesn't doesn't - * register a URL, which causes the "state" of DOI to transition from - * "draft" to "findable". Here are some DataCite docs on the matter: - * - * "DOIs can exist in three states: draft, registered, and findable. DOIs - * are in the draft state when metadata have been registered, and will - * transition to the findable state when registering a URL." -- - * https://support.datacite.org/docs/mds-api-guide#doi-states - */ - public String reserveIdentifier(String identifier, Map metadata, DvObject dvObject) throws IOException { - String retString = ""; - String xmlMetadata = getMetadataFromDvObject(identifier, metadata, dvObject); - DOIDataCiteRegisterCache rc = findByDOI(identifier); - String target = metadata.get("_target"); - if (rc != null) { - rc.setDoi(identifier); - rc.setXml(xmlMetadata); - // DataCite uses the term "draft" instead of "reserved". - rc.setStatus("reserved"); - if (target == null || target.trim().length() == 0) { - target = rc.getUrl(); - } else { - rc.setUrl(target); - } - } - - DataCiteRESTfullClient client = getClient(); - retString = client.postMetadata(xmlMetadata); - - return retString; - } - - public String registerIdentifier(String identifier, Map metadata, DvObject dvObject) throws IOException { - String retString = ""; - String xmlMetadata = getMetadataFromDvObject(identifier, metadata, dvObject); - DOIDataCiteRegisterCache rc = findByDOI(identifier); - String target = metadata.get("_target"); - if (rc != null) { - rc.setDoi(identifier); - rc.setXml(xmlMetadata); - rc.setStatus("public"); - if (target == null || target.trim().length() == 0) { - target = rc.getUrl(); - } else { - rc.setUrl(target); - } - } - - DataCiteRESTfullClient client = getClient(); - retString = client.postMetadata(xmlMetadata); - client.postUrl(identifier.substring(identifier.indexOf(":") + 1), target); - - return retString; - } - - public String deactivateIdentifier(String identifier, Map metadata, DvObject dvObject) throws IOException { - String retString = ""; - - String metadataString = getMetadataForDeactivateIdentifier(identifier, metadata, dvObject); - retString = client.postMetadata(metadataString); - retString = client.inactiveDataset(identifier.substring(identifier.indexOf(":") + 1)); - - return retString; - } - - public static String getMetadataFromDvObject(String identifier, Map metadata, DvObject dvObject) { - - Dataset dataset = null; - - if (dvObject instanceof Dataset) { - dataset = (Dataset) dvObject; - } else { - dataset = (Dataset) dvObject.getOwner(); - } - - DataCiteMetadataTemplate metadataTemplate = new DataCiteMetadataTemplate(); - metadataTemplate.setIdentifier(identifier.substring(identifier.indexOf(':') + 1)); - metadataTemplate.setCreators(Util.getListFromStr(metadata.get("datacite.creator"))); - metadataTemplate.setAuthors(dataset.getLatestVersion().getDatasetAuthors()); - if (dvObject.isInstanceofDataset()) { - //While getDescriptionPlainText strips < and > from HTML, it leaves '&' (at least so we need to xml escape as well - String description = StringEscapeUtils.escapeXml10(dataset.getLatestVersion().getDescriptionPlainText()); - if (description.isEmpty() || description.equals(DatasetField.NA_VALUE)) { - description = AbstractGlobalIdServiceBean.UNAVAILABLE; - } - metadataTemplate.setDescription(description); - } - if (dvObject.isInstanceofDataFile()) { - DataFile df = (DataFile) dvObject; - //Note: File metadata is not escaped like dataset metadata is, so adding an xml escape here. - //This could/should be removed if the datafile methods add escaping - String fileDescription = StringEscapeUtils.escapeXml10(df.getDescription()); - metadataTemplate.setDescription(fileDescription == null ? AbstractGlobalIdServiceBean.UNAVAILABLE : fileDescription); - String datasetPid = df.getOwner().getGlobalId().asString(); - metadataTemplate.setDatasetIdentifier(datasetPid); - } else { - metadataTemplate.setDatasetIdentifier(""); - } - - metadataTemplate.setContacts(dataset.getLatestVersion().getDatasetContacts()); - metadataTemplate.setProducers(dataset.getLatestVersion().getDatasetProducers()); - String title = dvObject.getCurrentName(); - if(dvObject.isInstanceofDataFile()) { - //Note file title is not currently escaped the way the dataset title is, so adding it here. - title = StringEscapeUtils.escapeXml10(title); - } - - if (title.isEmpty() || title.equals(DatasetField.NA_VALUE)) { - title = AbstractGlobalIdServiceBean.UNAVAILABLE; - } - - metadataTemplate.setTitle(title); - String producerString = BrandingUtil.getRootDataverseCollectionName(); - if (producerString.isEmpty() || producerString.equals(DatasetField.NA_VALUE)) { - producerString = AbstractGlobalIdServiceBean.UNAVAILABLE; - } - metadataTemplate.setPublisher(producerString); - metadataTemplate.setPublisherYear(metadata.get("datacite.publicationyear")); - - String xmlMetadata = metadataTemplate.generateXML(dvObject); - logger.log(Level.FINE, "XML to send to DataCite: {0}", xmlMetadata); - return xmlMetadata; - } - - public static String getMetadataForDeactivateIdentifier(String identifier, Map metadata, DvObject dvObject) { - - DataCiteMetadataTemplate metadataTemplate = new DataCiteMetadataTemplate(); - metadataTemplate.setIdentifier(identifier.substring(identifier.indexOf(':') + 1)); - metadataTemplate.setCreators(Util.getListFromStr(metadata.get("datacite.creator"))); - - metadataTemplate.setDescription(AbstractGlobalIdServiceBean.UNAVAILABLE); - - String title =metadata.get("datacite.title"); - - System.out.print("Map metadata title: "+ metadata.get("datacite.title")); - - metadataTemplate.setAuthors(null); - - metadataTemplate.setTitle(title); - String producerString = AbstractGlobalIdServiceBean.UNAVAILABLE; - - metadataTemplate.setPublisher(producerString); - metadataTemplate.setPublisherYear(metadata.get("datacite.publicationyear")); - - String xmlMetadata = metadataTemplate.generateXML(dvObject); - logger.log(Level.FINE, "XML to send to DataCite: {0}", xmlMetadata); - return xmlMetadata; - } - - public String modifyIdentifier(String identifier, HashMap metadata, DvObject dvObject) throws IOException { - - String xmlMetadata = getMetadataFromDvObject(identifier, metadata, dvObject); - - logger.fine("XML to send to DataCite: " + xmlMetadata); - - String status = metadata.get("_status").trim(); - String target = metadata.get("_target"); - String retString = ""; - if (status.equals("reserved")) { - DOIDataCiteRegisterCache rc = findByDOI(identifier); - if (rc == null) { - rc = new DOIDataCiteRegisterCache(); - rc.setDoi(identifier); - rc.setXml(xmlMetadata); - rc.setStatus("reserved"); - rc.setUrl(target); - em.persist(rc); - } else { - rc.setDoi(identifier); - rc.setXml(xmlMetadata); - rc.setStatus("reserved"); - rc.setUrl(target); - } - retString = "success to reserved " + identifier; - } else if (status.equals("public")) { - DOIDataCiteRegisterCache rc = findByDOI(identifier); - if (rc != null) { - rc.setDoi(identifier); - rc.setXml(xmlMetadata); - rc.setStatus("public"); - if (target == null || target.trim().length() == 0) { - target = rc.getUrl(); - } else { - rc.setUrl(target); - } - try { - DataCiteRESTfullClient client = getClient(); - retString = client.postMetadata(xmlMetadata); - client.postUrl(identifier.substring(identifier.indexOf(":") + 1), target); - - } catch (UnsupportedEncodingException ex) { - logger.log(Level.SEVERE, null, ex); - - } catch (RuntimeException rte) { - logger.log(Level.SEVERE, "Error creating DOI at DataCite: {0}", rte.getMessage()); - logger.log(Level.SEVERE, "Exception", rte); - - } - } - } else if (status.equals("unavailable")) { - DOIDataCiteRegisterCache rc = findByDOI(identifier); - try { - DataCiteRESTfullClient client = getClient(); - if (rc != null) { - rc.setStatus("unavailable"); - retString = client.inactiveDataset(identifier.substring(identifier.indexOf(":") + 1)); - } - } catch (IOException io) { - - } - } - return retString; - } - - public boolean testDOIExists(String identifier) { - boolean doiExists; - try { - DataCiteRESTfullClient client = getClient(); - doiExists = client.testDOIExists(identifier.substring(identifier.indexOf(":") + 1)); - } catch (Exception e) { - logger.log(Level.INFO, identifier, e); - return false; - } - return doiExists; - } - - public HashMap getMetadata(String identifier) throws IOException { - HashMap metadata = new HashMap<>(); - try { - DataCiteRESTfullClient client = getClient(); - String xmlMetadata = client.getMetadata(identifier.substring(identifier.indexOf(":") + 1)); - DOIDataCiteServiceBean.GlobalIdMetadataTemplate template = doiDataCiteServiceBean.new GlobalIdMetadataTemplate(xmlMetadata); - metadata.put("datacite.creator", Util.getStrFromList(template.getCreators())); - metadata.put("datacite.title", template.getTitle()); - metadata.put("datacite.publisher", template.getPublisher()); - metadata.put("datacite.publicationyear", template.getPublisherYear()); - DOIDataCiteRegisterCache rc = findByDOI(identifier); - if (rc != null) { - metadata.put("_status", rc.getStatus()); - } else { - metadata.put("_status", "public"); - } - } catch (RuntimeException e) { - logger.log(Level.INFO, identifier, e); - } - return metadata; - } - - public DOIDataCiteRegisterCache findByDOI(String doi) { - TypedQuery query = em.createNamedQuery("DOIDataCiteRegisterCache.findByDoi", - DOIDataCiteRegisterCache.class); - query.setParameter("doi", doi); - List rc = query.getResultList(); - if (rc.size() == 1) { - return rc.get(0); - } - return null; - } - - public void deleteIdentifier(String identifier) { - DOIDataCiteRegisterCache rc = findByDOI(identifier); - if (rc != null) { - em.remove(rc); - } - } - -} - -class DataCiteMetadataTemplate { - - private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.DataCiteMetadataTemplate"); - private static String template; - - static { - try (InputStream in = DataCiteMetadataTemplate.class.getResourceAsStream("datacite_metadata_template.xml")) { - template = Util.readAndClose(in, "utf-8"); - } catch (Exception e) { - logger.log(Level.SEVERE, "datacite metadata template load error"); - logger.log(Level.SEVERE, "String " + e.toString()); - logger.log(Level.SEVERE, "localized message " + e.getLocalizedMessage()); - logger.log(Level.SEVERE, "cause " + e.getCause()); - logger.log(Level.SEVERE, "message " + e.getMessage()); - } - } - - private String xmlMetadata; - private String identifier; - private String datasetIdentifier; - private List datafileIdentifiers; - private List creators; - private String title; - private String publisher; - private String publisherYear; - private List authors; - private String description; - private List contacts; - private List producers; - - public List getProducers() { - return producers; - } - - public void setProducers(List producers) { - this.producers = producers; - } - - public List getContacts() { - return contacts; - } - - public void setContacts(List contacts) { - this.contacts = contacts; - } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - - public List getAuthors() { - return authors; - } - - public void setAuthors(List authors) { - this.authors = authors; - } - - public DataCiteMetadataTemplate() { - } - - public List getDatafileIdentifiers() { - return datafileIdentifiers; - } - - public void setDatafileIdentifiers(List datafileIdentifiers) { - this.datafileIdentifiers = datafileIdentifiers; - } - - public DataCiteMetadataTemplate(String xmlMetaData) { - this.xmlMetadata = xmlMetaData; - Document doc = Jsoup.parseBodyFragment(xmlMetaData); - Elements identifierElements = doc.select("identifier"); - if (identifierElements.size() > 0) { - identifier = identifierElements.get(0).html(); - } - Elements creatorElements = doc.select("creatorName"); - creators = new ArrayList<>(); - for (Element creatorElement : creatorElements) { - creators.add(creatorElement.html()); - } - Elements titleElements = doc.select("title"); - if (titleElements.size() > 0) { - title = titleElements.get(0).html(); - } - Elements publisherElements = doc.select("publisher"); - if (publisherElements.size() > 0) { - publisher = publisherElements.get(0).html(); - } - Elements publisherYearElements = doc.select("publicationYear"); - if (publisherYearElements.size() > 0) { - publisherYear = publisherYearElements.get(0).html(); - } - } - - public String generateXML(DvObject dvObject) { - // Can't use "UNKNOWN" here because DataCite will respond with "[facet 'pattern'] the value 'unknown' is not accepted by the pattern '[\d]{4}'" - String publisherYearFinal = "9999"; - // FIXME: Investigate why this.publisherYear is sometimes null now that pull request #4606 has been merged. - if (this.publisherYear != null) { - // Added to prevent a NullPointerException when trying to destroy datasets when using DataCite rather than EZID. - publisherYearFinal = this.publisherYear; - } - xmlMetadata = template.replace("${identifier}", this.identifier.trim()) - .replace("${title}", this.title) - .replace("${publisher}", this.publisher) - .replace("${publisherYear}", publisherYearFinal) - .replace("${description}", this.description); - - StringBuilder creatorsElement = new StringBuilder(); - if (authors!= null && !authors.isEmpty()) { - for (DatasetAuthor author : authors) { - creatorsElement.append(""); - creatorsElement.append(author.getName().getDisplayValue()); - creatorsElement.append(""); - - if (author.getIdType() != null && author.getIdValue() != null && !author.getIdType().isEmpty() && !author.getIdValue().isEmpty() && author.getAffiliation() != null && !author.getAffiliation().getDisplayValue().isEmpty()) { - - if (author.getIdType().equals("ORCID")) { - creatorsElement.append("" + author.getIdValue() + ""); - } - if (author.getIdType().equals("ISNI")) { - creatorsElement.append("" + author.getIdValue() + ""); - } - if (author.getIdType().equals("LCNA")) { - creatorsElement.append("" + author.getIdValue() + ""); - } - } - if (author.getAffiliation() != null && !author.getAffiliation().getDisplayValue().isEmpty()) { - creatorsElement.append("" + author.getAffiliation().getDisplayValue() + ""); - } - creatorsElement.append(""); - } - - } else { - creatorsElement.append("").append(AbstractGlobalIdServiceBean.UNAVAILABLE).append(""); - } - - xmlMetadata = xmlMetadata.replace("${creators}", creatorsElement.toString()); - - StringBuilder contributorsElement = new StringBuilder(); - if (this.getContacts() != null) { - for (String[] contact : this.getContacts()) { - if (!contact[0].isEmpty()) { - contributorsElement.append("" + contact[0] + ""); - if (!contact[1].isEmpty()) { - contributorsElement.append("" + contact[1] + ""); - } - contributorsElement.append(""); - } - } - } - - if (this.getProducers() != null) { - for (String[] producer : this.getProducers()) { - contributorsElement.append("" + producer[0] + ""); - if (!producer[1].isEmpty()) { - contributorsElement.append("" + producer[1] + ""); - } - contributorsElement.append(""); - } - } - - String relIdentifiers = generateRelatedIdentifiers(dvObject); - - xmlMetadata = xmlMetadata.replace("${relatedIdentifiers}", relIdentifiers); - - xmlMetadata = xmlMetadata.replace("{$contributors}", contributorsElement.toString()); - return xmlMetadata; - } - - private String generateRelatedIdentifiers(DvObject dvObject) { - - StringBuilder sb = new StringBuilder(); - if (dvObject.isInstanceofDataset()) { - Dataset dataset = (Dataset) dvObject; - if (!dataset.getFiles().isEmpty() && !(dataset.getFiles().get(0).getIdentifier() == null)) { - - datafileIdentifiers = new ArrayList<>(); - for (DataFile dataFile : dataset.getFiles()) { - if (dataFile.getGlobalId() != null) { - if (sb.toString().isEmpty()) { - sb.append(""); - } - sb.append("" + dataFile.getGlobalId() + ""); - } - } - - if (!sb.toString().isEmpty()) { - sb.append(""); - } - } - } else if (dvObject.isInstanceofDataFile()) { - DataFile df = (DataFile) dvObject; - sb.append(""); - sb.append("" + df.getOwner().getGlobalId() + ""); - sb.append(""); - } - return sb.toString(); - } - - public void generateFileIdentifiers(DvObject dvObject) { - - if (dvObject.isInstanceofDataset()) { - Dataset dataset = (Dataset) dvObject; - - if (!dataset.getFiles().isEmpty() && !(dataset.getFiles().get(0).getIdentifier() == null)) { - - datafileIdentifiers = new ArrayList<>(); - for (DataFile dataFile : dataset.getFiles()) { - datafileIdentifiers.add(dataFile.getIdentifier()); - int x = xmlMetadata.indexOf("") - 1; - xmlMetadata = xmlMetadata.replace("{relatedIdentifier}", dataFile.getIdentifier()); - xmlMetadata = xmlMetadata.substring(0, x) + "${relatedIdentifier}" + template.substring(x, template.length() - 1); - - } - - } else { - xmlMetadata = xmlMetadata.replace("${relatedIdentifier}", ""); - } - } - } - - public static String getTemplate() { - return template; - } - - public static void setTemplate(String template) { - DataCiteMetadataTemplate.template = template; - } - - public String getIdentifier() { - return identifier; - } - - public void setIdentifier(String identifier) { - this.identifier = identifier; - } - - public void setDatasetIdentifier(String datasetIdentifier) { - this.datasetIdentifier = datasetIdentifier; - } - - public List getCreators() { - return creators; - } - - public void setCreators(List creators) { - this.creators = creators; - } - - public String getTitle() { - return title; - } - - public void setTitle(String title) { - this.title = title; - } - - public String getPublisher() { - return publisher; - } - - public void setPublisher(String publisher) { - this.publisher = publisher; - } - - public String getPublisherYear() { - return publisherYear; - } - - public void setPublisherYear(String publisherYear) { - this.publisherYear = publisherYear; - } -} - -class Util { - - public static void close(InputStream in) { - if (in != null) { - try { - in.close(); - } catch (IOException e) { - throw new RuntimeException("Fail to close InputStream"); - } - } - } - - public static String readAndClose(InputStream inStream, String encoding) { - ByteArrayOutputStream outStream = new ByteArrayOutputStream(); - byte[] buf = new byte[128]; - String data; - try { - int cnt; - while ((cnt = inStream.read(buf)) >= 0) { - outStream.write(buf, 0, cnt); - } - data = outStream.toString(encoding); - } catch (IOException ioe) { - throw new RuntimeException("IOException"); - } finally { - close(inStream); - } - return data; - } - - public static List getListFromStr(String str) { - return Arrays.asList(str.split("; ")); -// List authors = new ArrayList(); -// int preIdx = 0; -// for(int i=0;i authors) { - StringBuilder str = new StringBuilder(); - for (String author : authors) { - if (str.length() > 0) { - str.append("; "); - } - str.append(author); - } - return str.toString(); - } - -} diff --git a/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteServiceBean.java deleted file mode 100644 index 48786b41824..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteServiceBean.java +++ /dev/null @@ -1,248 +0,0 @@ -package edu.harvard.iq.dataverse; - -import java.io.IOException; -import java.net.HttpURLConnection; -import java.net.URL; -import java.util.Base64; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.logging.Level; -import java.util.logging.Logger; - -import jakarta.ejb.EJB; -import jakarta.ejb.Stateless; - -import edu.harvard.iq.dataverse.settings.JvmSettings; -import org.apache.commons.httpclient.HttpException; -import org.apache.commons.httpclient.HttpStatus; - - -/** - * - * @author luopc - */ -@Stateless -public class DOIDataCiteServiceBean extends DOIServiceBean { - - private static final Logger logger = Logger.getLogger(DOIDataCiteServiceBean.class.getCanonicalName()); - - private static final String PUBLIC = "public"; - private static final String FINDABLE = "findable"; - private static final String RESERVED = "reserved"; - private static final String DRAFT = "draft"; - - @EJB - DOIDataCiteRegisterService doiDataCiteRegisterService; - - @Override - public boolean registerWhenPublished() { - return false; - } - - - - @Override - public boolean alreadyRegistered(GlobalId pid, boolean noProviderDefault) { - logger.log(Level.FINE,"alreadyRegistered"); - if(pid==null || pid.asString().isEmpty()) { - logger.fine("No identifier sent."); - return false; - } - boolean alreadyRegistered; - String identifier = pid.asString(); - try{ - alreadyRegistered = doiDataCiteRegisterService.testDOIExists(identifier); - } catch (Exception e){ - logger.log(Level.WARNING, "alreadyRegistered failed"); - return false; - } - return alreadyRegistered; - } - - @Override - public String createIdentifier(DvObject dvObject) throws Exception { - logger.log(Level.FINE,"createIdentifier"); - if(dvObject.getIdentifier() == null || dvObject.getIdentifier().isEmpty() ){ - dvObject = generateIdentifier(dvObject); - } - String identifier = getIdentifier(dvObject); - Map metadata = getMetadataForCreateIndicator(dvObject); - metadata.put("_status", "reserved"); - try { - String retString = doiDataCiteRegisterService.reserveIdentifier(identifier, metadata, dvObject); - logger.log(Level.FINE, "create DOI identifier retString : " + retString); - return retString; - } catch (Exception e) { - logger.log(Level.WARNING, "Identifier not created: create failed", e); - throw e; - } - } - - @Override - public Map getIdentifierMetadata(DvObject dvObject) { - logger.log(Level.FINE,"getIdentifierMetadata"); - String identifier = getIdentifier(dvObject); - Map metadata = new HashMap<>(); - try { - metadata = doiDataCiteRegisterService.getMetadata(identifier); - } catch (Exception e) { - logger.log(Level.WARNING, "getIdentifierMetadata failed", e); - } - return metadata; - } - - - /** - * Modifies the DOI metadata for a Dataset - * @param dvObject the dvObject whose metadata needs to be modified - * @return the Dataset identifier, or null if the modification failed - * @throws java.lang.Exception - */ - @Override - public String modifyIdentifierTargetURL(DvObject dvObject) throws Exception { - logger.log(Level.FINE,"modifyIdentifier"); - String identifier = getIdentifier(dvObject); - try { - HashMap metadata = doiDataCiteRegisterService.getMetadata(identifier); - doiDataCiteRegisterService.modifyIdentifier(identifier, metadata, dvObject); - } catch (Exception e) { - logger.log(Level.WARNING, "modifyMetadata failed", e); - throw e; - } - return identifier; - } - - public void deleteRecordFromCache(Dataset datasetIn){ - logger.log(Level.FINE,"deleteRecordFromCache"); - String identifier = getIdentifier(datasetIn); - HashMap doiMetadata = new HashMap(); - try { - doiMetadata = doiDataCiteRegisterService.getMetadata(identifier); - } catch (Exception e) { - logger.log(Level.WARNING, "get matadata failed cannot delete"); - logger.log(Level.WARNING, "String {0}", e.toString()); - logger.log(Level.WARNING, "localized message {0}", e.getLocalizedMessage()); - logger.log(Level.WARNING, "cause", e.getCause()); - logger.log(Level.WARNING, "message {0}", e.getMessage()); - } - - String idStatus = (String) doiMetadata.get("_status"); - - if (idStatus == null || idStatus.equals("reserved")) { - logger.log(Level.WARNING, "Delete status is reserved.."); - try { - doiDataCiteRegisterService.deleteIdentifier(identifier); - } catch (Exception e) { - logger.log(Level.WARNING, "delete failed"); - logger.log(Level.WARNING, "String {0}", e.toString()); - logger.log(Level.WARNING, "localized message {0}", e.getLocalizedMessage()); - logger.log(Level.WARNING, "cause", e.getCause()); - logger.log(Level.WARNING, "message {0}", e.getMessage()); - throw new RuntimeException(e); - } - } - } - - /* - * Deletes a DOI if it is in DRAFT/RESERVED state or removes metadata and changes it from PUBLIC/FINDABLE to REGISTERED. - */ - @Override - public void deleteIdentifier(DvObject dvObject) throws IOException, HttpException { - logger.log(Level.FINE,"deleteIdentifier"); - String identifier = getIdentifier(dvObject); - //ToDo - PidUtils currently has a DataCite API call that would get the status at DataCite for this identifier - that could be more accurate than assuming based on whether the dvObject has been published - String idStatus = DRAFT; - if(dvObject.isReleased()) { - idStatus = PUBLIC; - } - if ( idStatus != null ) { - switch ( idStatus ) { - case RESERVED: - case DRAFT: - logger.log(Level.INFO, "Delete status is reserved.."); - //service only removes the identifier from the cache (since it was written before DOIs could be registered in draft state) - doiDataCiteRegisterService.deleteIdentifier(identifier); - //So we call the deleteDraftIdentifier method below until things are refactored - deleteDraftIdentifier(dvObject); - break; - - case PUBLIC: - case FINDABLE: - //if public then it has been released set to unavailable and reset target to n2t url - Map metadata = addDOIMetadataForDestroyedDataset(dvObject); - metadata.put("_status", "registered"); - metadata.put("_target", getTargetUrl(dvObject)); - doiDataCiteRegisterService.deactivateIdentifier(identifier, metadata, dvObject); - break; - } - } - } - - /** - * Deletes DOI from the DataCite side, if possible. Only "draft" DOIs can be - * deleted. - */ - private void deleteDraftIdentifier(DvObject dvObject) throws IOException { - - //ToDo - incorporate into DataCiteRESTfulClient - String baseUrl = JvmSettings.DATACITE_REST_API_URL.lookup(); - String username = JvmSettings.DATACITE_USERNAME.lookup(); - String password = JvmSettings.DATACITE_PASSWORD.lookup(); - GlobalId doi = dvObject.getGlobalId(); - /** - * Deletes the DOI from DataCite if it can. Returns 204 if PID was deleted - * (only possible for "draft" DOIs), 405 (method not allowed) if the DOI - * wasn't deleted (because it's in "findable" state, for example, 404 if the - * DOI wasn't found, and possibly other status codes such as 500 if DataCite - * is down. - */ - - URL url = new URL(baseUrl + "/dois/" + doi.getAuthority() + "/" + doi.getIdentifier()); - HttpURLConnection connection = null; - connection = (HttpURLConnection) url.openConnection(); - connection.setRequestMethod("DELETE"); - String userpass = username + ":" + password; - String basicAuth = "Basic " + new String(Base64.getEncoder().encode(userpass.getBytes())); - connection.setRequestProperty("Authorization", basicAuth); - int status = connection.getResponseCode(); - if(status!=HttpStatus.SC_NO_CONTENT) { - logger.warning("Incorrect Response Status from DataCite: " + status + " : " + connection.getResponseMessage()); - throw new HttpException("Status: " + status); - } - logger.fine("deleteDoi status for " + doi.asString() + ": " + status); - } - - @Override - public boolean publicizeIdentifier(DvObject dvObject) { - logger.log(Level.FINE,"updateIdentifierStatus"); - if(dvObject.getIdentifier() == null || dvObject.getIdentifier().isEmpty() ){ - dvObject = generateIdentifier(dvObject); - } - String identifier = getIdentifier(dvObject); - Map metadata = getUpdateMetadata(dvObject); - metadata.put("_status", PUBLIC); - metadata.put("datacite.publicationyear", generateYear(dvObject)); - metadata.put("_target", getTargetUrl(dvObject)); - try { - doiDataCiteRegisterService.registerIdentifier(identifier, metadata, dvObject); - return true; - } catch (Exception e) { - logger.log(Level.WARNING, "modifyMetadata failed: " + e.getMessage(), e); - return false; - } - } - - - @Override - public List getProviderInformation(){ - return List.of("DataCite", "https://status.datacite.org"); - } - - - - @Override - protected String getProviderKeyName() { - return "DataCite"; - } -} diff --git a/src/main/java/edu/harvard/iq/dataverse/DOIServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DOIServiceBean.java deleted file mode 100644 index 0182c745cd0..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/DOIServiceBean.java +++ /dev/null @@ -1,78 +0,0 @@ -package edu.harvard.iq.dataverse; - -import edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key; - -public abstract class DOIServiceBean extends AbstractGlobalIdServiceBean { - - public static final String DOI_PROTOCOL = "doi"; - public static final String DOI_RESOLVER_URL = "https://doi.org/"; - public static final String HTTP_DOI_RESOLVER_URL = "http://doi.org/"; - public static final String DXDOI_RESOLVER_URL = "https://dx.doi.org/"; - public static final String HTTP_DXDOI_RESOLVER_URL = "http://dx.doi.org/"; - - public DOIServiceBean() { - super(); - } - - @Override - public GlobalId parsePersistentId(String pidString) { - if (pidString.startsWith(DOI_RESOLVER_URL)) { - pidString = pidString.replace(DOI_RESOLVER_URL, - (DOI_PROTOCOL + ":")); - } else if (pidString.startsWith(HTTP_DOI_RESOLVER_URL)) { - pidString = pidString.replace(HTTP_DOI_RESOLVER_URL, - (DOI_PROTOCOL + ":")); - } else if (pidString.startsWith(DXDOI_RESOLVER_URL)) { - pidString = pidString.replace(DXDOI_RESOLVER_URL, - (DOI_PROTOCOL + ":")); - } - return super.parsePersistentId(pidString); - } - - @Override - public GlobalId parsePersistentId(String protocol, String identifierString) { - - if (!DOI_PROTOCOL.equals(protocol)) { - return null; - } - GlobalId globalId = super.parsePersistentId(protocol, identifierString); - if (globalId!=null && !GlobalIdServiceBean.checkDOIAuthority(globalId.getAuthority())) { - return null; - } - return globalId; - } - - @Override - public GlobalId parsePersistentId(String protocol, String authority, String identifier) { - - if (!DOI_PROTOCOL.equals(protocol)) { - return null; - } - return super.parsePersistentId(protocol, authority, identifier); - } - - public String getUrlPrefix() { - return DOI_RESOLVER_URL; - } - - @Override - public boolean isConfigured() { - if (configured == null) { - if (getProviderKeyName() == null) { - configured = false; - } else { - String doiProvider = settingsService.getValueForKey(Key.DoiProvider, ""); - if (getProviderKeyName().equals(doiProvider)) { - configured = true; - } else if (!doiProvider.isEmpty()) { - configured = false; - } - } - } - return super.isConfigured(); - } - - protected String getProviderKeyName() { - return null; - } -} \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/DataCitation.java b/src/main/java/edu/harvard/iq/dataverse/DataCitation.java index 9b4b89db44f..a012175deae 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataCitation.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataCitation.java @@ -7,6 +7,7 @@ import edu.harvard.iq.dataverse.branding.BrandingUtil; import edu.harvard.iq.dataverse.harvest.client.HarvestingClient; +import edu.harvard.iq.dataverse.pidproviders.AbstractPidProvider; import java.io.BufferedWriter; import java.io.ByteArrayOutputStream; @@ -635,12 +636,12 @@ public Map getDataCiteMetadata() { String authorString = getAuthorsString(); if (authorString.isEmpty()) { - authorString = AbstractGlobalIdServiceBean.UNAVAILABLE; + authorString = AbstractPidProvider.UNAVAILABLE; } String producerString = getPublisher(); if (producerString.isEmpty()) { - producerString = AbstractGlobalIdServiceBean.UNAVAILABLE; + producerString = AbstractPidProvider.UNAVAILABLE; } metadata.put("datacite.creator", authorString); diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFile.java b/src/main/java/edu/harvard/iq/dataverse/DataFile.java index 3d8086b142b..29a4a14c021 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataFile.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataFile.java @@ -242,6 +242,18 @@ public void setEmbargo(Embargo embargo) { this.embargo = embargo; } + @ManyToOne + @JoinColumn(name="retention_id") + private Retention retention; + + public Retention getRetention() { + return retention; + } + + public void setRetention(Retention retention) { + this.retention = retention; + } + public DataFile() { this.fileMetadatas = new ArrayList<>(); initFileReplaceAttributes(); @@ -545,61 +557,61 @@ public void setDescription(String description) { fmd.setDescription(description); } } + + public FileMetadata getDraftFileMetadata() { + FileMetadata latestFileMetadata = getLatestFileMetadata(); + if (latestFileMetadata.getDatasetVersion().isDraft()) { + return latestFileMetadata; + } + return null; + } public FileMetadata getFileMetadata() { return getLatestFileMetadata(); } - + public FileMetadata getLatestFileMetadata() { - FileMetadata fmd = null; + FileMetadata resultFileMetadata = null; - // for newly added or harvested, just return the one fmd if (fileMetadatas.size() == 1) { return fileMetadatas.get(0); } - + for (FileMetadata fileMetadata : fileMetadatas) { - // if it finds a draft, return it if (fileMetadata.getDatasetVersion().getVersionState().equals(VersionState.DRAFT)) { return fileMetadata; - } - - // otherwise return the one with the latest version number - // duplicate logic in getLatestPublishedFileMetadata() - if (fmd == null || fileMetadata.getDatasetVersion().getVersionNumber().compareTo( fmd.getDatasetVersion().getVersionNumber() ) > 0 ) { - fmd = fileMetadata; - } else if ((fileMetadata.getDatasetVersion().getVersionNumber().compareTo( fmd.getDatasetVersion().getVersionNumber())==0 )&& - ( fileMetadata.getDatasetVersion().getMinorVersionNumber().compareTo( fmd.getDatasetVersion().getMinorVersionNumber()) > 0 ) ) { - fmd = fileMetadata; } + resultFileMetadata = getTheNewerFileMetadata(resultFileMetadata, fileMetadata); } - return fmd; + + return resultFileMetadata; } - -// //Returns null if no published version + public FileMetadata getLatestPublishedFileMetadata() throws UnsupportedOperationException { - FileMetadata fmd = null; - - for (FileMetadata fileMetadata : fileMetadatas) { - // if it finds a draft, skip - if (fileMetadata.getDatasetVersion().getVersionState().equals(VersionState.DRAFT)) { - continue; - } - - // otherwise return the one with the latest version number - // duplicate logic in getLatestFileMetadata() - if (fmd == null || fileMetadata.getDatasetVersion().getVersionNumber().compareTo( fmd.getDatasetVersion().getVersionNumber() ) > 0 ) { - fmd = fileMetadata; - } else if ((fileMetadata.getDatasetVersion().getVersionNumber().compareTo( fmd.getDatasetVersion().getVersionNumber())==0 )&& - ( fileMetadata.getDatasetVersion().getMinorVersionNumber().compareTo( fmd.getDatasetVersion().getMinorVersionNumber()) > 0 ) ) { - fmd = fileMetadata; - } - } - if(fmd == null) { + FileMetadata resultFileMetadata = fileMetadatas.stream() + .filter(metadata -> !metadata.getDatasetVersion().getVersionState().equals(VersionState.DRAFT)) + .reduce(null, DataFile::getTheNewerFileMetadata); + + if (resultFileMetadata == null) { throw new UnsupportedOperationException("No published metadata version for DataFile " + this.getId()); } - return fmd; + return resultFileMetadata; + } + + public static FileMetadata getTheNewerFileMetadata(FileMetadata current, FileMetadata candidate) { + if (current == null) { + return candidate; + } + + DatasetVersion currentVersion = current.getDatasetVersion(); + DatasetVersion candidateVersion = candidate.getDatasetVersion(); + + if (DatasetVersion.compareByVersion.compare(candidateVersion, currentVersion) > 0) { + return candidate; + } + + return current; } /** @@ -610,7 +622,7 @@ public long getFilesize() { if (this.filesize == null) { // -1 means "unknown" return -1; - } + } return this.filesize; } diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java index c9d50bbed9d..41ea6ae39f0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java @@ -959,6 +959,7 @@ public boolean isThumbnailAvailable (DataFile file) { return true; } file.setPreviewImageFail(true); + file.setPreviewImageAvailable(false); this.save(file); return false; } @@ -1242,9 +1243,8 @@ public List selectFilesWithMissingOriginalSizes() { * Check that a identifier entered by the user is unique (not currently used * for any other study in this Dataverse Network). Also check for duplicate * in the remote PID service if needed - * @param userIdentifier - * @param datafile - * @param idServiceBean + * @param datafileId + * @param storageLocation * @return {@code true} iff the global identifier is unique. */ public void finalizeFileDelete(Long dataFileId, String storageLocation) throws IOException { @@ -1366,7 +1366,10 @@ public Embargo findEmbargo(Long id) { DataFile d = find(id); return d.getEmbargo(); } - + + public boolean isRetentionExpired(FileMetadata fm) { + return FileUtil.isRetentionExpired(fm); + } /** * Checks if the supplied DvObjectContainer (Dataset or Collection; although * only collection-level storage quotas are officially supported as of now) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataTable.java b/src/main/java/edu/harvard/iq/dataverse/DataTable.java index a17d8c65138..95f3aed0f40 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataTable.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataTable.java @@ -112,6 +112,16 @@ public DataTable() { @Column( nullable = true ) private String originalFileName; + + /** + * The physical tab-delimited file is in storage with the list of variable + * names saved as the 1st line. This means that we do not need to generate + * this line on the fly. (Also means that direct download mechanism can be + * used for this file!) + */ + @Column(nullable = false) + private boolean storedWithVariableHeader = false; + /* * Getter and Setter methods: */ @@ -206,6 +216,14 @@ public void setOriginalFileName(String originalFileName) { this.originalFileName = originalFileName; } + public boolean isStoredWithVariableHeader() { + return storedWithVariableHeader; + } + + public void setStoredWithVariableHeader(boolean storedWithVariableHeader) { + this.storedWithVariableHeader = storedWithVariableHeader; + } + /* * Custom overrides for hashCode(), equals() and toString() methods: */ diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataset.java b/src/main/java/edu/harvard/iq/dataverse/Dataset.java index a2f560bc959..eaf406d01bf 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Dataset.java +++ b/src/main/java/edu/harvard/iq/dataverse/Dataset.java @@ -317,6 +317,7 @@ public boolean isDeaccessioned() { } return hasDeaccessionedVersions; // since any published version would have already returned } + public DatasetVersion getLatestVersion() { return getVersions().get(0); @@ -852,6 +853,23 @@ public String getRemoteArchiveURL() { if (StringUtil.nonEmpty(this.getProtocol()) && StringUtil.nonEmpty(this.getAuthority()) && StringUtil.nonEmpty(this.getIdentifier())) { + + // If there is a custom archival url for this Harvesting + // Source, we'll use that + String harvestingUrl = this.getHarvestedFrom().getHarvestingUrl(); + String archivalUrl = this.getHarvestedFrom().getArchiveUrl(); + if (!harvestingUrl.contains(archivalUrl)) { + // When a Harvesting Client is created, the “archive url” is set to + // just the host part of the OAI url automatically. + // For example, if the OAI url was "https://remote.edu/oai", + // the archive url will default to "https://remote.edu/". + // If this is no longer true, we know it means the admin + // went to the trouble of setting it to something else - + // so we should use this url for the redirects back to source, + // instead of the global id resolver. + return archivalUrl + this.getAuthority() + "/" + this.getIdentifier(); + } + // ... if not, we'll redirect to the resolver for the global id: return this.getPersistentURL(); } diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetField.java b/src/main/java/edu/harvard/iq/dataverse/DatasetField.java index c836a20893f..31e7758c7d5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetField.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetField.java @@ -595,7 +595,8 @@ public boolean removeBlankDatasetFieldValues() { return true; } } else { // controlled vocab - if (this.getControlledVocabularyValues().isEmpty()) { + // during harvesting some CVV are put in getDatasetFieldValues. we don't want to remove those + if (this.getControlledVocabularyValues().isEmpty() && this.getDatasetFieldValues().isEmpty()) { return true; } } diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldConstant.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldConstant.java index 1621b80df55..d91aa101eb5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldConstant.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldConstant.java @@ -91,8 +91,9 @@ public class DatasetFieldConstant implements java.io.Serializable { public final static String datasetVersionValue="datasetVersionValue"; public final static String versionDate="versionDate"; public final static String keywordValue="keywordValue"; - public final static String keywordVocab="keywordVocabulary"; //SEK 6/10/2016 to match what is in the db - public final static String keywordVocabURI="keywordVocabularyURI"; //SEK 6/10/2016 to match what is in the db + public final static String keywordTermURI="keywordTermURI"; + public final static String keywordVocab="keywordVocabulary"; + public final static String keywordVocabURI="keywordVocabularyURI"; public final static String topicClassValue="topicClassValue"; public final static String topicClassVocab="topicClassVocab"; public final static String topicClassVocabURI="topicClassVocabURI"; @@ -112,8 +113,8 @@ public class DatasetFieldConstant implements java.io.Serializable { public final static String geographicUnit="geographicUnit"; public final static String westLongitude="westLongitude"; public final static String eastLongitude="eastLongitude"; - public final static String northLatitude="northLongitude"; //Changed to match DB - incorrectly entered into DB: https://github.com/IQSS/dataverse/issues/5645 - public final static String southLatitude="southLongitude"; //Incorrect in DB: https://github.com/IQSS/dataverse/issues/5645 + public final static String northLatitude="northLatitude"; + public final static String southLatitude="southLatitude"; public final static String unitOfAnalysis="unitOfAnalysis"; public final static String universe="universe"; public final static String kindOfData="kindOfData"; diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java index ce2b00086ec..34595728fa7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java @@ -4,7 +4,9 @@ import java.io.StringReader; import java.net.URI; import java.net.URISyntaxException; +import java.net.URLEncoder; import java.nio.charset.StandardCharsets; +import java.security.InvalidParameterException; import java.sql.Timestamp; import java.text.MessageFormat; import java.time.Instant; @@ -34,10 +36,12 @@ import jakarta.persistence.NoResultException; import jakarta.persistence.NonUniqueResultException; import jakarta.persistence.PersistenceContext; +import jakarta.persistence.PersistenceException; import jakarta.persistence.TypedQuery; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.httpclient.HttpException; +import org.apache.commons.lang3.StringUtils; import org.apache.http.HttpResponse; import org.apache.http.HttpResponseInterceptor; import org.apache.http.client.methods.HttpGet; @@ -46,7 +50,6 @@ import org.apache.http.impl.client.HttpClients; import org.apache.http.protocol.HttpContext; import org.apache.http.util.EntityUtils; - import edu.harvard.iq.dataverse.settings.SettingsServiceBean; /** @@ -319,14 +322,15 @@ public Map getCVocConf(boolean byTermUriField){ + jo.getString("term-uri-field")); } } - if (jo.containsKey("child-fields")) { - JsonArray childFields = jo.getJsonArray("child-fields"); - for (JsonString elm : childFields.getValuesAs(JsonString.class)) { - dft = findByNameOpt(elm.getString()); - logger.info("Found: " + dft.getName()); + if (jo.containsKey("managed-fields")) { + JsonObject managedFields = jo.getJsonObject("managed-fields"); + for (String s : managedFields.keySet()) { + dft = findByNameOpt(managedFields.getString(s)); if (dft == null) { logger.warning("Ignoring External Vocabulary setting for non-existent child field: " - + elm.getString()); + + managedFields.getString(s)); + } else { + logger.fine("Found: " + dft.getName()); } } } @@ -343,12 +347,16 @@ public Map getCVocConf(boolean byTermUriField){ * @param df - the primitive/parent compound field containing a newly saved value */ public void registerExternalVocabValues(DatasetField df) { - DatasetFieldType dft =df.getDatasetFieldType(); + DatasetFieldType dft = df.getDatasetFieldType(); logger.fine("Registering for field: " + dft.getName()); JsonObject cvocEntry = getCVocConf(true).get(dft.getId()); if (dft.isPrimitive()) { + List siblingsDatasetFields = new ArrayList<>(); + if(dft.getParentDatasetFieldType()!=null) { + siblingsDatasetFields = df.getParentDatasetFieldCompoundValue().getChildDatasetFields(); + } for (DatasetFieldValue dfv : df.getDatasetFieldValues()) { - registerExternalTerm(cvocEntry, dfv.getValue()); + registerExternalTerm(cvocEntry, dfv.getValue(), siblingsDatasetFields); } } else { if (df.getDatasetFieldType().isCompound()) { @@ -357,45 +365,55 @@ public void registerExternalVocabValues(DatasetField df) { for (DatasetField cdf : cv.getChildDatasetFields()) { logger.fine("Found term uri field type id: " + cdf.getDatasetFieldType().getId()); if (cdf.getDatasetFieldType().equals(termdft)) { - registerExternalTerm(cvocEntry, cdf.getValue()); + registerExternalTerm(cvocEntry, cdf.getValue(), cv.getChildDatasetFields()); } } } } } } - + /** - * Retrieves indexable strings from a cached externalvocabularyvalue entry. - * - * This method assumes externalvocabularyvalue entries have been filtered and - * the externalvocabularyvalue entry contain a single JsonObject whose "personName" or "termName" values - * are either Strings or an array of objects with "lang" and ("value" or "content") keys. The - * string, or the "value/content"s for each language are added to the set. - * + * Retrieves indexable strings from a cached externalvocabularyvalue entry filtered through retrieval-filtering configuration. + *

+ * This method assumes externalvocabularyvalue entries have been filtered and that they contain a single JsonObject. + * Cases Handled : A String, an Array of Strings, an Array of Objects with "value" or "content" keys, an Object with one or more entries that have String values or Array values with a set of String values. + * The string(s), or the "value/content"s for each language are added to the set. + * Retrieved string values are indexed in the term-uri-field (parameter defined in CVOC configuration) by default, or in the field specified by an optional "indexIn" parameter in the retrieval-filtering defined in the CVOC configuration. + *

* Any parsing error results in no entries (there can be unfiltered entries with * unknown structure - getting some strings from such an entry could give fairly * random info that would be bad to addd for searches, etc.) - * - * @param termUri + * + * @param termUri unique identifier to search in database + * @param cvocEntry related cvoc configuration + * @param indexingField name of solr field that will be filled with getStringsFor while indexing * @return - a set of indexable strings */ - public Set getStringsFor(String termUri) { - Set strings = new HashSet(); + public Set getIndexableStringsByTermUri(String termUri, JsonObject cvocEntry, String indexingField) { + Set strings = new HashSet<>(); JsonObject jo = getExternalVocabularyValue(termUri); + JsonObject filtering = cvocEntry.getJsonObject("retrieval-filtering"); + String termUriField = cvocEntry.getJsonString("term-uri-field").getString(); if (jo != null) { try { for (String key : jo.keySet()) { - if (key.equals("termName") || key.equals("personName")) { + String indexIn = filtering.getJsonObject(key).getString("indexIn", null); + // Either we are in mapping mode so indexingField (solr field) equals indexIn (cvoc config) + // Or we are in default mode indexingField is termUriField, indexIn is not defined then only termName and personName keys are used + if (indexingField.equals(indexIn) || + (indexIn == null && termUriField.equals(indexingField) && (key.equals("termName")) || key.equals("personName"))) { JsonValue jv = jo.get(key); if (jv.getValueType().equals(JsonValue.ValueType.STRING)) { logger.fine("adding " + jo.getString(key) + " for " + termUri); strings.add(jo.getString(key)); - } else { - if (jv.getValueType().equals(JsonValue.ValueType.ARRAY)) { - JsonArray jarr = jv.asJsonArray(); - for (int i = 0; i < jarr.size(); i++) { + } else if (jv.getValueType().equals(JsonValue.ValueType.ARRAY)) { + JsonArray jarr = jv.asJsonArray(); + for (int i = 0; i < jarr.size(); i++) { + if (jarr.get(i).getValueType().equals(JsonValue.ValueType.STRING)) { + strings.add(jarr.getString(i)); + } else if (jarr.get(i).getValueType().equals(ValueType.OBJECT)) { // This condition handles SKOMOS format like [{"lang": "en","value": "non-apis bee"},{"lang": "fr","value": "abeille non apis"}] JsonObject entry = jarr.getJsonObject(i); if (entry.containsKey("value")) { logger.fine("adding " + entry.getString("value") + " for " + termUri); @@ -407,6 +425,22 @@ public Set getStringsFor(String termUri) { } } } + } else if (jv.getValueType().equals(JsonValue.ValueType.OBJECT)) { + JsonObject joo = jv.asJsonObject(); + for (Map.Entry entry : joo.entrySet()) { + if (entry.getValue().getValueType().equals(JsonValue.ValueType.STRING)) { // This condition handles format like { "fr": "association de quartier", "en": "neighborhood associations"} + logger.fine("adding " + joo.getString(entry.getKey()) + " for " + termUri); + strings.add(joo.getString(entry.getKey())); + } else if (entry.getValue().getValueType().equals(ValueType.ARRAY)) { // This condition handles format like {"en": ["neighbourhood societies"]} + JsonArray jarr = entry.getValue().asJsonArray(); + for (int i = 0; i < jarr.size(); i++) { + if (jarr.get(i).getValueType().equals(JsonValue.ValueType.STRING)) { + logger.fine("adding " + jarr.getString(i) + " for " + termUri); + strings.add(jarr.getString(i)); + } + } + } + } } } } @@ -418,7 +452,7 @@ public Set getStringsFor(String termUri) { } logger.fine("Returning " + String.join(",", strings) + " for " + termUri); return strings; - } + } /** * Perform a query to retrieve a cached value from the externalvocabularvalue table @@ -445,16 +479,20 @@ public JsonObject getExternalVocabularyValue(String termUri) { /** * Perform a call to the external service to retrieve information about the term URI - * @param cvocEntry - the configuration for the DatasetFieldType associated with this term - * @param term - the term uri as a string + * + * @param cvocEntry - the configuration for the DatasetFieldType associated with this term + * @param term - the term uri as a string + * @param relatedDatasetFields - siblings or childs of the term */ - public void registerExternalTerm(JsonObject cvocEntry, String term) { + public void registerExternalTerm(JsonObject cvocEntry, String term, List relatedDatasetFields) { String retrievalUri = cvocEntry.getString("retrieval-uri"); + String termUriFieldName = cvocEntry.getString("term-uri-field"); String prefix = cvocEntry.getString("prefix", null); - if(term.isBlank()) { - logger.fine("Ingoring blank term"); + if(StringUtils.isBlank(term)) { + logger.fine("Ignoring blank term"); return; } + boolean isExternal = false; JsonObject vocabs = cvocEntry.getJsonObject("vocabs"); for (String key: vocabs.keySet()) { @@ -483,7 +521,22 @@ public void registerExternalTerm(JsonObject cvocEntry, String term) { } if (evv.getValue() == null) { String adjustedTerm = (prefix==null)? term: term.replace(prefix, ""); - retrievalUri = retrievalUri.replace("{0}", adjustedTerm); + + try { + retrievalUri = tryToReplaceRetrievalUriParam(retrievalUri, "0", adjustedTerm); + retrievalUri = tryToReplaceRetrievalUriParam(retrievalUri, termUriFieldName, adjustedTerm); + for (DatasetField f : relatedDatasetFields) { + retrievalUri = tryToReplaceRetrievalUriParam(retrievalUri, f.getDatasetFieldType().getName(), f.getValue()); + } + } catch (InvalidParameterException e) { + logger.warning("InvalidParameterException in tryReplaceRetrievalUriParam : " + e.getMessage()); + return; + } + if (retrievalUri.contains("{")) { + logger.severe("Retrieval URI still contains unreplaced parameter :" + retrievalUri); + return; + } + logger.fine("Didn't find " + term + ", calling " + retrievalUri); try (CloseableHttpClient httpClient = HttpClients.custom() .addInterceptorLast(new HttpResponseInterceptor() { @@ -502,14 +555,21 @@ public void process(HttpResponse response, HttpContext context) throws HttpExcep HttpGet httpGet = new HttpGet(retrievalUri); //application/json+ld is for backward compatibility httpGet.addHeader("Accept", "application/ld+json, application/json+ld, application/json"); - + //Adding others custom HTTP request headers if exists + final JsonObject headers = cvocEntry.getJsonObject("headers"); + if (headers != null) { + final Set headerKeys = headers.keySet(); + for (final String hKey: headerKeys) { + httpGet.addHeader(hKey, headers.getString(hKey)); + } + } HttpResponse response = httpClient.execute(httpGet); String data = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8); int statusCode = response.getStatusLine().getStatusCode(); if (statusCode == 200) { logger.fine("Returned data: " + data); try (JsonReader jsonReader = Json.createReader(new StringReader(data))) { - String dataObj =filterResponse(cvocEntry, jsonReader.readObject(), term).toString(); + String dataObj = filterResponse(cvocEntry, jsonReader.readObject(), term).toString(); evv.setValue(dataObj); evv.setLastUpdateDate(Timestamp.from(Instant.now())); logger.fine("JsonObject: " + dataObj); @@ -518,6 +578,8 @@ public void process(HttpResponse response, HttpContext context) throws HttpExcep logger.fine("Wrote value for term: " + term); } catch (JsonException je) { logger.severe("Error retrieving: " + retrievalUri + " : " + je.getMessage()); + } catch (PersistenceException e) { + logger.fine("Problem persisting: " + retrievalUri + " : " + e.getMessage()); } } else { logger.severe("Received response code : " + statusCode + " when retrieving " + retrievalUri @@ -526,19 +588,42 @@ public void process(HttpResponse response, HttpContext context) throws HttpExcep } catch (IOException ioe) { logger.severe("IOException when retrieving url: " + retrievalUri + " : " + ioe.getMessage()); } - } } catch (URISyntaxException e) { logger.fine("Term is not a URI: " + term); } + } + + private String tryToReplaceRetrievalUriParam(String retrievalUri, String paramName, String value) throws InvalidParameterException { + + if(StringUtils.isBlank(paramName)) { + throw new InvalidParameterException("Empty or null paramName is not allowed while replacing retrieval uri parameter"); + } + + if(retrievalUri.contains(paramName)) { + logger.fine("Parameter {" + paramName + "} found in retrievalUri"); + + if(StringUtils.isBlank(value)) { + throw new InvalidParameterException("Empty or null value is not allowed while replacing retrieval uri parameter"); + } + + if(retrievalUri.contains("encodeUrl:" + paramName)) { + retrievalUri = retrievalUri.replace("{encodeUrl:"+paramName+"}", URLEncoder.encode(value, StandardCharsets.UTF_8)); + } else { + retrievalUri = retrievalUri.replace("{"+paramName+"}", value); + } + } else { + logger.fine("Parameter {" + paramName + "} not found in retrievalUri"); + } + return retrievalUri; } /** * Parse the raw value returned by an external service for a give term uri and * filter it according to the 'retrieval-filtering' configuration for this * DatasetFieldType, creating a Json value with the specified structure - * + * * @param cvocEntry - the config for this DatasetFieldType * @param readObject - the raw response from the service * @param termUri - the term uri @@ -597,6 +682,8 @@ private JsonObject filterResponse(JsonObject cvocEntry, JsonObject readObject, S if (pattern.equals("{0}")) { if (vals.get(0) instanceof JsonArray) { job.add(filterKey, (JsonArray) vals.get(0)); + } else if (vals.get(0) instanceof JsonObject) { + job.add(filterKey, (JsonObject) vals.get(0)); } else { job.add(filterKey, (String) vals.get(0)); } @@ -634,7 +721,7 @@ Object processPathSegment(int index, String[] pathParts, JsonValue curPath, Stri String[] keyVal = pathParts[index].split("="); logger.fine("Looking for object where " + keyVal[0] + " is " + keyVal[1]); String expected = keyVal[1]; - + if (!expected.equals("*")) { if (expected.equals("@id")) { expected = termUri; @@ -663,7 +750,7 @@ Object processPathSegment(int index, String[] pathParts, JsonValue curPath, Stri } return parts.build(); } - + } else { curPath = ((JsonObject) curPath).get(pathParts[index]); logger.fine("Found next Path object " + curPath.toString()); diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldType.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldType.java index 824b486a42d..01785359e0e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldType.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldType.java @@ -284,7 +284,7 @@ public void setDisplayOnCreate(boolean displayOnCreate) { } public boolean isControlledVocabulary() { - return controlledVocabularyValues != null && !controlledVocabularyValues.isEmpty(); + return allowControlledVocabulary; } /** diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetKeyword.java b/src/main/java/edu/harvard/iq/dataverse/DatasetKeyword.java deleted file mode 100644 index 747e3c068f1..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetKeyword.java +++ /dev/null @@ -1,68 +0,0 @@ -package edu.harvard.iq.dataverse; - -/** - * - * @author skraffmiller - */ - -public class DatasetKeyword { - - private int displayOrder; - public int getDisplayOrder() { - return this.displayOrder; - } - public void setDisplayOrder(int displayOrder) { - this.displayOrder = displayOrder; - } - - private DatasetField value; - public DatasetField getValue() { - return this.value; - } - public void setValue(DatasetField value) { - this.value = value; - } - - private DatasetVersion datasetVersion; - public DatasetVersion getDatasetVersion() { - return datasetVersion; - } - public void setDatasetVersion(DatasetVersion metadata) { - this.datasetVersion = metadata; - } - /* - @Version - private Long version; - public Long getVersion() { - return this.version; - } - public void setVersion(Long version) { - this.version = version; - } */ - - private DatasetField vocab; - public DatasetField getVocab() { - return this.vocab; - } - public void setVocab(DatasetField vocab) { - this.vocab = vocab; - } - - private DatasetField vocabURI; - public DatasetField getVocabURI() { - return this.vocabURI; - } - public void setVocabURI(DatasetField vocabURI) { - this.vocabURI = vocabURI; - } - - - public boolean isEmpty() { - /*return ((value==null || value.getValue().trim().equals("")) - && (vocab==null || vocab.getValue().trim().equals("")) - && (vocabURI==null || vocabURI.getValue().trim().equals("")));*/ - return false; - } - - -} diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index b79f387f20b..9c7e951254a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -11,7 +11,6 @@ import edu.harvard.iq.dataverse.authorization.users.User; import edu.harvard.iq.dataverse.branding.BrandingUtil; import edu.harvard.iq.dataverse.dataaccess.StorageIO; -import edu.harvard.iq.dataverse.dataaccess.AbstractRemoteOverlayAccessIO; import edu.harvard.iq.dataverse.dataaccess.DataAccess; import edu.harvard.iq.dataverse.dataaccess.GlobusAccessibleStore; import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter; @@ -25,6 +24,7 @@ import edu.harvard.iq.dataverse.engine.command.Command; import edu.harvard.iq.dataverse.engine.command.CommandContext; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; +import edu.harvard.iq.dataverse.engine.command.impl.CheckRateLimitForDatasetPageCommand; import edu.harvard.iq.dataverse.engine.command.impl.CreatePrivateUrlCommand; import edu.harvard.iq.dataverse.engine.command.impl.CuratePublishedDatasetVersionCommand; import edu.harvard.iq.dataverse.engine.command.impl.DeaccessionDatasetVersionCommand; @@ -37,13 +37,17 @@ import edu.harvard.iq.dataverse.engine.command.impl.PublishDataverseCommand; import edu.harvard.iq.dataverse.engine.command.impl.UpdateDatasetVersionCommand; import edu.harvard.iq.dataverse.export.ExportService; +import edu.harvard.iq.dataverse.util.cache.CacheFactoryBean; import io.gdcc.spi.export.ExportException; import io.gdcc.spi.export.Exporter; import edu.harvard.iq.dataverse.ingest.IngestRequest; import edu.harvard.iq.dataverse.ingest.IngestServiceBean; import edu.harvard.iq.dataverse.license.LicenseServiceBean; import edu.harvard.iq.dataverse.metadataimport.ForeignMetadataImportServiceBean; +import edu.harvard.iq.dataverse.pidproviders.PidProvider; import edu.harvard.iq.dataverse.pidproviders.PidUtil; +import edu.harvard.iq.dataverse.pidproviders.doi.AbstractDOIProvider; +import edu.harvard.iq.dataverse.pidproviders.doi.datacite.DataCiteDOIProvider; import edu.harvard.iq.dataverse.privateurl.PrivateUrl; import edu.harvard.iq.dataverse.privateurl.PrivateUrlServiceBean; import edu.harvard.iq.dataverse.privateurl.PrivateUrlUtil; @@ -73,6 +77,7 @@ import java.lang.reflect.Method; import java.sql.Timestamp; import java.text.SimpleDateFormat; +import java.time.LocalDate; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; @@ -240,12 +245,16 @@ public enum DisplayMode { SolrClientService solrClientService; @EJB DvObjectServiceBean dvObjectService; + @EJB + CacheFactoryBean cacheFactory; @Inject DataverseRequestServiceBean dvRequestService; @Inject DatasetVersionUI datasetVersionUI; @Inject PermissionsWrapper permissionsWrapper; + @Inject + NavigationWrapper navigationWrapper; @Inject FileDownloadHelper fileDownloadHelper; @Inject @@ -261,6 +270,8 @@ public enum DisplayMode { @Inject EmbargoServiceBean embargoService; @Inject + RetentionServiceBean retentionService; + @Inject LicenseServiceBean licenseServiceBean; @Inject DataFileCategoryServiceBean dataFileCategoryService; @@ -706,6 +717,16 @@ public void setNumberOfFilesToShow(Long numberOfFilesToShow) { this.numberOfFilesToShow = numberOfFilesToShow; } + private String returnReason = ""; + + public String getReturnReason() { + return returnReason; + } + + public void setReturnReason(String returnReason) { + this.returnReason = returnReason; + } + public void showAll(){ setNumberOfFilesToShow(new Long(fileMetadatasSearch.size())); } @@ -772,11 +793,17 @@ public boolean isIndexedVersion() { return isIndexedVersion = false; } // If this is the latest published version, we want to confirm that this - // version was successfully indexed after the last publication - + // version was successfully indexed after the last publication if (isThisLatestReleasedVersion()) { - return isIndexedVersion = (workingVersion.getDataset().getIndexTime() != null) - && workingVersion.getDataset().getIndexTime().after(workingVersion.getReleaseTime()); + if (workingVersion.getDataset().getIndexTime() == null) { + return isIndexedVersion = false; + } + // We add 3 hours to the indexed time to prevent false negatives + // when indexed time gets overwritten in finalizing the publication step + // by a value before the release time + final long duration = 3 * 60 * 60 * 1000; + final Timestamp movedIndexTime = new Timestamp(workingVersion.getDataset().getIndexTime().getTime() + duration); + return isIndexedVersion = movedIndexTime.after(workingVersion.getReleaseTime()); } // Drafts don't have the indextime stamps set/incremented when indexed, @@ -1210,8 +1237,17 @@ public boolean canDownloadFiles() { canDownloadFiles = false; for (FileMetadata fmd : workingVersion.getFileMetadatas()) { if (fileDownloadHelper.canDownloadFile(fmd)) { - canDownloadFiles = true; - break; + if (isVersionHasGlobus()) { + String driverId = DataAccess + .getStorageDriverFromIdentifier(fmd.getDataFile().getStorageIdentifier()); + if (StorageIO.isDataverseAccessible(driverId)) { + canDownloadFiles = true; + break; + } + } else { + canDownloadFiles = true; + break; + } } } } @@ -1918,15 +1954,15 @@ private void setIdByPersistentId() { } private String init(boolean initFull) { - + // Check for rate limit exceeded. Must be done before anything else to prevent unnecessary processing. + if (!cacheFactory.checkRate(session.getUser(), new CheckRateLimitForDatasetPageCommand(null,null))) { + return navigationWrapper.tooManyRequests(); + } //System.out.println("_YE_OLDE_QUERY_COUNTER_"); // for debug purposes setDataverseSiteUrl(systemConfig.getDataverseSiteUrl()); guestbookResponse = new GuestbookResponse(); - String nonNullDefaultIfKeyNotFound = ""; - protocol = settingsWrapper.getValueForKey(SettingsServiceBean.Key.Protocol, nonNullDefaultIfKeyNotFound); - authority = settingsWrapper.getValueForKey(SettingsServiceBean.Key.Authority, nonNullDefaultIfKeyNotFound); String sortOrder = getSortOrder(); if(sortOrder != null) { FileMetadata.setCategorySortOrder(sortOrder); @@ -2026,7 +2062,7 @@ private String init(boolean initFull) { // to the local 404 page, below. logger.warning("failed to issue a redirect to "+originalSourceURL); } - return originalSourceURL; + return null; } return permissionsWrapper.notFound(); @@ -2108,8 +2144,6 @@ private String init(boolean initFull) { editMode = EditMode.CREATE; selectedHostDataverse = dataverseService.find(ownerId); dataset.setOwner(selectedHostDataverse); - dataset.setProtocol(protocol); - dataset.setAuthority(authority); if (dataset.getOwner() == null) { return permissionsWrapper.notFound(); @@ -2119,9 +2153,9 @@ private String init(boolean initFull) { //Wait until the create command before actually getting an identifier, except if we're using directUpload //Need to assign an identifier prior to calls to requestDirectUploadUrl if direct upload is used. if ( isEmpty(dataset.getIdentifier()) && systemConfig.directUploadEnabled(dataset) ) { - CommandContext ctxt = commandEngine.getContext(); - GlobalIdServiceBean idServiceBean = GlobalIdServiceBean.getBean(ctxt); - dataset.setIdentifier(idServiceBean.generateDatasetIdentifier(dataset)); + CommandContext ctxt = commandEngine.getContext(); + PidProvider pidProvider = ctxt.dvObjects().getEffectivePidGenerator(dataset); + pidProvider.generatePid(dataset); } dataverseTemplates.addAll(dataverseService.find(ownerId).getTemplates()); if (!dataverseService.find(ownerId).isTemplateRoot()) { @@ -2181,6 +2215,11 @@ private String init(boolean initFull) { } } + LocalDate minRetentiondate = settingsWrapper.getMinRetentionDate(); + if (minRetentiondate != null){ + selectionRetention.setDateUnavailable(minRetentiondate.plusDays(1L)); + } + displayLockInfo(dataset); displayPublishMessage(); @@ -2257,13 +2296,11 @@ private void displayPublishMessage(){ public boolean isValid() { if (valid == null) { - DatasetVersion version = dataset.getLatestVersion(); - if (!version.isDraft()) { + if (workingVersion.isDraft() || (canUpdateDataset() && JvmSettings.UI_SHOW_VALIDITY_LABEL_WHEN_PUBLISHED.lookupOptional(Boolean.class).orElse(true))) { + valid = workingVersion.isValid(); + } else { valid = true; } - DatasetVersion newVersion = version.cloneDatasetVersion(); - newVersion.setDatasetFields(newVersion.initDatasetFields()); - valid = newVersion.isValid(); } return valid; } @@ -2326,14 +2363,17 @@ private void displayLockInfo(Dataset dataset) { lockedDueToIngestVar = true; } - // With DataCite, we try to reserve the DOI when the dataset is created. Sometimes this - // fails because DataCite is down. We show the message below to set expectations that the - // "Publish" button won't work until the DOI has been reserved using the "Reserve PID" API. - if (settingsWrapper.isDataCiteInstallation() && dataset.getGlobalIdCreateTime() == null && editMode != EditMode.CREATE) { - JH.addMessage(FacesMessage.SEVERITY_WARN, BundleUtil.getStringFromBundle("dataset.locked.pidNotReserved.message"), - BundleUtil.getStringFromBundle("dataset.locked.pidNotReserved.message.details")); + if (dataset.getGlobalIdCreateTime() == null && editMode != EditMode.CREATE) { + // With DataCite, we try to reserve the DOI when the dataset is created. Sometimes this + // fails because DataCite is down. We show the message below to set expectations that the + // "Publish" button won't work until the DOI has been reserved using the "Reserve PID" API. + PidProvider pidProvider = PidUtil.getPidProvider(dataset.getGlobalId().getProviderId()); + if (DataCiteDOIProvider.TYPE.equals(pidProvider.getProviderType())) { + JH.addMessage(FacesMessage.SEVERITY_WARN, + BundleUtil.getStringFromBundle("dataset.locked.pidNotReserved.message"), + BundleUtil.getStringFromBundle("dataset.locked.pidNotReserved.message.details")); + } } - //if necessary refresh publish message also displayPublishMessage(); @@ -2652,8 +2692,7 @@ public void edit(EditMode editMode) { public String sendBackToContributor() { try { - //FIXME - Get Return Comment from sendBackToContributor popup - Command cmd = new ReturnDatasetToAuthorCommand(dvRequestService.getDataverseRequest(), dataset, ""); + Command cmd = new ReturnDatasetToAuthorCommand(dvRequestService.getDataverseRequest(), dataset, returnReason); dataset = commandEngine.submit(cmd); JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataset.reject.success")); } catch (CommandException ex) { @@ -3236,7 +3275,7 @@ public void startDownloadSelectedOriginal() { private void startDownload(boolean downloadOriginal){ boolean guestbookRequired = isDownloadPopupRequired(); - boolean validate = validateFilesForDownload(downloadOriginal); + boolean validate = validateFilesForDownload(downloadOriginal, false); if (validate) { updateGuestbookResponse(guestbookRequired, downloadOriginal, false); if(!guestbookRequired && !getValidateFilesOutcome().equals("Mixed")){ @@ -3259,7 +3298,7 @@ public void setValidateFilesOutcome(String validateFilesOutcome) { this.validateFilesOutcome = validateFilesOutcome; } - public boolean validateFilesForDownload(boolean downloadOriginal){ + public boolean validateFilesForDownload(boolean downloadOriginal, boolean isGlobusTransfer){ if (this.selectedFiles.isEmpty()) { PrimeFaces.current().executeScript("PF('selectFilesForDownload').show()"); return false; @@ -3276,33 +3315,39 @@ public boolean validateFilesForDownload(boolean downloadOriginal){ return false; } - for (FileMetadata fmd : getSelectedDownloadableFiles()) { - DataFile dataFile = fmd.getDataFile(); - if (downloadOriginal && dataFile.isTabularData()) { - bytes += dataFile.getOriginalFileSize() == null ? 0 : dataFile.getOriginalFileSize(); - } else { - bytes += dataFile.getFilesize(); + if (!isGlobusTransfer) { + for (FileMetadata fmd : getSelectedDownloadableFiles()) { + DataFile dataFile = fmd.getDataFile(); + if (downloadOriginal && dataFile.isTabularData()) { + bytes += dataFile.getOriginalFileSize() == null ? 0 : dataFile.getOriginalFileSize(); + } else { + bytes += dataFile.getFilesize(); + } } - } - //if there are two or more files, with a total size - //over the zip limit, post a "too large" popup - if (bytes > settingsWrapper.getZipDownloadLimit() && selectedDownloadableFiles.size() > 1) { - setValidateFilesOutcome("FailSize"); - return false; + // if there are two or more files, with a total size + // over the zip limit, post a "too large" popup + if (bytes > settingsWrapper.getZipDownloadLimit() && selectedDownloadableFiles.size() > 1) { + setValidateFilesOutcome("FailSize"); + return false; + } } - + // If some of the files were restricted and we had to drop them off the // list, and NONE of the files are left on the downloadable list - // - we show them a "you're out of luck" popup: - if (getSelectedDownloadableFiles().isEmpty() && getSelectedGlobusTransferableFiles().isEmpty() && !getSelectedNonDownloadableFiles().isEmpty()) { + // - we show them a "you're out of luck" popup + // Same for globus transfer + if ((!isGlobusTransfer + && (getSelectedDownloadableFiles().isEmpty() && !getSelectedNonDownloadableFiles().isEmpty())) + || (isGlobusTransfer && (getSelectedGlobusTransferableFiles().isEmpty() + && !getSelectedNonGlobusTransferableFiles().isEmpty()))) { setValidateFilesOutcome("FailRestricted"); return false; } - //Some are selected and there are non-downloadable ones or there are both downloadable and globus transferable files - if ((!(getSelectedDownloadableFiles().isEmpty() && getSelectedGlobusTransferableFiles().isEmpty()) - && (!getSelectedNonDownloadableFiles().isEmpty()) || (!getSelectedDownloadableFiles().isEmpty() && !getSelectedGlobusTransferableFiles().isEmpty()))) { + //For download or transfer, there are some that can be downloaded/transferred and some that can't + if ((!isGlobusTransfer && (!getSelectedNonDownloadableFiles().isEmpty() && !getSelectedDownloadableFiles().isEmpty())) || + (isGlobusTransfer && (!getSelectedNonGlobusTransferableFiles().isEmpty() && !getSelectedGlobusTransferableFiles().isEmpty()))) { setValidateFilesOutcome("Mixed"); return true; } @@ -3362,7 +3407,7 @@ private boolean filterSelectedFiles(){ if(globusDownloadEnabled) { String driverId = DataAccess.getStorageDriverFromIdentifier(fmd.getDataFile().getStorageIdentifier()); globusTransferable = GlobusAccessibleStore.isGlobusAccessible(driverId); - downloadable = downloadable && !AbstractRemoteOverlayAccessIO.isNotDataverseAccessible(driverId); + downloadable = downloadable && StorageIO.isDataverseAccessible(driverId); } if(downloadable){ getSelectedDownloadableFiles().add(fmd); @@ -3660,6 +3705,25 @@ public String deleteFiles() throws CommandException{ } } + //Remove retentions that are no longer referenced + //Identify which ones are involved here + List orphanedRetentions = new ArrayList(); + if (selectedFiles != null && selectedFiles.size() > 0) { + for (FileMetadata fmd : workingVersion.getFileMetadatas()) { + for (FileMetadata fm : selectedFiles) { + if (fm.getDataFile().equals(fmd.getDataFile()) && !fmd.getDataFile().isReleased()) { + Retention ret = fmd.getDataFile().getRetention(); + if (ret != null) { + ret.getDataFiles().remove(fmd.getDataFile()); + if (ret.getDataFiles().isEmpty()) { + orphanedRetentions.add(ret); + } + } + } + } + } + } + deleteFiles(filesToDelete); String retVal; @@ -3669,12 +3733,14 @@ public String deleteFiles() throws CommandException{ } else { retVal = save(); } - - - //And delete them only after the dataset is updated + + // And delete them only after the dataset is updated for(Embargo emb: orphanedEmbargoes) { embargoService.deleteById(emb.getId(), ((AuthenticatedUser)session.getUser()).getUserIdentifier()); } + for(Retention ret: orphanedRetentions) { + retentionService.delete(ret, ((AuthenticatedUser)session.getUser()).getUserIdentifier()); + } return retVal; } @@ -3864,12 +3930,6 @@ public String save() { ((UpdateDatasetVersionCommand) cmd).setValidateLenient(true); } dataset = commandEngine.submit(cmd); - for (DatasetField df : dataset.getLatestVersion().getFlatDatasetFields()) { - logger.fine("Found id: " + df.getDatasetFieldType().getId()); - if (fieldService.getCVocConf(true).containsKey(df.getDatasetFieldType().getId())) { - fieldService.registerExternalVocabValues(df); - } - } if (editMode == EditMode.CREATE) { if (session.getUser() instanceof AuthenticatedUser) { userNotificationService.sendNotification((AuthenticatedUser) session.getUser(), dataset.getCreateDate(), UserNotification.Type.CREATEDS, dataset.getLatestVersion().getId()); @@ -5341,7 +5401,7 @@ public boolean isFileAccessRequestMultiSignUpButtonEnabled(){ return false; } for (FileMetadata fmd : this.selectedRestrictedFiles){ - if (!this.fileDownloadHelper.canDownloadFile(fmd)&& !FileUtil.isActivelyEmbargoed(fmd)){ + if (!this.fileDownloadHelper.canDownloadFile(fmd) && !FileUtil.isActivelyEmbargoed(fmd)){ return true; } } @@ -5702,7 +5762,10 @@ public boolean isShowPreviewButton(Long fileId) { public boolean isShowQueryButton(Long fileId) { DataFile dataFile = datafileService.find(fileId); - if(dataFile.isRestricted() || !dataFile.isReleased() || FileUtil.isActivelyEmbargoed(dataFile)){ + if(dataFile.isRestricted() + || !dataFile.isReleased() + || FileUtil.isActivelyEmbargoed(dataFile) + || FileUtil.isRetentionExpired(dataFile)){ return false; } @@ -5800,6 +5863,19 @@ public boolean isThisLatestReleasedVersion() { } + public String getCroissant() { + if (isThisLatestReleasedVersion()) { + final String CROISSANT_SCHEMA_NAME = "croissant"; + ExportService instance = ExportService.getInstance(); + String croissant = instance.getExportAsString(dataset, CROISSANT_SCHEMA_NAME); + if (croissant != null && !croissant.isEmpty()) { + logger.fine("Returning cached CROISSANT."); + return croissant; + } + } + return null; + } + public String getJsonLd() { if (isThisLatestReleasedVersion()) { ExportService instance = ExportService.getInstance(); @@ -6247,12 +6323,18 @@ public void clearSelectionEmbargo() { PrimeFaces.current().resetInputs("datasetForm:embargoInputs"); } - public boolean isCantDownloadDueToEmbargo() { + public boolean isCantDownloadDueToEmbargoOrDVAccess() { if (getSelectedNonDownloadableFiles() != null) { for (FileMetadata fmd : getSelectedNonDownloadableFiles()) { if (FileUtil.isActivelyEmbargoed(fmd)) { return true; } + if (isVersionHasGlobus()) { + if (StorageIO.isDataverseAccessible( + DataAccess.getStorageDriverFromIdentifier(fmd.getDataFile().getStorageIdentifier()))) { + return true; + } + } } } return false; @@ -6278,6 +6360,195 @@ private boolean containsOnlyActivelyEmbargoedFiles(List selectedFi return true; } + public Retention getSelectionRetention() { + return selectionRetention; + } + + public void setSelectionRetention(Retention selectionRetention) { + this.selectionRetention = selectionRetention; + } + + + private Retention selectionRetention = new Retention(); + + public boolean isValidRetentionSelection() { + //If fileMetadataForAction is set, someone is using the kebab/single file menu + if (fileMetadataForAction != null) { + if (!fileMetadataForAction.getDataFile().isReleased()) { + return true; + } else { + return false; + } + } + //Otherwise we check the selected files + for (FileMetadata fmd : selectedFiles) { + if (!fmd.getDataFile().isReleased()) { + return true; + } + } + return false; + } + + /* + * This method checks to see if the selected file/files have a retention that could be removed. It doesn't return true of a released file has a retention. + */ + public boolean isExistingRetention() { + if (fileMetadataForAction != null) { + if (!fileMetadataForAction.getDataFile().isReleased() + && (fileMetadataForAction.getDataFile().getRetention() != null)) { + return true; + } else { + return false; + } + } + for (FileMetadata fmd : selectedFiles) { + if (!fmd.getDataFile().isReleased() && (fmd.getDataFile().getRetention() != null)) { + return true; + } + } + + return false; + } + + public boolean isRetentionExpired(List fmdList) { + return FileUtil.isRetentionExpired(fmdList); + } + + public boolean isRetentionForWholeSelection() { + for (FileMetadata fmd : selectedFiles) { + if (fmd.getDataFile().isReleased()) { + return false; + } + } + return true; + } + + private boolean removeRetention=false; + + public boolean isRemoveRetention() { + return removeRetention; + } + + public void setRemoveRetention(boolean removeRetention) { + boolean existing = this.removeRetention; + this.removeRetention = removeRetention; + //If we flipped the state, update the selectedRetention. Otherwise (e.g. when save is hit) don't make changes + if(existing != this.removeRetention) { + logger.fine("State flip"); + selectionRetention= new Retention(); + if(removeRetention) { + logger.fine("Setting empty retention"); + selectionRetention= new Retention(null, null); + } + PrimeFaces.current().resetInputs("datasetForm:retentionInputs"); + } + } + + public String saveRetention() { + if (workingVersion.isReleased()) { + refreshSelectedFiles(selectedFiles); + } + + if(isRemoveRetention() || (selectionRetention.getDateUnavailable()==null && selectionRetention.getReason()==null)) { + selectionRetention=null; + } + + if(!(selectionRetention==null || (selectionRetention!=null && settingsWrapper.isValidRetentionDate(selectionRetention)))) { + logger.fine("Validation error: " + selectionRetention.getFormattedDateUnavailable()); + FacesContext.getCurrentInstance().validationFailed(); + return ""; + } + List orphanedRetentions = new ArrayList(); + List retentionFMs = null; + if (fileMetadataForAction != null) { + retentionFMs = new ArrayList(); + retentionFMs.add(fileMetadataForAction); + } else if (selectedFiles != null && selectedFiles.size() > 0) { + retentionFMs = selectedFiles; + } + + if(retentionFMs!=null && !retentionFMs.isEmpty()) { + if(selectionRetention!=null) { + selectionRetention = retentionService.merge(selectionRetention); + } + for (FileMetadata fmd : workingVersion.getFileMetadatas()) { + for (FileMetadata fm : retentionFMs) { + if (fm.getDataFile().equals(fmd.getDataFile()) && (isSuperUser()||!fmd.getDataFile().isReleased())) { + Retention ret = fmd.getDataFile().getRetention(); + if (ret != null) { + logger.fine("Before: " + ret.getDataFiles().size()); + ret.getDataFiles().remove(fmd.getDataFile()); + if (ret.getDataFiles().isEmpty()) { + orphanedRetentions.add(ret); + } + logger.fine("After: " + ret.getDataFiles().size()); + } + fmd.getDataFile().setRetention(selectionRetention); + } + } + } + } + if (selectionRetention != null) { + retentionService.save(selectionRetention, ((AuthenticatedUser) session.getUser()).getIdentifier()); + } + // success message: + String successMessage = BundleUtil.getStringFromBundle("file.assignedRetention.success"); + logger.fine(successMessage); + successMessage = successMessage.replace("{0}", "Selected Files"); + JsfHelper.addFlashMessage(successMessage); + selectionRetention = new Retention(); + + save(); + for(Retention ret: orphanedRetentions) { + retentionService.delete(ret, ((AuthenticatedUser)session.getUser()).getUserIdentifier()); + } + return returnToDraftVersion(); + } + + public void clearRetentionPopup() { + logger.fine("clearRetentionPopup called"); + selectionRetention= new Retention(); + setRemoveRetention(false); + PrimeFaces.current().resetInputs("datasetForm:retentionInputs"); + } + + public void clearSelectionRetention() { + logger.fine("clearSelectionRetention called"); + selectionRetention= new Retention(); + PrimeFaces.current().resetInputs("datasetForm:retentionInputs"); + } + + public boolean isCantDownloadDueToRetention() { + if (getSelectedNonDownloadableFiles() != null) { + for (FileMetadata fmd : getSelectedNonDownloadableFiles()) { + if (FileUtil.isRetentionExpired(fmd)) { + return true; + } + } + } + return false; + } + + public boolean isCantRequestDueToRetention() { + if (fileDownloadHelper.getFilesForRequestAccess() != null) { + for (DataFile df : fileDownloadHelper.getFilesForRequestAccess()) { + if (FileUtil.isRetentionExpired(df)) { + return true; + } + } + } + return false; + } + + private boolean containsOnlyRetentionExpiredFiles(List selectedFiles) { + for (FileMetadata fmd : selectedFiles) { + if (!FileUtil.isRetentionExpired(fmd)) { + return false; + } + } + return true; + } + public String getIngestMessage() { return BundleUtil.getStringFromBundle("file.ingestFailed.message", Arrays.asList(settingsWrapper.getGuidesBaseUrl(), settingsWrapper.getGuidesVersion())); } @@ -6344,7 +6615,8 @@ public void startGlobusTransfer(boolean transferAll, boolean popupShown) { } boolean guestbookRequired = isDownloadPopupRequired(); - boolean validated = validateFilesForDownload(true); + boolean validated = validateFilesForDownload(true, true); + if (validated) { globusTransferRequested = true; boolean mixed = "Mixed".equals(getValidateFilesOutcome()); @@ -6395,5 +6667,9 @@ public String getSignpostingLinkHeader() { } return signpostingLinkHeader; } + + public boolean isDOI() { + return AbstractDOIProvider.DOI_PROTOCOL.equals(dataset.getGlobalId().getProtocol()); + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java index c6df2a2e1ab..dab0ff43fcf 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java @@ -39,11 +39,10 @@ import jakarta.ejb.TransactionAttributeType; import jakarta.inject.Named; import jakarta.persistence.EntityManager; -import jakarta.persistence.LockModeType; import jakarta.persistence.NoResultException; +import jakarta.persistence.NonUniqueResultException; import jakarta.persistence.PersistenceContext; import jakarta.persistence.Query; -import jakarta.persistence.StoredProcedureQuery; import jakarta.persistence.TypedQuery; import org.apache.commons.lang3.StringUtils; @@ -61,9 +60,6 @@ public class DatasetServiceBean implements java.io.Serializable { @EJB IndexServiceBean indexService; - @EJB - DOIEZIdServiceBean doiEZIdServiceBean; - @EJB SettingsServiceBean settingsService; @@ -116,26 +112,32 @@ public Dataset find(Object pk) { * @return a dataset with pre-fetched file objects */ public Dataset findDeep(Object pk) { - return (Dataset) em.createNamedQuery("Dataset.findById") - .setParameter("id", pk) - // Optimization hints: retrieve all data in one query; this prevents point queries when iterating over the files - .setHint("eclipselink.left-join-fetch", "o.files.ingestRequest") - .setHint("eclipselink.left-join-fetch", "o.files.thumbnailForDataset") - .setHint("eclipselink.left-join-fetch", "o.files.dataTables") - .setHint("eclipselink.left-join-fetch", "o.files.auxiliaryFiles") - .setHint("eclipselink.left-join-fetch", "o.files.ingestReports") - .setHint("eclipselink.left-join-fetch", "o.files.dataFileTags") - .setHint("eclipselink.left-join-fetch", "o.files.fileMetadatas") - .setHint("eclipselink.left-join-fetch", "o.files.fileMetadatas.fileCategories") - //.setHint("eclipselink.left-join-fetch", "o.files.guestbookResponses") - .setHint("eclipselink.left-join-fetch", "o.files.embargo") - .setHint("eclipselink.left-join-fetch", "o.files.fileAccessRequests") - .setHint("eclipselink.left-join-fetch", "o.files.owner") - .setHint("eclipselink.left-join-fetch", "o.files.releaseUser") - .setHint("eclipselink.left-join-fetch", "o.files.creator") - .setHint("eclipselink.left-join-fetch", "o.files.alternativePersistentIndentifiers") - .setHint("eclipselink.left-join-fetch", "o.files.roleAssignments") - .getSingleResult(); + try { + return (Dataset) em.createNamedQuery("Dataset.findById") + .setParameter("id", pk) + // Optimization hints: retrieve all data in one query; this prevents point queries when iterating over the files + .setHint("eclipselink.left-join-fetch", "o.files.ingestRequest") + .setHint("eclipselink.left-join-fetch", "o.files.thumbnailForDataset") + .setHint("eclipselink.left-join-fetch", "o.files.dataTables") + .setHint("eclipselink.left-join-fetch", "o.files.auxiliaryFiles") + .setHint("eclipselink.left-join-fetch", "o.files.ingestReports") + .setHint("eclipselink.left-join-fetch", "o.files.dataFileTags") + .setHint("eclipselink.left-join-fetch", "o.files.fileMetadatas") + .setHint("eclipselink.left-join-fetch", "o.files.fileMetadatas.fileCategories") + .setHint("eclipselink.left-join-fetch", "o.files.fileMetadatas.varGroups") + //.setHint("eclipselink.left-join-fetch", "o.files.guestbookResponses + .setHint("eclipselink.left-join-fetch", "o.files.embargo") + .setHint("eclipselink.left-join-fetch", "o.files.retention") + .setHint("eclipselink.left-join-fetch", "o.files.fileAccessRequests") + .setHint("eclipselink.left-join-fetch", "o.files.owner") + .setHint("eclipselink.left-join-fetch", "o.files.releaseUser") + .setHint("eclipselink.left-join-fetch", "o.files.creator") + .setHint("eclipselink.left-join-fetch", "o.files.alternativePersistentIndentifiers") + .setHint("eclipselink.left-join-fetch", "o.files.roleAssignments") + .getSingleResult(); + } catch (NoResultException | NonUniqueResultException ex) { + return null; + } } public List findByOwnerId(Long ownerId) { @@ -700,7 +702,7 @@ public void exportAllDatasets(boolean forceReExport) { Integer countError = 0; String logTimestamp = logFormatter.format(new Date()); Logger exportLogger = Logger.getLogger("edu.harvard.iq.dataverse.harvest.client.DatasetServiceBean." + "ExportAll" + logTimestamp); - String logFileName = "../logs" + File.separator + "export_" + logTimestamp + ".log"; + String logFileName = System.getProperty("com.sun.aas.instanceRoot") + File.separator + "logs" + File.separator + "export_" + logTimestamp + ".log"; FileHandler fileHandler; boolean fileHandlerSuceeded; try { @@ -861,18 +863,33 @@ public Dataset setDatasetFileAsThumbnail(Dataset dataset, DataFile datasetFileTh logger.fine("In setDatasetFileAsThumbnail but dataset is null! Returning null."); return null; } + // Just in case the previously designated thumbnail for the dataset was + // a "custom" kind, i.e. an uploaded "dataset_logo" file, the following method + // will try to delete it, and all the associated caches here (because there + // are no other uses for the file). This method is apparently called in all + // cases, without trying to check if the dataset was in fact using a custom + // logo; probably under the assumption that it can't hurt. DatasetUtil.deleteDatasetLogo(dataset); dataset.setThumbnailFile(datasetFileThumbnailToSwitchTo); dataset.setUseGenericThumbnail(false); return merge(dataset); } - public Dataset removeDatasetThumbnail(Dataset dataset) { + public Dataset clearDatasetLevelThumbnail(Dataset dataset) { if (dataset == null) { - logger.fine("In removeDatasetThumbnail but dataset is null! Returning null."); + logger.fine("In clearDatasetLevelThumbnail but dataset is null! Returning null."); return null; } + + // Just in case the thumbnail that was designated for the dataset was + // a "custom logo" kind, i.e. an uploaded "dataset_logo" file, the following method + // will try to delete it, and all the associated caches here (because there + // are no other uses for the file). This method is apparently called in all + // cases, without trying to check if the dataset was in fact using a custom + // logo; probably under the assumption that it can't hurt. DatasetUtil.deleteDatasetLogo(dataset); + + // Clear any designated thumbnails for the dataset: dataset.setThumbnailFile(null); dataset.setUseGenericThumbnail(true); return merge(dataset); @@ -940,80 +957,6 @@ public void callFinalizePublishCommandAsynchronously(Long datasetId, CommandCont } } - /* - Experimental asynchronous method for requesting persistent identifiers for - datafiles. We decided not to run this method on upload/create (so files - will not have persistent ids while in draft; when the draft is published, - we will force obtaining persistent ids for all the files in the version. - - If we go back to trying to register global ids on create, care will need to - be taken to make sure the asynchronous changes below are not conflicting with - the changes from file ingest (which may be happening in parallel, also - asynchronously). We would also need to lock the dataset (similarly to how - tabular ingest logs the dataset), to prevent the user from publishing the - version before all the identifiers get assigned - otherwise more conflicts - are likely. (It sounds like it would make sense to treat these two tasks - - persistent identifiers for files and ingest - as one post-upload job, so that - they can be run in sequence). -- L.A. Mar. 2018 - */ - @Asynchronous - public void obtainPersistentIdentifiersForDatafiles(Dataset dataset) { - GlobalIdServiceBean idServiceBean = GlobalIdServiceBean.getBean(dataset.getProtocol(), commandEngine.getContext()); - - //If the Id type is sequential and Dependent then write file idenitifiers outside the command - String datasetIdentifier = dataset.getIdentifier(); - Long maxIdentifier = null; - - if (systemConfig.isDataFilePIDSequentialDependent()) { - maxIdentifier = getMaximumExistingDatafileIdentifier(dataset); - } - - for (DataFile datafile : dataset.getFiles()) { - logger.info("Obtaining persistent id for datafile id=" + datafile.getId()); - - if (datafile.getIdentifier() == null || datafile.getIdentifier().isEmpty()) { - - logger.info("Obtaining persistent id for datafile id=" + datafile.getId()); - - if (maxIdentifier != null) { - maxIdentifier++; - datafile.setIdentifier(datasetIdentifier + "/" + maxIdentifier.toString()); - } else { - datafile.setIdentifier(idServiceBean.generateDataFileIdentifier(datafile)); - } - - if (datafile.getProtocol() == null) { - datafile.setProtocol(settingsService.getValueForKey(SettingsServiceBean.Key.Protocol, "")); - } - if (datafile.getAuthority() == null) { - datafile.setAuthority(settingsService.getValueForKey(SettingsServiceBean.Key.Authority, "")); - } - - logger.info("identifier: " + datafile.getIdentifier()); - - String doiRetString; - - try { - logger.log(Level.FINE, "creating identifier"); - doiRetString = idServiceBean.createIdentifier(datafile); - } catch (Throwable e) { - logger.log(Level.WARNING, "Exception while creating Identifier: " + e.getMessage(), e); - doiRetString = ""; - } - - // Check return value to make sure registration succeeded - if (!idServiceBean.registerWhenPublished() && doiRetString.contains(datafile.getIdentifier())) { - datafile.setIdentifierRegistered(true); - datafile.setGlobalIdCreateTime(new Date()); - } - - DataFile merged = em.merge(datafile); - merged = null; - } - - } - } - public long findStorageSize(Dataset dataset) throws IOException { return findStorageSize(dataset, false, GetDatasetStorageSizeCommand.Mode.STORAGE, null); } diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java index 5fd963f3931..943693355a3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java @@ -1728,7 +1728,36 @@ public List> validateRequired() { } public boolean isValid() { - return validate().isEmpty(); + // first clone to leave the original untouched + final DatasetVersion newVersion = this.cloneDatasetVersion(); + // initDatasetFields + newVersion.setDatasetFields(newVersion.initDatasetFields()); + // remove special "N/A" values and empty values + newVersion.removeEmptyValues(); + // check validity of present fields and detect missing mandatory fields + return newVersion.validate().isEmpty(); + } + + private void removeEmptyValues() { + if (this.getDatasetFields() != null) { + for (DatasetField dsf : this.getDatasetFields()) { + removeEmptyValues(dsf); + } + } + } + + private void removeEmptyValues(DatasetField dsf) { + if (dsf.getDatasetFieldType().isPrimitive()) { // primitive + final Iterator i = dsf.getDatasetFieldValues().iterator(); + while (i.hasNext()) { + final String v = i.next().getValue(); + if (StringUtils.isBlank(v) || DatasetField.NA_VALUE.equals(v)) { + i.remove(); + } + } + } else { + dsf.getDatasetFieldCompoundValues().forEach(cv -> cv.getChildDatasetFields().forEach(v -> removeEmptyValues(v))); + } } public Set validate() { diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java index 99c3c65e3b8..afcfafe976c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java @@ -42,6 +42,16 @@ public enum FileDownloadSizeMode { All, Original, Archival } + /** + * Given a DatasetVersion, returns its total file metadata count + * + * @param datasetVersion the DatasetVersion to access + * @return long value of total file metadata count + */ + public long getFileMetadataCount(DatasetVersion datasetVersion) { + return getFileMetadataCount(datasetVersion, new FileSearchCriteria(null, null, null, null, null)); + } + /** * Given a DatasetVersion, returns its total file metadata count * @@ -189,6 +199,32 @@ public long getFilesDownloadSize(DatasetVersion datasetVersion, FileSearchCriter }; } + /** + * Determines whether or not a DataFile is present in a DatasetVersion + * + * @param datasetVersion the DatasetVersion to check + * @param dataFile the DataFile to check + * @return boolean value + */ + public boolean isDataFilePresentInDatasetVersion(DatasetVersion datasetVersion, DataFile dataFile) { + CriteriaBuilder criteriaBuilder = em.getCriteriaBuilder(); + CriteriaQuery criteriaQuery = criteriaBuilder.createQuery(Long.class); + Root dataFileRoot = criteriaQuery.from(DataFile.class); + Root fileMetadataRoot = criteriaQuery.from(FileMetadata.class); + Root datasetVersionRoot = criteriaQuery.from(DatasetVersion.class); + criteriaQuery + .select(criteriaBuilder.count(dataFileRoot)) + .where(criteriaBuilder.and( + criteriaBuilder.equal(dataFileRoot.get("id"), dataFile.getId()), + criteriaBuilder.equal(datasetVersionRoot.get("id"), datasetVersion.getId()), + fileMetadataRoot.in(dataFileRoot.get("fileMetadatas")), + fileMetadataRoot.in(datasetVersionRoot.get("fileMetadatas")) + ) + ); + Long count = em.createQuery(criteriaQuery).getSingleResult(); + return count != null && count > 0; + } + private void addAccessStatusCountToTotal(DatasetVersion datasetVersion, Map totalCounts, FileAccessStatus dataFileAccessStatus, FileSearchCriteria searchCriteria) { long fileMetadataCount = getFileMetadataCountByAccessStatus(datasetVersion, dataFileAccessStatus, searchCriteria); if (fileMetadataCount > 0) { @@ -210,6 +246,8 @@ private long getFileMetadataCountByAccessStatus(DatasetVersion datasetVersion, F private Predicate createSearchCriteriaAccessStatusPredicate(FileAccessStatus accessStatus, CriteriaBuilder criteriaBuilder, Root fileMetadataRoot) { Path dataFile = fileMetadataRoot.get("dataFile"); + Path retention = dataFile.get("retention"); + Predicate retentionExpiredPredicate = criteriaBuilder.lessThan(retention.get("dateUnavailable"), criteriaBuilder.currentDate()); Path embargo = dataFile.get("embargo"); Predicate activelyEmbargoedPredicate = criteriaBuilder.greaterThanOrEqualTo(embargo.get("dateAvailable"), criteriaBuilder.currentDate()); Predicate inactivelyEmbargoedPredicate = criteriaBuilder.isNull(embargo); @@ -217,6 +255,7 @@ private Predicate createSearchCriteriaAccessStatusPredicate(FileAccessStatus acc Predicate isRestrictedPredicate = criteriaBuilder.isTrue(isRestricted); Predicate isUnrestrictedPredicate = criteriaBuilder.isFalse(isRestricted); return switch (accessStatus) { + case RetentionPeriodExpired -> criteriaBuilder.and(retentionExpiredPredicate); case EmbargoedThenRestricted -> criteriaBuilder.and(activelyEmbargoedPredicate, isRestrictedPredicate); case EmbargoedThenPublic -> criteriaBuilder.and(activelyEmbargoedPredicate, isUnrestrictedPredicate); case Restricted -> criteriaBuilder.and(inactivelyEmbargoedPredicate, isRestrictedPredicate); diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java index 1ee517c9831..ab23fa779d5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java @@ -163,6 +163,7 @@ public DatasetVersion findDeep(Object pk) { .setHint("eclipselink.left-join-fetch", "o.fileMetadatas.dataFile.dataTables") .setHint("eclipselink.left-join-fetch", "o.fileMetadatas.fileCategories") .setHint("eclipselink.left-join-fetch", "o.fileMetadatas.dataFile.embargo") + .setHint("eclipselink.left-join-fetch", "o.fileMetadatas.dataFile.retention") .setHint("eclipselink.left-join-fetch", "o.fileMetadatas.datasetVersion") .setHint("eclipselink.left-join-fetch", "o.fileMetadatas.dataFile.releaseUser") .setHint("eclipselink.left-join-fetch", "o.fileMetadatas.dataFile.creator") @@ -802,6 +803,7 @@ public Long getThumbnailByVersionId(Long versionId) { + "AND fm.datafile_id = df.id " + "AND df.restricted = false " + "AND df.embargo_id is null " + + "AND df.retention_id is null " + "AND o.previewImageAvailable = true " + "ORDER BY df.id LIMIT 1;").getSingleResult(); } catch (Exception ex) { @@ -828,6 +830,7 @@ public Long getThumbnailByVersionId(Long versionId) { + "AND o.previewimagefail = false " + "AND df.restricted = false " + "AND df.embargo_id is null " + + "AND df.retention_id is null " + "AND df.contenttype LIKE 'image/%' " + "AND NOT df.contenttype = 'image/fits' " + "AND df.filesize < " + imageThumbnailSizeLimit + " " @@ -862,6 +865,7 @@ public Long getThumbnailByVersionId(Long versionId) { + "AND o.previewimagefail = false " + "AND df.restricted = false " + "AND df.embargo_id is null " + + "AND df.retention_id is null " + "AND df.contenttype = 'application/pdf' " + "AND df.filesize < " + imageThumbnailSizeLimit + " " + "ORDER BY df.filesize ASC LIMIT 1;").getSingleResult(); diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java index c1de9d63410..78b1827c798 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java +++ b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java @@ -411,6 +411,14 @@ public List getDataverseFieldTypeInputLevels() { return dataverseFieldTypeInputLevels; } + public boolean isDatasetFieldTypeRequiredAsInputLevel(Long datasetFieldTypeId) { + for(DataverseFieldTypeInputLevel dataverseFieldTypeInputLevel : dataverseFieldTypeInputLevels) { + if (dataverseFieldTypeInputLevel.getDatasetFieldType().getId().equals(datasetFieldTypeId) && dataverseFieldTypeInputLevel.isRequired()) { + return true; + } + } + return false; + } public Template getDefaultTemplate() { return defaultTemplate; @@ -466,9 +474,6 @@ public void setTemplateRoot(boolean templateRoot) { this.templateRoot = templateRoot; } - - - public List getMetadataBlocks() { return getMetadataBlocks(false); } diff --git a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java index 943a74327d5..351d304bad3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java @@ -9,12 +9,16 @@ import edu.harvard.iq.dataverse.dataverse.DataverseUtil; import edu.harvard.iq.dataverse.engine.command.Command; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; +import edu.harvard.iq.dataverse.engine.command.impl.CheckRateLimitForCollectionPageCommand; import edu.harvard.iq.dataverse.engine.command.impl.CreateDataverseCommand; import edu.harvard.iq.dataverse.engine.command.impl.CreateSavedSearchCommand; import edu.harvard.iq.dataverse.engine.command.impl.DeleteDataverseCommand; import edu.harvard.iq.dataverse.engine.command.impl.LinkDataverseCommand; import edu.harvard.iq.dataverse.engine.command.impl.PublishDataverseCommand; import edu.harvard.iq.dataverse.engine.command.impl.UpdateDataverseCommand; +import edu.harvard.iq.dataverse.pidproviders.PidProvider; +import edu.harvard.iq.dataverse.pidproviders.PidProviderFactoryBean; +import edu.harvard.iq.dataverse.pidproviders.PidUtil; import edu.harvard.iq.dataverse.search.FacetCategory; import edu.harvard.iq.dataverse.search.IndexServiceBean; import edu.harvard.iq.dataverse.search.SearchFields; @@ -28,15 +32,20 @@ import static edu.harvard.iq.dataverse.util.JsfHelper.JH; import edu.harvard.iq.dataverse.util.SystemConfig; import java.util.List; + +import edu.harvard.iq.dataverse.util.cache.CacheFactoryBean; import jakarta.ejb.EJB; import jakarta.faces.application.FacesMessage; import jakarta.faces.context.FacesContext; import jakarta.faces.view.ViewScoped; import jakarta.inject.Inject; import jakarta.inject.Named; + +import java.util.AbstractMap; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; import java.util.Map.Entry; import java.util.Set; @@ -109,7 +118,13 @@ public enum LinkMode { @EJB DataverseLinkingServiceBean linkingService; @Inject PermissionsWrapper permissionsWrapper; - @Inject DataverseHeaderFragment dataverseHeaderFragment; + @Inject + NavigationWrapper navigationWrapper; + @Inject DataverseHeaderFragment dataverseHeaderFragment; + @EJB + PidProviderFactoryBean pidProviderFactoryBean; + @EJB + CacheFactoryBean cacheFactory; private Dataverse dataverse = new Dataverse(); @@ -310,7 +325,10 @@ public void updateOwnerDataverse() { public String init() { //System.out.println("_YE_OLDE_QUERY_COUNTER_"); // for debug purposes - + // Check for rate limit exceeded. Must be done before anything else to prevent unnecessary processing. + if (!cacheFactory.checkRate(session.getUser(), new CheckRateLimitForCollectionPageCommand(null,null))) { + return navigationWrapper.tooManyRequests(); + } if (this.getAlias() != null || this.getId() != null || this.getOwnerId() == null) {// view mode for a dataverse if (this.getAlias() != null) { dataverse = dataverseService.findByAlias(this.getAlias()); @@ -362,7 +380,7 @@ public void initFeaturedDataverses() { List featuredSource = new ArrayList<>(); List featuredTarget = new ArrayList<>(); featuredSource.addAll(dataverseService.findAllPublishedByOwnerId(dataverse.getId())); - featuredSource.addAll(linkingService.findLinkingDataverses(dataverse.getId())); + featuredSource.addAll(linkingService.findLinkedDataverses(dataverse.getId())); List featuredList = featuredDataverseService.findByDataverseId(dataverse.getId()); for (DataverseFeaturedDataverse dfd : featuredList) { Dataverse fd = dfd.getFeaturedDataverse(); @@ -1289,4 +1307,34 @@ public String getCurationLabelSetNameLabel() { public Set> getGuestbookEntryOptions() { return settingsWrapper.getGuestbookEntryOptions(this.dataverse).entrySet(); } + + public Set> getPidProviderOptions() { + PidProvider defaultPidProvider = pidProviderFactoryBean.getDefaultPidGenerator(); + Set providerIds = PidUtil.getManagedProviderIds(); + Set> options = new HashSet>(); + if (providerIds.size() > 1) { + + String label = null; + if (this.dataverse.getOwner() != null && this.dataverse.getOwner().getEffectivePidGenerator()!= null) { + PidProvider inheritedPidProvider = this.dataverse.getOwner().getEffectivePidGenerator(); + label = inheritedPidProvider.getLabel() + " " + BundleUtil.getStringFromBundle("dataverse.inherited") + ": " + + inheritedPidProvider.getProtocol() + ":" + inheritedPidProvider.getAuthority() + + inheritedPidProvider.getSeparator() + inheritedPidProvider.getShoulder(); + } else { + label = defaultPidProvider.getLabel() + " " + BundleUtil.getStringFromBundle("dataverse.default") + ": " + + defaultPidProvider.getProtocol() + ":" + defaultPidProvider.getAuthority() + + defaultPidProvider.getSeparator() + defaultPidProvider.getShoulder(); + } + Entry option = new AbstractMap.SimpleEntry("default", label); + options.add(option); + } + for (String providerId : providerIds) { + PidProvider pidProvider = PidUtil.getPidProvider(providerId); + String label = pidProvider.getLabel() + ": " + pidProvider.getProtocol() + ":" + pidProvider.getAuthority() + + pidProvider.getSeparator() + pidProvider.getShoulder(); + Entry option = new AbstractMap.SimpleEntry(providerId, label); + options.add(option); + } + return options; + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java b/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java index 82057315fbb..56d26a7260d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java +++ b/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java @@ -1,14 +1,20 @@ package edu.harvard.iq.dataverse; import edu.harvard.iq.dataverse.dataaccess.DataAccess; +import edu.harvard.iq.dataverse.pidproviders.PidProvider; +import edu.harvard.iq.dataverse.pidproviders.PidUtil; import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.storageuse.StorageUse; import edu.harvard.iq.dataverse.util.SystemConfig; +import edu.harvard.iq.dataverse.util.json.JsonUtil; +import jakarta.json.JsonObject; +import jakarta.json.JsonObjectBuilder; import jakarta.persistence.CascadeType; import java.util.Optional; - import jakarta.persistence.MappedSuperclass; import jakarta.persistence.OneToOne; +import jakarta.persistence.Transient; + import org.apache.commons.lang3.StringUtils; /** @@ -42,6 +48,11 @@ public boolean isEffectivelyPermissionRoot() { private String metadataLanguage=null; private Boolean guestbookAtRequest = null; + + private String pidGeneratorSpecs = null; + + @Transient + private PidProvider pidGenerator = null; @OneToOne(mappedBy = "dvObjectContainer",cascade={ CascadeType.REMOVE, CascadeType.PERSIST}, orphanRemoval=true) private StorageUse storageUse; @@ -175,4 +186,79 @@ public void setCurationLabelSetName(String setName) { public void setStorageUse(StorageUse storageUse) { this.storageUse = storageUse; } + + + /* Dataverse collections and dataset can be configured to use different PidProviders as PID generators for contained objects (datasets or data files). + * This mechanism is similar to others except that the stored value is a JSON object defining the protocol, authority, shoulder, and, optionally, the separator for the PidProvider. + */ + + public String getPidGeneratorSpecs() { + return pidGeneratorSpecs; + } + + public void setPidGeneratorSpecs(String pidGeneratorSpecs) { + this.pidGeneratorSpecs = pidGeneratorSpecs; + } + + // Used in JSF when selecting the PidGenerator + // It only returns an id if this dvObjectContainer has PidGenerator specs set on it, otherwise it returns "default" + public String getPidGeneratorId() { + if (StringUtils.isBlank(getPidGeneratorSpecs())) { + return "default"; + } else { + return getEffectivePidGenerator().getId(); + } + } + + //Used in JSF when setting the PidGenerator + public void setPidGeneratorId(String pidGeneratorId) { + // Note that the "default" provider will not be found so will result in + // setPidGenerator(null), which unsets the pidGenerator/Specs as desired + setPidGenerator(PidUtil.getPidProvider(pidGeneratorId)); + } + + public void setPidGenerator(PidProvider pidGenerator) { + this.pidGenerator = pidGenerator; + if (pidGenerator != null) { + JsonObjectBuilder job = jakarta.json.Json.createObjectBuilder(); + this.pidGeneratorSpecs = job.add("protocol", pidGenerator.getProtocol()) + .add("authority", pidGenerator.getAuthority()).add("shoulder", pidGenerator.getShoulder()) + .add("separator", pidGenerator.getSeparator()).build().toString(); + } else { + this.pidGeneratorSpecs = null; + } + } + + public PidProvider getEffectivePidGenerator() { + if (pidGenerator == null) { + String specs = getPidGeneratorSpecs(); + if (StringUtils.isBlank(specs)) { + GlobalId pid = getGlobalId(); + if ((pid != null) && PidUtil.getPidProvider(pid.getProviderId()).canCreatePidsLike(pid)) { + pidGenerator = PidUtil.getPidProvider(pid.getProviderId()); + } else { + if (getOwner() != null) { + pidGenerator = getOwner().getEffectivePidGenerator(); + } + } + } else { + JsonObject providerSpecs = JsonUtil.getJsonObject(specs); + if (providerSpecs.containsKey("separator")) { + pidGenerator = PidUtil.getPidProvider(providerSpecs.getString("protocol"), + providerSpecs.getString("authority"), providerSpecs.getString("shoulder"), + providerSpecs.getString("separator")); + } else { + pidGenerator = PidUtil.getPidProvider(providerSpecs.getString("protocol"), + providerSpecs.getString("authority"), providerSpecs.getString("shoulder")); + } + } + if(pidGenerator!=null && pidGenerator.canManagePID()) { + setPidGenerator(pidGenerator); + } else { + setPidGenerator(null); + } + } + return pidGenerator; + } + } diff --git a/src/main/java/edu/harvard/iq/dataverse/DvObjectServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DvObjectServiceBean.java index d4219c36149..bd7fbeaff10 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DvObjectServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DvObjectServiceBean.java @@ -1,8 +1,9 @@ package edu.harvard.iq.dataverse; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import edu.harvard.iq.dataverse.pidproviders.PidProvider; +import edu.harvard.iq.dataverse.pidproviders.PidProviderFactoryBean; import edu.harvard.iq.dataverse.pidproviders.PidUtil; - import java.sql.Timestamp; import java.util.ArrayList; import java.util.Date; @@ -12,6 +13,8 @@ import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; + +import jakarta.ejb.EJB; import jakarta.ejb.Stateless; import jakarta.ejb.TransactionAttribute; import static jakarta.ejb.TransactionAttributeType.REQUIRES_NEW; @@ -38,6 +41,9 @@ public class DvObjectServiceBean implements java.io.Serializable { @PersistenceContext(unitName = "VDCNet-ejbPU") private EntityManager em; + @EJB + PidProviderFactoryBean pidProviderFactoryBean; + private static final Logger logger = Logger.getLogger(DvObjectServiceBean.class.getCanonicalName()); /** * @param dvoc The object we check @@ -389,4 +395,19 @@ public String generateNewIdentifierByStoredProcedure() { return (String) query.getOutputParameterValue(1); } + /** @deprecated Backward-compatibility method to get the effective pid generator for a DvObjectContainer. + * If the dvObjectContainer method fails, this method will check for the old global default settings. + * If/when those are no longer supported, this method can be removed and replaced with calls directly + * to dvObjectContainer.getEffectivePidGenerator(); + * + */ + @Deprecated(forRemoval = true, since = "2024-02-09") + public PidProvider getEffectivePidGenerator(DvObjectContainer dvObjectContainer) { + PidProvider pidGenerator = dvObjectContainer.getEffectivePidGenerator(); + if (pidGenerator == null) { + pidGenerator = pidProviderFactoryBean.getDefaultPidGenerator(); + } + return pidGenerator; + } + } diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index a6f31e24764..993cb02b66b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -1237,9 +1237,6 @@ public String save() { - We decided not to bother obtaining persistent ids for new files as they are uploaded and created. The identifiers will be assigned later, when the version is published. - - logger.info("starting async job for obtaining persistent ids for files."); - datasetService.obtainPersistentIdentifiersForDatafiles(dataset); */ } diff --git a/src/main/java/edu/harvard/iq/dataverse/EjbDataverseEngine.java b/src/main/java/edu/harvard/iq/dataverse/EjbDataverseEngine.java index 5a689c06019..c8537f2a424 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EjbDataverseEngine.java +++ b/src/main/java/edu/harvard/iq/dataverse/EjbDataverseEngine.java @@ -4,6 +4,7 @@ import edu.harvard.iq.dataverse.actionlogging.ActionLogServiceBean; import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; import edu.harvard.iq.dataverse.authorization.providers.builtin.BuiltinUserServiceBean; +import edu.harvard.iq.dataverse.util.cache.CacheFactoryBean; import edu.harvard.iq.dataverse.engine.DataverseEngine; import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.authorization.groups.GroupServiceBean; @@ -16,9 +17,9 @@ import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.exception.PermissionException; +import edu.harvard.iq.dataverse.engine.command.exception.RateLimitCommandException; import edu.harvard.iq.dataverse.ingest.IngestServiceBean; -import edu.harvard.iq.dataverse.pidproviders.FakePidProviderServiceBean; -import edu.harvard.iq.dataverse.pidproviders.PermaLinkPidProviderServiceBean; +import edu.harvard.iq.dataverse.pidproviders.PidProviderFactoryBean; import edu.harvard.iq.dataverse.privateurl.PrivateUrlServiceBean; import edu.harvard.iq.dataverse.search.IndexBatchServiceBean; import edu.harvard.iq.dataverse.search.IndexServiceBean; @@ -49,7 +50,6 @@ import static jakarta.ejb.TransactionAttributeType.SUPPORTS; import jakarta.persistence.EntityManager; import jakarta.persistence.PersistenceContext; -import jakarta.validation.ConstraintViolation; import jakarta.validation.ConstraintViolationException; /** @@ -114,20 +114,8 @@ public class EjbDataverseEngine { DataverseFieldTypeInputLevelServiceBean fieldTypeInputLevels; @EJB - DOIEZIdServiceBean doiEZId; - - @EJB - DOIDataCiteServiceBean doiDataCite; + PidProviderFactoryBean pidProviderFactory; - @EJB - FakePidProviderServiceBean fakePidProvider; - - @EJB - HandlenetServiceBean handleNet; - - @EJB - PermaLinkPidProviderServiceBean permaLinkProvider; - @EJB SettingsServiceBean settings; @@ -136,6 +124,9 @@ public class EjbDataverseEngine { @EJB GuestbookResponseServiceBean responses; + + @EJB + MetadataBlockServiceBean metadataBlockService; @EJB DataverseLinkingServiceBean dvLinking; @@ -143,6 +134,9 @@ public class EjbDataverseEngine { @EJB DatasetLinkingServiceBean dsLinking; + @EJB + DatasetFieldServiceBean dsField; + @EJB ExplicitGroupServiceBean explicitGroups; @@ -190,7 +184,9 @@ public class EjbDataverseEngine { @EJB EjbDataverseEngineInner innerEngine; - + + @EJB + CacheFactoryBean cacheFactory; @Resource EJBContext ejbCtxt; @@ -216,7 +212,11 @@ public R submit(Command aCommand) throws CommandException { try { logRec.setUserIdentifier( aCommand.getRequest().getUser().getIdentifier() ); - + // Check for rate limit exceeded. Must be done before anything else to prevent unnecessary processing. + if (!cacheFactory.checkRate(aCommand.getRequest().getUser(), aCommand)) { + throw new RateLimitCommandException(BundleUtil.getStringFromBundle("command.exception.user.ratelimited", Arrays.asList(aCommand.getClass().getSimpleName())), aCommand); + } + // Check permissions - or throw an exception Map> requiredMap = aCommand.getRequiredPermissions(); if (requiredMap == null) { @@ -484,28 +484,8 @@ public DataverseFieldTypeInputLevelServiceBean fieldTypeInputLevels() { } @Override - public DOIEZIdServiceBean doiEZId() { - return doiEZId; - } - - @Override - public DOIDataCiteServiceBean doiDataCite() { - return doiDataCite; - } - - @Override - public FakePidProviderServiceBean fakePidProvider() { - return fakePidProvider; - } - - @Override - public HandlenetServiceBean handleNet() { - return handleNet; - } - - @Override - public PermaLinkPidProviderServiceBean permaLinkProvider() { - return permaLinkProvider; + public PidProviderFactoryBean pidProviderFactory() { + return pidProviderFactory; } @Override @@ -532,7 +512,12 @@ public DataverseLinkingServiceBean dvLinking() { public DatasetLinkingServiceBean dsLinking() { return dsLinking; } - + + @Override + public DatasetFieldServiceBean dsField() { + return dsField; + } + @Override public StorageUseServiceBean storageUse() { return storageUseService; @@ -613,6 +598,11 @@ public ActionLogServiceBean actionLog() { return logSvc; } + @Override + public MetadataBlockServiceBean metadataBlocks() { + return metadataBlockService; + } + @Override public void beginCommandSequence() { this.commandsCalled = new Stack(); diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java index 4d8100124ec..80cf3db8d53 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java @@ -8,6 +8,8 @@ import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.authorization.users.PrivateUrlUser; +import edu.harvard.iq.dataverse.dataaccess.DataAccess; +import edu.harvard.iq.dataverse.dataaccess.StorageIO; import edu.harvard.iq.dataverse.externaltools.ExternalTool; import edu.harvard.iq.dataverse.globus.GlobusServiceBean; import edu.harvard.iq.dataverse.util.BundleUtil; @@ -221,7 +223,10 @@ public boolean canDownloadFile(FileMetadata fileMetadata){ // Always allow download for PrivateUrlUser return true; } - + + // Retention expired files are always made unavailable, because they might be destroyed + if (FileUtil.isRetentionExpired(fileMetadata)) return false; + Long fid = fileMetadata.getId(); //logger.info("calling candownloadfile on filemetadata "+fid); // Note that `isRestricted` at the FileMetadata level is for expressing intent by version. Enforcement is done with `isRestricted` at the DataFile level. @@ -244,7 +249,9 @@ public boolean canDownloadFile(FileMetadata fileMetadata){ } } - if (!isRestrictedFile && !FileUtil.isActivelyEmbargoed(fileMetadata)){ + if (!isRestrictedFile + && !FileUtil.isActivelyEmbargoed(fileMetadata) + && !FileUtil.isRetentionExpired(fileMetadata)) { // Yes, save answer and return true this.fileDownloadPermissionMap.put(fid, true); return true; diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java index de3f4d2ab56..ab9e4f9be66 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java @@ -354,7 +354,8 @@ public void explore(GuestbookResponse guestbookResponse, FileMetadata fmd, Exter ApiToken apiToken = null; User user = session.getUser(); DatasetVersion version = fmd.getDatasetVersion(); - if (version.isDraft() || fmd.getDatasetVersion().isDeaccessioned() || (fmd.getDataFile().isRestricted()) || (FileUtil.isActivelyEmbargoed(fmd))) { + if (version.isDraft() || fmd.getDatasetVersion().isDeaccessioned() || (fmd.getDataFile().isRestricted()) + || (FileUtil.isActivelyEmbargoed(fmd)) || (FileUtil.isRetentionExpired(fmd))) { apiToken = authService.getValidApiTokenForUser(user); } DataFile dataFile = null; @@ -382,28 +383,26 @@ public void explore(GuestbookResponse guestbookResponse, FileMetadata fmd, Exter } } - public void downloadDatasetCitationXML(Dataset dataset) { - downloadCitationXML(null, dataset, false); + public void downloadDatasetCitationXML(DatasetVersion version) { + // DatasetVersion-level citation: + DataCitation citation=null; + citation = new DataCitation(version); + String fileNameString; + fileNameString = "attachment;filename=" + getFileNameFromPid(citation.getPersistentId()) + ".xml"; + downloadXML(citation, fileNameString); } public void downloadDatafileCitationXML(FileMetadata fileMetadata) { - downloadCitationXML(fileMetadata, null, false); + downloadCitationXML(fileMetadata, false); } public void downloadDirectDatafileCitationXML(FileMetadata fileMetadata) { - downloadCitationXML(fileMetadata, null, true); + downloadCitationXML(fileMetadata, true); } - public void downloadCitationXML(FileMetadata fileMetadata, Dataset dataset, boolean direct) { - DataCitation citation=null; - if (dataset != null){ - citation = new DataCitation(dataset.getLatestVersion()); - } else { - citation= new DataCitation(fileMetadata, direct); - } - FacesContext ctx = FacesContext.getCurrentInstance(); - HttpServletResponse response = (HttpServletResponse) ctx.getExternalContext().getResponse(); - response.setContentType("text/xml"); + public void downloadCitationXML(FileMetadata fileMetadata, boolean direct) { + DataCitation citation=null; + citation= new DataCitation(fileMetadata, direct); String fileNameString; if (fileMetadata == null || fileMetadata.getLabel() == null) { // Dataset-level citation: @@ -412,43 +411,46 @@ public void downloadCitationXML(FileMetadata fileMetadata, Dataset dataset, bool // Datafile-level citation: fileNameString = "attachment;filename=" + getFileNameFromPid(citation.getPersistentId()) + "-" + FileUtil.getCiteDataFileFilename(citation.getFileTitle(), FileUtil.FileCitationExtension.ENDNOTE); } + downloadXML(citation, fileNameString); + } + + public void downloadXML(DataCitation citation, String fileNameString) { + FacesContext ctx = FacesContext.getCurrentInstance(); + HttpServletResponse response = (HttpServletResponse) ctx.getExternalContext().getResponse(); + response.setContentType("text/xml"); response.setHeader("Content-Disposition", fileNameString); + try { ServletOutputStream out = response.getOutputStream(); citation.writeAsEndNoteCitation(out); out.flush(); ctx.responseComplete(); } catch (IOException e) { - } } - - public void downloadDatasetCitationRIS(Dataset dataset) { - downloadCitationRIS(null, dataset, false); + public void downloadDatasetCitationRIS(DatasetVersion version) { + // DatasetVersion-level citation: + DataCitation citation=null; + citation = new DataCitation(version); + String fileNameString; + fileNameString = "attachment;filename=" + getFileNameFromPid(citation.getPersistentId()) + ".ris"; + downloadRIS(citation, fileNameString); } public void downloadDatafileCitationRIS(FileMetadata fileMetadata) { - downloadCitationRIS(fileMetadata, null, false); + downloadCitationRIS(fileMetadata, false); } public void downloadDirectDatafileCitationRIS(FileMetadata fileMetadata) { - downloadCitationRIS(fileMetadata, null, true); + downloadCitationRIS(fileMetadata, true); } - public void downloadCitationRIS(FileMetadata fileMetadata, Dataset dataset, boolean direct) { - DataCitation citation=null; - if (dataset != null){ - citation = new DataCitation(dataset.getLatestVersion()); - } else { - citation= new DataCitation(fileMetadata, direct); - } - - FacesContext ctx = FacesContext.getCurrentInstance(); - HttpServletResponse response = (HttpServletResponse) ctx.getExternalContext().getResponse(); - response.setContentType("application/download"); - + public void downloadCitationRIS(FileMetadata fileMetadata, boolean direct) { + DataCitation citation=null; + citation= new DataCitation(fileMetadata, direct); + String fileNameString; if (fileMetadata == null || fileMetadata.getLabel() == null) { // Dataset-level citation: @@ -457,6 +459,14 @@ public void downloadCitationRIS(FileMetadata fileMetadata, Dataset dataset, bool // Datafile-level citation: fileNameString = "attachment;filename=" + getFileNameFromPid(citation.getPersistentId()) + "-" + FileUtil.getCiteDataFileFilename(citation.getFileTitle(), FileUtil.FileCitationExtension.RIS); } + downloadRIS(citation, fileNameString); + } + + public void downloadRIS(DataCitation citation, String fileNameString) { + //SEK 12/3/2018 changing this to open the json in a new tab. + FacesContext ctx = FacesContext.getCurrentInstance(); + HttpServletResponse response = (HttpServletResponse) ctx.getExternalContext().getResponse(); + response.setContentType("application/download"); response.setHeader("Content-Disposition", fileNameString); try { @@ -468,38 +478,33 @@ public void downloadCitationRIS(FileMetadata fileMetadata, Dataset dataset, bool } } - + private String getFileNameFromPid(GlobalId id) { return id.asString(); } - public void downloadDatasetCitationBibtex(Dataset dataset) { - downloadCitationBibtex(null, dataset, false); + public void downloadDatasetCitationBibtex(DatasetVersion version) { + // DatasetVersion-level citation: + DataCitation citation=null; + citation = new DataCitation(version); + String fileNameString; + fileNameString = "inline;filename=" + getFileNameFromPid(citation.getPersistentId()) + ".bib"; + downloadBibtex(citation, fileNameString); } public void downloadDatafileCitationBibtex(FileMetadata fileMetadata) { - downloadCitationBibtex(fileMetadata, null, false); + downloadCitationBibtex(fileMetadata, false); } public void downloadDirectDatafileCitationBibtex(FileMetadata fileMetadata) { - downloadCitationBibtex(fileMetadata, null, true); + downloadCitationBibtex(fileMetadata, true); } - public void downloadCitationBibtex(FileMetadata fileMetadata, Dataset dataset, boolean direct) { - DataCitation citation=null; - if (dataset != null){ - citation = new DataCitation(dataset.getLatestVersion()); - } else { - citation= new DataCitation(fileMetadata, direct); - } - //SEK 12/3/2018 changing this to open the json in a new tab. - FacesContext ctx = FacesContext.getCurrentInstance(); - HttpServletResponse response = (HttpServletResponse) ctx.getExternalContext().getResponse(); - - //Fix for 6029 FireFox was failing to parse it when content type was set to json - response.setContentType("text/plain"); + public void downloadCitationBibtex(FileMetadata fileMetadata, boolean direct) { + DataCitation citation=null; + citation= new DataCitation(fileMetadata, direct); String fileNameString; if (fileMetadata == null || fileMetadata.getLabel() == null) { @@ -509,6 +514,16 @@ public void downloadCitationBibtex(FileMetadata fileMetadata, Dataset dataset, b // Datafile-level citation: fileNameString = "inline;filename=" + getFileNameFromPid(citation.getPersistentId()) + "-" + FileUtil.getCiteDataFileFilename(citation.getFileTitle(), FileUtil.FileCitationExtension.BIBTEX); } + downloadBibtex(citation, fileNameString); + } + + public void downloadBibtex(DataCitation citation, String fileNameString) { + //SEK 12/3/2018 changing this to open the json in a new tab. + FacesContext ctx = FacesContext.getCurrentInstance(); + HttpServletResponse response = (HttpServletResponse) ctx.getExternalContext().getResponse(); + + //Fix for 6029 FireFox was failing to parse it when content type was set to json + response.setContentType("text/plain"); response.setHeader("Content-Disposition", fileNameString); try { diff --git a/src/main/java/edu/harvard/iq/dataverse/FilePage.java b/src/main/java/edu/harvard/iq/dataverse/FilePage.java index 479c8a429c6..afede00f3eb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FilePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/FilePage.java @@ -21,6 +21,7 @@ import edu.harvard.iq.dataverse.engine.command.impl.CreateNewDatasetCommand; import edu.harvard.iq.dataverse.engine.command.impl.PersistProvFreeFormCommand; import edu.harvard.iq.dataverse.engine.command.impl.RestrictFileCommand; +import edu.harvard.iq.dataverse.engine.command.impl.UningestFileCommand; import edu.harvard.iq.dataverse.engine.command.impl.UpdateDatasetVersionCommand; import edu.harvard.iq.dataverse.export.ExportService; import io.gdcc.spi.export.ExportException; @@ -28,24 +29,33 @@ import edu.harvard.iq.dataverse.externaltools.ExternalTool; import edu.harvard.iq.dataverse.externaltools.ExternalToolHandler; import edu.harvard.iq.dataverse.externaltools.ExternalToolServiceBean; +import edu.harvard.iq.dataverse.ingest.IngestRequest; +import edu.harvard.iq.dataverse.ingest.IngestServiceBean; import edu.harvard.iq.dataverse.makedatacount.MakeDataCountLoggingServiceBean; import edu.harvard.iq.dataverse.makedatacount.MakeDataCountLoggingServiceBean.MakeDataCountEntry; import edu.harvard.iq.dataverse.privateurl.PrivateUrlServiceBean; +import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.FileUtil; import edu.harvard.iq.dataverse.util.JsfHelper; +import edu.harvard.iq.dataverse.util.StringUtil; + import static edu.harvard.iq.dataverse.util.JsfHelper.JH; import edu.harvard.iq.dataverse.util.SystemConfig; import java.io.IOException; +import java.time.LocalDate; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Set; +import java.util.logging.Level; import java.util.logging.Logger; +import java.util.stream.Collectors; + import jakarta.ejb.EJB; import jakarta.ejb.EJBException; import jakarta.faces.application.FacesMessage; @@ -112,10 +122,10 @@ public class FilePage implements java.io.Serializable { GuestbookResponseServiceBean guestbookResponseService; @EJB AuthenticationServiceBean authService; - @EJB DatasetServiceBean datasetService; - + @EJB + IngestServiceBean ingestService; @EJB SystemConfig systemConfig; @@ -144,6 +154,9 @@ public class FilePage implements java.io.Serializable { @Inject EmbargoServiceBean embargoService; + @Inject + RetentionServiceBean retentionService; + private static final Logger logger = Logger.getLogger(FilePage.class.getCanonicalName()); private boolean fileDeleteInProgress = false; @@ -209,7 +222,7 @@ public String init() { // If this DatasetVersion is unpublished and permission is doesn't have permissions: // > Go to the Login page // - // Check permisisons + // Check permissions Boolean authorized = (fileMetadata.getDatasetVersion().isReleased()) || (!fileMetadata.getDatasetVersion().isReleased() && this.canViewUnpublishedDataset()); @@ -238,12 +251,10 @@ public String init() { if (file.isTabularData()) { contentType=DataFileServiceBean.MIME_TYPE_TSV_ALT; } - configureTools = externalToolService.findFileToolsByTypeAndContentType(ExternalTool.Type.CONFIGURE, contentType); - exploreTools = externalToolService.findFileToolsByTypeAndContentType(ExternalTool.Type.EXPLORE, contentType); - queryTools = externalToolService.findFileToolsByTypeAndContentType(ExternalTool.Type.QUERY, contentType); - Collections.sort(exploreTools, CompareExternalToolName); - toolsWithPreviews = sortExternalTools(); - + loadExternalTools(); + + + if (toolType != null) { if (toolType.equals("PREVIEW")) { if (!toolsWithPreviews.isEmpty()) { @@ -271,11 +282,32 @@ public String init() { if(!hasValidTermsOfAccess && canUpdateDataset() ){ JsfHelper.addWarningMessage(BundleUtil.getStringFromBundle("dataset.message.editMetadata.invalid.TOUA.message")); } - + + LocalDate minRetentiondate = settingsWrapper.getMinRetentionDate(); + if (minRetentiondate != null){ + selectionRetention.setDateUnavailable(minRetentiondate.plusDays(1L)); + } + displayPublishMessage(); return null; } + private void loadExternalTools() { + String contentType= file.getContentType(); + configureTools = externalToolService.findFileToolsByTypeAndContentType(ExternalTool.Type.CONFIGURE, contentType); + exploreTools = externalToolService.findFileToolsByTypeAndContentType(ExternalTool.Type.EXPLORE, contentType); + queryTools = externalToolService.findFileToolsByTypeAndContentType(ExternalTool.Type.QUERY, contentType); + Collections.sort(exploreTools, CompareExternalToolName); + toolsWithPreviews = sortExternalTools(); + //For inaccessible files, only show the tools that have access to aux files (which are currently always accessible) + if(!StorageIO.isDataverseAccessible(DataAccess.getStorageDriverFromIdentifier(file.getStorageIdentifier()))) { + configureTools = configureTools.stream().filter(tool ->tool.accessesAuxFiles()).collect(Collectors.toList()); + exploreTools = exploreTools.stream().filter(tool ->tool.accessesAuxFiles()).collect(Collectors.toList()); + queryTools = queryTools.stream().filter(tool ->tool.accessesAuxFiles()).collect(Collectors.toList()); + toolsWithPreviews = toolsWithPreviews.stream().filter(tool ->tool.accessesAuxFiles()).collect(Collectors.toList()); + } + } + private void displayPublishMessage(){ if (fileMetadata.getDatasetVersion().isDraft() && canUpdateDataset() && (canPublishDataset() || !fileMetadata.getDatasetVersion().getDataset().isLockedFor(DatasetLock.Reason.InReview))){ @@ -283,13 +315,18 @@ private void displayPublishMessage(){ } } + Boolean valid = null; + public boolean isValid() { - if (!fileMetadata.getDatasetVersion().isDraft()) { - return true; + if (valid == null) { + final DatasetVersion workingVersion = fileMetadata.getDatasetVersion(); + if (workingVersion.isDraft() || (canUpdateDataset() && JvmSettings.UI_SHOW_VALIDITY_LABEL_WHEN_PUBLISHED.lookupOptional(Boolean.class).orElse(true))) { + valid = workingVersion.isValid(); + } else { + valid = true; + } } - DatasetVersion newVersion = fileMetadata.getDatasetVersion().cloneDatasetVersion(); - newVersion.setDatasetFields(newVersion.initDatasetFields()); - return newVersion.isValid(); + return valid; } private boolean canViewUnpublishedDataset() { @@ -475,6 +512,119 @@ public String restrictFile(boolean restricted) throws CommandException{ return returnToDraftVersion(); } + public String ingestFile() throws CommandException{ + + User u = session.getUser(); + if(!u.isAuthenticated() || !u.isSuperuser()) { + //Shouldn't happen (choice not displayed for users who don't have the right permission), but check anyway + logger.warning("User: " + u.getIdentifier() + " tried to ingest a file"); + JH.addMessage(FacesMessage.SEVERITY_WARN, BundleUtil.getStringFromBundle("file.ingest.cantIngestFileWarning")); + return null; + } + + editDataset = file.getOwner(); + + if (file.isTabularData()) { + JH.addMessage(FacesMessage.SEVERITY_WARN, BundleUtil.getStringFromBundle("file.ingest.alreadyIngestedWarning")); + return null; + } + + boolean ingestLock = dataset.isLockedFor(DatasetLock.Reason.Ingest); + + if (ingestLock) { + JH.addMessage(FacesMessage.SEVERITY_WARN, BundleUtil.getStringFromBundle("file.ingest.ingestInProgressWarning")); + return null; + } + + if (!FileUtil.canIngestAsTabular(file)) { + JH.addMessage(FacesMessage.SEVERITY_WARN, BundleUtil.getStringFromBundle("file.ingest.cantIngestFileWarning")); + return null; + + } + + file.SetIngestScheduled(); + + if (file.getIngestRequest() == null) { + file.setIngestRequest(new IngestRequest(file)); + } + + file.getIngestRequest().setForceTypeCheck(true); + + // update the datafile, to save the newIngest request in the database: + datafileService.save(file); + + // queue the data ingest job for asynchronous execution: + String status = ingestService.startIngestJobs(editDataset.getId(), new ArrayList<>(Arrays.asList(file)), (AuthenticatedUser) session.getUser()); + + if (!StringUtil.isEmpty(status)) { + // This most likely indicates some sort of a problem (for example, + // the ingest job was not put on the JMS queue because of the size + // of the file). But we are still returning the OK status - because + // from the point of view of the API, it's a success - we have + // successfully gone through the process of trying to schedule the + // ingest job... + + logger.warning("Ingest Status for file: " + file.getId() + " : " + status); + } + logger.fine("File: " + file.getId() + " ingest queued"); + + init(); + JsfHelper.addInfoMessage(BundleUtil.getStringFromBundle("file.ingest.ingestQueued")); + return returnToDraftVersion(); + } + + public String uningestFile() throws CommandException { + + if (!file.isTabularData()) { + //Ingest never succeeded, either there was a failure or this is not a tabular data file + if (file.isIngestProblem()) { + //We allow anyone who can publish to uningest in order to clear a problem + User u = session.getUser(); + if (!u.isAuthenticated() || !(permissionService.permissionsFor(u, file).contains(Permission.PublishDataset))) { + logger.warning("User: " + u.getIdentifier() + " tried to uningest a file"); + // Shouldn't happen (choice not displayed for users who don't have the right + // permission), but check anyway + JH.addMessage(FacesMessage.SEVERITY_WARN, + BundleUtil.getStringFromBundle("file.ingest.cantUningestFileWarning")); + return null; + } + file.setIngestDone(); + file.setIngestReport(null); + } else { + //Shouldn't happen - got called when there is no tabular data or an ingest problem + JH.addMessage(FacesMessage.SEVERITY_WARN, + BundleUtil.getStringFromBundle("file.ingest.cantUningestFileWarning")); + return null; + } + } else { + //Superuser required to uningest after a success + //Uningest command does it's own check for isSuperuser + commandEngine.submit(new UningestFileCommand(dvRequestService.getDataverseRequest(), file)); + Long dataFileId = file.getId(); + file = datafileService.find(dataFileId); + } + editDataset = file.getOwner(); + if (editDataset.isReleased()) { + try { + ExportService instance = ExportService.getInstance(); + instance.exportAllFormats(editDataset); + + } catch (ExportException ex) { + // Something went wrong! + // Just like with indexing, a failure to export is not a fatal + // condition. We'll just log the error as a warning and keep + // going: + logger.log(Level.WARNING, "Uningest: Exception while exporting:{0}", ex.getMessage()); + } + } + datafileService.save(file); + + // Refresh filemetadata with file title, etc. + init(); + JH.addMessage(FacesMessage.SEVERITY_INFO, BundleUtil.getStringFromBundle("file.uningest.complete")); + return returnToDraftVersion(); + } + private List filesToBeDeleted = new ArrayList<>(); public String deleteFile() { @@ -948,6 +1098,12 @@ public boolean isPubliclyDownloadable() { return FileUtil.isPubliclyDownloadable(fileMetadata); } + public boolean isIngestable() { + DataFile f = fileMetadata.getDataFile(); + //Datafile is an ingestable type and hasn't been ingested yet or had an ingest fail + return (FileUtil.canIngestAsTabular(f)&&!(f.isTabularData() || f.isIngestProblem())); + } + private Boolean lockedFromEditsVar; private Boolean lockedFromDownloadVar; @@ -1247,7 +1403,129 @@ public String getEmbargoPhrase() { return BundleUtil.getStringFromBundle("embargoed.willbeuntil"); } } - + + public boolean isValidRetentionSelection() { + if (!fileMetadata.getDataFile().isReleased()) { + return true; + } + return false; + } + + public boolean isExistingRetention() { + if (!fileMetadata.getDataFile().isReleased() && (fileMetadata.getDataFile().getRetention() != null)) { + return true; + } + return false; + } + + public boolean isRetentionForWholeSelection() { + return isValidRetentionSelection(); + } + + public Retention getSelectionRetention() { + return selectionRetention; + } + + public void setSelectionRetention(Retention selectionRetention) { + this.selectionRetention = selectionRetention; + } + + private Retention selectionRetention = new Retention(); + + private boolean removeRetention=false; + + public boolean isRemoveRetention() { + return removeRetention; + } + + public void setRemoveRetention(boolean removeRetention) { + boolean existing = this.removeRetention; + this.removeRetention = removeRetention; + if (existing != this.removeRetention) { + logger.info("State flip"); + selectionRetention = new Retention(); + if (removeRetention) { + selectionRetention = new Retention(null, null); + } + } + PrimeFaces.current().resetInputs("fileForm:retentionInputs"); + } + + public String saveRetention() { + + if(isRemoveRetention() || (selectionRetention.getDateUnavailable()==null && selectionRetention.getReason()==null)) { + selectionRetention=null; + } + + Retention ret = null; + // Note: this.fileMetadata.getDataFile() is not the same object as this.file. + // (Not sure there's a good reason for this other than that's the way it is.) + // So changes to this.fileMetadata.getDataFile() will not be saved with + // editDataset = this.file.getOwner() set as it is below. + if (!file.isReleased()) { + ret = file.getRetention(); + if (ret != null) { + logger.fine("Before: " + ret.getDataFiles().size()); + ret.getDataFiles().remove(fileMetadata.getDataFile()); + logger.fine("After: " + ret.getDataFiles().size()); + } + if (selectionRetention != null) { + retentionService.merge(selectionRetention); + } + file.setRetention(selectionRetention); + if (ret != null && !ret.getDataFiles().isEmpty()) { + ret = null; + } + } + if(selectionRetention!=null) { + retentionService.save(selectionRetention, ((AuthenticatedUser)session.getUser()).getIdentifier()); + } + // success message: + String successMessage = BundleUtil.getStringFromBundle("file.assignedRetention.success"); + logger.fine(successMessage); + successMessage = successMessage.replace("{0}", "Selected Files"); + JsfHelper.addFlashMessage(successMessage); + selectionRetention = new Retention(); + + //Caller has to set editDataset before calling save() + editDataset = this.file.getOwner(); + + save(); + init(); + if(ret!=null) { + retentionService.delete(ret,((AuthenticatedUser)session.getUser()).getIdentifier()); + } + return returnToDraftVersion(); + } + + public void clearRetentionPopup() { + setRemoveRetention(false); + selectionRetention = new Retention(); + PrimeFaces.current().resetInputs("fileForm:retentionInputs"); + } + + public void clearSelectionRetention() { + selectionRetention = new Retention(); + PrimeFaces.current().resetInputs("fileForm:retentionInputs"); + } + + public boolean isCantRequestDueToRetention() { + return FileUtil.isRetentionExpired(fileMetadata); + } + + public String getRetentionPhrase() { + //Should only be getting called when there is a retention + if(file.isReleased()) { + if(FileUtil.isRetentionExpired(file)) { + return BundleUtil.getStringFromBundle("retention.after"); + } else { + return BundleUtil.getStringFromBundle("retention.isfrom"); + } + } else { + return BundleUtil.getStringFromBundle("retention.willbeafter"); + } + } + public String getToolTabTitle(){ if (getAllAvailableTools().size() > 1) { return BundleUtil.getStringFromBundle("file.toolTab.header"); diff --git a/src/main/java/edu/harvard/iq/dataverse/FileSearchCriteria.java b/src/main/java/edu/harvard/iq/dataverse/FileSearchCriteria.java index 62f10c18bdf..e3ed507a9c2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FileSearchCriteria.java +++ b/src/main/java/edu/harvard/iq/dataverse/FileSearchCriteria.java @@ -12,7 +12,7 @@ public class FileSearchCriteria { * Status of the particular DataFile based on active embargoes and restriction state */ public enum FileAccessStatus { - Public, Restricted, EmbargoedThenRestricted, EmbargoedThenPublic + Public, Restricted, EmbargoedThenRestricted, EmbargoedThenPublic, RetentionPeriodExpired } public FileSearchCriteria(String contentType, FileAccessStatus accessStatus, String categoryName, String tabularTagName, String searchText) { diff --git a/src/main/java/edu/harvard/iq/dataverse/GlobalId.java b/src/main/java/edu/harvard/iq/dataverse/GlobalId.java index 890b146a61c..a542cb52ac0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/GlobalId.java +++ b/src/main/java/edu/harvard/iq/dataverse/GlobalId.java @@ -6,7 +6,7 @@ package edu.harvard.iq.dataverse; -import edu.harvard.iq.dataverse.pidproviders.PermaLinkPidProviderServiceBean; +import edu.harvard.iq.dataverse.pidproviders.perma.PermaLinkPidProvider; import edu.harvard.iq.dataverse.util.BundleUtil; import static edu.harvard.iq.dataverse.util.StringUtil.isEmpty; import java.net.MalformedURLException; @@ -33,7 +33,7 @@ public GlobalId(String protocol, String authority, String identifier, String sep this.separator = separator; } this.urlPrefix = urlPrefix; - this.managingProviderName = providerName; + this.managingProviderId = providerName; } // protocol the identifier system, e.g. "doi" @@ -42,7 +42,7 @@ public GlobalId(String protocol, String authority, String identifier, String sep private String protocol; private String authority; private String identifier; - private String managingProviderName; + private String managingProviderId; private String separator = "/"; private String urlPrefix; @@ -67,8 +67,8 @@ public String getIdentifier() { return identifier; } - public String getProvider() { - return managingProviderName; + public String getProviderId() { + return managingProviderId; } public String toString() { diff --git a/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java index 72fc6ee6d64..1eee9c65501 100644 --- a/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java @@ -11,6 +11,7 @@ import edu.harvard.iq.dataverse.branding.BrandingUtil; import edu.harvard.iq.dataverse.confirmemail.ConfirmEmailServiceBean; import edu.harvard.iq.dataverse.dataset.DatasetUtil; +import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key; import edu.harvard.iq.dataverse.util.BundleUtil; @@ -24,11 +25,15 @@ import java.util.Arrays; import java.util.Date; import java.util.List; +import java.util.Objects; +import java.util.Optional; import java.util.Set; +import java.util.logging.Level; import java.util.logging.Logger; -import jakarta.annotation.Resource; import jakarta.ejb.EJB; import jakarta.ejb.Stateless; +import jakarta.inject.Inject; +import jakarta.inject.Named; import jakarta.mail.Address; import jakarta.mail.Message; import jakarta.mail.MessagingException; @@ -78,77 +83,133 @@ public class MailServiceBean implements java.io.Serializable { */ public MailServiceBean() { } + + /** + * Creates a new instance of MailServiceBean with explicit injection, as used during testing. + */ + public MailServiceBean(Session session, SettingsServiceBean settingsService) { + this.session = session; + this.settingsService = settingsService; + } - @Resource(name = "mail/notifyMailSession") + @Inject + @Named("mail/systemSession") private Session session; public boolean sendSystemEmail(String to, String subject, String messageText) { return sendSystemEmail(to, subject, messageText, false); } - + + /** + * Send a system notification to one or multiple recipients by email. + * Will skip sending when {@link #getSystemAddress()} doesn't return a configured "from" address. + * @param to A comma separated list of one or multiple recipients' addresses. May contain a "personal name" and + * the recipients address in <>. See also {@link InternetAddress}. + * @param subject The message's subject + * @param messageText The message's text + * @param isHtmlContent Determine if the message text is formatted using HTML or plain text. + * @return Status: true if sent successfully, false otherwise + */ public boolean sendSystemEmail(String to, String subject, String messageText, boolean isHtmlContent) { + Optional optionalAddress = getSystemAddress(); + if (optionalAddress.isEmpty()) { + logger.fine(() -> "Skipping sending mail to " + to + ", because no system address has been set."); + return false; + } + InternetAddress systemAddress = optionalAddress.get(); - boolean sent = false; - InternetAddress systemAddress = getSystemAddress(); - - String body = messageText - + (isHtmlContent ? BundleUtil.getStringFromBundle("notification.email.closing.html", Arrays.asList(BrandingUtil.getSupportTeamEmailAddress(systemAddress), BrandingUtil.getSupportTeamName(systemAddress))) - : BundleUtil.getStringFromBundle("notification.email.closing", Arrays.asList(BrandingUtil.getSupportTeamEmailAddress(systemAddress), BrandingUtil.getSupportTeamName(systemAddress)))); + String body = messageText + + BundleUtil.getStringFromBundle(isHtmlContent ? "notification.email.closing.html" : "notification.email.closing", + List.of(BrandingUtil.getSupportTeamEmailAddress(systemAddress), BrandingUtil.getSupportTeamName(systemAddress))); - logger.fine("Sending email to " + to + ". Subject: <<<" + subject + ">>>. Body: " + body); + logger.fine(() -> "Sending email to %s. Subject: <<<%s>>>. Body: %s".formatted(to, subject, body)); try { + // Since JavaMail 1.6, we have support for UTF-8 mail addresses and do not need to handle these ourselves. + InternetAddress[] recipients = InternetAddress.parse(to); + MimeMessage msg = new MimeMessage(session); - if (systemAddress != null) { - msg.setFrom(systemAddress); - msg.setSentDate(new Date()); - String[] recipientStrings = to.split(","); - InternetAddress[] recipients = new InternetAddress[recipientStrings.length]; - for (int i = 0; i < recipients.length; i++) { - try { - recipients[i] = new InternetAddress(recipientStrings[i], "", charset); - } catch (UnsupportedEncodingException ex) { - logger.severe(ex.getMessage()); - } - } - msg.setRecipients(Message.RecipientType.TO, recipients); - msg.setSubject(subject, charset); - if (isHtmlContent) { - msg.setText(body, charset, "html"); - } else { - msg.setText(body, charset); - } - - try { - Transport.send(msg, recipients); - sent = true; - } catch (MessagingException ssfe) { - logger.warning("Failed to send mail to: " + to); - logger.warning("MessagingException Message: " + ssfe); - } + msg.setFrom(systemAddress); + msg.setSentDate(new Date()); + msg.setRecipients(Message.RecipientType.TO, recipients); + msg.setSubject(subject, charset); + if (isHtmlContent) { + msg.setText(body, charset, "html"); } else { - logger.fine("Skipping sending mail to " + to + ", because the \"no-reply\" address not set (" + Key.SystemEmail + " setting)."); + msg.setText(body, charset); } - } catch (AddressException ae) { - logger.warning("Failed to send mail to " + to); - ae.printStackTrace(System.out); - } catch (MessagingException me) { - logger.warning("Failed to send mail to " + to); - me.printStackTrace(System.out); + + Transport.send(msg, recipients); + return true; + } catch (MessagingException ae) { + logger.log(Level.WARNING, "Failed to send mail to %s: %s".formatted(to, ae.getMessage()), ae); + logger.info("When UTF-8 characters in recipients: make sure MTA supports it and JVM option " + JvmSettings.MAIL_MTA_SUPPORT_UTF8.getScopedKey() + "=true"); } - return sent; + return false; } - - public InternetAddress getSystemAddress() { - String systemEmail = settingsService.getValueForKey(Key.SystemEmail); - return MailUtil.parseSystemAddress(systemEmail); + + /** + * Lookup the system mail address ({@code InternetAddress} may contain personal and actual address). + * @return The system mail address or an empty {@code Optional} if not configured. + */ + public Optional getSystemAddress() { + boolean providedByDB = false; + String mailAddress = JvmSettings.SYSTEM_EMAIL.lookupOptional().orElse(null); + + // Try lookup of (deprecated) database setting only if not configured via MPCONFIG + if (mailAddress == null) { + mailAddress = settingsService.getValueForKey(Key.SystemEmail); + // Encourage people to migrate from deprecated setting + if (mailAddress != null) { + providedByDB = true; + logger.warning("The :SystemMail DB setting has been deprecated, please reconfigure using JVM option " + JvmSettings.SYSTEM_EMAIL.getScopedKey()); + } + } + + try { + // Parse and return. + return Optional.of(new InternetAddress(Objects.requireNonNull(mailAddress), true)); + } catch (AddressException e) { + logger.log(Level.WARNING, "Could not parse system mail address '%s' provided by %s: " + .formatted(providedByDB ? "DB setting" : "JVM option", mailAddress), e); + } catch (NullPointerException e) { + // Do not pester the logs - no configuration may mean someone wants to disable mail notifications + logger.fine("Could not find a system mail setting in database (key :SystemEmail, deprecated) or JVM option '" + JvmSettings.SYSTEM_EMAIL.getScopedKey() + "'"); + } + // We define the system email address as an optional setting, in case people do not want to enable mail + // notifications (like in a development context, but might be useful elsewhere, too). + return Optional.empty(); + } + + /** + * Lookup the support team mail address ({@code InternetAddress} may contain personal and actual address). + * Will default to return {@code #getSystemAddress} if not configured. + * @return Support team mail address + */ + public Optional getSupportAddress() { + Optional supportMailAddress = JvmSettings.SUPPORT_EMAIL.lookupOptional(); + if (supportMailAddress.isPresent()) { + try { + return Optional.of(new InternetAddress(supportMailAddress.get(), true)); + } catch (AddressException e) { + logger.log(Level.WARNING, "Could not parse support mail address '%s', defaulting to system address: ".formatted(supportMailAddress.get()), e); + } + } + return getSystemAddress(); } //@Resource(name="mail/notifyMailSession") public void sendMail(String reply, String to, String cc, String subject, String messageText) { + Optional optionalAddress = getSystemAddress(); + if (optionalAddress.isEmpty()) { + logger.fine(() -> "Skipping sending mail to " + to + ", because no system address has been set."); + return; + } + // Always send from system address to avoid email being blocked + InternetAddress fromAddress = optionalAddress.get(); + try { MimeMessage msg = new MimeMessage(session); - // Always send from system address to avoid email being blocked - InternetAddress fromAddress = getSystemAddress(); + try { setContactDelegation(reply, fromAddress); } catch (UnsupportedEncodingException ex) { @@ -466,18 +527,24 @@ public String getMessageTextBasedOnNotification(UserNotification userNotificatio case RETURNEDDS: version = (DatasetVersion) targetObject; pattern = BundleUtil.getStringFromBundle("notification.email.wasReturnedByReviewer"); - String optionalReturnReason = ""; - /* - FIXME - Setting up to add single comment when design completed - optionalReturnReason = "."; - if (comment != null && !comment.isEmpty()) { - optionalReturnReason = ".\n\n" + BundleUtil.getStringFromBundle("wasReturnedReason") + "\n\n" + comment; - } - */ + String[] paramArrayReturnedDataset = {version.getDataset().getDisplayName(), getDatasetDraftLink(version.getDataset()), - version.getDataset().getOwner().getDisplayName(), getDataverseLink(version.getDataset().getOwner()), optionalReturnReason}; + version.getDataset().getOwner().getDisplayName(), getDataverseLink(version.getDataset().getOwner())}; messageText += MessageFormat.format(pattern, paramArrayReturnedDataset); + + if (comment != null && !comment.isEmpty()) { + messageText += "\n\n" + MessageFormat.format(BundleUtil.getStringFromBundle("notification.email.wasReturnedByReviewerReason"), comment); + } + + Dataverse d = (Dataverse) version.getDataset().getOwner(); + List contactEmailList = new ArrayList(); + for (DataverseContact dc : d.getDataverseContacts()) { + contactEmailList.add(dc.getContactEmail()); + } + if (!contactEmailList.isEmpty()) { + String contactEmails = String.join(", ", contactEmailList); + messageText += "\n\n" + MessageFormat.format(BundleUtil.getStringFromBundle("notification.email.wasReturnedByReviewer.collectionContacts"), contactEmails); + } return messageText; case WORKFLOW_SUCCESS: @@ -505,13 +572,12 @@ public String getMessageTextBasedOnNotification(UserNotification userNotificatio messageText += MessageFormat.format(pattern, paramArrayStatus); return messageText; case CREATEACC: - InternetAddress systemAddress = getSystemAddress(); String accountCreatedMessage = BundleUtil.getStringFromBundle("notification.email.welcome", Arrays.asList( BrandingUtil.getInstallationBrandName(), systemConfig.getGuidesBaseUrl(), systemConfig.getGuidesVersion(), - BrandingUtil.getSupportTeamName(systemAddress), - BrandingUtil.getSupportTeamEmailAddress(systemAddress) + BrandingUtil.getSupportTeamName(getSystemAddress().orElse(null)), + BrandingUtil.getSupportTeamEmailAddress(getSystemAddress().orElse(null)) )); String optionalConfirmEmailAddon = confirmEmailService.optionalConfirmEmailAddonMsg(userNotification.getUser()); accountCreatedMessage += optionalConfirmEmailAddon; diff --git a/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java b/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java index ca2f6145cba..1ead0b13cdc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java @@ -422,7 +422,10 @@ public void grantAccess(ActionEvent evt) { // set request(s) granted, if they exist for (AuthenticatedUser au : roleAssigneeService.getExplicitUsers(roleAssignee)) { FileAccessRequest far = file.getAccessRequestForAssignee(au); - far.setStateGranted(); + //There may not be a request, so do the null check + if (far != null) { + far.setStateGranted(); + } } datafileService.save(file); } diff --git a/src/main/java/edu/harvard/iq/dataverse/MetadataBlockServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/MetadataBlockServiceBean.java index bb6daa264ba..1e2a34f5472 100644 --- a/src/main/java/edu/harvard/iq/dataverse/MetadataBlockServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/MetadataBlockServiceBean.java @@ -1,43 +1,82 @@ package edu.harvard.iq.dataverse; -import java.util.List; import jakarta.ejb.Stateless; import jakarta.inject.Named; import jakarta.persistence.EntityManager; import jakarta.persistence.NoResultException; import jakarta.persistence.PersistenceContext; +import jakarta.persistence.TypedQuery; +import jakarta.persistence.criteria.*; + +import java.util.List; /** - * * @author michael */ @Stateless @Named public class MetadataBlockServiceBean { - + @PersistenceContext(unitName = "VDCNet-ejbPU") private EntityManager em; - + public MetadataBlock save(MetadataBlock mdb) { - return em.merge(mdb); - } - - + return em.merge(mdb); + } + public List listMetadataBlocks() { + return listMetadataBlocks(false); + } + + public List listMetadataBlocks(boolean onlyDisplayedOnCreate) { + if (onlyDisplayedOnCreate) { + return listMetadataBlocksDisplayedOnCreate(null); + } return em.createNamedQuery("MetadataBlock.listAll", MetadataBlock.class).getResultList(); } - - public MetadataBlock findById( Long id ) { + + public MetadataBlock findById(Long id) { return em.find(MetadataBlock.class, id); } - - public MetadataBlock findByName( String name ) { + + public MetadataBlock findByName(String name) { try { return em.createNamedQuery("MetadataBlock.findByName", MetadataBlock.class) - .setParameter("name", name) - .getSingleResult(); - } catch ( NoResultException nre ) { + .setParameter("name", name) + .getSingleResult(); + } catch (NoResultException nre) { return null; } } + + public List listMetadataBlocksDisplayedOnCreate(Dataverse ownerDataverse) { + CriteriaBuilder criteriaBuilder = em.getCriteriaBuilder(); + CriteriaQuery criteriaQuery = criteriaBuilder.createQuery(MetadataBlock.class); + Root metadataBlockRoot = criteriaQuery.from(MetadataBlock.class); + Join datasetFieldTypeJoin = metadataBlockRoot.join("datasetFieldTypes"); + Predicate displayOnCreatePredicate = criteriaBuilder.isTrue(datasetFieldTypeJoin.get("displayOnCreate")); + + if (ownerDataverse != null) { + Root dataverseRoot = criteriaQuery.from(Dataverse.class); + Join datasetFieldTypeInputLevelJoin = dataverseRoot.join("dataverseFieldTypeInputLevels", JoinType.LEFT); + + Predicate requiredPredicate = criteriaBuilder.and( + datasetFieldTypeInputLevelJoin.get("datasetFieldType").in(metadataBlockRoot.get("datasetFieldTypes")), + criteriaBuilder.isTrue(datasetFieldTypeInputLevelJoin.get("required"))); + + Predicate unionPredicate = criteriaBuilder.or(displayOnCreatePredicate, requiredPredicate); + + criteriaQuery.where(criteriaBuilder.and( + criteriaBuilder.equal(dataverseRoot.get("id"), ownerDataverse.getId()), + metadataBlockRoot.in(dataverseRoot.get("metadataBlocks")), + unionPredicate + )); + } else { + criteriaQuery.where(displayOnCreatePredicate); + } + + criteriaQuery.select(metadataBlockRoot).distinct(true); + TypedQuery typedQuery = em.createQuery(criteriaQuery); + return typedQuery.getResultList(); + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/NavigationWrapper.java b/src/main/java/edu/harvard/iq/dataverse/NavigationWrapper.java index 832d7ec19ef..54fb8f211a6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/NavigationWrapper.java +++ b/src/main/java/edu/harvard/iq/dataverse/NavigationWrapper.java @@ -16,6 +16,7 @@ import java.util.logging.Logger; import jakarta.faces.context.FacesContext; import jakarta.faces.view.ViewScoped; +import jakarta.ws.rs.core.Response.Status; import jakarta.inject.Inject; import jakarta.inject.Named; import jakarta.servlet.http.HttpServletRequest; @@ -87,6 +88,10 @@ public String notAuthorized(){ } } + public String tooManyRequests() { + return sendError(Status.TOO_MANY_REQUESTS.getStatusCode()); + } + public String notFound() { return sendError(HttpServletResponse.SC_NOT_FOUND); } diff --git a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java index 8fb762e3e5b..a389cbc735b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java @@ -97,6 +97,9 @@ public class PermissionServiceBean { @Inject DataverseRequestServiceBean dvRequestService; + @Inject + DatasetVersionFilesServiceBean datasetVersionFilesServiceBean; + /** * A request-level permission query (e.g includes IP ras). */ @@ -442,23 +445,14 @@ private Set getInferredPermissions(DvObject dvo) { * download permission for everybody: */ private boolean isPublicallyDownloadable(DvObject dvo) { - if (dvo instanceof DataFile) { + if (dvo instanceof DataFile df) { // unrestricted files that are part of a release dataset // automatically get download permission for everybody: // -- L.A. 4.0 beta12 - - DataFile df = (DataFile) dvo; - if (!df.isRestricted()) { - if (df.getOwner().getReleasedVersion() != null) { - List fileMetadatas = df.getOwner().getReleasedVersion().getFileMetadatas(); - if (fileMetadatas != null) { - for (FileMetadata fm : fileMetadatas) { - if (df.equals(fm.getDataFile())) { - return true; - } - } - } + DatasetVersion releasedVersion = df.getOwner().getReleasedVersion(); + if (releasedVersion != null) { + return datasetVersionFilesServiceBean.isDataFilePresentInDatasetVersion(releasedVersion, df); } } } diff --git a/src/main/java/edu/harvard/iq/dataverse/Retention.java b/src/main/java/edu/harvard/iq/dataverse/Retention.java new file mode 100644 index 00000000000..e1bd2231570 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/Retention.java @@ -0,0 +1,102 @@ +package edu.harvard.iq.dataverse; + +import edu.harvard.iq.dataverse.util.BundleUtil; +import jakarta.persistence.*; + +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.util.List; +import java.util.Objects; + +@NamedQueries({ + @NamedQuery( name="Retention.findAll", + query = "SELECT r FROM Retention r"), + @NamedQuery( name="Retention.findById", + query = "SELECT r FROM Retention r WHERE r.id=:id"), + @NamedQuery( name="Retention.findByDateUnavailable", + query = "SELECT r FROM Retention r WHERE r.dateUnavailable=:dateUnavailable"), + @NamedQuery( name="Retention.deleteById", + query = "DELETE FROM Retention r WHERE r.id=:id") +}) +@Entity +public class Retention { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column(nullable = false) + private LocalDate dateUnavailable; + + @Column(columnDefinition="TEXT") + private String reason; + + @OneToMany(mappedBy="retention", cascade={ CascadeType.REMOVE, CascadeType.PERSIST}) + private List dataFiles; + + public Retention(){ + dateUnavailable = LocalDate.now().plusYears(1000); // Most likely valid with respect to configuration + } + + public Retention(LocalDate dateUnavailable, String reason) { + this.dateUnavailable = dateUnavailable; + this.reason = reason; + } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public LocalDate getDateUnavailable() { + return dateUnavailable; + } + + public void setDateUnavailable(LocalDate dateUnavailable) { + this.dateUnavailable = dateUnavailable; + } + + public String getFormattedDateUnavailable() { + return getDateUnavailable().format(DateTimeFormatter.ISO_LOCAL_DATE.withLocale(BundleUtil.getCurrentLocale())); + } + + public String getReason() { + return reason; + } + + public void setReason(String reason) { + this.reason = reason; + } + + public List getDataFiles() { + return dataFiles; + } + + public void setDataFiles(List dataFiles) { + this.dataFiles = dataFiles; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Retention retention = (Retention) o; + return id.equals(retention.id) && dateUnavailable.equals(retention.dateUnavailable) && Objects.equals(reason, retention.reason); + } + + @Override + public int hashCode() { + return Objects.hash(id, dateUnavailable, reason); + } + + @Override + public String toString() { + return "Retention{" + + "id=" + id + + ", dateUnavailable=" + dateUnavailable + + ", reason='" + reason + '\'' + + '}'; + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/RetentionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/RetentionServiceBean.java new file mode 100644 index 00000000000..1421ac61120 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/RetentionServiceBean.java @@ -0,0 +1,66 @@ +package edu.harvard.iq.dataverse; + +import edu.harvard.iq.dataverse.actionlogging.ActionLogRecord; +import edu.harvard.iq.dataverse.actionlogging.ActionLogServiceBean; +import jakarta.ejb.EJB; +import jakarta.ejb.Stateless; +import jakarta.inject.Named; +import jakarta.persistence.EntityManager; +import jakarta.persistence.PersistenceContext; +import jakarta.persistence.Query; + +import java.util.List; + + +@Stateless +@Named +public class RetentionServiceBean { + + @PersistenceContext + EntityManager em; + + @EJB + ActionLogServiceBean actionLogSvc; + + public List findAllRetentions() { + return em.createNamedQuery("Retention.findAll", Retention.class).getResultList(); + } + + public Retention findByRetentionId(Long id) { + Query query = em.createNamedQuery("Retention.findById", Retention.class); + query.setParameter("id", id); + try { + return (Retention) query.getSingleResult(); + } catch (Exception ex) { + return null; + } + } + + public Retention merge(Retention r) { + return em.merge(r); + } + + public Long save(Retention retention, String userIdentifier) { + if (retention.getId() == null) { + em.persist(retention); + em.flush(); + } + //Not quite from a command, but this action can be done by anyone, so command seems better than Admin or other alternatives + actionLogSvc.log(new ActionLogRecord(ActionLogRecord.ActionType.Command, "retentionCreate") + .setInfo("id: " + retention.getId() + " date unavailable: " + retention.getDateUnavailable() + " reason: " + retention.getReason()).setUserIdentifier(userIdentifier)); + return retention.getId(); + } + + private int deleteById(long id, String userIdentifier) { + //Not quite from a command, but this action can be done by anyone, so command seems better than Admin or other alternatives + actionLogSvc.log(new ActionLogRecord(ActionLogRecord.ActionType.Command, "retentionDelete") + .setInfo(Long.toString(id)) + .setUserIdentifier(userIdentifier)); + return em.createNamedQuery("Retention.deleteById") + .setParameter("id", id) + .executeUpdate(); + } + public int delete(Retention retention, String userIdentifier) { + return deleteById(retention.getId(), userIdentifier); + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/S3PackageImporter.java b/src/main/java/edu/harvard/iq/dataverse/S3PackageImporter.java index 71318a0184a..a387b27d98b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/S3PackageImporter.java +++ b/src/main/java/edu/harvard/iq/dataverse/S3PackageImporter.java @@ -17,6 +17,7 @@ import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectSummary; import edu.harvard.iq.dataverse.api.AbstractApiBean; +import edu.harvard.iq.dataverse.pidproviders.PidProvider; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.FileUtil; import java.io.BufferedReader; @@ -203,35 +204,21 @@ public DataFile createPackageDataFile(Dataset dataset, String folderName, long t fmd.setDatasetVersion(dataset.getLatestVersion()); FileUtil.generateS3PackageStorageIdentifier(packageFile); - - GlobalIdServiceBean idServiceBean = GlobalIdServiceBean.getBean(packageFile.getProtocol(), commandEngine.getContext()); + PidProvider pidProvider = commandEngine.getContext().dvObjects().getEffectivePidGenerator(dataset); if (packageFile.getIdentifier() == null || packageFile.getIdentifier().isEmpty()) { - String packageIdentifier = idServiceBean.generateDataFileIdentifier(packageFile); - packageFile.setIdentifier(packageIdentifier); - } - - String nonNullDefaultIfKeyNotFound = ""; - String protocol = commandEngine.getContext().settings().getValueForKey(SettingsServiceBean.Key.Protocol, nonNullDefaultIfKeyNotFound); - String authority = commandEngine.getContext().settings().getValueForKey(SettingsServiceBean.Key.Authority, nonNullDefaultIfKeyNotFound); - - if (packageFile.getProtocol() == null) { - packageFile.setProtocol(protocol); - } - if (packageFile.getAuthority() == null) { - packageFile.setAuthority(authority); + pidProvider.generatePid(packageFile); } if (!packageFile.isIdentifierRegistered()) { String doiRetString = ""; - idServiceBean = GlobalIdServiceBean.getBean(commandEngine.getContext()); try { - doiRetString = idServiceBean.createIdentifier(packageFile); + doiRetString = pidProvider.createIdentifier(packageFile); } catch (Throwable e) { } // Check return value to make sure registration succeeded - if (!idServiceBean.registerWhenPublished() && doiRetString.contains(packageFile.getIdentifier())) { + if (!pidProvider.registerWhenPublished() && doiRetString.contains(packageFile.getIdentifier())) { packageFile.setIdentifierRegistered(true); packageFile.setGlobalIdCreateTime(new Date()); } diff --git a/src/main/java/edu/harvard/iq/dataverse/SendFeedbackDialog.java b/src/main/java/edu/harvard/iq/dataverse/SendFeedbackDialog.java index 68912969003..46941c8b5b6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/SendFeedbackDialog.java +++ b/src/main/java/edu/harvard/iq/dataverse/SendFeedbackDialog.java @@ -7,7 +7,6 @@ import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.JsfHelper; -import edu.harvard.iq.dataverse.util.MailUtil; import edu.harvard.iq.dataverse.util.SystemConfig; import java.util.Optional; import java.util.Random; @@ -102,8 +101,7 @@ public void initUserInput(ActionEvent ae) { op1 = Long.valueOf(random.nextInt(10)); op2 = Long.valueOf(random.nextInt(10)); userSum = null; - String supportEmail = JvmSettings.SUPPORT_EMAIL.lookupOptional().orElse(settingsService.getValueForKey(SettingsServiceBean.Key.SystemEmail)); - systemAddress = MailUtil.parseSystemAddress(supportEmail); + systemAddress = mailService.getSupportAddress().orElse(null); } public Long getOp1() { @@ -131,6 +129,10 @@ public void setUserSum(Long userSum) { } public String getMessageTo() { + if (op1 == null || op2 == null) { + // Fix for 403 error page: initUserInput method doesn't call before + initUserInput(null); + } if (feedbackTarget == null) { return BrandingUtil.getSupportTeamName(systemAddress); } else if (feedbackTarget.isInstanceofDataverse()) { diff --git a/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java b/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java index 8ab1e87aef2..48196591b19 100644 --- a/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java +++ b/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java @@ -6,9 +6,9 @@ package edu.harvard.iq.dataverse; import edu.harvard.iq.dataverse.branding.BrandingUtil; -import edu.harvard.iq.dataverse.dataaccess.AbstractRemoteOverlayAccessIO; import edu.harvard.iq.dataverse.dataaccess.DataAccess; import edu.harvard.iq.dataverse.dataaccess.GlobusAccessibleStore; +import edu.harvard.iq.dataverse.dataaccess.StorageIO; import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.settings.Setting; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; @@ -66,6 +66,9 @@ public class SettingsWrapper implements java.io.Serializable { @EJB MetadataBlockServiceBean mdbService; + + @EJB + MailServiceBean mailServiceBean; private Map settingsMap; @@ -75,6 +78,9 @@ public class SettingsWrapper implements java.io.Serializable { private boolean embargoDateChecked = false; private LocalDate maxEmbargoDate = null; + private boolean retentionDateChecked = false; + private LocalDate minRetentionDate = null; + private String siteUrl = null; private Dataverse rootDataverse = null; @@ -299,14 +305,16 @@ public boolean isPublicInstall(){ } return publicInstall; } - + + @Deprecated(forRemoval = true, since = "2024-07-07") public boolean isRsyncUpload() { if (rsyncUpload == null) { rsyncUpload = getUploadMethodAvailable(SystemConfig.FileUploadMethods.RSYNC.toString()); } return rsyncUpload; } - + + @Deprecated(forRemoval = true, since = "2024-07-07") public boolean isRsyncDownload() { if (rsyncDownload == null) { rsyncDownload = systemConfig.isRsyncDownload(); @@ -344,7 +352,7 @@ public boolean isDownloadable(FileMetadata fmd) { if(isGlobusFileDownload()) { String driverId = DataAccess.getStorageDriverFromIdentifier(fmd.getDataFile().getStorageIdentifier()); - downloadable = downloadable && !AbstractRemoteOverlayAccessIO.isNotDataverseAccessible(driverId); + downloadable = downloadable && StorageIO.isDataverseAccessible(driverId); } return downloadable; } @@ -373,7 +381,8 @@ public boolean isWebloaderUpload() { } return webloaderUpload; } - + + @Deprecated(forRemoval = true, since = "2024-07-07") public boolean isRsyncOnly() { if (rsyncOnly == null) { String downloadMethods = getValueForKey(SettingsServiceBean.Key.DownloadMethods); @@ -392,7 +401,7 @@ public boolean isRsyncOnly() { } return rsyncOnly; } - + public boolean isHTTPUpload(){ if (httpUpload == null) { httpUpload = getUploadMethodAvailable(SystemConfig.FileUploadMethods.NATIVE.toString()); @@ -400,22 +409,15 @@ public boolean isHTTPUpload(){ return httpUpload; } - public boolean isDataFilePIDSequentialDependent(){ - if (dataFilePIDSequentialDependent == null) { - dataFilePIDSequentialDependent = systemConfig.isDataFilePIDSequentialDependent(); - } - return dataFilePIDSequentialDependent; - } - public String getSupportTeamName() { - String systemEmail = getValueForKey(SettingsServiceBean.Key.SystemEmail); - InternetAddress systemAddress = MailUtil.parseSystemAddress(systemEmail); + // TODO: should this be replaced with mailServiceBean.getSupportAddress() to expose a configured support team? + InternetAddress systemAddress = mailServiceBean.getSystemAddress().orElse(null); return BrandingUtil.getSupportTeamName(systemAddress); } public String getSupportTeamEmail() { - String systemEmail = getValueForKey(SettingsServiceBean.Key.SystemEmail); - InternetAddress systemAddress = MailUtil.parseSystemAddress(systemEmail); + // TODO: should this be replaced with mailServiceBean.getSupportAddress() to expose a configured support team? + InternetAddress systemAddress = mailServiceBean.getSystemAddress().orElse(null); return BrandingUtil.getSupportTeamEmailAddress(systemAddress) != null ? BrandingUtil.getSupportTeamEmailAddress(systemAddress) : BrandingUtil.getSupportTeamName(systemAddress); } @@ -470,23 +472,6 @@ public Map getConfiguredLocales() { return configuredLocales; } - public boolean isDoiInstallation() { - String protocol = getValueForKey(SettingsServiceBean.Key.Protocol); - if ("doi".equals(protocol)) { - return true; - } else { - return false; - } - } - - public boolean isDataCiteInstallation() { - String protocol = getValueForKey(SettingsServiceBean.Key.DoiProvider); - if ("DataCite".equals(protocol)) { - return true; - } else { - return false; - } - } public boolean isMakeDataCountDisplayEnabled() { boolean safeDefaultIfKeyNotFound = (getValueForKey(SettingsServiceBean.Key.MDCLogPath)!=null); //Backward compatible @@ -603,6 +588,89 @@ public void validateEmbargoDate(FacesContext context, UIComponent component, Obj } } + public LocalDate getMinRetentionDate() { + if (!retentionDateChecked) { + String months = getValueForKey(Key.MinRetentionDurationInMonths); + Long minMonths = null; + if (months != null) { + try { + minMonths = Long.parseLong(months); + } catch (NumberFormatException nfe) { + logger.warning("Cant interpret :MinRetentionDurationInMonths as a long"); + } + } + + if (minMonths != null && minMonths != 0) { + if (minMonths == -1) { + minMonths = 0l; // Absolute minimum is 0 + } + minRetentionDate = LocalDate.now().plusMonths(minMonths); + } + retentionDateChecked = true; + } + return minRetentionDate; + } + + public LocalDate getMaxRetentionDate() { + Long maxMonths = 12000l; // Arbitrary cutoff at 1000 years - needs to keep maxDate < year 999999999 and + // somehwere 1K> x >10K years the datepicker widget stops showing a popup + // calendar + return LocalDate.now().plusMonths(maxMonths); + } + + public boolean isValidRetentionDate(Retention r) { + + if (r.getDateUnavailable()==null || + isRetentionAllowed() && r.getDateUnavailable().isAfter(getMinRetentionDate())) { + return true; + } + + return false; + } + + public boolean isRetentionAllowed() { + //Need a valid :MinRetentionDurationInMonths setting to allow retentions + return getMinRetentionDate()!=null; + } + + public void validateRetentionDate(FacesContext context, UIComponent component, Object value) + throws ValidatorException { + if (isRetentionAllowed()) { + UIComponent cb = component.findComponent("retentionCheckbox"); + UIInput endComponent = (UIInput) cb; + boolean removedState = false; + if (endComponent != null) { + try { + removedState = (Boolean) endComponent.getSubmittedValue(); + } catch (NullPointerException npe) { + // Do nothing - checkbox is not being shown (and is therefore not checked) + } + } + if (!removedState && value == null) { + String msgString = BundleUtil.getStringFromBundle("retention.date.required"); + FacesMessage msg = new FacesMessage(msgString); + msg.setSeverity(FacesMessage.SEVERITY_ERROR); + throw new ValidatorException(msg); + } + Retention newR = new Retention(((LocalDate) value), null); + if (!isValidRetentionDate(newR)) { + String minDate = getMinRetentionDate().format(DateTimeFormatter.ofPattern("yyyy-MM-dd")); + String maxDate = getMaxRetentionDate().format(DateTimeFormatter.ofPattern("yyyy-MM-dd")); + String msgString = BundleUtil.getStringFromBundle("retention.date.invalid", + Arrays.asList(minDate, maxDate)); + // If we don't throw an exception here, the datePicker will use it's own + // vaidator and display a default message. The value for that can be set by + // adding validatorMessage="#{bundle['retention.date.invalid']}" (a version with + // no params) to the datepicker + // element in file-edit-popup-fragment.html, but it would be better to catch all + // problems here (so we can show a message with the min/max dates). + FacesMessage msg = new FacesMessage(msgString); + msg.setSeverity(FacesMessage.SEVERITY_ERROR); + throw new ValidatorException(msg); + } + } + } + Map languageMap = null; public Map getBaseMetadataLanguageMap(boolean refresh) { diff --git a/src/main/java/edu/harvard/iq/dataverse/Shib.java b/src/main/java/edu/harvard/iq/dataverse/Shib.java index 24c0f9d7926..f9cf061e771 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Shib.java +++ b/src/main/java/edu/harvard/iq/dataverse/Shib.java @@ -59,6 +59,8 @@ public class Shib implements java.io.Serializable { SettingsServiceBean settingsService; @EJB SystemConfig systemConfig; + @EJB + UserServiceBean userService; HttpServletRequest request; @@ -259,6 +261,7 @@ else if (ShibAffiliationOrder.equals("firstAffiliation")) { state = State.REGULAR_LOGIN_INTO_EXISTING_SHIB_ACCOUNT; logger.fine("Found user based on " + userPersistentId + ". Logging in."); logger.fine("Updating display info for " + au.getName()); + userService.updateLastLogin(au); authSvc.updateAuthenticatedUser(au, displayInfo); logInUserAndSetShibAttributes(au); String prettyFacesHomePageString = getPrettyFacesHomePageString(false); diff --git a/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java b/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java index b6ab23848e2..d31fdd4e380 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java +++ b/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java @@ -177,7 +177,7 @@ public String getDatasetCardImageAsUrl(Dataset dataset, Long versionId, boolean StorageIO storageIO = null; try { storageIO = DataAccess.getStorageIO(dataset); - if (storageIO.isAuxObjectCached(DatasetUtil.datasetLogoFilenameFinal)) { + if (storageIO != null && storageIO.isAuxObjectCached(DatasetUtil.datasetLogoFilenameFinal)) { // If not, return null/use the default, otherwise pass the logo URL hasDatasetLogo = true; } diff --git a/src/main/java/edu/harvard/iq/dataverse/UserServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/UserServiceBean.java index 93892376edc..d63fcfa3e34 100644 --- a/src/main/java/edu/harvard/iq/dataverse/UserServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/UserServiceBean.java @@ -147,6 +147,8 @@ private AuthenticatedUser createAuthenticatedUserForView (Object[] dbRowValues, user.setMutedEmails(Type.tokenizeToSet((String) dbRowValues[15])); user.setMutedNotifications(Type.tokenizeToSet((String) dbRowValues[15])); + user.setRateLimitTier((int)dbRowValues[17]); + user.setRoles(roles); return user; } @@ -419,7 +421,7 @@ private List getUserListCore(String searchTerm, qstr += " u.createdtime, u.lastlogintime, u.lastapiusetime, "; qstr += " prov.id, prov.factoryalias, "; qstr += " u.deactivated, u.deactivatedtime, "; - qstr += " u.mutedEmails, u.mutedNotifications "; + qstr += " u.mutedEmails, u.mutedNotifications, u.rateLimitTier "; qstr += " FROM authenticateduser u,"; qstr += " authenticateduserlookup prov_lookup,"; qstr += " authenticationproviderrow prov"; diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index bc94d7f0bcc..19df6d8c1c7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -2,6 +2,7 @@ import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.actionlogging.ActionLogServiceBean; +import static edu.harvard.iq.dataverse.api.Datasets.handleVersion; import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; import edu.harvard.iq.dataverse.authorization.DataverseRole; import edu.harvard.iq.dataverse.authorization.RoleAssignee; @@ -15,8 +16,14 @@ import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; import edu.harvard.iq.dataverse.engine.command.exception.PermissionException; +import edu.harvard.iq.dataverse.engine.command.impl.GetDraftDatasetVersionCommand; +import edu.harvard.iq.dataverse.engine.command.impl.GetLatestAccessibleDatasetVersionCommand; +import edu.harvard.iq.dataverse.engine.command.impl.GetLatestPublishedDatasetVersionCommand; +import edu.harvard.iq.dataverse.engine.command.impl.GetSpecificPublishedDatasetVersionCommand; +import edu.harvard.iq.dataverse.engine.command.exception.RateLimitCommandException; import edu.harvard.iq.dataverse.externaltools.ExternalToolServiceBean; import edu.harvard.iq.dataverse.license.LicenseServiceBean; +import edu.harvard.iq.dataverse.pidproviders.PidUtil; import edu.harvard.iq.dataverse.locality.StorageSiteServiceBean; import edu.harvard.iq.dataverse.metrics.MetricsServiceBean; import edu.harvard.iq.dataverse.search.savedsearch.SavedSearchServiceBean; @@ -36,6 +43,7 @@ import jakarta.persistence.NoResultException; import jakarta.persistence.PersistenceContext; import jakarta.servlet.http.HttpServletRequest; +import jakarta.validation.constraints.NotNull; import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.MediaType; @@ -364,6 +372,11 @@ protected DataverseLinkingDataverse findDataverseLinkingDataverseOrDie(String da } protected Dataset findDatasetOrDie(String id) throws WrappedResponse { + return findDatasetOrDie(id, false); + } + + protected Dataset findDatasetOrDie(String id, boolean deep) throws WrappedResponse { + Long datasetId; Dataset dataset; if (id.equals(PERSISTENT_ID_KEY)) { String persistentId = getRequestParameter(PERSISTENT_ID_KEY.substring(1)); @@ -371,28 +384,67 @@ protected Dataset findDatasetOrDie(String id) throws WrappedResponse { throw new WrappedResponse( badRequest(BundleUtil.getStringFromBundle("find.dataset.error.dataset_id_is_null", Collections.singletonList(PERSISTENT_ID_KEY.substring(1))))); } - dataset = datasetSvc.findByGlobalId(persistentId); - if (dataset == null) { - throw new WrappedResponse(notFound(BundleUtil.getStringFromBundle("find.dataset.error.dataset.not.found.persistentId", Collections.singletonList(persistentId)))); + GlobalId globalId; + try { + globalId = PidUtil.parseAsGlobalID(persistentId); + } catch (IllegalArgumentException e) { + throw new WrappedResponse( + badRequest(BundleUtil.getStringFromBundle("find.dataset.error.dataset.not.found.bad.id", Collections.singletonList(persistentId)))); + } + datasetId = dvObjSvc.findIdByGlobalId(globalId, DvObject.DType.Dataset); + if (datasetId == null) { + datasetId = dvObjSvc.findIdByAltGlobalId(globalId, DvObject.DType.Dataset); + } + if (datasetId == null) { + throw new WrappedResponse( + notFound(BundleUtil.getStringFromBundle("find.dataset.error.dataset_id_is_null", Collections.singletonList(PERSISTENT_ID_KEY.substring(1))))); } - return dataset; - } else { try { - dataset = datasetSvc.find(Long.parseLong(id)); - if (dataset == null) { - throw new WrappedResponse(notFound(BundleUtil.getStringFromBundle("find.dataset.error.dataset.not.found.id", Collections.singletonList(id)))); - } - return dataset; + datasetId = Long.parseLong(id); } catch (NumberFormatException nfe) { throw new WrappedResponse( badRequest(BundleUtil.getStringFromBundle("find.dataset.error.dataset.not.found.bad.id", Collections.singletonList(id)))); } } + if (deep) { + dataset = datasetSvc.findDeep(datasetId); + } else { + dataset = datasetSvc.find(datasetId); + } + if (dataset == null) { + throw new WrappedResponse(notFound(BundleUtil.getStringFromBundle("find.dataset.error.dataset.not.found.id", Collections.singletonList(id)))); + } + return dataset; } - + + protected DatasetVersion findDatasetVersionOrDie(final DataverseRequest req, String versionNumber, final Dataset ds, boolean includeDeaccessioned, boolean checkPermsWhenDeaccessioned) throws WrappedResponse { + DatasetVersion dsv = execCommand(handleVersion(versionNumber, new Datasets.DsVersionHandler>() { + + @Override + public Command handleLatest() { + return new GetLatestAccessibleDatasetVersionCommand(req, ds, includeDeaccessioned, checkPermsWhenDeaccessioned); + } + + @Override + public Command handleDraft() { + return new GetDraftDatasetVersionCommand(req, ds); + } + + @Override + public Command handleSpecific(long major, long minor) { + return new GetSpecificPublishedDatasetVersionCommand(req, ds, major, minor, includeDeaccessioned, checkPermsWhenDeaccessioned); + } + + @Override + public Command handleLatestPublished() { + return new GetLatestPublishedDatasetVersionCommand(req, ds, includeDeaccessioned, checkPermsWhenDeaccessioned); + } + })); + return dsv; + } + protected DataFile findDataFileOrDie(String id) throws WrappedResponse { - DataFile datafile; if (id.equals(PERSISTENT_ID_KEY)) { String persistentId = getRequestParameter(PERSISTENT_ID_KEY.substring(1)); @@ -500,17 +552,21 @@ protected DvObject findDvo( Long id ) { * with that alias. If that fails, tries to get a {@link Dataset} with that global id. * @param id a value identifying the DvObject, either numeric of textual. * @return A DvObject, or {@code null} + * @throws WrappedResponse */ - protected DvObject findDvo( String id ) { - if ( isNumeric(id) ) { - return findDvo( Long.valueOf(id)) ; + @NotNull + protected DvObject findDvo(@NotNull final String id) throws WrappedResponse { + DvObject d = null; + if (isNumeric(id)) { + d = findDvo(Long.valueOf(id)); } else { - Dataverse d = dataverseSvc.findByAlias(id); - return ( d != null ) ? - d : datasetSvc.findByGlobalId(id); - + d = dataverseSvc.findByAlias(id); } - } + if (d == null) { + return findDatasetOrDie(id); + } + return d; + } protected T failIfNull( T t, String errorMessage ) throws WrappedResponse { if ( t != null ) return t; @@ -545,6 +601,8 @@ protected T execCommand( Command cmd ) throws WrappedResponse { try { return engineSvc.submit(cmd); + } catch (RateLimitCommandException ex) { + throw new WrappedResponse(rateLimited(ex.getMessage())); } catch (IllegalCommandException ex) { //for 8859 for api calls that try to update datasets with TOA out of compliance if (ex.getMessage().toLowerCase().contains("terms of use")){ @@ -746,11 +804,15 @@ protected Response notFound( String msg ) { protected Response badRequest( String msg ) { return error( Status.BAD_REQUEST, msg ); } - + protected Response forbidden( String msg ) { return error( Status.FORBIDDEN, msg ); } - + + protected Response rateLimited( String msg ) { + return error( Status.TOO_MANY_REQUESTS, msg ); + } + protected Response conflict( String msg ) { return error( Status.CONFLICT, msg ); } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java index 297ec2d3681..00da4990996 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java @@ -130,6 +130,14 @@ import jakarta.ws.rs.core.MediaType; import static jakarta.ws.rs.core.Response.Status.FORBIDDEN; import static jakarta.ws.rs.core.Response.Status.UNAUTHORIZED; + +import org.eclipse.microprofile.openapi.annotations.Operation; +import org.eclipse.microprofile.openapi.annotations.media.Content; +import org.eclipse.microprofile.openapi.annotations.media.Schema; +import org.eclipse.microprofile.openapi.annotations.parameters.RequestBody; +import org.eclipse.microprofile.openapi.annotations.responses.APIResponse; +import org.eclipse.microprofile.openapi.annotations.responses.APIResponses; +import org.eclipse.microprofile.openapi.annotations.tags.Tag; import org.glassfish.jersey.media.multipart.FormDataBodyPart; import org.glassfish.jersey.media.multipart.FormDataParam; @@ -466,7 +474,9 @@ public String tabularDatafileMetadataDDI(@Context ContainerRequestContext crc, @ if (!dataFile.isTabularData()) { throw new BadRequestException("tabular data required"); } - + if (FileUtil.isRetentionExpired(dataFile)) { + throw new BadRequestException("unable to download file with expired retention"); + } if (dataFile.isRestricted() || FileUtil.isActivelyEmbargoed(dataFile)) { boolean hasPermissionToDownloadFile = false; DataverseRequest dataverseRequest; @@ -921,14 +931,15 @@ public void write(OutputStream os) throws IOException, } } else { boolean embargoed = FileUtil.isActivelyEmbargoed(file); - if (file.isRestricted() || embargoed) { + boolean retentionExpired = FileUtil.isRetentionExpired(file); + if (file.isRestricted() || embargoed || retentionExpired) { if (zipper == null) { fileManifest = fileManifest + file.getFileMetadata().getLabel() + " IS " - + (embargoed ? "EMBARGOED" : "RESTRICTED") + + (embargoed ? "EMBARGOED" : retentionExpired ? "RETENTIONEXPIRED" : "RESTRICTED") + " AND CANNOT BE DOWNLOADED\r\n"; } else { zipper.addToManifest(file.getFileMetadata().getLabel() + " IS " - + (embargoed ? "EMBARGOED" : "RESTRICTED") + + (embargoed ? "EMBARGOED" : retentionExpired ? "RETENTIONEXPIRED" : "RESTRICTED") + " AND CANNOT BE DOWNLOADED\r\n"); } } else { @@ -1245,6 +1256,20 @@ private String getWebappImageResource(String imageName) { @AuthRequired @Path("datafile/{fileId}/auxiliary/{formatTag}/{formatVersion}") @Consumes(MediaType.MULTIPART_FORM_DATA) + @Produces("application/json") + @Operation(summary = "Save auxiliary file with version", + description = "Saves an auxiliary file") + @APIResponses(value = { + @APIResponse(responseCode = "200", + description = "File saved response"), + @APIResponse(responseCode = "403", + description = "User not authorized to edit the dataset."), + @APIResponse(responseCode = "400", + description = "File not found based on id.") + }) + @Tag(name = "saveAuxiliaryFileWithVersion", + description = "Save Auxiliary File With Version") + @RequestBody(content = @Content(mediaType = MediaType.MULTIPART_FORM_DATA)) public Response saveAuxiliaryFileWithVersion(@Context ContainerRequestContext crc, @PathParam("fileId") Long fileId, @PathParam("formatTag") String formatTag, @@ -1402,6 +1427,10 @@ public Response requestFileAccess(@Context ContainerRequestContext crc, @PathPar return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.requestAccess.fileNotFound", args)); } + if (FileUtil.isRetentionExpired(dataFile)) { + return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.requestAccess.failure.retentionExpired")); + } + if (!dataFile.getOwner().isFileAccessRequest()) { return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.requestAccess.requestsNotAccepted")); } @@ -1735,8 +1764,11 @@ private boolean isAccessAuthorized(User requestUser, DataFile df) { //True if there's an embargo that hasn't yet expired //In this state, we block access as though the file is restricted (even if it is not restricted) boolean embargoed = FileUtil.isActivelyEmbargoed(df); - - + // access is also blocked for retention expired files + boolean retentionExpired = FileUtil.isRetentionExpired(df); + // No access ever if retention is expired + if(retentionExpired) return false; + /* SEK 7/26/2018 for 3661 relying on the version state of the dataset versions to which this file is attached check to see if at least one is RELEASED @@ -1801,7 +1833,7 @@ private boolean isAccessAuthorized(User requestUser, DataFile df) { //The one case where we don't need to check permissions - if (!restricted && !embargoed && published) { + if (!restricted && !embargoed && !retentionExpired && published) { // If they are not published, they can still be downloaded, if the user // has the permission to view unpublished versions! (this case will // be handled below) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index 48f9e19d835..d60884bad2f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -14,11 +14,12 @@ import edu.harvard.iq.dataverse.DataverseServiceBean; import edu.harvard.iq.dataverse.DataverseSession; import edu.harvard.iq.dataverse.DvObject; +import edu.harvard.iq.dataverse.DvObjectServiceBean; import edu.harvard.iq.dataverse.api.auth.AuthRequired; import edu.harvard.iq.dataverse.settings.JvmSettings; +import edu.harvard.iq.dataverse.util.StringUtil; import edu.harvard.iq.dataverse.validation.EMailValidator; import edu.harvard.iq.dataverse.EjbDataverseEngine; -import edu.harvard.iq.dataverse.HandlenetServiceBean; import edu.harvard.iq.dataverse.Template; import edu.harvard.iq.dataverse.TemplateServiceBean; import edu.harvard.iq.dataverse.UserServiceBean; @@ -97,6 +98,7 @@ import edu.harvard.iq.dataverse.engine.command.impl.DeleteTemplateCommand; import edu.harvard.iq.dataverse.engine.command.impl.RegisterDvObjectCommand; import edu.harvard.iq.dataverse.ingest.IngestServiceBean; +import edu.harvard.iq.dataverse.pidproviders.handle.HandlePidProvider; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.userdata.UserListMaker; import edu.harvard.iq.dataverse.userdata.UserListResult; @@ -136,46 +138,48 @@ public class Admin extends AbstractApiBean { private static final Logger logger = Logger.getLogger(Admin.class.getName()); - @EJB - AuthenticationProvidersRegistrationServiceBean authProvidersRegistrationSvc; - @EJB - BuiltinUserServiceBean builtinUserService; - @EJB - ShibServiceBean shibService; - @EJB - AuthTestDataServiceBean authTestDataService; - @EJB - UserServiceBean userService; - @EJB - IngestServiceBean ingestService; - @EJB - DataFileServiceBean fileService; - @EJB - DatasetServiceBean datasetService; - @EJB - DataverseServiceBean dataverseService; - @EJB - DatasetVersionServiceBean datasetversionService; - @Inject - DataverseRequestServiceBean dvRequestService; - @EJB - EjbDataverseEngine commandEngine; - @EJB - GroupServiceBean groupService; - @EJB - SettingsServiceBean settingsService; - @EJB - DatasetVersionServiceBean datasetVersionService; - @EJB - ExplicitGroupServiceBean explicitGroupService; - @EJB - BannerMessageServiceBean bannerMessageService; - @EJB - TemplateServiceBean templateService; - - // Make the session available - @Inject - DataverseSession session; + @EJB + AuthenticationProvidersRegistrationServiceBean authProvidersRegistrationSvc; + @EJB + BuiltinUserServiceBean builtinUserService; + @EJB + ShibServiceBean shibService; + @EJB + AuthTestDataServiceBean authTestDataService; + @EJB + UserServiceBean userService; + @EJB + IngestServiceBean ingestService; + @EJB + DataFileServiceBean fileService; + @EJB + DatasetServiceBean datasetService; + @EJB + DataverseServiceBean dataverseService; + @EJB + DvObjectServiceBean dvObjectService; + @EJB + DatasetVersionServiceBean datasetversionService; + @Inject + DataverseRequestServiceBean dvRequestService; + @EJB + EjbDataverseEngine commandEngine; + @EJB + GroupServiceBean groupService; + @EJB + SettingsServiceBean settingsService; + @EJB + DatasetVersionServiceBean datasetVersionService; + @EJB + ExplicitGroupServiceBean explicitGroupService; + @EJB + BannerMessageServiceBean bannerMessageService; + @EJB + TemplateServiceBean templateService; + + // Make the session available + @Inject + DataverseSession session; public static final String listUsersPartialAPIPath = "list-users"; public static final String listUsersFullAPIPath = "/api/admin/" + listUsersPartialAPIPath; @@ -197,7 +201,7 @@ public Response putSetting(@PathParam("name") String name, String content) { @Path("settings/{name}/lang/{lang}") @PUT - public Response putSetting(@PathParam("name") String name, @PathParam("lang") String lang, String content) { + public Response putSettingLang(@PathParam("name") String name, @PathParam("lang") String lang, String content) { Setting s = settingsSvc.set(name, lang, content); return ok("Setting " + name + " - " + lang + " - added."); } @@ -220,7 +224,7 @@ public Response deleteSetting(@PathParam("name") String name) { @Path("settings/{name}/lang/{lang}") @DELETE - public Response deleteSetting(@PathParam("name") String name, @PathParam("lang") String lang) { + public Response deleteSettingLang(@PathParam("name") String name, @PathParam("lang") String lang) { settingsSvc.delete(name, lang); return ok("Setting " + name + " - " + lang + " deleted."); } @@ -1026,29 +1030,49 @@ public Response deleteRole(@Context ContainerRequestContext crc, @PathParam("id" }, getRequestUser(crc)); } - @Path("superuser/{identifier}") - @POST - public Response toggleSuperuser(@PathParam("identifier") String identifier) { - ActionLogRecord alr = new ActionLogRecord(ActionLogRecord.ActionType.Admin, "toggleSuperuser") - .setInfo(identifier); - try { - AuthenticatedUser user = authSvc.getAuthenticatedUser(identifier); - if (user.isDeactivated()) { - return error(Status.BAD_REQUEST, "You cannot make a deactivated user a superuser."); - } + @Path("superuser/{identifier}") + @Deprecated + @POST + public Response toggleSuperuser(@PathParam("identifier") String identifier) { + ActionLogRecord alr = new ActionLogRecord(ActionLogRecord.ActionType.Admin, "toggleSuperuser") + .setInfo(identifier); + try { + final AuthenticatedUser user = authSvc.getAuthenticatedUser(identifier); + return setSuperuserStatus(user, !user.isSuperuser()); + } catch (Exception e) { + alr.setActionResult(ActionLogRecord.Result.InternalError); + alr.setInfo(alr.getInfo() + "// " + e.getMessage()); + return error(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage()); + } finally { + actionLogSvc.log(alr); + } + } - user.setSuperuser(!user.isSuperuser()); + private Response setSuperuserStatus(AuthenticatedUser user, Boolean isSuperuser) { + if (user.isDeactivated()) { + return error(Status.BAD_REQUEST, "You cannot make a deactivated user a superuser."); + } + user.setSuperuser(isSuperuser); + return ok("User " + user.getIdentifier() + " " + (user.isSuperuser() ? "set" : "removed") + + " as a superuser."); + } - return ok("User " + user.getIdentifier() + " " + (user.isSuperuser() ? "set" : "removed") - + " as a superuser."); - } catch (Exception e) { - alr.setActionResult(ActionLogRecord.Result.InternalError); - alr.setInfo(alr.getInfo() + "// " + e.getMessage()); - return error(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage()); - } finally { - actionLogSvc.log(alr); - } - } + @Path("superuser/{identifier}") + @PUT + // Using string instead of boolean so user doesn't need to add a Content-type header in their request + public Response setSuperuserStatus(@PathParam("identifier") String identifier, String isSuperuser) { + ActionLogRecord alr = new ActionLogRecord(ActionLogRecord.ActionType.Admin, "setSuperuserStatus") + .setInfo(identifier + ":" + isSuperuser); + try { + return setSuperuserStatus(authSvc.getAuthenticatedUser(identifier), StringUtil.isTrue(isSuperuser)); + } catch (Exception e) { + alr.setActionResult(ActionLogRecord.Result.InternalError); + alr.setInfo(alr.getInfo() + "// " + e.getMessage()); + return error(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage()); + } finally { + actionLogSvc.log(alr); + } + } @GET @Path("validate/datasets") @@ -1329,26 +1353,24 @@ public Response convertUserFromBcryptToSha1(String json) { } - @Path("permissions/{dvo}") - @AuthRequired - @GET - public Response findPermissonsOn(@Context ContainerRequestContext crc, @PathParam("dvo") String dvo) { - try { - DvObject dvObj = findDvo(dvo); - if (dvObj == null) { - return notFound("DvObject " + dvo + " not found"); - } - User aUser = getRequestUser(crc); - JsonObjectBuilder bld = Json.createObjectBuilder(); - bld.add("user", aUser.getIdentifier()); - bld.add("permissions", json(permissionSvc.permissionsFor(createDataverseRequest(aUser), dvObj))); - return ok(bld); - - } catch (Exception e) { - logger.log(Level.SEVERE, "Error while testing permissions", e); - return error(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage()); - } - } + @Path("permissions/{dvo}") + @AuthRequired + @GET + public Response findPermissonsOn(@Context final ContainerRequestContext crc, @PathParam("dvo") final String dvo) { + try { + final DvObject dvObj = findDvo(dvo); + final User aUser = getRequestUser(crc); + final JsonObjectBuilder bld = Json.createObjectBuilder(); + bld.add("user", aUser.getIdentifier()); + bld.add("permissions", json(permissionSvc.permissionsFor(createDataverseRequest(aUser), dvObj))); + return ok(bld); + } catch (WrappedResponse r) { + return r.getResponse(); + } catch (Exception e) { + logger.log(Level.SEVERE, "Error while testing permissions", e); + return error(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage()); + } + } @Path("assignee/{idtf}") @GET @@ -1474,10 +1496,7 @@ public Response isOrcidEnabled() { public Response reregisterHdlToPID(@Context ContainerRequestContext crc, @PathParam("id") String id) { logger.info("Starting to reregister " + id + " Dataset Id. (from hdl to doi)" + new Date()); try { - if (settingsSvc.get(SettingsServiceBean.Key.Protocol.toString()).equals(HandlenetServiceBean.HDL_PROTOCOL)) { - logger.info("Bad Request protocol set to handle " ); - return error(Status.BAD_REQUEST, BundleUtil.getStringFromBundle("admin.api.migrateHDL.failure.must.be.set.for.doi")); - } + User u = getRequestUser(crc); if (!u.isSuperuser()) { @@ -1487,7 +1506,12 @@ public Response reregisterHdlToPID(@Context ContainerRequestContext crc, @PathPa DataverseRequest r = createDataverseRequest(u); Dataset ds = findDatasetOrDie(id); - if (ds.getIdentifier() != null && !ds.getIdentifier().isEmpty() && ds.getProtocol().equals(HandlenetServiceBean.HDL_PROTOCOL)) { + + if (HandlePidProvider.HDL_PROTOCOL.equals(dvObjectService.getEffectivePidGenerator(ds).getProtocol())) { + logger.info("Bad Request protocol set to handle " ); + return error(Status.BAD_REQUEST, BundleUtil.getStringFromBundle("admin.api.migrateHDL.failure.must.be.set.for.doi")); + } + if (ds.getIdentifier() != null && !ds.getIdentifier().isEmpty() && ds.getProtocol().equals(HandlePidProvider.HDL_PROTOCOL)) { execCommand(new RegisterDvObjectCommand(r, ds, true)); } else { return error(Status.BAD_REQUEST, BundleUtil.getStringFromBundle("admin.api.migrateHDL.failure.must.be.hdl.dataset")); @@ -2313,6 +2337,7 @@ public Response addBannerMessage(JsonObject jsonObject) throws WrappedResponse { BannerMessage toAdd = new BannerMessage(); try { + String dismissible = jsonObject.getString("dismissibleByUser"); boolean dismissibleByUser = false; @@ -2333,12 +2358,17 @@ public Response addBannerMessage(JsonObject jsonObject) throws WrappedResponse { messageText.setBannerMessage(toAdd); toAdd.getBannerMessageTexts().add(messageText); } - bannerMessageService.save(toAdd); - return ok("Banner Message added successfully."); + bannerMessageService.save(toAdd); + + JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder() + .add("message", "Banner Message added successfully.") + .add("id", toAdd.getId()); + + return ok(jsonObjectBuilder); } catch (Exception e) { logger.warning("Unexpected Exception: " + e.getMessage()); - return error(Status.BAD_REQUEST, "Add Banner Message unexpected exception: " + e.getMessage()); + return error(Status.BAD_REQUEST, "Add Banner Message unexpected exception: invalid or missing JSON object."); } } @@ -2374,10 +2404,19 @@ public Response deactivateBannerMessage(@PathParam("id") Long id) throws Wrapped @Path("/bannerMessage") public Response getBannerMessages(@PathParam("id") Long id) throws WrappedResponse { - return ok(bannerMessageService.findAllBannerMessages().stream() - .map(m -> jsonObjectBuilder().add("id", m.getId()).add("displayValue", m.getDisplayValue())) - .collect(toJsonArray())); + List messagesList = bannerMessageService.findAllBannerMessages(); + for (BannerMessage message : messagesList) { + if ("".equals(message.getDisplayValue())) { + return error(Response.Status.INTERNAL_SERVER_ERROR, "No banner messages found for this locale."); + } + } + + JsonArrayBuilder messages = messagesList.stream() + .map(m -> jsonObjectBuilder().add("id", m.getId()).add("displayValue", m.getDisplayValue())) + .collect(toJsonArray()); + + return ok(messages); } @POST diff --git a/src/main/java/edu/harvard/iq/dataverse/api/BatchServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/BatchServiceBean.java index daddc447117..fda698d6f5c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/BatchServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/BatchServiceBean.java @@ -46,8 +46,8 @@ public void processFilePath(String fileDir, String parentIdtf, DataverseRequest SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd_HH:mm:ss"); - validationLog = new PrintWriter(new FileWriter( "../logs/validationLog"+ formatter.format(timestamp)+".txt")); - cleanupLog = new PrintWriter(new FileWriter( "../logs/cleanupLog"+ formatter.format(timestamp)+".txt")); + validationLog = new PrintWriter(new FileWriter( System.getProperty("com.sun.aas.instanceRoot") + File.separator + "logs" + File.separator + "validationLog"+ formatter.format(timestamp)+".txt")); + cleanupLog = new PrintWriter(new FileWriter( System.getProperty("com.sun.aas.instanceRoot") + File.separator + "logs" + File.separator + "cleanupLog"+ formatter.format(timestamp)+".txt")); File dir = new File(fileDir); if (dir.isDirectory()) { for (File file : dir.listFiles()) { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/BuiltinUsers.java b/src/main/java/edu/harvard/iq/dataverse/api/BuiltinUsers.java index 50862bc0d35..ba99cf33c5b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/BuiltinUsers.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/BuiltinUsers.java @@ -119,7 +119,7 @@ public Response create(BuiltinUser user, @PathParam("password") String password, */ @POST @Path("{password}/{key}/{sendEmailNotification}") - public Response create(BuiltinUser user, @PathParam("password") String password, @PathParam("key") String key, @PathParam("sendEmailNotification") Boolean sendEmailNotification) { + public Response createWithNotification(BuiltinUser user, @PathParam("password") String password, @PathParam("key") String key, @PathParam("sendEmailNotification") Boolean sendEmailNotification) { return internalSave(user, password, key, sendEmailNotification); } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java b/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java index 00b7dfa6e36..01c51dc2b4c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java @@ -24,7 +24,6 @@ import jakarta.ejb.EJBException; import jakarta.json.Json; import jakarta.json.JsonArrayBuilder; -import jakarta.validation.ConstraintViolation; import jakarta.validation.ConstraintViolationException; import jakarta.ws.rs.Consumes; import jakarta.ws.rs.GET; @@ -488,9 +487,7 @@ private String parseControlledVocabulary(String[] values) { @Consumes("application/zip") @Path("loadpropertyfiles") public Response loadLanguagePropertyFile(File inputFile) { - try - { - ZipFile file = new ZipFile(inputFile); + try (ZipFile file = new ZipFile(inputFile)) { //Get file entries Enumeration entries = file.entries(); @@ -502,20 +499,26 @@ public Response loadLanguagePropertyFile(File inputFile) { { ZipEntry entry = entries.nextElement(); String dataverseLangFileName = dataverseLangDirectory + "/" + entry.getName(); - FileOutputStream fileOutput = new FileOutputStream(dataverseLangFileName); + File entryFile = new File(dataverseLangFileName); + String canonicalPath = entryFile.getCanonicalPath(); + if (canonicalPath.startsWith(dataverseLangDirectory + "/")) { + try (FileOutputStream fileOutput = new FileOutputStream(dataverseLangFileName)) { - InputStream is = file.getInputStream(entry); - BufferedInputStream bis = new BufferedInputStream(is); + InputStream is = file.getInputStream(entry); + BufferedInputStream bis = new BufferedInputStream(is); - while (bis.available() > 0) { - fileOutput.write(bis.read()); + while (bis.available() > 0) { + fileOutput.write(bis.read()); + } + } + } else { + logger.log(Level.SEVERE, "Zip Slip prevented: uploaded zip file tried to write to {}", canonicalPath); + return Response.status(400).entity("The zip file includes an illegal file path").build(); } - fileOutput.close(); } } - catch(IOException e) - { - e.printStackTrace(); + catch(IOException e) { + logger.log(Level.SEVERE, "Reading the language property zip file failed", e); return Response.status(500).entity("Internal server error. More details available at the server logs.").build(); } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 1a5eba52a62..78e551780dc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -1,6 +1,7 @@ package edu.harvard.iq.dataverse.api; import com.amazonaws.services.s3.model.PartETag; + import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.DatasetLock.Reason; import edu.harvard.iq.dataverse.actionlogging.ActionLogRecord; @@ -39,11 +40,14 @@ import edu.harvard.iq.dataverse.makedatacount.*; import edu.harvard.iq.dataverse.makedatacount.MakeDataCountLoggingServiceBean.MakeDataCountEntry; import edu.harvard.iq.dataverse.metrics.MetricsUtil; +import edu.harvard.iq.dataverse.pidproviders.PidProvider; +import edu.harvard.iq.dataverse.pidproviders.PidUtil; import edu.harvard.iq.dataverse.privateurl.PrivateUrl; import edu.harvard.iq.dataverse.privateurl.PrivateUrlServiceBean; import edu.harvard.iq.dataverse.search.IndexServiceBean; import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; +import edu.harvard.iq.dataverse.storageuse.UploadSessionQuotaLimit; import edu.harvard.iq.dataverse.util.*; import edu.harvard.iq.dataverse.util.bagit.OREMap; import edu.harvard.iq.dataverse.util.json.*; @@ -63,6 +67,12 @@ import jakarta.ws.rs.core.*; import jakarta.ws.rs.core.Response.Status; import org.apache.commons.lang3.StringUtils; +import org.eclipse.microprofile.openapi.annotations.Operation; +import org.eclipse.microprofile.openapi.annotations.media.Content; +import org.eclipse.microprofile.openapi.annotations.media.Schema; +import org.eclipse.microprofile.openapi.annotations.parameters.RequestBody; +import org.eclipse.microprofile.openapi.annotations.responses.APIResponse; +import org.eclipse.microprofile.openapi.annotations.tags.Tag; import org.glassfish.jersey.media.multipart.FormDataBodyPart; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; import org.glassfish.jersey.media.multipart.FormDataParam; @@ -77,6 +87,7 @@ import java.time.LocalDateTime; import java.time.ZoneId; import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; import java.util.*; import java.util.Map.Entry; import java.util.concurrent.ExecutionException; @@ -151,6 +162,9 @@ public class Datasets extends AbstractApiBean { @EJB EmbargoServiceBean embargoService; + @EJB + RetentionServiceBean retentionService; + @Inject MakeDataCountLoggingServiceBean mdcLogService; @@ -186,11 +200,11 @@ public interface DsVersionHandler { @GET @AuthRequired @Path("{id}") - public Response getDataset(@Context ContainerRequestContext crc, @PathParam("id") String id, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) { + public Response getDataset(@Context ContainerRequestContext crc, @PathParam("id") String id, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response, @QueryParam("returnOwners") boolean returnOwners) { return response( req -> { - final Dataset retrieved = execCommand(new GetDatasetCommand(req, findDatasetOrDie(id))); + final Dataset retrieved = execCommand(new GetDatasetCommand(req, findDatasetOrDie(id, true))); final DatasetVersion latest = execCommand(new GetLatestAccessibleDatasetVersionCommand(req, retrieved)); - final JsonObjectBuilder jsonbuilder = json(retrieved); + final JsonObjectBuilder jsonbuilder = json(retrieved, returnOwners); //Report MDC if this is a released version (could be draft if user has access, or user may not have access at all and is not getting metadata beyond the minimum) if((latest != null) && latest.isReleased()) { MakeDataCountLoggingServiceBean.MakeDataCountEntry entry = new MakeDataCountEntry(uriInfo, headers, dvRequestService, retrieved); @@ -421,25 +435,37 @@ public Response getVersion(@Context ContainerRequestContext crc, @PathParam("versionId") String versionId, @QueryParam("excludeFiles") Boolean excludeFiles, @QueryParam("includeDeaccessioned") boolean includeDeaccessioned, + @QueryParam("returnOwners") boolean returnOwners, @Context UriInfo uriInfo, @Context HttpHeaders headers) { return response( req -> { - //If excludeFiles is null the default is to provide the files and because of this we need to check permissions. boolean checkPerms = excludeFiles == null ? true : !excludeFiles; - - Dataset dst = findDatasetOrDie(datasetId); - DatasetVersion dsv = getDatasetVersionOrDie(req, versionId, dst, uriInfo, headers, includeDeaccessioned, checkPerms); - - if (dsv == null || dsv.getId() == null) { + + Dataset dataset = findDatasetOrDie(datasetId); + DatasetVersion requestedDatasetVersion = getDatasetVersionOrDie(req, + versionId, + dataset, + uriInfo, + headers, + includeDeaccessioned, + checkPerms); + + if (requestedDatasetVersion == null || requestedDatasetVersion.getId() == null) { return notFound("Dataset version not found"); } if (excludeFiles == null ? true : !excludeFiles) { - dsv = datasetversionService.findDeep(dsv.getId()); + requestedDatasetVersion = datasetversionService.findDeep(requestedDatasetVersion.getId()); } - return ok(json(dsv, excludeFiles == null ? true : !excludeFiles)); + + JsonObjectBuilder jsonBuilder = json(requestedDatasetVersion, + null, + excludeFiles == null ? true : !excludeFiles, + returnOwners); + return ok(jsonBuilder); + }, getRequestUser(crc)); } @@ -461,7 +487,7 @@ public Response getVersionFiles(@Context ContainerRequestContext crc, @Context UriInfo uriInfo, @Context HttpHeaders headers) { return response(req -> { - DatasetVersion datasetVersion = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers, includeDeaccessioned); + DatasetVersion datasetVersion = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId, false), uriInfo, headers, includeDeaccessioned); DatasetVersionFilesServiceBean.FileOrderCriteria fileOrderCriteria; try { fileOrderCriteria = orderCriteria != null ? DatasetVersionFilesServiceBean.FileOrderCriteria.valueOf(orderCriteria) : DatasetVersionFilesServiceBean.FileOrderCriteria.NameAZ; @@ -777,8 +803,8 @@ public Response getVersionJsonLDMetadata(@Context ContainerRequestContext crc, @ @AuthRequired @Path("{id}/metadata") @Produces("application/ld+json, application/json-ld") - public Response getVersionJsonLDMetadata(@Context ContainerRequestContext crc, @PathParam("id") String id, @Context UriInfo uriInfo, @Context HttpHeaders headers) { - return getVersionJsonLDMetadata(crc, id, DS_VERSION_DRAFT, uriInfo, headers); + public Response getJsonLDMetadata(@Context ContainerRequestContext crc, @PathParam("id") String id, @Context UriInfo uriInfo, @Context HttpHeaders headers) { + return getVersionJsonLDMetadata(crc, id, DS_VERSION_LATEST, uriInfo, headers); } @PUT @@ -1675,6 +1701,306 @@ public Response removeFileEmbargo(@Context ContainerRequestContext crc, @PathPar } } + @POST + @AuthRequired + @Path("{id}/files/actions/:set-retention") + public Response createFileRetention(@Context ContainerRequestContext crc, @PathParam("id") String id, String jsonBody){ + + // user is authenticated + AuthenticatedUser authenticatedUser = null; + try { + authenticatedUser = getRequestAuthenticatedUserOrDie(crc); + } catch (WrappedResponse ex) { + return error(Status.UNAUTHORIZED, "Authentication is required."); + } + + Dataset dataset; + try { + dataset = findDatasetOrDie(id); + } catch (WrappedResponse ex) { + return ex.getResponse(); + } + + boolean hasValidTerms = TermsOfUseAndAccessValidator.isTOUAValid(dataset.getLatestVersion().getTermsOfUseAndAccess(), null); + + if (!hasValidTerms){ + return error(Status.CONFLICT, BundleUtil.getStringFromBundle("dataset.message.toua.invalid")); + } + + // client is superadmin or (client has EditDataset permission on these files and files are unreleased) + // check if files are unreleased(DRAFT?) + if ((!authenticatedUser.isSuperuser() && (dataset.getLatestVersion().getVersionState() != DatasetVersion.VersionState.DRAFT) ) || !permissionService.userOn(authenticatedUser, dataset).has(Permission.EditDataset)) { + return error(Status.FORBIDDEN, "Either the files are released and user is not a superuser or user does not have EditDataset permissions"); + } + + // check if retentions are allowed(:MinRetentionDurationInMonths), gets the :MinRetentionDurationInMonths setting variable, if 0 or not set(null) return 400 + long minRetentionDurationInMonths = 0; + try { + minRetentionDurationInMonths = Long.parseLong(settingsService.get(SettingsServiceBean.Key.MinRetentionDurationInMonths.toString())); + } catch (NumberFormatException nfe){ + if (nfe.getMessage().contains("null")) { + return error(Status.BAD_REQUEST, "No Retention periods allowed"); + } + } + if (minRetentionDurationInMonths == 0){ + return error(Status.BAD_REQUEST, "No Retention periods allowed"); + } + + JsonObject json; + try { + json = JsonUtil.getJsonObject(jsonBody); + } catch (JsonException ex) { + return error(Status.BAD_REQUEST, "Invalid JSON; error message: " + ex.getMessage()); + } + + Retention retention = new Retention(); + + + LocalDate currentDateTime = LocalDate.now(); + + // Extract the dateUnavailable - check if specified and valid + String dateUnavailableStr = ""; + LocalDate dateUnavailable; + try { + dateUnavailableStr = json.getString("dateUnavailable"); + dateUnavailable = LocalDate.parse(dateUnavailableStr); + } catch (NullPointerException npex) { + return error(Status.BAD_REQUEST, "Invalid retention period; no dateUnavailable specified"); + } catch (ClassCastException ccex) { + return error(Status.BAD_REQUEST, "Invalid retention period; dateUnavailable must be a string"); + } catch (DateTimeParseException dtpex) { + return error(Status.BAD_REQUEST, "Invalid date format for dateUnavailable: " + dateUnavailableStr); + } + + // check :MinRetentionDurationInMonths if -1 + LocalDate minRetentionDateTime = minRetentionDurationInMonths != -1 ? LocalDate.now().plusMonths(minRetentionDurationInMonths) : null; + // dateUnavailable is not in the past + if (dateUnavailable.isAfter(currentDateTime)){ + retention.setDateUnavailable(dateUnavailable); + } else { + return error(Status.BAD_REQUEST, "Date unavailable can not be in the past"); + } + + // dateAvailable is within limits + if (minRetentionDateTime != null){ + if (dateUnavailable.isBefore(minRetentionDateTime)){ + return error(Status.BAD_REQUEST, "Date unavailable can not be earlier than MinRetentionDurationInMonths: "+minRetentionDurationInMonths + " from now"); + } + } + + try { + String reason = json.getString("reason"); + retention.setReason(reason); + } catch (NullPointerException npex) { + // ignoring; no reason specified is OK, it is optional + } catch (ClassCastException ccex) { + return error(Status.BAD_REQUEST, "Invalid retention period; reason must be a string"); + } + + + List datasetFiles = dataset.getFiles(); + List filesToRetention = new LinkedList<>(); + + // extract fileIds from json, find datafiles and add to list + if (json.containsKey("fileIds")){ + try { + JsonArray fileIds = json.getJsonArray("fileIds"); + for (JsonValue jsv : fileIds) { + try { + DataFile dataFile = findDataFileOrDie(jsv.toString()); + filesToRetention.add(dataFile); + } catch (WrappedResponse ex) { + return ex.getResponse(); + } + } + } catch (ClassCastException ccex) { + return error(Status.BAD_REQUEST, "Invalid retention period; fileIds must be an array of id strings"); + } catch (NullPointerException npex) { + return error(Status.BAD_REQUEST, "Invalid retention period; no fileIds specified"); + } + } else { + return error(Status.BAD_REQUEST, "No fileIds specified"); + } + + List orphanedRetentions = new ArrayList(); + // check if files belong to dataset + if (datasetFiles.containsAll(filesToRetention)) { + JsonArrayBuilder restrictedFiles = Json.createArrayBuilder(); + boolean badFiles = false; + for (DataFile datafile : filesToRetention) { + // superuser can overrule an existing retention, even on released files + if (datafile.isReleased() && !authenticatedUser.isSuperuser()) { + restrictedFiles.add(datafile.getId()); + badFiles = true; + } + } + if (badFiles) { + return Response.status(Status.FORBIDDEN) + .entity(NullSafeJsonBuilder.jsonObjectBuilder().add("status", ApiConstants.STATUS_ERROR) + .add("message", "You do not have permission to set a retention period for the following files") + .add("files", restrictedFiles).build()) + .type(MediaType.APPLICATION_JSON_TYPE).build(); + } + retention=retentionService.merge(retention); + // Good request, so add the retention. Track any existing retentions so we can + // delete them if there are no files left that reference them. + for (DataFile datafile : filesToRetention) { + Retention ret = datafile.getRetention(); + if (ret != null) { + ret.getDataFiles().remove(datafile); + if (ret.getDataFiles().isEmpty()) { + orphanedRetentions.add(ret); + } + } + // Save merges the datafile with an retention into the context + datafile.setRetention(retention); + fileService.save(datafile); + } + //Call service to get action logged + long retentionId = retentionService.save(retention, authenticatedUser.getIdentifier()); + if (orphanedRetentions.size() > 0) { + for (Retention ret : orphanedRetentions) { + retentionService.delete(ret, authenticatedUser.getIdentifier()); + } + } + //If superuser, report changes to any released files + if (authenticatedUser.isSuperuser()) { + String releasedFiles = filesToRetention.stream().filter(d -> d.isReleased()) + .map(d -> d.getId().toString()).collect(Collectors.joining(",")); + if (!releasedFiles.isBlank()) { + actionLogSvc + .log(new ActionLogRecord(ActionLogRecord.ActionType.Admin, "retentionAddedTo") + .setInfo("Retention id: " + retention.getId() + " added for released file(s), id(s) " + + releasedFiles + ".") + .setUserIdentifier(authenticatedUser.getIdentifier())); + } + } + return ok(Json.createObjectBuilder().add("message", "File(s) retention period has been set or updated")); + } else { + return error(BAD_REQUEST, "Not all files belong to dataset"); + } + } + + @POST + @AuthRequired + @Path("{id}/files/actions/:unset-retention") + public Response removeFileRetention(@Context ContainerRequestContext crc, @PathParam("id") String id, String jsonBody){ + + // user is authenticated + AuthenticatedUser authenticatedUser = null; + try { + authenticatedUser = getRequestAuthenticatedUserOrDie(crc); + } catch (WrappedResponse ex) { + return error(Status.UNAUTHORIZED, "Authentication is required."); + } + + Dataset dataset; + try { + dataset = findDatasetOrDie(id); + } catch (WrappedResponse ex) { + return ex.getResponse(); + } + + // client is superadmin or (client has EditDataset permission on these files and files are unreleased) + // check if files are unreleased(DRAFT?) + //ToDo - here and below - check the release status of files and not the dataset state (draft dataset version still can have released files) + if ((!authenticatedUser.isSuperuser() && (dataset.getLatestVersion().getVersionState() != DatasetVersion.VersionState.DRAFT) ) || !permissionService.userOn(authenticatedUser, dataset).has(Permission.EditDataset)) { + return error(Status.FORBIDDEN, "Either the files are released and user is not a superuser or user does not have EditDataset permissions"); + } + + // check if retentions are allowed(:MinRetentionDurationInMonths), gets the :MinRetentionDurationInMonths setting variable, if 0 or not set(null) return 400 + int minRetentionDurationInMonths = 0; + try { + minRetentionDurationInMonths = Integer.parseInt(settingsService.get(SettingsServiceBean.Key.MinRetentionDurationInMonths.toString())); + } catch (NumberFormatException nfe){ + if (nfe.getMessage().contains("null")) { + return error(Status.BAD_REQUEST, "No Retention periods allowed"); + } + } + if (minRetentionDurationInMonths == 0){ + return error(Status.BAD_REQUEST, "No Retention periods allowed"); + } + + JsonObject json; + try { + json = JsonUtil.getJsonObject(jsonBody); + } catch (JsonException ex) { + return error(Status.BAD_REQUEST, "Invalid JSON; error message: " + ex.getMessage()); + } + + List datasetFiles = dataset.getFiles(); + List retentionFilesToUnset = new LinkedList<>(); + + // extract fileIds from json, find datafiles and add to list + if (json.containsKey("fileIds")){ + try { + JsonArray fileIds = json.getJsonArray("fileIds"); + for (JsonValue jsv : fileIds) { + try { + DataFile dataFile = findDataFileOrDie(jsv.toString()); + retentionFilesToUnset.add(dataFile); + } catch (WrappedResponse ex) { + return ex.getResponse(); + } + } + } catch (ClassCastException ccex) { + return error(Status.BAD_REQUEST, "fileIds must be an array of id strings"); + } catch (NullPointerException npex) { + return error(Status.BAD_REQUEST, "No fileIds specified"); + } + } else { + return error(Status.BAD_REQUEST, "No fileIds specified"); + } + + List orphanedRetentions = new ArrayList(); + // check if files belong to dataset + if (datasetFiles.containsAll(retentionFilesToUnset)) { + JsonArrayBuilder restrictedFiles = Json.createArrayBuilder(); + boolean badFiles = false; + for (DataFile datafile : retentionFilesToUnset) { + // superuser can overrule an existing retention, even on released files + if (datafile.getRetention()==null || ((datafile.isReleased() && datafile.getRetention() != null) && !authenticatedUser.isSuperuser())) { + restrictedFiles.add(datafile.getId()); + badFiles = true; + } + } + if (badFiles) { + return Response.status(Status.FORBIDDEN) + .entity(NullSafeJsonBuilder.jsonObjectBuilder().add("status", ApiConstants.STATUS_ERROR) + .add("message", "The following files do not have retention periods or you do not have permission to remove their retention periods") + .add("files", restrictedFiles).build()) + .type(MediaType.APPLICATION_JSON_TYPE).build(); + } + // Good request, so remove the retention from the files. Track any existing retentions so we can + // delete them if there are no files left that reference them. + for (DataFile datafile : retentionFilesToUnset) { + Retention ret = datafile.getRetention(); + if (ret != null) { + ret.getDataFiles().remove(datafile); + if (ret.getDataFiles().isEmpty()) { + orphanedRetentions.add(ret); + } + } + // Save merges the datafile with an retention into the context + datafile.setRetention(null); + fileService.save(datafile); + } + if (orphanedRetentions.size() > 0) { + for (Retention ret : orphanedRetentions) { + retentionService.delete(ret, authenticatedUser.getIdentifier()); + } + } + String releasedFiles = retentionFilesToUnset.stream().filter(d -> d.isReleased()).map(d->d.getId().toString()).collect(Collectors.joining(",")); + if(!releasedFiles.isBlank()) { + ActionLogRecord removeRecord = new ActionLogRecord(ActionLogRecord.ActionType.Admin, "retentionRemovedFrom").setInfo("Retention removed from released file(s), id(s) " + releasedFiles + "."); + removeRecord.setUserIdentifier(authenticatedUser.getIdentifier()); + actionLogSvc.log(removeRecord); + } + return ok(Json.createObjectBuilder().add("message", "Retention periods were removed from file(s)")); + } else { + return error(BAD_REQUEST, "Not all files belong to dataset"); + } + } @PUT @AuthRequired @@ -1942,6 +2268,14 @@ public Response setDataFileAsThumbnail(@Context ContainerRequestContext crc, @Pa @AuthRequired @Path("{id}/thumbnail") @Consumes(MediaType.MULTIPART_FORM_DATA) + @Produces("application/json") + @Operation(summary = "Uploads a logo for a dataset", + description = "Uploads a logo for a dataset") + @APIResponse(responseCode = "200", + description = "Dataset logo uploaded successfully") + @Tag(name = "uploadDatasetLogo", + description = "Uploads a logo for a dataset") + @RequestBody(content = @Content(mediaType = MediaType.MULTIPART_FORM_DATA)) public Response uploadDatasetLogo(@Context ContainerRequestContext crc, @PathParam("id") String idSupplied, @FormDataParam("file") InputStream inputStream) { try { DatasetThumbnail datasetThumbnail = execCommand(new UpdateDatasetThumbnailCommand(createDataverseRequest(getRequestUser(crc)), findDatasetOrDie(idSupplied), UpdateDatasetThumbnailCommand.UserIntent.setNonDatasetFileAsThumbnail, null, inputStream)); @@ -1963,6 +2297,7 @@ public Response removeDatasetLogo(@Context ContainerRequestContext crc, @PathPar } } + @Deprecated(forRemoval = true, since = "2024-07-07") @GET @AuthRequired @Path("{identifier}/dataCaptureModule/rsync") @@ -2143,9 +2478,8 @@ public Response returnToAuthor(@Context ContainerRequestContext crc, @PathParam( Dataset dataset = findDatasetOrDie(idSupplied); String reasonForReturn = null; reasonForReturn = json.getString("reasonForReturn"); - // TODO: Once we add a box for the curator to type into, pass the reason for return to the ReturnDatasetToAuthorCommand and delete this check and call to setReturnReason on the API side. if (reasonForReturn == null || reasonForReturn.isEmpty()) { - return error(Response.Status.BAD_REQUEST, "You must enter a reason for returning a dataset to the author(s)."); + return error(Response.Status.BAD_REQUEST, BundleUtil.getStringFromBundle("dataset.reject.datasetNotInReview")); } AuthenticatedUser authenticatedUser = getRequestAuthenticatedUserOrDie(crc); Dataset updatedDataset = execCommand(new ReturnDatasetToAuthorCommand(createDataverseRequest(authenticatedUser), dataset, reasonForReturn )); @@ -2219,42 +2553,6 @@ public Response deleteCurationStatus(@Context ContainerRequestContext crc, @Path } } - @GET - @AuthRequired - @Path("{id}/uploadsid") - @Deprecated - public Response getUploadUrl(@Context ContainerRequestContext crc, @PathParam("id") String idSupplied) { - try { - Dataset dataset = findDatasetOrDie(idSupplied); - - boolean canUpdateDataset = false; - canUpdateDataset = permissionSvc.requestOn(createDataverseRequest(getRequestUser(crc)), dataset).canIssue(UpdateDatasetVersionCommand.class); - if (!canUpdateDataset) { - return error(Response.Status.FORBIDDEN, "You are not permitted to upload files to this dataset."); - } - S3AccessIO s3io = FileUtil.getS3AccessForDirectUpload(dataset); - if (s3io == null) { - return error(Response.Status.NOT_FOUND, "Direct upload not supported for files in this dataset: " + dataset.getId()); - } - String url = null; - String storageIdentifier = null; - try { - url = s3io.generateTemporaryS3UploadUrl(); - storageIdentifier = FileUtil.getStorageIdentifierFromLocation(s3io.getStorageLocation()); - } catch (IOException io) { - logger.warning(io.getMessage()); - throw new WrappedResponse(io, error(Response.Status.INTERNAL_SERVER_ERROR, "Could not create process direct upload request")); - } - - JsonObjectBuilder response = Json.createObjectBuilder() - .add("url", url) - .add("storageIdentifier", storageIdentifier); - return ok(response); - } catch (WrappedResponse wr) { - return wr.getResponse(); - } - } - @GET @AuthRequired @Path("{id}/uploadurls") @@ -2273,6 +2571,22 @@ public Response getMPUploadUrls(@Context ContainerRequestContext crc, @PathParam return error(Response.Status.NOT_FOUND, "Direct upload not supported for files in this dataset: " + dataset.getId()); } + Long maxSize = systemConfig.getMaxFileUploadSizeForStore(dataset.getEffectiveStorageDriverId()); + if (maxSize != null) { + if(fileSize > maxSize) { + return error(Response.Status.BAD_REQUEST, + "The file you are trying to upload is too large to be uploaded to this dataset. " + + "The maximum allowed file size is " + maxSize + " bytes."); + } + } + UploadSessionQuotaLimit limit = fileService.getUploadSessionQuotaLimit(dataset); + if (limit != null) { + if(fileSize > limit.getRemainingQuotaInBytes()) { + return error(Response.Status.BAD_REQUEST, + "The file you are trying to upload is too large to be uploaded to this dataset. " + + "The remaing file size quota is " + limit.getRemainingQuotaInBytes() + " bytes."); + } + } JsonObjectBuilder response = null; String storageIdentifier = null; try { @@ -2435,6 +2749,14 @@ public Response completeMPUpload(@Context ContainerRequestContext crc, String pa @AuthRequired @Path("{id}/add") @Consumes(MediaType.MULTIPART_FORM_DATA) + @Produces("application/json") + @Operation(summary = "Uploads a file for a dataset", + description = "Uploads a file for a dataset") + @APIResponse(responseCode = "200", + description = "File uploaded successfully to dataset") + @Tag(name = "addFileToDataset", + description = "Uploads a file for a dataset") + @RequestBody(content = @Content(mediaType = MediaType.MULTIPART_FORM_DATA)) public Response addFileToDataset(@Context ContainerRequestContext crc, @PathParam("id") String idSupplied, @FormDataParam("jsonData") String jsonData, @@ -2711,46 +3033,49 @@ public static T handleVersion(String versionId, DsVersionHandler hdl) * includeDeaccessioned default to false and checkPermsWhenDeaccessioned to false. Use it only when you are sure that the you don't need to work with * a deaccessioned dataset. */ - private DatasetVersion getDatasetVersionOrDie(final DataverseRequest req, String versionNumber, final Dataset ds, UriInfo uriInfo, HttpHeaders headers) throws WrappedResponse { + private DatasetVersion getDatasetVersionOrDie(final DataverseRequest req, + String versionNumber, + final Dataset ds, + UriInfo uriInfo, + HttpHeaders headers) throws WrappedResponse { //The checkPerms was added to check the permissions ONLY when the dataset is deaccessioned. - return getDatasetVersionOrDie(req, versionNumber, ds, uriInfo, headers, false, false); + boolean checkFilePerms = false; + boolean includeDeaccessioned = false; + return getDatasetVersionOrDie(req, versionNumber, ds, uriInfo, headers, includeDeaccessioned, checkFilePerms); } /* * checkPermsWhenDeaccessioned default to true. Be aware that the version will be only be obtainable if the user has edit permissions. */ - private DatasetVersion getDatasetVersionOrDie(final DataverseRequest req, String versionNumber, final Dataset ds, UriInfo uriInfo, HttpHeaders headers, boolean includeDeaccessioned) throws WrappedResponse{ - return getDatasetVersionOrDie(req, versionNumber, ds, uriInfo, headers, includeDeaccessioned, true); + private DatasetVersion getDatasetVersionOrDie(final DataverseRequest req, String versionNumber, final Dataset ds, + UriInfo uriInfo, HttpHeaders headers, boolean includeDeaccessioned) throws WrappedResponse { + boolean checkPermsWhenDeaccessioned = true; + boolean bypassAccessCheck = false; + return getDatasetVersionOrDie(req, versionNumber, ds, uriInfo, headers, includeDeaccessioned, checkPermsWhenDeaccessioned, bypassAccessCheck); } /* - * Will allow to define when the permissions should be checked when a deaccesioned dataset is requested. If the user doesn't have edit permissions will result in an error. + * checkPermsWhenDeaccessioned default to true. Be aware that the version will be only be obtainable if the user has edit permissions. */ - private DatasetVersion getDatasetVersionOrDie(final DataverseRequest req, String versionNumber, final Dataset ds, UriInfo uriInfo, HttpHeaders headers, boolean includeDeaccessioned, boolean checkPermsWhenDeaccessioned) throws WrappedResponse { - DatasetVersion dsv = execCommand(handleVersion(versionNumber, new DsVersionHandler>() { - - @Override - public Command handleLatest() { - return new GetLatestAccessibleDatasetVersionCommand(req, ds, includeDeaccessioned, checkPermsWhenDeaccessioned); - } + private DatasetVersion getDatasetVersionOrDie(final DataverseRequest req, String versionNumber, final Dataset ds, + UriInfo uriInfo, HttpHeaders headers, boolean includeDeaccessioned, boolean checkPermsWhenDeaccessioned) throws WrappedResponse { + boolean bypassAccessCheck = false; + return getDatasetVersionOrDie(req, versionNumber, ds, uriInfo, headers, includeDeaccessioned, checkPermsWhenDeaccessioned, bypassAccessCheck); + } - @Override - public Command handleDraft() { - return new GetDraftDatasetVersionCommand(req, ds); - } + /* + * Will allow to define when the permissions should be checked when a deaccesioned dataset is requested. If the user doesn't have edit permissions will result in an error. + */ + private DatasetVersion getDatasetVersionOrDie(final DataverseRequest req, String versionNumber, final Dataset ds, + UriInfo uriInfo, HttpHeaders headers, boolean includeDeaccessioned, boolean checkPermsWhenDeaccessioned, + boolean bypassAccessCheck) + throws WrappedResponse { - @Override - public Command handleSpecific(long major, long minor) { - return new GetSpecificPublishedDatasetVersionCommand(req, ds, major, minor, includeDeaccessioned, checkPermsWhenDeaccessioned); - } + DatasetVersion dsv = findDatasetVersionOrDie(req, versionNumber, ds, includeDeaccessioned, checkPermsWhenDeaccessioned); - @Override - public Command handleLatestPublished() { - return new GetLatestPublishedDatasetVersionCommand(req, ds, includeDeaccessioned, checkPermsWhenDeaccessioned); - } - })); if (dsv == null || dsv.getId() == null) { - throw new WrappedResponse(notFound("Dataset version " + versionNumber + " of dataset " + ds.getId() + " not found")); + throw new WrappedResponse( + notFound("Dataset version " + versionNumber + " of dataset " + ds.getId() + " not found")); } if (dsv.isReleased()&& uriInfo!=null) { MakeDataCountLoggingServiceBean.MakeDataCountEntry entry = new MakeDataCountEntry(uriInfo, headers, dvRequestService, ds); @@ -2758,7 +3083,7 @@ public Command handleLatestPublished() { } return dsv; } - + @GET @Path("{identifier}/locks") public Response getLocksForDataset(@PathParam("identifier") String id, @QueryParam("type") DatasetLock.Reason lockType) { @@ -3505,6 +3830,16 @@ public Response getGlobusUploadParams(@Context ContainerRequestContext crc, @Pat params.add(key, substitutedParams.get(key)); }); params.add("managed", Boolean.toString(managed)); + if (managed) { + Long maxSize = systemConfig.getMaxFileUploadSizeForStore(storeId); + if (maxSize != null) { + params.add("fileSizeLimit", maxSize); + } + UploadSessionQuotaLimit limit = fileService.getUploadSessionQuotaLimit(dataset); + if (limit != null) { + params.add("remainingQuota", limit.getRemainingQuotaInBytes()); + } + } if (transferEndpoint != null) { params.add("endpoint", transferEndpoint); } else { @@ -3647,6 +3982,14 @@ public Response requestGlobusUpload(@Context ContainerRequestContext crc, @PathP @AuthRequired @Path("{id}/addGlobusFiles") @Consumes(MediaType.MULTIPART_FORM_DATA) + @Produces("application/json") + @Operation(summary = "Uploads a Globus file for a dataset", + description = "Uploads a Globus file for a dataset") + @APIResponse(responseCode = "200", + description = "Globus file uploaded successfully to dataset") + @Tag(name = "addGlobusFilesToDataset", + description = "Uploads a Globus file for a dataset") + @RequestBody(content = @Content(mediaType = MediaType.MULTIPART_FORM_DATA)) public Response addGlobusFilesToDataset(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @FormDataParam("jsonData") String jsonData, @@ -3749,11 +4092,11 @@ public Response getGlobusDownloadParams(@Context ContainerRequestContext crc, @P // ------------------------------------- // (1) Get the user from the ContainerRequestContext // ------------------------------------- - AuthenticatedUser authUser; + AuthenticatedUser authUser = null; try { authUser = getRequestAuthenticatedUserOrDie(crc); } catch (WrappedResponse e) { - return e.getResponse(); + logger.fine("guest user globus download"); } // ------------------------------------- // (2) Get the Dataset Id @@ -4029,6 +4372,14 @@ public Response monitorGlobusDownload(@Context ContainerRequestContext crc, @Pat @AuthRequired @Path("{id}/addFiles") @Consumes(MediaType.MULTIPART_FORM_DATA) + @Produces("application/json") + @Operation(summary = "Uploads a set of files to a dataset", + description = "Uploads a set of files to a dataset") + @APIResponse(responseCode = "200", + description = "Files uploaded successfully to dataset") + @Tag(name = "addFilesToDataset", + description = "Uploads a set of files to a dataset") + @RequestBody(content = @Content(mediaType = MediaType.MULTIPART_FORM_DATA)) public Response addFilesToDataset(@Context ContainerRequestContext crc, @PathParam("id") String idSupplied, @FormDataParam("jsonData") String jsonData) { @@ -4096,6 +4447,14 @@ public Response addFilesToDataset(@Context ContainerRequestContext crc, @PathPar @AuthRequired @Path("{id}/replaceFiles") @Consumes(MediaType.MULTIPART_FORM_DATA) + @Produces("application/json") + @Operation(summary = "Replace a set of files to a dataset", + description = "Replace a set of files to a dataset") + @APIResponse(responseCode = "200", + description = "Files replaced successfully to dataset") + @Tag(name = "replaceFilesInDataset", + description = "Replace a set of files to a dataset") + @RequestBody(content = @Content(mediaType = MediaType.MULTIPART_FORM_DATA)) public Response replaceFilesInDataset(@Context ContainerRequestContext crc, @PathParam("id") String idSupplied, @FormDataParam("jsonData") String jsonData) { @@ -4407,7 +4766,7 @@ public Response getDatasetSummaryFieldNames() { @GET @Path("privateUrlDatasetVersion/{privateUrlToken}") - public Response getPrivateUrlDatasetVersion(@PathParam("privateUrlToken") String privateUrlToken) { + public Response getPrivateUrlDatasetVersion(@PathParam("privateUrlToken") String privateUrlToken, @QueryParam("returnOwners") boolean returnOwners) { PrivateUrlUser privateUrlUser = privateUrlService.getPrivateUrlUserFromToken(privateUrlToken); if (privateUrlUser == null) { return notFound("Private URL user not found"); @@ -4424,9 +4783,9 @@ public Response getPrivateUrlDatasetVersion(@PathParam("privateUrlToken") String JsonObjectBuilder responseJson; if (isAnonymizedAccess) { List anonymizedFieldTypeNamesList = new ArrayList<>(Arrays.asList(anonymizedFieldTypeNames.split(",\\s"))); - responseJson = json(dsv, anonymizedFieldTypeNamesList, true); + responseJson = json(dsv, anonymizedFieldTypeNamesList, true, returnOwners); } else { - responseJson = json(dsv, true); + responseJson = json(dsv, null, true, returnOwners); } return ok(responseJson); } @@ -4452,8 +4811,11 @@ public Response getDatasetVersionCitation(@Context ContainerRequestContext crc, @QueryParam("includeDeaccessioned") boolean includeDeaccessioned, @Context UriInfo uriInfo, @Context HttpHeaders headers) { + boolean checkFilePerms = false; return response(req -> ok( - getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers, includeDeaccessioned, false).getCitation(true, false)), getRequestUser(crc)); + getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers, + includeDeaccessioned, checkFilePerms).getCitation(true, false)), + getRequestUser(crc)); } @POST @@ -4477,7 +4839,10 @@ public Response deaccessionDataset(@Context ContainerRequestContext crc, @PathPa } } execCommand(new DeaccessionDatasetVersionCommand(req, datasetVersion, false)); - return ok("Dataset " + datasetId + " deaccessioned for version " + versionId); + + return ok("Dataset " + + (":persistentId".equals(datasetId) ? datasetVersion.getDataset().getGlobalId().asString() : datasetId) + + " deaccessioned for version " + versionId); } catch (JsonParsingException jpe) { return error(Response.Status.BAD_REQUEST, "Error parsing Json: " + jpe.getMessage()); } @@ -4603,4 +4968,97 @@ public Response getCanDownloadAtLeastOneFile(@Context ContainerRequestContext cr return ok(permissionService.canDownloadAtLeastOneFile(req, datasetVersion)); }, getRequestUser(crc)); } + + /** + * Get the PidProvider that will be used for generating new DOIs in this dataset + * + * @return - the id of the effective PID generator for the given dataset + * @throws WrappedResponse + */ + @GET + @AuthRequired + @Path("{identifier}/pidGenerator") + public Response getPidGenerator(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf, + @Context HttpHeaders headers) throws WrappedResponse { + + Dataset dataset; + + try { + dataset = findDatasetOrDie(dvIdtf); + } catch (WrappedResponse ex) { + return error(Response.Status.NOT_FOUND, "No such dataset"); + } + PidProvider pidProvider = dataset.getEffectivePidGenerator(); + if(pidProvider == null) { + //This is basically a config error, e.g. if a valid pid provider was removed after this dataset used it + return error(Response.Status.NOT_FOUND, BundleUtil.getStringFromBundle("datasets.api.pidgenerator.notfound")); + } + String pidGeneratorId = pidProvider.getId(); + return ok(pidGeneratorId); + } + + @PUT + @AuthRequired + @Path("{identifier}/pidGenerator") + public Response setPidGenerator(@Context ContainerRequestContext crc, @PathParam("identifier") String datasetId, + String generatorId, @Context HttpHeaders headers) throws WrappedResponse { + + // Superuser-only: + AuthenticatedUser user; + try { + user = getRequestAuthenticatedUserOrDie(crc); + } catch (WrappedResponse ex) { + return error(Response.Status.UNAUTHORIZED, "Authentication is required."); + } + if (!user.isSuperuser()) { + return error(Response.Status.FORBIDDEN, "Superusers only."); + } + + Dataset dataset; + + try { + dataset = findDatasetOrDie(datasetId); + } catch (WrappedResponse ex) { + return error(Response.Status.NOT_FOUND, "No such dataset"); + } + if (PidUtil.getManagedProviderIds().contains(generatorId)) { + dataset.setPidGeneratorId(generatorId); + datasetService.merge(dataset); + return ok("PID Generator set to: " + generatorId); + } else { + return error(Response.Status.NOT_FOUND, "No PID Generator found for the give id"); + } + + } + + @DELETE + @AuthRequired + @Path("{identifier}/pidGenerator") + public Response resetPidGenerator(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf, + @Context HttpHeaders headers) throws WrappedResponse { + + // Superuser-only: + AuthenticatedUser user; + try { + user = getRequestAuthenticatedUserOrDie(crc); + } catch (WrappedResponse ex) { + return error(Response.Status.BAD_REQUEST, "Authentication is required."); + } + if (!user.isSuperuser()) { + return error(Response.Status.FORBIDDEN, "Superusers only."); + } + + Dataset dataset; + + try { + dataset = findDatasetOrDie(dvIdtf); + } catch (WrappedResponse ex) { + return error(Response.Status.NOT_FOUND, "No such dataset"); + } + + dataset.setPidGenerator(null); + datasetService.merge(dataset); + return ok("Pid Generator reset to default: " + dataset.getEffectivePidGenerator().getId()); + } + } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index 6c1bf42c02a..02b60fdb32a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -1,25 +1,10 @@ package edu.harvard.iq.dataverse.api; -import edu.harvard.iq.dataverse.DataFile; -import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.DatasetFieldType; -import edu.harvard.iq.dataverse.DatasetVersion; -import edu.harvard.iq.dataverse.Dataverse; -import edu.harvard.iq.dataverse.DataverseFacet; -import edu.harvard.iq.dataverse.DataverseContact; -import edu.harvard.iq.dataverse.DataverseMetadataBlockFacet; -import edu.harvard.iq.dataverse.DataverseServiceBean; +import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.api.auth.AuthRequired; import edu.harvard.iq.dataverse.api.datadeposit.SwordServiceBean; import edu.harvard.iq.dataverse.api.dto.DataverseMetadataBlockFacetDTO; import edu.harvard.iq.dataverse.authorization.DataverseRole; -import edu.harvard.iq.dataverse.DvObject; -import edu.harvard.iq.dataverse.GlobalId; -import edu.harvard.iq.dataverse.GlobalIdServiceBean; -import edu.harvard.iq.dataverse.GuestbookResponseServiceBean; -import edu.harvard.iq.dataverse.GuestbookServiceBean; -import edu.harvard.iq.dataverse.MetadataBlock; -import edu.harvard.iq.dataverse.RoleAssignment; import edu.harvard.iq.dataverse.api.dto.ExplicitGroupDTO; import edu.harvard.iq.dataverse.api.dto.RoleAssignmentDTO; @@ -35,45 +20,9 @@ import edu.harvard.iq.dataverse.authorization.users.User; import edu.harvard.iq.dataverse.dataverse.DataverseUtil; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; -import edu.harvard.iq.dataverse.engine.command.impl.AddRoleAssigneesToExplicitGroupCommand; -import edu.harvard.iq.dataverse.engine.command.impl.AssignRoleCommand; -import edu.harvard.iq.dataverse.engine.command.impl.CreateDataverseCommand; -import edu.harvard.iq.dataverse.engine.command.impl.CreateExplicitGroupCommand; -import edu.harvard.iq.dataverse.engine.command.impl.CreateNewDatasetCommand; -import edu.harvard.iq.dataverse.engine.command.impl.CreateRoleCommand; -import edu.harvard.iq.dataverse.engine.command.impl.DeleteCollectionQuotaCommand; -import edu.harvard.iq.dataverse.engine.command.impl.DeleteDataverseCommand; -import edu.harvard.iq.dataverse.engine.command.impl.DeleteDataverseLinkingDataverseCommand; -import edu.harvard.iq.dataverse.engine.command.impl.DeleteExplicitGroupCommand; -import edu.harvard.iq.dataverse.engine.command.impl.GetDatasetSchemaCommand; -import edu.harvard.iq.dataverse.engine.command.impl.GetCollectionQuotaCommand; -import edu.harvard.iq.dataverse.engine.command.impl.GetCollectionStorageUseCommand; -import edu.harvard.iq.dataverse.engine.command.impl.UpdateMetadataBlockFacetRootCommand; -import edu.harvard.iq.dataverse.engine.command.impl.GetDataverseCommand; -import edu.harvard.iq.dataverse.engine.command.impl.GetDataverseStorageSizeCommand; -import edu.harvard.iq.dataverse.engine.command.impl.GetExplicitGroupCommand; -import edu.harvard.iq.dataverse.engine.command.impl.ImportDatasetCommand; -import edu.harvard.iq.dataverse.engine.command.impl.LinkDataverseCommand; -import edu.harvard.iq.dataverse.engine.command.impl.ListDataverseContentCommand; -import edu.harvard.iq.dataverse.engine.command.impl.ListExplicitGroupsCommand; -import edu.harvard.iq.dataverse.engine.command.impl.ListFacetsCommand; -import edu.harvard.iq.dataverse.engine.command.impl.ListMetadataBlockFacetsCommand; -import edu.harvard.iq.dataverse.engine.command.impl.ListMetadataBlocksCommand; -import edu.harvard.iq.dataverse.engine.command.impl.ListRoleAssignments; -import edu.harvard.iq.dataverse.engine.command.impl.ListRolesCommand; -import edu.harvard.iq.dataverse.engine.command.impl.PublishDatasetCommand; -import edu.harvard.iq.dataverse.engine.command.impl.PublishDatasetResult; -import edu.harvard.iq.dataverse.engine.command.impl.MoveDataverseCommand; -import edu.harvard.iq.dataverse.engine.command.impl.PublishDataverseCommand; -import edu.harvard.iq.dataverse.engine.command.impl.RemoveRoleAssigneesFromExplicitGroupCommand; -import edu.harvard.iq.dataverse.engine.command.impl.RevokeRoleCommand; -import edu.harvard.iq.dataverse.engine.command.impl.SetCollectionQuotaCommand; -import edu.harvard.iq.dataverse.engine.command.impl.UpdateDataverseCommand; -import edu.harvard.iq.dataverse.engine.command.impl.UpdateDataverseDefaultContributorRoleCommand; -import edu.harvard.iq.dataverse.engine.command.impl.UpdateDataverseMetadataBlocksCommand; -import edu.harvard.iq.dataverse.engine.command.impl.UpdateExplicitGroupCommand; -import edu.harvard.iq.dataverse.engine.command.impl.UpdateMetadataBlockFacetsCommand; -import edu.harvard.iq.dataverse.engine.command.impl.ValidateDatasetJsonCommand; +import edu.harvard.iq.dataverse.engine.command.impl.*; +import edu.harvard.iq.dataverse.pidproviders.PidProvider; +import edu.harvard.iq.dataverse.pidproviders.PidUtil; import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.BundleUtil; @@ -86,23 +35,14 @@ import edu.harvard.iq.dataverse.util.json.JsonPrinter; import edu.harvard.iq.dataverse.util.json.JsonUtil; -import static edu.harvard.iq.dataverse.util.json.JsonPrinter.brief; -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; -import java.util.TreeSet; +import java.io.StringReader; +import java.util.*; import java.util.logging.Level; import java.util.logging.Logger; import jakarta.ejb.EJB; import jakarta.ejb.EJBException; import jakarta.ejb.Stateless; -import jakarta.json.Json; -import jakarta.json.JsonArrayBuilder; -import jakarta.json.JsonNumber; -import jakarta.json.JsonObject; -import jakarta.json.JsonObjectBuilder; -import jakarta.json.JsonString; -import jakarta.json.JsonValue; +import jakarta.json.*; import jakarta.json.JsonValue.ValueType; import jakarta.json.stream.JsonParsingException; import jakarta.validation.ConstraintViolationException; @@ -126,10 +66,6 @@ import java.io.OutputStream; import java.text.MessageFormat; import java.text.SimpleDateFormat; -import java.util.Arrays; -import java.util.Date; -import java.util.Map; -import java.util.Optional; import java.util.stream.Collectors; import jakarta.servlet.http.HttpServletResponse; import jakarta.ws.rs.WebApplicationException; @@ -167,6 +103,12 @@ public class Dataverses extends AbstractApiBean { @EJB DataverseServiceBean dataverseService; + @EJB + DataverseLinkingServiceBean linkingService; + + @EJB + FeaturedDataverseServiceBean featuredDataverseService; + @EJB SwordServiceBean swordService; @@ -421,7 +363,7 @@ public Response importDataset(@Context ContainerRequestContext crc, String jsonB if (!GlobalId.verifyImportCharacters(pidParam)) { return badRequest("PID parameter contains characters that are not allowed by the Dataverse application. On import, the PID must only contain characters specified in this regex: " + BundleUtil.getStringFromBundle("pid.allowedCharacters")); } - Optional maybePid = GlobalIdServiceBean.parse(pidParam); + Optional maybePid = PidProvider.parse(pidParam); if (maybePid.isPresent()) { ds.setGlobalId(maybePid.get()); } else { @@ -496,7 +438,7 @@ public Response importDatasetDdi(@Context ContainerRequestContext crc, String xm if (!GlobalId.verifyImportCharacters(pidParam)) { return badRequest("PID parameter contains characters that are not allowed by the Dataverse application. On import, the PID must only contain characters specified in this regex: " + BundleUtil.getStringFromBundle("pid.allowedCharacters")); } - Optional maybePid = GlobalIdServiceBean.parse(pidParam); + Optional maybePid = PidProvider.parse(pidParam); if (maybePid.isPresent()) { ds.setGlobalId(maybePid.get()); } else { @@ -559,12 +501,10 @@ public Response recreateDataset(@Context ContainerRequestContext crc, String jso ds.setOwner(owner); ds = JSONLDUtil.updateDatasetMDFromJsonLD(ds, jsonLDBody, metadataBlockSvc, datasetFieldSvc, false, true, licenseSvc); //ToDo - verify PID is one Dataverse can manage (protocol/authority/shoulder match) - if(! - (ds.getAuthority().equals(settingsService.getValueForKey(SettingsServiceBean.Key.Authority))&& - ds.getProtocol().equals(settingsService.getValueForKey(SettingsServiceBean.Key.Protocol))&& - ds.getIdentifier().startsWith(settingsService.getValueForKey(SettingsServiceBean.Key.Shoulder)))) { - throw new BadRequestException("Cannot recreate a dataset that has a PID that doesn't match the server's settings"); - } + if (!PidUtil.getPidProvider(ds.getGlobalId().getProviderId()).canManagePID()) { + throw new BadRequestException( + "Cannot recreate a dataset that has a PID that doesn't match the server's settings"); + } if(!dvObjectSvc.isGlobalIdLocallyUnique(ds.getGlobalId())) { throw new BadRequestException("Cannot recreate a dataset whose PID is already in use"); } @@ -610,10 +550,11 @@ private Dataset parseDataset(String datasetJson) throws WrappedResponse { @GET @AuthRequired @Path("{identifier}") - public Response viewDataverse(@Context ContainerRequestContext crc, @PathParam("identifier") String idtf) { + public Response getDataverse(@Context ContainerRequestContext crc, @PathParam("identifier") String idtf, @QueryParam("returnOwners") boolean returnOwners) { return response(req -> ok( json(execCommand(new GetDataverseCommand(req, findDataverseOrDie(idtf))), - settingsService.isTrueForKey(SettingsServiceBean.Key.ExcludeEmailFromExport, false) + settingsService.isTrueForKey(SettingsServiceBean.Key.ExcludeEmailFromExport, false), + returnOwners )), getRequestUser(crc)); } @@ -696,6 +637,43 @@ public Response updateAttribute(@Context ContainerRequestContext crc, @PathParam } } + @PUT + @AuthRequired + @Path("{identifier}/inputLevels") + public Response updateInputLevels(@Context ContainerRequestContext crc, @PathParam("identifier") String identifier, String jsonBody) { + try { + Dataverse dataverse = findDataverseOrDie(identifier); + List newInputLevels = parseInputLevels(jsonBody, dataverse); + execCommand(new UpdateDataverseInputLevelsCommand(dataverse, createDataverseRequest(getRequestUser(crc)), newInputLevels)); + return ok(BundleUtil.getStringFromBundle("dataverse.update.success"), JsonPrinter.json(dataverse)); + } catch (WrappedResponse e) { + return e.getResponse(); + } + } + + private List parseInputLevels(String jsonBody, Dataverse dataverse) throws WrappedResponse { + JsonArray inputLevelsArray = Json.createReader(new StringReader(jsonBody)).readArray(); + + List newInputLevels = new ArrayList<>(); + for (JsonValue value : inputLevelsArray) { + JsonObject inputLevel = (JsonObject) value; + String datasetFieldTypeName = inputLevel.getString("datasetFieldTypeName"); + DatasetFieldType datasetFieldType = datasetFieldSvc.findByName(datasetFieldTypeName); + + if (datasetFieldType == null) { + String errorMessage = MessageFormat.format(BundleUtil.getStringFromBundle("dataverse.updateinputlevels.error.invalidfieldtypename"), datasetFieldTypeName); + throw new WrappedResponse(badRequest(errorMessage)); + } + + boolean required = inputLevel.getBoolean("required"); + boolean include = inputLevel.getBoolean("include"); + + newInputLevels.add(new DataverseFieldTypeInputLevel(datasetFieldType, dataverse, required, include)); + } + + return newInputLevels; + } + @DELETE @AuthRequired @Path("{linkingDataverseId}/deleteLink/{linkedDataverseId}") @@ -710,14 +688,20 @@ public Response deleteDataverseLinkingDataverse(@Context ContainerRequestContext @GET @AuthRequired @Path("{identifier}/metadatablocks") - public Response listMetadataBlocks(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf) { + public Response listMetadataBlocks(@Context ContainerRequestContext crc, + @PathParam("identifier") String dvIdtf, + @QueryParam("onlyDisplayedOnCreate") boolean onlyDisplayedOnCreate, + @QueryParam("returnDatasetFieldTypes") boolean returnDatasetFieldTypes) { try { - JsonArrayBuilder arr = Json.createArrayBuilder(); - final List blocks = execCommand(new ListMetadataBlocksCommand(createDataverseRequest(getRequestUser(crc)), findDataverseOrDie(dvIdtf))); - for (MetadataBlock mdb : blocks) { - arr.add(brief.json(mdb)); - } - return ok(arr); + Dataverse dataverse = findDataverseOrDie(dvIdtf); + final List metadataBlocks = execCommand( + new ListMetadataBlocksCommand( + createDataverseRequest(getRequestUser(crc)), + dataverse, + onlyDisplayedOnCreate + ) + ); + return ok(json(metadataBlocks, returnDatasetFieldTypes, onlyDisplayedOnCreate, dataverse)); } catch (WrappedResponse we) { return we.getResponse(); } @@ -821,6 +805,111 @@ public Response listFacets(@Context ContainerRequestContext crc, @PathParam("ide } } + + @GET + @AuthRequired + @Path("{identifier}/featured") + /* + Allows user to get the collections that are featured by a given collection + probably more for SPA than end user + */ + public Response getFeaturedDataverses(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf, String dvAliases) { + + try { + User u = getRequestUser(crc); + DataverseRequest r = createDataverseRequest(u); + Dataverse dataverse = findDataverseOrDie(dvIdtf); + JsonArrayBuilder fs = Json.createArrayBuilder(); + for (Dataverse f : execCommand(new ListFeaturedCollectionsCommand(r, dataverse))) { + fs.add(f.getAlias()); + } + return ok(fs); + } catch (WrappedResponse e) { + return e.getResponse(); + } + } + + + @POST + @AuthRequired + @Path("{identifier}/featured") + /** + * Allows user to set featured dataverses - must have edit dataverse permission + * + */ + public Response setFeaturedDataverses(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf, String dvAliases) { + List dvsFromInput = new LinkedList<>(); + + + try { + + for (JsonString dvAlias : Util.asJsonArray(dvAliases).getValuesAs(JsonString.class)) { + Dataverse dvToBeFeatured = dataverseService.findByAlias(dvAlias.getString()); + if (dvToBeFeatured == null) { + return error(Response.Status.BAD_REQUEST, "Can't find dataverse collection with alias '" + dvAlias + "'"); + } + dvsFromInput.add(dvToBeFeatured); + } + + if (dvsFromInput.isEmpty()) { + return error(Response.Status.BAD_REQUEST, "Please provide a valid Json array of dataverse collection aliases to be featured."); + } + + Dataverse dataverse = findDataverseOrDie(dvIdtf); + List featuredSource = new ArrayList<>(); + List featuredTarget = new ArrayList<>(); + featuredSource.addAll(dataverseService.findAllPublishedByOwnerId(dataverse.getId())); + featuredSource.addAll(linkingService.findLinkedDataverses(dataverse.getId())); + List featuredList = featuredDataverseService.findByDataverseId(dataverse.getId()); + + if (featuredSource.isEmpty()) { + return error(Response.Status.BAD_REQUEST, "There are no collections avaialble to be featured in Dataverse collection '" + dataverse.getDisplayName() + "'."); + } + + for (DataverseFeaturedDataverse dfd : featuredList) { + Dataverse fd = dfd.getFeaturedDataverse(); + featuredTarget.add(fd); + featuredSource.remove(fd); + } + + for (Dataverse test : dvsFromInput) { + if (featuredTarget.contains(test)) { + return error(Response.Status.BAD_REQUEST, "Dataverse collection '" + test.getDisplayName() + "' is already featured in Dataverse collection '" + dataverse.getDisplayName() + "'."); + } + + if (featuredSource.contains(test)) { + featuredTarget.add(test); + } else { + return error(Response.Status.BAD_REQUEST, "Dataverse collection '" + test.getDisplayName() + "' may not be featured in Dataverse collection '" + dataverse.getDisplayName() + "'."); + } + + } + // by passing null for Facets and DataverseFieldTypeInputLevel, those are not changed + execCommand(new UpdateDataverseCommand(dataverse, null, featuredTarget, createDataverseRequest(getRequestUser(crc)), null)); + return ok("Featured Dataverses of dataverse " + dvIdtf + " updated."); + + } catch (WrappedResponse ex) { + return ex.getResponse(); + } catch (JsonParsingException jpe){ + return error(Response.Status.BAD_REQUEST, "Please provide a valid Json array of dataverse collection aliases to be featured."); + } + + } + + @DELETE + @AuthRequired + @Path("{identifier}/featured") + public Response deleteFeaturedCollections(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf) throws WrappedResponse { + try { + Dataverse dataverse = findDataverseOrDie(dvIdtf); + List featuredTarget = new ArrayList<>(); + execCommand(new UpdateDataverseCommand(dataverse, null, featuredTarget, createDataverseRequest(getRequestUser(crc)), null)); + return ok(BundleUtil.getStringFromBundle("dataverses.api.delete.featured.collections.successful")); + } catch (WrappedResponse ex) { + return ex.getResponse(); + } + } + @POST @AuthRequired @Path("{identifier}/facets") diff --git a/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java b/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java index bcb8799ec9e..c815caa09eb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java @@ -22,7 +22,6 @@ import jakarta.ws.rs.ext.Provider; import edu.harvard.iq.dataverse.DataFile; -import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.dataaccess.*; import edu.harvard.iq.dataverse.datavariable.DataVariable; import edu.harvard.iq.dataverse.engine.command.Command; @@ -104,8 +103,10 @@ public void writeTo(DownloadInstance di, Class clazz, Type type, Annotation[] String auxiliaryTag = null; String auxiliaryType = null; String auxiliaryFileName = null; + // Before we do anything else, check if this download can be handled // by a redirect to remote storage (only supported on S3, as of 5.4): + if (storageIO.downloadRedirectEnabled()) { // Even if the above is true, there are a few cases where a @@ -159,7 +160,7 @@ public void writeTo(DownloadInstance di, Class clazz, Type type, Annotation[] } } else if (dataFile.isTabularData()) { - // Many separate special cases here. + // Many separate special cases here. if (di.getConversionParam() != null) { if (di.getConversionParam().equals("format")) { @@ -180,12 +181,26 @@ public void writeTo(DownloadInstance di, Class clazz, Type type, Annotation[] redirectSupported = false; } } - } else if (!di.getConversionParam().equals("noVarHeader")) { - // This is a subset request - can't do. + } else if (di.getConversionParam().equals("noVarHeader")) { + // This will work just fine, if the tab. file is + // stored without the var. header. Throw "unavailable" + // exception otherwise. + // @todo: should we actually drop support for this "noVarHeader" flag? + if (dataFile.getDataTable().isStoredWithVariableHeader()) { + throw new ServiceUnavailableException(); + } + // ... defaults to redirectSupported = true + } else { + // This must be a subset request then - can't do. + redirectSupported = false; + } + } else { + // "straight" download of the full tab-delimited file. + // can redirect, but only if stored with the variable + // header already added: + if (!dataFile.getDataTable().isStoredWithVariableHeader()) { redirectSupported = false; } - } else { - redirectSupported = false; } } } @@ -247,11 +262,16 @@ public void writeTo(DownloadInstance di, Class clazz, Type type, Annotation[] // finally, issue the redirect: Response response = Response.seeOther(redirect_uri).build(); logger.fine("Issuing redirect to the file location."); + // Yes, this throws an exception. It's not an exception + // as in, "bummer, something went wrong". This is how a + // redirect is produced here! throw new RedirectionException(response); } throw new ServiceUnavailableException(); } + // Past this point, this is a locally served/streamed download + if (di.getConversionParam() != null) { // Image Thumbnail and Tabular data conversion: // NOTE: only supported on local files, as of 4.0.2! @@ -263,7 +283,7 @@ public void writeTo(DownloadInstance di, Class clazz, Type type, Annotation[] storageIO = ImageThumbConverter.getImageThumbnailAsInputStream(storageIO, ImageThumbConverter.DEFAULT_THUMBNAIL_SIZE); } else { try { - int size = new Integer(di.getConversionParamValue()); + int size = Integer.parseInt(di.getConversionParamValue()); if (size > 0) { storageIO = ImageThumbConverter.getImageThumbnailAsInputStream(storageIO, size); } @@ -274,8 +294,10 @@ public void writeTo(DownloadInstance di, Class clazz, Type type, Annotation[] // and, since we now have tabular data files that can // have thumbnail previews... obviously, we don't want to // add the variable header to the image stream! - storageIO.setNoVarHeader(Boolean.TRUE); - storageIO.setVarHeader(null); + if (storageIO != null) { // ImageThumbConverter returns null if thumbnail conversion fails + storageIO.setNoVarHeader(Boolean.TRUE); + storageIO.setVarHeader(null); + } } } else if (dataFile.isTabularData()) { logger.fine("request for tabular data download;"); @@ -285,9 +307,14 @@ public void writeTo(DownloadInstance di, Class clazz, Type type, Annotation[] // request any tabular-specific services. if (di.getConversionParam().equals("noVarHeader")) { - logger.fine("tabular data with no var header requested"); - storageIO.setNoVarHeader(Boolean.TRUE); - storageIO.setVarHeader(null); + if (!dataFile.getDataTable().isStoredWithVariableHeader()) { + logger.fine("tabular data with no var header requested"); + storageIO.setNoVarHeader(Boolean.TRUE); + storageIO.setVarHeader(null); + } else { + logger.fine("can't serve request for tabular data without varheader, since stored with it"); + throw new ServiceUnavailableException(); + } } else if (di.getConversionParam().equals("format")) { // Conversions, and downloads of "stored originals" are // now supported on all DataFiles for which StorageIO @@ -329,11 +356,10 @@ public void writeTo(DownloadInstance di, Class clazz, Type type, Annotation[] if (variable.getDataTable().getDataFile().getId().equals(dataFile.getId())) { logger.fine("adding variable id " + variable.getId() + " to the list."); variablePositionIndex.add(variable.getFileOrder()); - if (subsetVariableHeader == null) { - subsetVariableHeader = variable.getName(); - } else { - subsetVariableHeader = subsetVariableHeader.concat("\t"); - subsetVariableHeader = subsetVariableHeader.concat(variable.getName()); + if (!dataFile.getDataTable().isStoredWithVariableHeader()) { + subsetVariableHeader = subsetVariableHeader == null + ? variable.getName() + : subsetVariableHeader.concat("\t" + variable.getName()); } } else { logger.warning("variable does not belong to this data file."); @@ -346,7 +372,17 @@ public void writeTo(DownloadInstance di, Class clazz, Type type, Annotation[] try { File tempSubsetFile = File.createTempFile("tempSubsetFile", ".tmp"); TabularSubsetGenerator tabularSubsetGenerator = new TabularSubsetGenerator(); - tabularSubsetGenerator.subsetFile(storageIO.getInputStream(), tempSubsetFile.getAbsolutePath(), variablePositionIndex, dataFile.getDataTable().getCaseQuantity(), "\t"); + + long numberOfLines = dataFile.getDataTable().getCaseQuantity(); + if (dataFile.getDataTable().isStoredWithVariableHeader()) { + numberOfLines++; + } + + tabularSubsetGenerator.subsetFile(storageIO.getInputStream(), + tempSubsetFile.getAbsolutePath(), + variablePositionIndex, + numberOfLines, + "\t"); if (tempSubsetFile.exists()) { FileInputStream subsetStream = new FileInputStream(tempSubsetFile); @@ -354,8 +390,11 @@ public void writeTo(DownloadInstance di, Class clazz, Type type, Annotation[] InputStreamIO subsetStreamIO = new InputStreamIO(subsetStream, subsetSize); logger.fine("successfully created subset output stream."); - subsetVariableHeader = subsetVariableHeader.concat("\n"); - subsetStreamIO.setVarHeader(subsetVariableHeader); + + if (subsetVariableHeader != null) { + subsetVariableHeader = subsetVariableHeader.concat("\n"); + subsetStreamIO.setVarHeader(subsetVariableHeader); + } String tabularFileName = storageIO.getFileName(); @@ -380,8 +419,13 @@ public void writeTo(DownloadInstance di, Class clazz, Type type, Annotation[] } else { logger.fine("empty list of extra arguments."); } + // end of tab. data subset case + } else if (dataFile.getDataTable().isStoredWithVariableHeader()) { + logger.fine("tabular file stored with the var header included, no need to generate it on the fly"); + storageIO.setNoVarHeader(Boolean.TRUE); + storageIO.setVarHeader(null); } - } + } // end of tab. data file case if (storageIO == null) { //throw new WebApplicationException(Response.Status.SERVICE_UNAVAILABLE); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/FeedbackApi.java b/src/main/java/edu/harvard/iq/dataverse/api/FeedbackApi.java index 8a178f8da62..56c5ca95ce6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/FeedbackApi.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/FeedbackApi.java @@ -7,9 +7,6 @@ import edu.harvard.iq.dataverse.branding.BrandingUtil; import edu.harvard.iq.dataverse.feedback.Feedback; import edu.harvard.iq.dataverse.feedback.FeedbackUtil; -import edu.harvard.iq.dataverse.settings.JvmSettings; -import edu.harvard.iq.dataverse.settings.SettingsServiceBean; -import edu.harvard.iq.dataverse.util.MailUtil; import jakarta.ejb.EJB; import jakarta.json.Json; @@ -40,7 +37,7 @@ public class FeedbackApi extends AbstractApiBean { * user input (e.g. to strip potentially malicious html, etc.)!!!! **/ @POST - public Response submitFeedback(JsonObject jsonObject) throws AddressException { + public Response submitFeedback(JsonObject jsonObject) { JsonNumber jsonNumber = jsonObject.getJsonNumber("targetId"); DvObject feedbackTarget = null; if (jsonNumber != null) { @@ -51,8 +48,7 @@ public Response submitFeedback(JsonObject jsonObject) throws AddressException { } DataverseSession dataverseSession = null; String userMessage = jsonObject.getString("body"); - String systemEmail = JvmSettings.SUPPORT_EMAIL.lookupOptional().orElse(settingsSvc.getValueForKey(SettingsServiceBean.Key.SystemEmail)); - InternetAddress systemAddress = MailUtil.parseSystemAddress(systemEmail); + InternetAddress systemAddress = mailService.getSupportAddress().orElse(null); String userEmail = jsonObject.getString("fromEmail"); String messageSubject = jsonObject.getString("subject"); String baseUrl = systemConfig.getDataverseSiteUrl(); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java index 5d400ee1438..d786aab35a8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java @@ -2,22 +2,7 @@ import com.google.gson.Gson; import com.google.gson.JsonObject; -import edu.harvard.iq.dataverse.DataFile; -import edu.harvard.iq.dataverse.DataFileServiceBean; -import edu.harvard.iq.dataverse.DataFileTag; -import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.DatasetLock; -import edu.harvard.iq.dataverse.DatasetServiceBean; -import edu.harvard.iq.dataverse.DatasetVersion; -import edu.harvard.iq.dataverse.DatasetVersionServiceBean; -import edu.harvard.iq.dataverse.DataverseRequestServiceBean; -import edu.harvard.iq.dataverse.DataverseServiceBean; -import edu.harvard.iq.dataverse.EjbDataverseEngine; -import edu.harvard.iq.dataverse.FileDownloadServiceBean; -import edu.harvard.iq.dataverse.FileMetadata; -import edu.harvard.iq.dataverse.GuestbookResponseServiceBean; -import edu.harvard.iq.dataverse.TermsOfUseAndAccessValidator; -import edu.harvard.iq.dataverse.UserNotificationServiceBean; +import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.api.auth.AuthRequired; import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.authorization.users.ApiToken; @@ -27,15 +12,11 @@ import edu.harvard.iq.dataverse.datasetutility.DataFileTagException; import edu.harvard.iq.dataverse.datasetutility.NoFilesException; import edu.harvard.iq.dataverse.datasetutility.OptionalFileParams; +import edu.harvard.iq.dataverse.engine.command.Command; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; -import edu.harvard.iq.dataverse.engine.command.impl.GetDataFileCommand; -import edu.harvard.iq.dataverse.engine.command.impl.GetDraftFileMetadataIfAvailableCommand; -import edu.harvard.iq.dataverse.engine.command.impl.RedetectFileTypeCommand; -import edu.harvard.iq.dataverse.engine.command.impl.RestrictFileCommand; -import edu.harvard.iq.dataverse.engine.command.impl.UningestFileCommand; -import edu.harvard.iq.dataverse.engine.command.impl.UpdateDatasetVersionCommand; +import edu.harvard.iq.dataverse.engine.command.impl.*; import edu.harvard.iq.dataverse.export.ExportService; import io.gdcc.spi.export.ExportException; import edu.harvard.iq.dataverse.externaltools.ExternalTool; @@ -51,6 +32,8 @@ import edu.harvard.iq.dataverse.util.SystemConfig; import edu.harvard.iq.dataverse.util.URLTokenUtil; +import static edu.harvard.iq.dataverse.api.ApiConstants.*; +import static edu.harvard.iq.dataverse.api.Datasets.handleVersion; import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json; import edu.harvard.iq.dataverse.util.json.JsonUtil; import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder; @@ -66,10 +49,7 @@ import jakarta.ejb.EJB; import jakarta.ejb.EJBException; import jakarta.inject.Inject; -import jakarta.json.Json; -import jakarta.json.JsonArray; -import jakarta.json.JsonString; -import jakarta.json.JsonValue; +import jakarta.json.*; import jakarta.json.stream.JsonParsingException; import jakarta.servlet.http.HttpServletResponse; import jakarta.ws.rs.*; @@ -84,6 +64,13 @@ import static jakarta.ws.rs.core.Response.Status.FORBIDDEN; import jakarta.ws.rs.core.UriInfo; + +import org.eclipse.microprofile.openapi.annotations.Operation; +import org.eclipse.microprofile.openapi.annotations.media.Content; +import org.eclipse.microprofile.openapi.annotations.media.Schema; +import org.eclipse.microprofile.openapi.annotations.parameters.RequestBody; +import org.eclipse.microprofile.openapi.annotations.responses.APIResponse; +import org.eclipse.microprofile.openapi.annotations.tags.Tag; import org.glassfish.jersey.media.multipart.FormDataBodyPart; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; import org.glassfish.jersey.media.multipart.FormDataParam; @@ -196,6 +183,14 @@ public Response restrictFileInDataset(@Context ContainerRequestContext crc, @Pat @AuthRequired @Path("{id}/replace") @Consumes(MediaType.MULTIPART_FORM_DATA) + @Produces("application/json") + @Operation(summary = "Replace a file on a dataset", + description = "Replace a file to a dataset") + @APIResponse(responseCode = "200", + description = "File replaced successfully on the dataset") + @Tag(name = "replaceFilesInDataset", + description = "Replace a file to a dataset") + @RequestBody(content = @Content(mediaType = MediaType.MULTIPART_FORM_DATA)) public Response replaceFileInDataset( @Context ContainerRequestContext crc, @PathParam("id") String fileIdOrPersistentId, @@ -230,10 +225,10 @@ public Response replaceFileInDataset( // - Will skip extra attributes which includes fileToReplaceId and forceReplace optionalFileParams = new OptionalFileParams(jsonData); } catch (DataFileTagException ex) { - return error(Response.Status.BAD_REQUEST, ex.getMessage()); + return error(BAD_REQUEST, ex.getMessage()); } } catch (ClassCastException | com.google.gson.JsonParseException ex) { - return error(Response.Status.BAD_REQUEST, BundleUtil.getStringFromBundle("file.addreplace.error.parsing")); + return error(BAD_REQUEST, BundleUtil.getStringFromBundle("file.addreplace.error.parsing")); } } @@ -322,7 +317,7 @@ public Response replaceFileInDataset( //"Look at that! You added a file! (hey hey, it may have worked)"); } catch (NoFilesException ex) { Logger.getLogger(Files.class.getName()).log(Level.SEVERE, null, ex); - return error(Response.Status.BAD_REQUEST, "NoFileException! Serious Error! See administrator!"); + return error(BAD_REQUEST, "NoFileException! Serious Error! See administrator!"); } } @@ -412,7 +407,7 @@ public Response updateFileMetadata(@Context ContainerRequestContext crc, @FormDa //we get the data file to do a permissions check, if this fails it'll go to the WrappedResponse below for an ugly unpermitted error execCommand(new GetDataFileCommand(req, findDataFileOrDie(result.get(0).toString()))); - return error(Response.Status.BAD_REQUEST, "You cannot edit metadata on a dataFile that has been replaced. Please try again with the newest file id."); + return error(BAD_REQUEST, "You cannot edit metadata on a dataFile that has been replaced. Please try again with the newest file id."); } // (2) Check/Parse the JSON (if uploaded) @@ -434,10 +429,10 @@ public Response updateFileMetadata(@Context ContainerRequestContext crc, @FormDa // - Will skip extra attributes which includes fileToReplaceId and forceReplace optionalFileParams = new OptionalFileParams(jsonData); } catch (DataFileTagException ex) { - return error(Response.Status.BAD_REQUEST, ex.getMessage()); + return error(BAD_REQUEST, ex.getMessage()); } } catch (ClassCastException | com.google.gson.JsonParseException ex) { - return error(Response.Status.BAD_REQUEST, BundleUtil.getStringFromBundle("file.addreplace.error.parsing")); + return error(BAD_REQUEST, BundleUtil.getStringFromBundle("file.addreplace.error.parsing")); } } @@ -458,7 +453,7 @@ public Response updateFileMetadata(@Context ContainerRequestContext crc, @FormDa } if (upFmd == null){ - return error(Response.Status.BAD_REQUEST, "An error has occurred attempting to update the requested DataFile. It is not part of the current version of the Dataset."); + return error(BAD_REQUEST, "An error has occurred attempting to update the requested DataFile. It is not part of the current version of the Dataset."); } jakarta.json.JsonObject jsonObject = JsonUtil.getJsonObject(jsonData); @@ -489,7 +484,7 @@ public Response updateFileMetadata(@Context ContainerRequestContext crc, @FormDa } } catch (WrappedResponse wr) { - return error(Response.Status.BAD_REQUEST, "An error has occurred attempting to update the requested DataFile, likely related to permissions."); + return error(BAD_REQUEST, "An error has occurred attempting to update the requested DataFile, likely related to permissions."); } String jsonString = upFmd.asGsonObject(true).toString(); @@ -500,79 +495,82 @@ public Response updateFileMetadata(@Context ContainerRequestContext crc, @FormDa .type(MediaType.TEXT_PLAIN) //Our plain text string is already json .build(); } - + @GET @AuthRequired - @Path("{id}/draft") - public Response getFileDataDraft(@Context ContainerRequestContext crc, @PathParam("id") String fileIdOrPersistentId, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) throws WrappedResponse, Exception { - return getFileDataResponse(getRequestUser(crc), fileIdOrPersistentId, uriInfo, headers, response, true); + @Path("{id}") + public Response getFileData(@Context ContainerRequestContext crc, + @PathParam("id") String fileIdOrPersistentId, + @QueryParam("includeDeaccessioned") boolean includeDeaccessioned, + @QueryParam("returnDatasetVersion") boolean returnDatasetVersion, + @QueryParam("returnOwners") boolean returnOwners, + @Context UriInfo uriInfo, + @Context HttpHeaders headers) { + return response( req -> getFileDataResponse(req, fileIdOrPersistentId, DS_VERSION_LATEST, includeDeaccessioned, returnDatasetVersion, returnOwners, uriInfo, headers), getRequestUser(crc)); } - + @GET @AuthRequired - @Path("{id}") - public Response getFileData(@Context ContainerRequestContext crc, @PathParam("id") String fileIdOrPersistentId, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) throws WrappedResponse, Exception { - return getFileDataResponse(getRequestUser(crc), fileIdOrPersistentId, uriInfo, headers, response, false); + @Path("{id}/versions/{datasetVersionId}") + public Response getFileDataForVersion(@Context ContainerRequestContext crc, + @PathParam("id") String fileIdOrPersistentId, + @PathParam("datasetVersionId") String datasetVersionId, + @QueryParam("includeDeaccessioned") boolean includeDeaccessioned, + @QueryParam("returnDatasetVersion") boolean returnDatasetVersion, + @QueryParam("returnOwners") boolean returnOwners, + @Context UriInfo uriInfo, + @Context HttpHeaders headers) { + return response( req -> getFileDataResponse(req, fileIdOrPersistentId, datasetVersionId, includeDeaccessioned, returnDatasetVersion, returnOwners, uriInfo, headers), getRequestUser(crc)); } - - private Response getFileDataResponse(User user, String fileIdOrPersistentId, UriInfo uriInfo, HttpHeaders headers, HttpServletResponse response, boolean draft ){ - - DataverseRequest req; - try { - req = createDataverseRequest(user); - } catch (Exception e) { - return error(BAD_REQUEST, "Error attempting to request information. Maybe a bad API token?"); - } - final DataFile df; - try { - df = execCommand(new GetDataFileCommand(req, findDataFileOrDie(fileIdOrPersistentId))); - } catch (Exception e) { - return error(BAD_REQUEST, "Error attempting get the requested data file."); - } - FileMetadata fm; + private Response getFileDataResponse(final DataverseRequest req, + String fileIdOrPersistentId, + String datasetVersionId, + boolean includeDeaccessioned, + boolean returnDatasetVersion, + boolean returnOwners, + UriInfo uriInfo, + HttpHeaders headers) throws WrappedResponse { + final DataFile dataFile = execCommand(new GetDataFileCommand(req, findDataFileOrDie(fileIdOrPersistentId))); + FileMetadata fileMetadata = execCommand(handleVersion(datasetVersionId, new Datasets.DsVersionHandler<>() { + @Override + public Command handleLatest() { + return new GetLatestAccessibleFileMetadataCommand(req, dataFile, includeDeaccessioned); + } - if (draft) { - try { - fm = execCommand(new GetDraftFileMetadataIfAvailableCommand(req, df)); - } catch (WrappedResponse w) { - return error(BAD_REQUEST, "An error occurred getting a draft version, you may not have permission to access unpublished data on this dataset."); + @Override + public Command handleDraft() { + return new GetDraftFileMetadataIfAvailableCommand(req, dataFile); } - if (null == fm) { - return error(BAD_REQUEST, BundleUtil.getStringFromBundle("files.api.no.draft")); + + @Override + public Command handleSpecific(long major, long minor) { + return new GetSpecificPublishedFileMetadataByDatasetVersionCommand(req, dataFile, major, minor, includeDeaccessioned); } - } else { - //first get latest published - //if not available get draft if permissible - try { - fm = df.getLatestPublishedFileMetadata(); - - } catch (UnsupportedOperationException e) { - try { - fm = execCommand(new GetDraftFileMetadataIfAvailableCommand(req, df)); - } catch (WrappedResponse w) { - return error(BAD_REQUEST, "An error occurred getting a draft version, you may not have permission to access unpublished data on this dataset."); - } - if (null == fm) { - return error(BAD_REQUEST, BundleUtil.getStringFromBundle("files.api.no.draft")); - } + @Override + public Command handleLatestPublished() { + return new GetLatestPublishedFileMetadataCommand(req, dataFile, includeDeaccessioned); } + })); + if (fileMetadata == null) { + throw new WrappedResponse(notFound(BundleUtil.getStringFromBundle("files.api.notFoundInVersion", Arrays.asList(fileIdOrPersistentId, datasetVersionId)))); } - - if (fm.getDatasetVersion().isReleased()) { - MakeDataCountLoggingServiceBean.MakeDataCountEntry entry = new MakeDataCountLoggingServiceBean.MakeDataCountEntry(uriInfo, headers, dvRequestService, df); + + if (fileMetadata.getDatasetVersion().isReleased()) { + MakeDataCountLoggingServiceBean.MakeDataCountEntry entry = new MakeDataCountLoggingServiceBean.MakeDataCountEntry(uriInfo, headers, dvRequestService, dataFile); mdcLogService.logEntry(entry); } - + return Response.ok(Json.createObjectBuilder() .add("status", ApiConstants.STATUS_OK) - .add("data", json(fm)).build()) + .add("data", json(fileMetadata, returnOwners, returnDatasetVersion)).build()) .type(MediaType.APPLICATION_JSON) .build(); } + @GET @AuthRequired @Path("{id}/metadata") @@ -637,23 +635,41 @@ public Response uningestDatafile(@Context ContainerRequestContext crc, @PathPara if (dataFile == null) { return error(Response.Status.NOT_FOUND, "File not found for given id."); } - if (!dataFile.isTabularData()) { - return error(Response.Status.BAD_REQUEST, "Cannot uningest non-tabular file."); - } - - try { - DataverseRequest req = createDataverseRequest(getRequestUser(crc)); - execCommand(new UningestFileCommand(req, dataFile)); - Long dataFileId = dataFile.getId(); - dataFile = fileService.find(dataFileId); - Dataset theDataset = dataFile.getOwner(); - exportDatasetMetadata(settingsService, theDataset); - return ok("Datafile " + dataFileId + " uningested."); - } catch (WrappedResponse wr) { - return wr.getResponse(); + // Ingest never succeeded, either there was a failure or this is not a tabular + // data file + // We allow anyone who can publish to uningest in order to clear a problem + if (dataFile.isIngestProblem()) { + try { + AuthenticatedUser au = getRequestAuthenticatedUserOrDie(crc); + if (!(permissionSvc.permissionsFor(au, dataFile).contains(Permission.PublishDataset))) { + return forbidden( + "Uningesting to remove an ingest problem can only be done by those who can publish the dataset"); + } + } catch (WrappedResponse wr) { + return wr.getResponse(); + } + dataFile.setIngestDone(); + dataFile.setIngestReport(null); + fileService.save(dataFile); + return ok("Datafile " + dataFile.getId() + " uningested."); + } else { + return error(BAD_REQUEST, + BundleUtil.getStringFromBundle("Cannot uningest non-tabular file.")); + } + } else { + try { + DataverseRequest req = createDataverseRequest(getRequestUser(crc)); + execCommand(new UningestFileCommand(req, dataFile)); + Long dataFileId = dataFile.getId(); + dataFile = fileService.find(dataFileId); + Dataset theDataset = dataFile.getOwner(); + exportDatasetMetadata(settingsService, theDataset); + return ok("Datafile " + dataFileId + " uningested."); + } catch (WrappedResponse wr) { + return wr.getResponse(); + } } - } // reingest attempts to queue an *existing* DataFile @@ -672,7 +688,7 @@ public Response reingest(@Context ContainerRequestContext crc, @PathParam("id") try { u = getRequestAuthenticatedUserOrDie(crc); if (!u.isSuperuser()) { - return error(Response.Status.FORBIDDEN, "This API call can be used by superusers only"); + return error(FORBIDDEN, "This API call can be used by superusers only"); } } catch (WrappedResponse wr) { return wr.getResponse(); @@ -688,21 +704,21 @@ public Response reingest(@Context ContainerRequestContext crc, @PathParam("id") Dataset dataset = dataFile.getOwner(); if (dataset == null) { - return error(Response.Status.BAD_REQUEST, "Failed to locate the parent dataset for the datafile."); + return error(BAD_REQUEST, "Failed to locate the parent dataset for the datafile."); } if (dataFile.isTabularData()) { - return error(Response.Status.BAD_REQUEST, "The datafile is already ingested as Tabular."); + return error(BAD_REQUEST, "The datafile is already ingested as Tabular."); } boolean ingestLock = dataset.isLockedFor(DatasetLock.Reason.Ingest); if (ingestLock) { - return error(Response.Status.FORBIDDEN, "Dataset already locked with an Ingest lock"); + return error(FORBIDDEN, "Dataset already locked with an Ingest lock"); } if (!FileUtil.canIngestAsTabular(dataFile)) { - return error(Response.Status.BAD_REQUEST, "Tabular ingest is not supported for this file type (id: "+id+", type: "+dataFile.getContentType()+")"); + return error(BAD_REQUEST, "Tabular ingest is not supported for this file type (id: "+id+", type: "+dataFile.getContentType()+")"); } dataFile.SetIngestScheduled(); @@ -742,7 +758,7 @@ public Response redetectDatafile(@Context ContainerRequestContext crc, @PathPara // Ingested Files have mimetype = text/tab-separated-values // No need to redetect if (dataFileIn.isTabularData()) { - return error(Response.Status.BAD_REQUEST, "The file is an ingested tabular file."); + return error(BAD_REQUEST, "The file is an ingested tabular file."); } String originalContentType = dataFileIn.getContentType(); DataFile dataFileOut = execCommand(new RedetectFileTypeCommand(createDataverseRequest(getRequestUser(crc)), dataFileIn, dryRun)); @@ -765,7 +781,7 @@ public Response extractNcml(@Context ContainerRequestContext crc, @PathParam("id if (!au.isSuperuser()) { // We can always make a command in the future if there's a need // for non-superusers to call this API. - return error(Response.Status.FORBIDDEN, "This API call can be used by superusers only"); + return error(FORBIDDEN, "This API call can be used by superusers only"); } DataFile dataFileIn = findDataFileOrDie(id); java.nio.file.Path tempLocationPath = null; @@ -931,4 +947,37 @@ public Response getHasBeenDeleted(@Context ContainerRequestContext crc, @PathPar return ok(dataFileServiceBean.hasBeenDeleted(dataFile)); }, getRequestUser(crc)); } + + /** + * @param fileIdOrPersistentId Database ID or PID of the data file. + * @param versionNumber The version of the dataset, such as 1.0, :draft, + * :latest-published, etc. + * @param includeDeaccessioned Defaults to false. + */ + @GET + @AuthRequired + @Path("{id}/versions/{dsVersionString}/citation") + public Response getFileCitationByVersion(@Context ContainerRequestContext crc, @PathParam("id") String fileIdOrPersistentId, @PathParam("dsVersionString") String versionNumber, @QueryParam("includeDeaccessioned") boolean includeDeaccessioned) { + try { + DataverseRequest req = createDataverseRequest(getRequestUser(crc)); + final DataFile df = execCommand(new GetDataFileCommand(req, findDataFileOrDie(fileIdOrPersistentId))); + Dataset ds = df.getOwner(); + DatasetVersion dsv = findDatasetVersionOrDie(req, versionNumber, ds, includeDeaccessioned, true); + if (dsv == null) { + return unauthorized(BundleUtil.getStringFromBundle("files.api.no.draftOrUnauth")); + } + + Long getDatasetVersionID = dsv.getId(); + FileMetadata fm = dataFileServiceBean.findFileMetadataByDatasetVersionIdAndDataFileId(getDatasetVersionID, df.getId()); + if (fm == null) { + return notFound(BundleUtil.getStringFromBundle("files.api.fileNotFound")); + } + boolean direct = df.isIdentifierRegistered(); + DataCitation citation = new DataCitation(fm, direct); + return ok(citation.toString(true)); + } catch (WrappedResponse ex) { + return ex.getResponse(); + } + } + } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Groups.java b/src/main/java/edu/harvard/iq/dataverse/api/Groups.java index d56a787c7ff..ed996b8ecf9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Groups.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Groups.java @@ -88,8 +88,8 @@ public Response postIpGroup( JsonObject dto ){ * that group from being created. */ @PUT - @Path("ip/{groupName}") - public Response putIpGroups( @PathParam("groupName") String groupName, JsonObject dto ){ + @Path("ip/{group}") + public Response putIpGroups( @PathParam("group") String groupName, JsonObject dto ){ try { if ( groupName == null || groupName.trim().isEmpty() ) { return badRequest("Group name cannot be empty"); @@ -118,8 +118,8 @@ public Response listIpGroups() { } @GET - @Path("ip/{groupIdtf}") - public Response getIpGroup( @PathParam("groupIdtf") String groupIdtf ) { + @Path("ip/{group}") + public Response getIpGroup( @PathParam("group") String groupIdtf ) { IpGroup grp; if ( isNumeric(groupIdtf) ) { grp = ipGroupPrv.get( Long.parseLong(groupIdtf) ); @@ -131,8 +131,8 @@ public Response getIpGroup( @PathParam("groupIdtf") String groupIdtf ) { } @DELETE - @Path("ip/{groupIdtf}") - public Response deleteIpGroup( @PathParam("groupIdtf") String groupIdtf ) { + @Path("ip/{group}") + public Response deleteIpGroup( @PathParam("group") String groupIdtf ) { IpGroup grp; if ( isNumeric(groupIdtf) ) { grp = ipGroupPrv.get( Long.parseLong(groupIdtf) ); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Index.java b/src/main/java/edu/harvard/iq/dataverse/api/Index.java index 4910c460b6a..c30a77acb58 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Index.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Index.java @@ -215,7 +215,7 @@ public Response clearSolrIndex() { return error(Status.INTERNAL_SERVER_ERROR, ex.getLocalizedMessage()); } } - + @GET @Path("{type}/{id}") public Response indexTypeById(@PathParam("type") String type, @PathParam("id") Long id) { @@ -326,6 +326,29 @@ public Response indexDatasetByPersistentId(@QueryParam("persistentId") String pe } } + /** + * Clears the entry for a dataset from Solr + * + * @param id numer id of the dataset + * @return response; + * will return 404 if no such dataset in the database; but will attempt to + * clear the entry from Solr regardless. + */ + @DELETE + @Path("datasets/{id}") + public Response clearDatasetFromIndex(@PathParam("id") Long id) { + Dataset dataset = datasetService.find(id); + // We'll attempt to delete the Solr document regardless of whether the + // dataset exists in the database: + String response = indexService.removeSolrDocFromIndex(IndexServiceBean.solrDocIdentifierDataset + id); + if (dataset != null) { + return ok("Sent request to clear Solr document for dataset " + id + ": " + response); + } else { + return notFound("Could not find dataset " + id + " in the database. Requested to clear from Solr anyway: " + response); + } + } + + /** * This is just a demo of the modular math logic we use for indexAll. */ diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Info.java b/src/main/java/edu/harvard/iq/dataverse/api/Info.java index 40ce6cd25b7..257519677d3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Info.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Info.java @@ -1,16 +1,35 @@ package edu.harvard.iq.dataverse.api; +import java.io.FileInputStream; +import java.io.InputStream; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +import jakarta.ws.rs.Produces; +import org.apache.commons.io.IOUtils; + import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; +import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.SystemConfig; import jakarta.ejb.EJB; import jakarta.json.Json; import jakarta.json.JsonValue; import jakarta.ws.rs.GET; import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; +import org.eclipse.microprofile.openapi.annotations.Operation; +import org.eclipse.microprofile.openapi.annotations.responses.APIResponse; +import org.eclipse.microprofile.openapi.annotations.tags.Tag; @Path("info") +@Tag(name = "info", description = "General information about the Dataverse installation.") public class Info extends AbstractApiBean { @EJB @@ -19,6 +38,8 @@ public class Info extends AbstractApiBean { @EJB SystemConfig systemConfig; + private static final Logger logger = Logger.getLogger(Info.class.getCanonicalName()); + @GET @Path("settings/:DatasetPublishPopupCustomText") public Response getDatasetPublishPopupCustomText() { @@ -33,6 +54,9 @@ public Response getMaxEmbargoDurationInMonths() { @GET @Path("version") + @Operation(summary = "Get version and build information", description = "Get version and build information") + @APIResponse(responseCode = "200", + description = "Version and build information") public Response getInfo() { String versionStr = systemConfig.getVersion(true); String[] comps = versionStr.split("build",2); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java b/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java index 05d12f1083c..6a9c608dc13 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java @@ -1,12 +1,9 @@ package edu.harvard.iq.dataverse.api; -import edu.harvard.iq.dataverse.DOIServiceBean; import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetServiceBean; import edu.harvard.iq.dataverse.DataverseRoleServiceBean; import edu.harvard.iq.dataverse.GlobalId; -import edu.harvard.iq.dataverse.GlobalIdServiceBean; -import edu.harvard.iq.dataverse.HandlenetServiceBean; import edu.harvard.iq.dataverse.MailServiceBean; import edu.harvard.iq.dataverse.RoleAssigneeServiceBean; import edu.harvard.iq.dataverse.RoleAssignment; @@ -15,6 +12,9 @@ import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IpAddress; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.pidproviders.PidProvider; +import edu.harvard.iq.dataverse.pidproviders.doi.AbstractDOIProvider; +import edu.harvard.iq.dataverse.pidproviders.handle.HandlePidProvider; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.json.JSONLDUtil; import edu.harvard.iq.dataverse.util.json.JsonLDNamespace; @@ -134,13 +134,13 @@ public Response acceptMessage(String body) { .getString("@id"); if (citedResource.getString("@type").equals(JsonLDTerm.schemaOrg("Dataset").getUrl())) { logger.fine("Raw PID: " + pid); - if (pid.startsWith(DOIServiceBean.DOI_RESOLVER_URL)) { - pid = pid.replace(DOIServiceBean.DOI_RESOLVER_URL, DOIServiceBean.DOI_PROTOCOL + ":"); - } else if (pid.startsWith(HandlenetServiceBean.HDL_RESOLVER_URL)) { - pid = pid.replace(HandlenetServiceBean.HDL_RESOLVER_URL, HandlenetServiceBean.HDL_PROTOCOL + ":"); + if (pid.startsWith(AbstractDOIProvider.DOI_RESOLVER_URL)) { + pid = pid.replace(AbstractDOIProvider.DOI_RESOLVER_URL, AbstractDOIProvider.DOI_PROTOCOL + ":"); + } else if (pid.startsWith(HandlePidProvider.HDL_RESOLVER_URL)) { + pid = pid.replace(HandlePidProvider.HDL_RESOLVER_URL, HandlePidProvider.HDL_PROTOCOL + ":"); } logger.fine("Protocol PID: " + pid); - Optional id = GlobalIdServiceBean.parse(pid); + Optional id = PidProvider.parse(pid); Dataset dataset = datasetSvc.findByGlobalId(pid); if (dataset != null) { JsonObject citingResource = Json.createObjectBuilder().add("@id", citingPID) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java b/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java index b2696757220..1f2f1039327 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java @@ -2,10 +2,16 @@ import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetServiceBean; +import edu.harvard.iq.dataverse.GlobalId; import edu.harvard.iq.dataverse.makedatacount.DatasetExternalCitations; import edu.harvard.iq.dataverse.makedatacount.DatasetExternalCitationsServiceBean; import edu.harvard.iq.dataverse.makedatacount.DatasetMetrics; import edu.harvard.iq.dataverse.makedatacount.DatasetMetricsServiceBean; +import edu.harvard.iq.dataverse.makedatacount.MakeDataCountProcessState; +import edu.harvard.iq.dataverse.makedatacount.MakeDataCountProcessStateServiceBean; +import edu.harvard.iq.dataverse.pidproviders.PidProvider; +import edu.harvard.iq.dataverse.pidproviders.PidUtil; +import edu.harvard.iq.dataverse.pidproviders.doi.datacite.DataCiteDOIProvider; import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.util.SystemConfig; import edu.harvard.iq.dataverse.util.json.JsonUtil; @@ -25,6 +31,8 @@ import jakarta.json.JsonObject; import jakarta.json.JsonObjectBuilder; import jakarta.json.JsonValue; +import jakarta.ws.rs.DELETE; +import jakarta.ws.rs.GET; import jakarta.ws.rs.POST; import jakarta.ws.rs.Path; import jakarta.ws.rs.PathParam; @@ -43,6 +51,8 @@ public class MakeDataCountApi extends AbstractApiBean { @EJB DatasetMetricsServiceBean datasetMetricsService; @EJB + MakeDataCountProcessStateServiceBean makeDataCountProcessStateService; + @EJB DatasetExternalCitationsServiceBean datasetExternalCitationsService; @EJB DatasetServiceBean datasetService; @@ -106,7 +116,7 @@ public Response addUsageMetricsFromSushiReport(@PathParam("id") String id, @Quer @POST @Path("/addUsageMetricsFromSushiReport") - public Response addUsageMetricsFromSushiReportAll(@PathParam("id") String id, @QueryParam("reportOnDisk") String reportOnDisk) { + public Response addUsageMetricsFromSushiReportAll(@QueryParam("reportOnDisk") String reportOnDisk) { try { JsonObject report = JsonUtil.getJsonObjectFromFile(reportOnDisk); @@ -131,8 +141,14 @@ public Response addUsageMetricsFromSushiReportAll(@PathParam("id") String id, @Q public Response updateCitationsForDataset(@PathParam("id") String id) throws IOException { try { Dataset dataset = findDatasetOrDie(id); - String persistentId = dataset.getGlobalId().toString(); - //ToDo - if this isn't a DOI? + GlobalId pid = dataset.getGlobalId(); + PidProvider pidProvider = PidUtil.getPidProvider(pid.getProviderId()); + // Only supported for DOIs and for DataCite DOI providers + if(!DataCiteDOIProvider.TYPE.equals(pidProvider.getProviderType())) { + return error(Status.BAD_REQUEST, "Only DataCite DOI providers are supported"); + } + String persistentId = pid.toString(); + // DataCite wants "doi=", not "doi:". String authorityPlusIdentifier = persistentId.replaceFirst("doi:", ""); // Request max page size and then loop to handle multiple pages @@ -190,5 +206,51 @@ public Response updateCitationsForDataset(@PathParam("id") String id) throws IOE return wr.getResponse(); } } + @GET + @Path("{yearMonth}/processingState") + public Response getProcessingState(@PathParam("yearMonth") String yearMonth) { + MakeDataCountProcessState mdcps; + try { + mdcps = makeDataCountProcessStateService.getMakeDataCountProcessState(yearMonth); + } catch (IllegalArgumentException e) { + return error(Status.BAD_REQUEST,e.getMessage()); + } + if (mdcps != null) { + JsonObjectBuilder output = Json.createObjectBuilder(); + output.add("yearMonth", mdcps.getYearMonth()); + output.add("state", mdcps.getState().name()); + output.add("stateChangeTimestamp", mdcps.getStateChangeTime().toString()); + return ok(output); + } else { + return error(Status.NOT_FOUND, "Could not find an existing process state for " + yearMonth); + } + } + @POST + @Path("{yearMonth}/processingState") + public Response updateProcessingState(@PathParam("yearMonth") String yearMonth, @QueryParam("state") String state) { + MakeDataCountProcessState mdcps; + try { + mdcps = makeDataCountProcessStateService.setMakeDataCountProcessState(yearMonth, state); + } catch (Exception e) { + return badRequest(e.getMessage()); + } + + JsonObjectBuilder output = Json.createObjectBuilder(); + output.add("yearMonth", mdcps.getYearMonth()); + output.add("state", mdcps.getState().name()); + output.add("stateChangeTimestamp", mdcps.getStateChangeTime().toString()); + return ok(output); + } + + @DELETE + @Path("{yearMonth}/processingState") + public Response deleteProcessingState(@PathParam("yearMonth") String yearMonth) { + boolean deleted = makeDataCountProcessStateService.deleteMakeDataCountProcessState(yearMonth); + if (deleted) { + return ok("Processing State deleted for " + yearMonth); + } else { + return notFound("Processing State not found for " + yearMonth); + } + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/MetadataBlocks.java b/src/main/java/edu/harvard/iq/dataverse/api/MetadataBlocks.java index 448fb48e389..8861abd4803 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/MetadataBlocks.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/MetadataBlocks.java @@ -1,34 +1,33 @@ package edu.harvard.iq.dataverse.api; import edu.harvard.iq.dataverse.MetadataBlock; -import jakarta.ws.rs.GET; -import jakarta.ws.rs.Path; -import jakarta.ws.rs.Produces; +import jakarta.ws.rs.*; import jakarta.ws.rs.core.Response; -import static edu.harvard.iq.dataverse.util.json.JsonPrinter.brief; -import jakarta.ws.rs.PathParam; + +import java.util.List; + import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json; -import static edu.harvard.iq.dataverse.util.json.JsonPrinter.toJsonArray; /** * Api bean for managing metadata blocks. + * * @author michael */ @Path("metadatablocks") @Produces("application/json") public class MetadataBlocks extends AbstractApiBean { - + @GET - public Response list() { - return ok(metadataBlockSvc.listMetadataBlocks().stream().map(brief::json).collect(toJsonArray())); + public Response listMetadataBlocks(@QueryParam("onlyDisplayedOnCreate") boolean onlyDisplayedOnCreate, + @QueryParam("returnDatasetFieldTypes") boolean returnDatasetFieldTypes) { + List metadataBlocks = metadataBlockSvc.listMetadataBlocks(onlyDisplayedOnCreate); + return ok(json(metadataBlocks, returnDatasetFieldTypes, onlyDisplayedOnCreate)); } - + @Path("{identifier}") @GET - public Response getBlock( @PathParam("identifier") String idtf ) { + public Response getMetadataBlock(@PathParam("identifier") String idtf) { MetadataBlock b = findMetadataBlock(idtf); - - return (b != null ) ? ok(json(b)) : notFound("Can't find metadata block '" + idtf + "'"); + return (b != null) ? ok(json(b)) : notFound("Can't find metadata block '" + idtf + "'"); } - } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Metrics.java b/src/main/java/edu/harvard/iq/dataverse/api/Metrics.java index 7bb2570334b..452e5df9f9a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Metrics.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Metrics.java @@ -547,6 +547,98 @@ public Response getDownloadsPastDays(@Context UriInfo uriInfo, @PathParam("days" return ok(jsonObj); } + /** Accounts */ + + @GET + @Path("accounts") + public Response getAccountsAllTime(@Context UriInfo uriInfo) { + return getAccountsToMonth(uriInfo, MetricsUtil.getCurrentMonth()); + } + + @GET + @Path("accounts/toMonth/{yyyymm}") + public Response getAccountsToMonth(@Context UriInfo uriInfo, @PathParam("yyyymm") String yyyymm) { + + try { + errorIfUnrecongizedQueryParamPassed(uriInfo, new String[] { }); + } catch (IllegalArgumentException ia) { + return error(BAD_REQUEST, ia.getLocalizedMessage()); + } + + String metricName = "accountsToMonth"; + String sanitizedyyyymm = MetricsUtil.sanitizeYearMonthUserInput(yyyymm); + JsonObject jsonObj = MetricsUtil.stringToJsonObject(metricsSvc.returnUnexpiredCacheMonthly(metricName, sanitizedyyyymm, null, null)); + + if (null == jsonObj) { // run query and save + Long count; + try { + count = metricsSvc.accountsToMonth(sanitizedyyyymm); + } catch (ParseException e) { + return error(BAD_REQUEST, "Unable to parse supplied date: " + e.getLocalizedMessage()); + } + jsonObj = MetricsUtil.countToJson(count).build(); + metricsSvc.save(new Metric(metricName, sanitizedyyyymm, null, null, jsonObj.toString())); + } + + return ok(jsonObj); + } + + @GET + @Path("accounts/pastDays/{days}") + public Response getAccountsPastDays(@Context UriInfo uriInfo, @PathParam("days") int days) { + + try { + errorIfUnrecongizedQueryParamPassed(uriInfo, new String[] { }); + } catch (IllegalArgumentException ia) { + return error(BAD_REQUEST, ia.getLocalizedMessage()); + } + + String metricName = "accountsPastDays"; + + if (days < 1) { + return error(BAD_REQUEST, "Invalid parameter for number of days."); + } + + JsonObject jsonObj = MetricsUtil.stringToJsonObject(metricsSvc.returnUnexpiredCacheDayBased(metricName, String.valueOf(days), null, null)); + + if (null == jsonObj) { // run query and save + Long count = metricsSvc.accountsPastDays(days); + jsonObj = MetricsUtil.countToJson(count).build(); + metricsSvc.save(new Metric(metricName, String.valueOf(days), null, null, jsonObj.toString())); + } + + return ok(jsonObj); + } + + @GET + @Path("accounts/monthly") + @Produces("text/csv, application/json") + public Response getAccountsTimeSeries(@Context Request req, @Context UriInfo uriInfo) { + + try { + errorIfUnrecongizedQueryParamPassed(uriInfo, new String[] { }); + } catch (IllegalArgumentException ia) { + return error(BAD_REQUEST, ia.getLocalizedMessage()); + } + + String metricName = "accounts"; + JsonArray jsonArray = MetricsUtil.stringToJsonArray(metricsSvc.returnUnexpiredCacheAllTime(metricName, null, null)); + + if (null == jsonArray) { // run query and save + // Only handling published right now + jsonArray = metricsSvc.accountsTimeSeries(); + metricsSvc.save(new Metric(metricName, null, null, null, jsonArray.toString())); + } + + MediaType requestedType = getVariant(req, MediaType.valueOf(FileUtil.MIME_TYPE_CSV), MediaType.APPLICATION_JSON_TYPE); + if ((requestedType != null) && (requestedType.equals(MediaType.APPLICATION_JSON_TYPE))) { + return ok(jsonArray); + } + return ok(FileUtil.jsonArrayOfObjectsToCSV(jsonArray, MetricsUtil.DATE, MetricsUtil.COUNT), MediaType.valueOf(FileUtil.MIME_TYPE_CSV), "accounts.timeseries.csv"); + } + + /** MakeDataCount */ + @GET @Path("makeDataCount/{metric}") public Response getMakeDataCountMetricCurrentMonth(@Context UriInfo uriInfo, @PathParam("metric") String metricSupplied, @QueryParam("country") String country, @QueryParam("parentAlias") String parentAlias) { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Notifications.java b/src/main/java/edu/harvard/iq/dataverse/api/Notifications.java index 37c894d3071..df172f36973 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Notifications.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Notifications.java @@ -55,7 +55,6 @@ public Response getAllNotificationsForUser(@Context ContainerRequestContext crc) notificationObjectBuilder.add("id", notification.getId()); notificationObjectBuilder.add("type", type.toString()); /* FIXME - Re-add reasons for return if/when they are added to the notifications page. - if (Type.RETURNEDDS.equals(type) || Type.SUBMITTEDDS.equals(type)) { JsonArrayBuilder reasons = getReasonsForReturn(notification); for (JsonValue reason : reasons.build()) { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Pids.java b/src/main/java/edu/harvard/iq/dataverse/api/Pids.java index 534e42fd505..4ad57bceb58 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Pids.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Pids.java @@ -130,4 +130,41 @@ public Response deletePid(@Context ContainerRequestContext crc, @PathParam("id") } } + @GET + @AuthRequired + @Path("providers") + @Produces(MediaType.APPLICATION_JSON) + public Response getPidProviders(@Context ContainerRequestContext crc) throws WrappedResponse { + try { + getRequestAuthenticatedUserOrDie(crc); + } catch (WrappedResponse ex) { + return ex.getResponse(); + } + return ok(PidUtil.getProviders()); + } + + @GET + @AuthRequired + // The :.+ suffix allows PIDs with a / char to be entered w/o escaping + @Path("providers/{persistentId:.+}") + @Produces(MediaType.APPLICATION_JSON) + public Response getPidProviderId(@Context ContainerRequestContext crc, @PathParam("persistentId") String persistentId) throws WrappedResponse { + try { + getRequestAuthenticatedUserOrDie(crc); + } catch (WrappedResponse ex) { + return ex.getResponse(); + } + GlobalId globalId = PidUtil.parseAsGlobalID(persistentId); + if(globalId== null) { + return error(Response.Status.NOT_FOUND, "No provider found for PID"); + } else { + String providerId = globalId.getProviderId(); + if(PidUtil.getManagedProviderIds().contains(providerId)) { + return ok(globalId.getProviderId()); + } else { + return ok("PID recognized as an unmanaged " + globalId.getProtocol()); + } + } + } + } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Search.java b/src/main/java/edu/harvard/iq/dataverse/api/Search.java index 71e2865ca4d..6b9fcb38305 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Search.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Search.java @@ -1,10 +1,8 @@ package edu.harvard.iq.dataverse.api; -import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.api.auth.AuthRequired; import edu.harvard.iq.dataverse.search.SearchFields; -import edu.harvard.iq.dataverse.DataverseServiceBean; -import edu.harvard.iq.dataverse.DvObjectServiceBean; import edu.harvard.iq.dataverse.search.FacetCategory; import edu.harvard.iq.dataverse.search.FacetLabel; import edu.harvard.iq.dataverse.search.SolrSearchResult; @@ -16,7 +14,6 @@ import edu.harvard.iq.dataverse.search.SearchConstants; import edu.harvard.iq.dataverse.search.SearchException; import edu.harvard.iq.dataverse.search.SearchUtil; -import edu.harvard.iq.dataverse.search.SolrIndexServiceBean; import edu.harvard.iq.dataverse.search.SortBy; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import java.io.IOException; @@ -26,6 +23,7 @@ import java.util.Map; import java.util.logging.Logger; import jakarta.ejb.EJB; +import jakarta.inject.Inject; import jakarta.json.Json; import jakarta.json.JsonArrayBuilder; import jakarta.json.JsonObjectBuilder; @@ -51,10 +49,8 @@ public class Search extends AbstractApiBean { SearchServiceBean searchService; @EJB DataverseServiceBean dataverseService; - @EJB - DvObjectServiceBean dvObjectService; - @EJB - SolrIndexServiceBean SolrIndexService; + @Inject + DatasetVersionFilesServiceBean datasetVersionFilesServiceBean; @GET @AuthRequired @@ -179,7 +175,7 @@ public Response search( JsonArrayBuilder itemsArrayBuilder = Json.createArrayBuilder(); List solrSearchResults = solrQueryResponse.getSolrSearchResults(); for (SolrSearchResult solrSearchResult : solrSearchResults) { - itemsArrayBuilder.add(solrSearchResult.toJsonObject(showRelevance, showEntityIds, showApiUrls, metadataFields)); + itemsArrayBuilder.add(solrSearchResult.json(showRelevance, showEntityIds, showApiUrls, metadataFields, getDatasetFileCount(solrSearchResult))); } JsonObjectBuilder spelling_alternatives = Json.createObjectBuilder(); @@ -187,31 +183,32 @@ public Response search( spelling_alternatives.add(entry.getKey(), entry.getValue().toString()); } - JsonArrayBuilder facets = Json.createArrayBuilder(); - JsonObjectBuilder facetCategoryBuilder = Json.createObjectBuilder(); - for (FacetCategory facetCategory : solrQueryResponse.getFacetCategoryList()) { - JsonObjectBuilder facetCategoryBuilderFriendlyPlusData = Json.createObjectBuilder(); - JsonArrayBuilder facetLabelBuilderData = Json.createArrayBuilder(); - for (FacetLabel facetLabel : facetCategory.getFacetLabel()) { - JsonObjectBuilder countBuilder = Json.createObjectBuilder(); - countBuilder.add(facetLabel.getName(), facetLabel.getCount()); - facetLabelBuilderData.add(countBuilder); - } - facetCategoryBuilderFriendlyPlusData.add("friendly", facetCategory.getFriendlyName()); - facetCategoryBuilderFriendlyPlusData.add("labels", facetLabelBuilderData); - facetCategoryBuilder.add(facetCategory.getName(), facetCategoryBuilderFriendlyPlusData); - } - facets.add(facetCategoryBuilder); - JsonObjectBuilder value = Json.createObjectBuilder() .add("q", query) .add("total_count", solrQueryResponse.getNumResultsFound()) .add("start", solrQueryResponse.getResultsStart()) .add("spelling_alternatives", spelling_alternatives) .add("items", itemsArrayBuilder.build()); + if (showFacets) { + JsonArrayBuilder facets = Json.createArrayBuilder(); + JsonObjectBuilder facetCategoryBuilder = Json.createObjectBuilder(); + for (FacetCategory facetCategory : solrQueryResponse.getFacetCategoryList()) { + JsonObjectBuilder facetCategoryBuilderFriendlyPlusData = Json.createObjectBuilder(); + JsonArrayBuilder facetLabelBuilderData = Json.createArrayBuilder(); + for (FacetLabel facetLabel : facetCategory.getFacetLabel()) { + JsonObjectBuilder countBuilder = Json.createObjectBuilder(); + countBuilder.add(facetLabel.getName(), facetLabel.getCount()); + facetLabelBuilderData.add(countBuilder); + } + facetCategoryBuilderFriendlyPlusData.add("friendly", facetCategory.getFriendlyName()); + facetCategoryBuilderFriendlyPlusData.add("labels", facetLabelBuilderData); + facetCategoryBuilder.add(facetCategory.getName(), facetCategoryBuilderFriendlyPlusData); + } + facets.add(facetCategoryBuilder); value.add("facets", facets); } + value.add("count_in_response", solrSearchResults.size()); /** * @todo Returning the fq might be useful as a troubleshooting aid @@ -232,6 +229,15 @@ public Response search( } } + private Long getDatasetFileCount(SolrSearchResult solrSearchResult) { + DvObject dvObject = solrSearchResult.getEntity(); + if (dvObject.isInstanceofDataset()) { + DatasetVersion datasetVersion = ((Dataset) dvObject).getVersionFromId(solrSearchResult.getDatasetVersionId()); + return datasetVersionFilesServiceBean.getFileMetadataCount(datasetVersion); + } + return null; + } + private User getUser(ContainerRequestContext crc) throws WrappedResponse { User userToExecuteSearchAs = GuestUser.get(); try { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/TestApi.java b/src/main/java/edu/harvard/iq/dataverse/api/TestApi.java index 87be1f14e05..46747b50c29 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/TestApi.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/TestApi.java @@ -21,7 +21,7 @@ public class TestApi extends AbstractApiBean { @GET @Path("datasets/{id}/externalTools") - public Response getExternalToolsforFile(@PathParam("id") String idSupplied, @QueryParam("type") String typeSupplied) { + public Response getDatasetExternalToolsforFile(@PathParam("id") String idSupplied, @QueryParam("type") String typeSupplied) { ExternalTool.Type type; try { type = ExternalTool.Type.fromString(typeSupplied); @@ -44,6 +44,34 @@ public Response getExternalToolsforFile(@PathParam("id") String idSupplied, @Que return wr.getResponse(); } } + + @GET + @Path("datasets/{id}/externalTool/{toolId}") + public Response getExternalToolforDatasetById(@PathParam("id") String idSupplied, @PathParam("toolId") String toolId, @QueryParam("type") String typeSupplied) { + ExternalTool.Type type; + try { + type = ExternalTool.Type.fromString(typeSupplied); + } catch (IllegalArgumentException ex) { + return error(BAD_REQUEST, ex.getLocalizedMessage()); + } + Dataset dataset; + try { + dataset = findDatasetOrDie(idSupplied); + JsonArrayBuilder tools = Json.createArrayBuilder(); + List datasetTools = externalToolService.findDatasetToolsByType(type); + for (ExternalTool tool : datasetTools) { + ApiToken apiToken = externalToolService.getApiToken(getRequestApiKey()); + ExternalToolHandler externalToolHandler = new ExternalToolHandler(tool, dataset, apiToken, null); + JsonObjectBuilder toolToJson = externalToolService.getToolAsJsonWithQueryParameters(externalToolHandler); + if (tool.getId().toString().equals(toolId)) { + return ok(toolToJson); + } + } + } catch (WrappedResponse wr) { + return wr.getResponse(); + } + return error(BAD_REQUEST, "Could not find external tool with id of " + toolId); + } @Path("files/{id}/externalTools") @GET @@ -71,5 +99,31 @@ public Response getExternalToolsForFile(@PathParam("id") String idSupplied, @Que return wr.getResponse(); } } + + @Path("files/{id}/externalTool/{toolId}") + @GET + public Response getExternalToolForFileById(@PathParam("id") String idSupplied, @QueryParam("type") String typeSupplied, @PathParam("toolId") String toolId) { + ExternalTool.Type type; + try { + type = ExternalTool.Type.fromString(typeSupplied); + } catch (IllegalArgumentException ex) { + return error(BAD_REQUEST, ex.getLocalizedMessage()); + } + try { + DataFile dataFile = findDataFileOrDie(idSupplied); + List datasetTools = externalToolService.findFileToolsByTypeAndContentType(type, dataFile.getContentType()); + for (ExternalTool tool : datasetTools) { + ApiToken apiToken = externalToolService.getApiToken(getRequestApiKey()); + ExternalToolHandler externalToolHandler = new ExternalToolHandler(tool, dataFile, apiToken, dataFile.getFileMetadata(), null); + JsonObjectBuilder toolToJson = externalToolService.getToolAsJsonWithQueryParameters(externalToolHandler); + if (externalToolService.meetsRequirements(tool, dataFile) && tool.getId().toString().equals(toolId)) { + return ok(toolToJson); + } + } + return error(BAD_REQUEST, "Could not find external tool with id of " + toolId); + } catch (WrappedResponse wr) { + return wr.getResponse(); + } + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/TestIngest.java b/src/main/java/edu/harvard/iq/dataverse/api/TestIngest.java index 05ba150df8e..add43ea2091 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/TestIngest.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/TestIngest.java @@ -100,7 +100,7 @@ public String datafile(@QueryParam("fileName") String fileName, @QueryParam("fil TabularDataIngest tabDataIngest = null; try { - tabDataIngest = ingestPlugin.read(fileInputStream, null); + tabDataIngest = ingestPlugin.read(fileInputStream, false, null); } catch (IOException ingestEx) { output = output.concat("Caught an exception trying to ingest file " + fileName + ": " + ingestEx.getLocalizedMessage()); return output; diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Users.java b/src/main/java/edu/harvard/iq/dataverse/api/Users.java index 791fc7aa774..1f5430340c2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Users.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Users.java @@ -234,7 +234,7 @@ public Response getTraces(@Context ContainerRequestContext crc, @PathParam("iden @AuthRequired @Path("{identifier}/traces/{element}") @Produces("text/csv, application/json") - public Response getTraces(@Context ContainerRequestContext crc, @Context Request req, @PathParam("identifier") String identifier, @PathParam("element") String element) { + public Response getTracesElement(@Context ContainerRequestContext crc, @Context Request req, @PathParam("identifier") String identifier, @PathParam("element") String element) { try { AuthenticatedUser userToQuery = authSvc.getAuthenticatedUser(identifier); if(!elements.contains(element)) { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/WorkflowsAdmin.java b/src/main/java/edu/harvard/iq/dataverse/api/WorkflowsAdmin.java index 8d5024c1c14..15478aacff7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/WorkflowsAdmin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/WorkflowsAdmin.java @@ -111,9 +111,9 @@ public Response deleteDefault(@PathParam("triggerType") String triggerType) { } } - @Path("/{identifier}") + @Path("/{id}") @GET - public Response getWorkflow(@PathParam("identifier") String identifier ) { + public Response getWorkflow(@PathParam("id") String identifier ) { try { long idtf = Long.parseLong(identifier); return workflows.getWorkflow(idtf) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/CollectionDepositManagerImpl.java b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/CollectionDepositManagerImpl.java index 5bc50903be8..a81848bd7af 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/CollectionDepositManagerImpl.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/CollectionDepositManagerImpl.java @@ -6,6 +6,7 @@ import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.Dataverse; import edu.harvard.iq.dataverse.DataverseServiceBean; +import edu.harvard.iq.dataverse.DvObjectServiceBean; import edu.harvard.iq.dataverse.EjbDataverseEngine; import edu.harvard.iq.dataverse.PermissionServiceBean; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; @@ -14,6 +15,7 @@ import edu.harvard.iq.dataverse.api.imports.ImportGenericServiceBean; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.impl.CreateNewDatasetCommand; +import edu.harvard.iq.dataverse.pidproviders.PidProvider; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.ConstraintViolationUtil; import java.util.logging.Level; @@ -44,6 +46,8 @@ public class CollectionDepositManagerImpl implements CollectionDepositManager { @EJB DatasetServiceBean datasetService; @EJB + DvObjectServiceBean dvObjectService; + @EJB PermissionServiceBean permissionService; @Inject SwordAuth swordAuth; @@ -96,13 +100,10 @@ public DepositReceipt createNew(String collectionUri, Deposit deposit, AuthCrede Dataset dataset = new Dataset(); dataset.setOwner(dvThatWillOwnDataset); - String nonNullDefaultIfKeyNotFound = ""; - String protocol = settingsService.getValueForKey(SettingsServiceBean.Key.Protocol, nonNullDefaultIfKeyNotFound); - String authority = settingsService.getValueForKey(SettingsServiceBean.Key.Authority, nonNullDefaultIfKeyNotFound); - - dataset.setProtocol(protocol); - dataset.setAuthority(authority); - //Wait until the create command before actually getting an identifier + PidProvider pidProvider = dvObjectService.getEffectivePidGenerator(dataset); + dataset.setProtocol(pidProvider.getProtocol()); + dataset.setAuthority(pidProvider.getAuthority()); + //Wait until the create command before actually getting an identifier logger.log(Level.FINE, "DS Deposit identifier: {0}", dataset.getIdentifier()); AbstractCreateDatasetCommand createDatasetCommand = new CreateNewDatasetCommand(dataset, dvReq); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java index 73a83035fc5..85d4868605d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java @@ -828,9 +828,9 @@ private HashSet processGeoBndBox(XMLStreamReader xmlr) throws XMLStrea } else if (xmlr.getLocalName().equals("eastBL")) { addToSet(set,"eastLongitude", parseText(xmlr)); } else if (xmlr.getLocalName().equals("southBL")) { - addToSet(set,"southLongitude", parseText(xmlr)); + addToSet(set,"southLatitude", parseText(xmlr)); } else if (xmlr.getLocalName().equals("northBL")) { - addToSet(set,"northLongitude", parseText(xmlr)); + addToSet(set,"northLatitude", parseText(xmlr)); } } else if (event == XMLStreamConstants.END_ELEMENT) { if (xmlr.getLocalName().equals("geoBndBox")) break; diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java index f7a6cf54dd5..6068ec45e4f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java @@ -2,7 +2,6 @@ import com.google.gson.Gson; -import edu.harvard.iq.dataverse.DOIServiceBean; import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetFieldCompoundValue; import edu.harvard.iq.dataverse.DatasetFieldConstant; @@ -11,13 +10,14 @@ import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.ForeignMetadataFieldMapping; import edu.harvard.iq.dataverse.ForeignMetadataFormatMapping; -import edu.harvard.iq.dataverse.HandlenetServiceBean; import edu.harvard.iq.dataverse.MetadataBlockServiceBean; import edu.harvard.iq.dataverse.api.dto.*; import edu.harvard.iq.dataverse.api.dto.FieldDTO; import edu.harvard.iq.dataverse.api.dto.MetadataBlockDTO; import edu.harvard.iq.dataverse.license.LicenseServiceBean; -import edu.harvard.iq.dataverse.pidproviders.PermaLinkPidProviderServiceBean; +import edu.harvard.iq.dataverse.pidproviders.doi.AbstractDOIProvider; +import edu.harvard.iq.dataverse.pidproviders.handle.HandlePidProvider; +import edu.harvard.iq.dataverse.pidproviders.perma.PermaLinkPidProvider; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.StringUtil; import edu.harvard.iq.dataverse.util.json.JsonParseException; @@ -352,7 +352,7 @@ private String getOtherIdFromDTO(DatasetVersionDTO datasetVersionDTO) { if (!otherIds.isEmpty()) { // We prefer doi or hdl identifiers like "doi:10.7910/DVN/1HE30F" for (String otherId : otherIds) { - if (otherId.startsWith(DOIServiceBean.DOI_PROTOCOL) || otherId.startsWith(HandlenetServiceBean.HDL_PROTOCOL) || otherId.startsWith(DOIServiceBean.DOI_RESOLVER_URL) || otherId.startsWith(HandlenetServiceBean.HDL_RESOLVER_URL) || otherId.startsWith(DOIServiceBean.HTTP_DOI_RESOLVER_URL) || otherId.startsWith(HandlenetServiceBean.HTTP_HDL_RESOLVER_URL) || otherId.startsWith(DOIServiceBean.DXDOI_RESOLVER_URL) || otherId.startsWith(DOIServiceBean.HTTP_DXDOI_RESOLVER_URL)) { + if (otherId.startsWith(AbstractDOIProvider.DOI_PROTOCOL) || otherId.startsWith(HandlePidProvider.HDL_PROTOCOL) || otherId.startsWith(AbstractDOIProvider.DOI_RESOLVER_URL) || otherId.startsWith(HandlePidProvider.HDL_RESOLVER_URL) || otherId.startsWith(AbstractDOIProvider.HTTP_DOI_RESOLVER_URL) || otherId.startsWith(HandlePidProvider.HTTP_HDL_RESOLVER_URL) || otherId.startsWith(AbstractDOIProvider.DXDOI_RESOLVER_URL) || otherId.startsWith(AbstractDOIProvider.HTTP_DXDOI_RESOLVER_URL)) { return otherId; } } @@ -361,7 +361,7 @@ private String getOtherIdFromDTO(DatasetVersionDTO datasetVersionDTO) { try { HandleResolver hr = new HandleResolver(); hr.resolveHandle(otherId); - return HandlenetServiceBean.HDL_PROTOCOL + ":" + otherId; + return HandlePidProvider.HDL_PROTOCOL + ":" + otherId; } catch (HandleException e) { logger.fine("Not a valid handle: " + e.toString()); } @@ -388,7 +388,7 @@ public String reassignIdentifierAsGlobalId(String identifierString, DatasetDTO d String protocol = identifierString.substring(0, index1); - if (DOIServiceBean.DOI_PROTOCOL.equals(protocol) || HandlenetServiceBean.HDL_PROTOCOL.equals(protocol) || PermaLinkPidProviderServiceBean.PERMA_PROTOCOL.equals(protocol)) { + if (AbstractDOIProvider.DOI_PROTOCOL.equals(protocol) || HandlePidProvider.HDL_PROTOCOL.equals(protocol) || PermaLinkPidProvider.PERMA_PROTOCOL.equals(protocol)) { logger.fine("Processing hdl:- or doi:- or perma:-style identifier : "+identifierString); } else if ("http".equalsIgnoreCase(protocol) || "https".equalsIgnoreCase(protocol)) { @@ -396,21 +396,21 @@ public String reassignIdentifierAsGlobalId(String identifierString, DatasetDTO d // We also recognize global identifiers formatted as global resolver URLs: //ToDo - refactor index1 always has -1 here so that we can use index1+1 later //ToDo - single map of protocol/url, are all three cases the same then? - if (identifierString.startsWith(HandlenetServiceBean.HDL_RESOLVER_URL) || identifierString.startsWith(HandlenetServiceBean.HTTP_HDL_RESOLVER_URL)) { + if (identifierString.startsWith(HandlePidProvider.HDL_RESOLVER_URL) || identifierString.startsWith(HandlePidProvider.HTTP_HDL_RESOLVER_URL)) { logger.fine("Processing Handle identifier formatted as a resolver URL: "+identifierString); - protocol = HandlenetServiceBean.HDL_PROTOCOL; - index1 = (identifierString.startsWith(HandlenetServiceBean.HDL_RESOLVER_URL)) ? HandlenetServiceBean.HDL_RESOLVER_URL.length() - 1 : HandlenetServiceBean.HTTP_HDL_RESOLVER_URL.length() - 1; + protocol = HandlePidProvider.HDL_PROTOCOL; + index1 = (identifierString.startsWith(HandlePidProvider.HDL_RESOLVER_URL)) ? HandlePidProvider.HDL_RESOLVER_URL.length() - 1 : HandlePidProvider.HTTP_HDL_RESOLVER_URL.length() - 1; index2 = identifierString.indexOf("/", index1 + 1); - } else if (identifierString.startsWith(DOIServiceBean.DOI_RESOLVER_URL) || identifierString.startsWith(DOIServiceBean.HTTP_DOI_RESOLVER_URL) || identifierString.startsWith(DOIServiceBean.DXDOI_RESOLVER_URL) || identifierString.startsWith(DOIServiceBean.HTTP_DXDOI_RESOLVER_URL)) { + } else if (identifierString.startsWith(AbstractDOIProvider.DOI_RESOLVER_URL) || identifierString.startsWith(AbstractDOIProvider.HTTP_DOI_RESOLVER_URL) || identifierString.startsWith(AbstractDOIProvider.DXDOI_RESOLVER_URL) || identifierString.startsWith(AbstractDOIProvider.HTTP_DXDOI_RESOLVER_URL)) { logger.fine("Processing DOI identifier formatted as a resolver URL: "+identifierString); - protocol = DOIServiceBean.DOI_PROTOCOL; - identifierString = identifierString.replace(DOIServiceBean.DXDOI_RESOLVER_URL, DOIServiceBean.DOI_RESOLVER_URL); - identifierString = identifierString.replace(DOIServiceBean.HTTP_DXDOI_RESOLVER_URL, DOIServiceBean.HTTP_DOI_RESOLVER_URL); - index1 = (identifierString.startsWith(DOIServiceBean.DOI_RESOLVER_URL)) ? DOIServiceBean.DOI_RESOLVER_URL.length() - 1 : DOIServiceBean.HTTP_DOI_RESOLVER_URL.length() - 1; + protocol = AbstractDOIProvider.DOI_PROTOCOL; + identifierString = identifierString.replace(AbstractDOIProvider.DXDOI_RESOLVER_URL, AbstractDOIProvider.DOI_RESOLVER_URL); + identifierString = identifierString.replace(AbstractDOIProvider.HTTP_DXDOI_RESOLVER_URL, AbstractDOIProvider.HTTP_DOI_RESOLVER_URL); + index1 = (identifierString.startsWith(AbstractDOIProvider.DOI_RESOLVER_URL)) ? AbstractDOIProvider.DOI_RESOLVER_URL.length() - 1 : AbstractDOIProvider.HTTP_DOI_RESOLVER_URL.length() - 1; index2 = identifierString.indexOf("/", index1 + 1); - } else if (identifierString.startsWith(PermaLinkPidProviderServiceBean.PERMA_RESOLVER_URL + Dataset.TARGET_URL)) { - protocol = PermaLinkPidProviderServiceBean.PERMA_PROTOCOL; - index1 = PermaLinkPidProviderServiceBean.PERMA_RESOLVER_URL.length() + + Dataset.TARGET_URL.length() - 1; + } else if (identifierString.startsWith(PermaLinkPidProvider.PERMA_RESOLVER_URL + Dataset.TARGET_URL)) { + protocol = PermaLinkPidProvider.PERMA_PROTOCOL; + index1 = PermaLinkPidProvider.PERMA_RESOLVER_URL.length() + + Dataset.TARGET_URL.length() - 1; index2 = identifierString.indexOf("/", index1 + 1); } else { logger.warning("HTTP Url in supplied as the identifier is neither a Handle nor DOI resolver: "+identifierString); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java index c17ba909230..39977190691 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java @@ -38,6 +38,8 @@ import edu.harvard.iq.dataverse.util.json.JsonParser; import edu.harvard.iq.dataverse.util.json.JsonUtil; import edu.harvard.iq.dataverse.license.LicenseServiceBean; +import edu.harvard.iq.dataverse.pidproviders.PidUtil; + import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -421,8 +423,9 @@ public JsonObjectBuilder doImport(DataverseRequest dataverseRequest, Dataverse o // For ImportType.NEW, if the user supplies a global identifier, and it's not a protocol // we support, it will be rejected. + if (importType.equals(ImportType.NEW)) { - if (ds.getGlobalId().asString() != null && !ds.getProtocol().equals(settingsService.getValueForKey(SettingsServiceBean.Key.Protocol, ""))) { + if (ds.getGlobalId().asString() != null && !PidUtil.getPidProvider(ds.getGlobalId().getProviderId()).canManagePID()) { throw new ImportException("Could not register id " + ds.getGlobalId().asString() + ", protocol not supported"); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/AuthFilter.java b/src/main/java/edu/harvard/iq/dataverse/authorization/AuthFilter.java index a2cf3082ae7..c93a1496c17 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/AuthFilter.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/AuthFilter.java @@ -29,9 +29,7 @@ public void init(FilterConfig filterConfig) throws ServletException { logger.info(AuthFilter.class.getName() + "initialized. filterConfig.getServletContext().getServerInfo(): " + filterConfig.getServletContext().getServerInfo()); try { - String glassfishLogsDirectory = "logs"; - - FileHandler logFile = new FileHandler(".." + File.separator + glassfishLogsDirectory + File.separator + "authfilter.log"); + FileHandler logFile = new FileHandler( System.getProperty("com.sun.aas.instanceRoot") + File.separator + "logs" + File.separator + "authfilter.log"); SimpleFormatter formatterTxt = new SimpleFormatter(); logFile.setFormatter(formatterTxt); logger.addHandler(logFile); diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java index 1c0f5010059..4a8fb123fd4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java @@ -307,11 +307,9 @@ public AuthenticatedUser getUpdateAuthenticatedUser( String authenticationProvid if (user != null && !user.isDeactivated()) { user = userService.updateLastLogin(user); } - + if ( user == null ) { throw new IllegalStateException("Authenticated user does not exist. The functionality to support creating one at this point in authentication has been removed."); - //return createAuthenticatedUser( - // new UserRecordIdentifier(authenticationProviderId, resp.getUserId()), resp.getUserId(), resp.getUserDisplayInfo(), true ); } else { if (BuiltinAuthenticationProvider.PROVIDER_ID.equals(user.getAuthenticatedUserLookup().getAuthenticationProviderId())) { return user; diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2LoginBackingBean.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2LoginBackingBean.java index 0fd0852b4df..8f3dc07fdea 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2LoginBackingBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2LoginBackingBean.java @@ -1,6 +1,7 @@ package edu.harvard.iq.dataverse.authorization.providers.oauth2; import edu.harvard.iq.dataverse.DataverseSession; +import edu.harvard.iq.dataverse.UserServiceBean; import edu.harvard.iq.dataverse.authorization.AuthenticationProvider; import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; import edu.harvard.iq.dataverse.authorization.UserRecordIdentifier; @@ -65,6 +66,9 @@ public class OAuth2LoginBackingBean implements Serializable { @EJB SystemConfig systemConfig; + @EJB + UserServiceBean userService; + @Inject DataverseSession session; @@ -128,6 +132,7 @@ public void exchangeCodeForToken() throws IOException { } else { // login the user and redirect to HOME of intended page (if any). // setUser checks for deactivated users. + dvUser = userService.updateLastLogin(dvUser); session.setUser(dvUser); final OAuth2TokenData tokenData = oauthUser.getTokenData(); if (tokenData != null) { diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java b/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java index b307c655798..d6d3e0317ed 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java @@ -16,6 +16,7 @@ import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.json.JsonPrinter; import static edu.harvard.iq.dataverse.util.StringUtil.nonEmpty; + import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder; import java.io.Serializable; import java.sql.Timestamp; @@ -42,6 +43,7 @@ import jakarta.persistence.PostLoad; import jakarta.persistence.PrePersist; import jakarta.persistence.Transient; +import jakarta.validation.constraints.Min; import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.NotNull; @@ -146,6 +148,10 @@ public class AuthenticatedUser implements User, Serializable { @Transient private Set mutedNotificationsSet = new HashSet<>(); + @Column(nullable=false) + @Min(value = 1, message = "Rate Limit Tier must be greater than 0.") + private int rateLimitTier = 1; + @PrePersist void prePersist() { mutedNotifications = Type.toStringValue(mutedNotificationsSet); @@ -397,6 +403,13 @@ public void setDeactivatedTime(Timestamp deactivatedTime) { this.deactivatedTime = deactivatedTime; } + public int getRateLimitTier() { + return rateLimitTier; + } + public void setRateLimitTier(int rateLimitTier) { + this.rateLimitTier = rateLimitTier; + } + @OneToOne(mappedBy = "authenticatedUser") private AuthenticatedUserLookup authenticatedUserLookup; @@ -435,7 +448,6 @@ public void setShibIdentityProvider(String shibIdentityProvider) { public JsonObjectBuilder toJson() { //JsonObjectBuilder authenicatedUserJson = Json.createObjectBuilder(); - NullSafeJsonBuilder authenicatedUserJson = NullSafeJsonBuilder.jsonObjectBuilder(); authenicatedUserJson.add("id", this.id); diff --git a/src/main/java/edu/harvard/iq/dataverse/batch/jobs/importer/filesystem/FileRecordWriter.java b/src/main/java/edu/harvard/iq/dataverse/batch/jobs/importer/filesystem/FileRecordWriter.java index ba34a3d1ed1..af1e9c6a294 100644 --- a/src/main/java/edu/harvard/iq/dataverse/batch/jobs/importer/filesystem/FileRecordWriter.java +++ b/src/main/java/edu/harvard/iq/dataverse/batch/jobs/importer/filesystem/FileRecordWriter.java @@ -33,6 +33,7 @@ import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.impl.UpdateDatasetVersionCommand; +import edu.harvard.iq.dataverse.pidproviders.PidProvider; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.FileUtil; @@ -58,7 +59,6 @@ import java.util.logging.Level; import java.util.logging.Logger; import jakarta.servlet.http.HttpServletRequest; -import edu.harvard.iq.dataverse.GlobalIdServiceBean; @Named @Dependent @@ -360,31 +360,22 @@ private DataFile createPackageDataFile(List files) { if (commandEngine.getContext().systemConfig().isFilePIDsEnabledForCollection(dataset.getOwner())) { - GlobalIdServiceBean idServiceBean = GlobalIdServiceBean.getBean(packageFile.getProtocol(), commandEngine.getContext()); + PidProvider pidProvider = commandEngine.getContext().dvObjects().getEffectivePidGenerator(dataset); if (packageFile.getIdentifier() == null || packageFile.getIdentifier().isEmpty()) { - packageFile.setIdentifier(idServiceBean.generateDataFileIdentifier(packageFile)); - } - String nonNullDefaultIfKeyNotFound = ""; - String protocol = commandEngine.getContext().settings().getValueForKey(SettingsServiceBean.Key.Protocol, nonNullDefaultIfKeyNotFound); - String authority = commandEngine.getContext().settings().getValueForKey(SettingsServiceBean.Key.Authority, nonNullDefaultIfKeyNotFound); - if (packageFile.getProtocol() == null) { - packageFile.setProtocol(protocol); - } - if (packageFile.getAuthority() == null) { - packageFile.setAuthority(authority); + pidProvider.generatePid(packageFile); } if (!packageFile.isIdentifierRegistered()) { String doiRetString = ""; - idServiceBean = GlobalIdServiceBean.getBean(commandEngine.getContext()); + try { - doiRetString = idServiceBean.createIdentifier(packageFile); + doiRetString = pidProvider.createIdentifier(packageFile); } catch (Throwable e) { } // Check return value to make sure registration succeeded - if (!idServiceBean.registerWhenPublished() && doiRetString.contains(packageFile.getIdentifier())) { + if (!pidProvider.registerWhenPublished() && doiRetString.contains(packageFile.getIdentifier())) { packageFile.setIdentifierRegistered(true); packageFile.setGlobalIdCreateTime(new Date()); } diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/AbstractRemoteOverlayAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/AbstractRemoteOverlayAccessIO.java index 10ff68a56f3..18269f6970e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/AbstractRemoteOverlayAccessIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/AbstractRemoteOverlayAccessIO.java @@ -334,11 +334,5 @@ protected String getStoragePath() throws IOException { logger.fine("fullStoragePath: " + fullStoragePath); return fullStoragePath; } - - public static boolean isNotDataverseAccessible(String storeId) { - return Boolean.parseBoolean(StorageIO.getConfigParamForDriver(storeId, FILES_NOT_ACCESSIBLE_BY_DATAVERSE)); - } - - } \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java index a1bcbe49327..bc4c69390cf 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java @@ -270,6 +270,10 @@ public static StorageIO createNewStorageIO(T dvObject, S logger.warning("Could not find storage driver for: " + storageTag); throw new IOException("createDataAccessObject: Unsupported storage method " + storageDriverId); } + if (storageIO == null) { + logger.warning("Could not find storage driver for: " + storageTag); + throw new IOException("createDataAccessObject: Unsupported storage method " + storageDriverId); + } // Note: All storageIO classes must assure that dvObject instances' storageIdentifiers are prepended with // the :// + any additional storageIO type information required (e.g. the bucketname for s3/swift) // This currently happens when the storageIO is opened for write access diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/FileAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/FileAccessIO.java index f2a1312a150..26637ec5742 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/FileAccessIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/FileAccessIO.java @@ -120,7 +120,8 @@ public void open (DataAccessOption... options) throws IOException { && dataFile.getContentType().equals("text/tab-separated-values") && dataFile.isTabularData() && dataFile.getDataTable() != null - && (!this.noVarHeader())) { + && (!this.noVarHeader()) + && (!dataFile.getDataTable().isStoredWithVariableHeader())) { List datavariables = dataFile.getDataTable().getDataVariables(); String varHeaderLine = generateVariableHeader(datavariables); diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java index 7a6809cb2ff..3bf2107e52b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java @@ -171,24 +171,25 @@ private static String findMatchingEndpoint(String path, String[] allowedEndpoint } protected void validatePath(String relPath) throws IOException { - if (isManaged()) { - if (!usesStandardNamePattern(relPath)) { - throw new IOException("Unacceptable identifier pattern in submitted identifier: " + relPath); - } - } else { - try { - String endpoint = findMatchingEndpoint(relPath, allowedEndpoints); - logger.fine(endpoint + " " + relPath); - - if (endpoint == null || !Paths.get(endpoint, relPath).normalize().startsWith(endpoint)) { - throw new IOException( - "storageidentifier doesn't start with one of " + this.driverId + "'s allowed endpoints"); + if (dvObject != null && dvObject.isInstanceofDataFile()) { + if (isManaged()) { + if (!usesStandardNamePattern(relPath)) { + throw new IOException("Unacceptable identifier pattern in submitted identifier: " + relPath); + } + } else { + try { + String endpoint = findMatchingEndpoint(relPath, allowedEndpoints); + logger.fine(endpoint + " " + relPath); + + if (endpoint == null || !Paths.get(endpoint, relPath).normalize().startsWith(endpoint)) { + throw new IOException("storageidentifier doesn't start with one of " + this.driverId + + "'s allowed endpoints"); + } + } catch (InvalidPathException e) { + throw new IOException("Could not interpret storageidentifier in globus store " + this.driverId); } - } catch (InvalidPathException e) { - throw new IOException("Could not interpret storageidentifier in globus store " + this.driverId); } } - } // Call the Globus API to get the file size @@ -214,7 +215,7 @@ public long retrieveSizeFromMedia() { JsonArray dataArray = responseJson.getJsonArray("DATA"); if (dataArray != null && dataArray.size() != 0) { //File found - return (long) responseJson.getJsonArray("DATA").getJsonObject(0).getInt("size"); + return (long) responseJson.getJsonArray("DATA").getJsonObject(0).getJsonNumber("size").longValueExact(); } } else { logger.warning("Response from " + get.getURI().toString() + " was " @@ -239,7 +240,7 @@ public long retrieveSizeFromMedia() { public InputStream getInputStream() throws IOException { //Currently only supported when using an S3 store with the Globus S3Connector. //ToDo: Support when using a managed Globus endpoint that supports http access - if(!AbstractRemoteOverlayAccessIO.isNotDataverseAccessible(endpoint)) { + if(StorageIO.isDataverseAccessible(endpoint)) { return baseStore.getInputStream(); } else { throw new IOException("Not implemented"); @@ -450,8 +451,12 @@ public void open(DataAccessOption... options) throws IOException { this.setSize(retrieveSizeFromMedia()); } // Only applies for the S3 Connector case (where we could have run an ingest) - if (dataFile.getContentType() != null && dataFile.getContentType().equals("text/tab-separated-values") - && dataFile.isTabularData() && dataFile.getDataTable() != null && (!this.noVarHeader())) { + if (dataFile.getContentType() != null + && dataFile.getContentType().equals("text/tab-separated-values") + && dataFile.isTabularData() + && dataFile.getDataTable() != null + && (!this.noVarHeader()) + && (!dataFile.getDataTable().isStoredWithVariableHeader())) { List datavariables = dataFile.getDataTable().getDataVariables(); String varHeaderLine = generateVariableHeader(datavariables); diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java index 2de37174a3b..2435e3f778a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java @@ -35,21 +35,21 @@ import javax.imageio.stream.ImageOutputStream; import edu.harvard.iq.dataverse.DataFile; +import edu.harvard.iq.dataverse.DataFileServiceBean; import edu.harvard.iq.dataverse.util.FileUtil; import edu.harvard.iq.dataverse.util.SystemConfig; import java.io.ByteArrayOutputStream; -import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.OutputStream; import java.nio.channels.Channel; import java.nio.channels.Channels; -import java.nio.channels.FileChannel; import java.nio.channels.ReadableByteChannel; import java.nio.channels.WritableByteChannel; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.logging.Level; import java.util.logging.Logger; + +import jakarta.enterprise.inject.spi.CDI; import org.apache.commons.io.IOUtils; //import org.primefaces.util.Base64; import java.util.Base64; @@ -110,6 +110,12 @@ private static boolean isThumbnailAvailable(StorageIO storageIO, int s return false; } + // check if thumbnail generation failed: + if (file.isPreviewImageFail()) { + logger.fine("Thumbnail failed to be generated for "+ file.getId()); + return false; + } + if (isThumbnailCached(storageIO, size)) { logger.fine("Found cached thumbnail for " + file.getId()); return true; @@ -119,22 +125,23 @@ private static boolean isThumbnailAvailable(StorageIO storageIO, int s } private static boolean generateThumbnail(DataFile file, StorageIO storageIO, int size) { - logger.log(Level.FINE, (file.isPreviewImageFail() ? "Not trying" : "Trying") + " to generate thumbnail, file id: " + file.getId()); + logger.fine((file.isPreviewImageFail() ? "Not trying" : "Trying") + " to generate thumbnail, file id: " + file.getId()); + boolean thumbnailGenerated = false; // Don't try to generate if there have been failures: if (!file.isPreviewImageFail()) { - boolean thumbnailGenerated = false; if (file.getContentType().substring(0, 6).equalsIgnoreCase("image/")) { thumbnailGenerated = generateImageThumbnail(storageIO, size); } else if (file.getContentType().equalsIgnoreCase("application/pdf")) { thumbnailGenerated = generatePDFThumbnail(storageIO, size); } if (!thumbnailGenerated) { + file.setPreviewImageFail(true); + file.setPreviewImageAvailable(false); logger.fine("No thumbnail generated for " + file.getId()); } - return thumbnailGenerated; } - return false; + return thumbnailGenerated; } // Note that this method works on ALL file types for which thumbnail @@ -165,15 +172,30 @@ public static InputStreamIO getImageThumbnailAsInputStream(StorageIO s return null; } int cachedThumbnailSize = (int) storageIO.getAuxObjectSize(THUMBNAIL_SUFFIX + size); + InputStreamIO inputStreamIO = cachedThumbnailSize > 0 ? new InputStreamIO(cachedThumbnailInputStream, cachedThumbnailSize) : null; - InputStreamIO inputStreamIO = new InputStreamIO(cachedThumbnailInputStream, cachedThumbnailSize); + if (inputStreamIO != null) { + inputStreamIO.setMimeType(THUMBNAIL_MIME_TYPE); - inputStreamIO.setMimeType(THUMBNAIL_MIME_TYPE); - - String fileName = storageIO.getFileName(); - if (fileName != null) { - fileName = fileName.replaceAll("\\.[^\\.]*$", THUMBNAIL_FILE_EXTENSION); - inputStreamIO.setFileName(fileName); + String fileName = storageIO.getFileName(); + if (fileName != null) { + fileName = fileName.replaceAll("\\.[^\\.]*$", THUMBNAIL_FILE_EXTENSION); + inputStreamIO.setFileName(fileName); + } + } else { + if (storageIO.getDataFile() != null && cachedThumbnailSize == 0) { + // We found an older 0 length thumbnail. Newer image uploads will not have this issue. + // Once cleaned up, this thumbnail will no longer have this issue + logger.warning("Cleaning up zero sized thumbnail ID: "+ storageIO.getDataFile().getId()); + storageIO.getDataFile().setPreviewImageFail(true); + storageIO.getDataFile().setPreviewImageAvailable(false); + DataFileServiceBean datafileService = CDI.current().select(DataFileServiceBean.class).get(); + datafileService.save(storageIO.getDataFile()); + + // Now that we have marked this File as a thumbnail failure, + // no reason not to try and delete this 0-size cache here: + storageIO.deleteAuxObject(THUMBNAIL_SUFFIX + size); + } } return inputStreamIO; } catch (Exception ioex) { @@ -208,6 +230,7 @@ private static boolean generatePDFThumbnail(StorageIO storageIO, int s // will run the ImageMagick on it, and will save its output in another temp // file, and will save it as an "auxiliary" file via the driver. boolean tempFilesRequired = false; + File tempFile = null; try { Path pdfFilePath = storageIO.getFileSystemPath(); @@ -225,7 +248,7 @@ private static boolean generatePDFThumbnail(StorageIO storageIO, int s } if (tempFilesRequired) { - InputStream inputStream = null; + InputStream inputStream = null; try { storageIO.open(); inputStream = storageIO.getInputStream(); @@ -234,12 +257,11 @@ private static boolean generatePDFThumbnail(StorageIO storageIO, int s return false; } - File tempFile; OutputStream outputStream = null; try { tempFile = File.createTempFile("tempFileToRescale", ".tmp"); outputStream = new FileOutputStream(tempFile); - //Reads/transfers all bytes from the input stream to the output stream. + //Reads/transfers all bytes from the input stream to the output stream. inputStream.transferTo(outputStream); } catch (IOException ioex) { logger.warning("GenerateImageThumb: failed to save pdf bytes in a temporary file."); @@ -270,6 +292,12 @@ private static boolean generatePDFThumbnail(StorageIO storageIO, int s logger.warning("failed to save generated pdf thumbnail, as AUX file " + THUMBNAIL_SUFFIX + size + "!"); return false; } + finally { + try { + tempFile.delete(); + } + catch (Exception e) {} + } } return true; @@ -301,6 +329,7 @@ private static boolean generateImageThumbnail(StorageIO storageIO, int private static boolean generateImageThumbnailFromInputStream(StorageIO storageIO, int size, InputStream inputStream) { BufferedImage fullSizeImage; + boolean thumbnailGenerated = false; try { logger.fine("attempting to read the image file with ImageIO.read(InputStream), " + storageIO.getDataFile().getStorageIdentifier()); @@ -353,26 +382,35 @@ private static boolean generateImageThumbnailFromInputStream(StorageIO try { rescaleImage(fullSizeImage, width, height, size, outputStream); - /* - // while we are at it, let's make sure other size thumbnails are - // generated too: - for (int s : (new int[]{DEFAULT_PREVIEW_SIZE, DEFAULT_THUMBNAIL_SIZE, DEFAULT_CARDIMAGE_SIZE})) { - if (size != s && !thumbnailFileExists(fileLocation, s)) { - rescaleImage(fullSizeImage, width, height, s, fileLocation); - } - } - */ if (tempFileRequired) { storageIO.savePathAsAux(Paths.get(tempFile.getAbsolutePath()), THUMBNAIL_SUFFIX + size); } + thumbnailGenerated = true; } catch (Exception ioex) { logger.warning("Failed to rescale and/or save the image: " + ioex.getMessage()); - return false; + thumbnailGenerated = false; + } + finally { + if(tempFileRequired) { + try { + tempFile.delete(); + } + catch (Exception e) {} + } else if (!thumbnailGenerated) { + // if it was a local file - let's make sure we are not leaving + // behind a half-baked, broken image - such as a 0-size file - + // if this was a failure. + try { + storageIO.deleteAuxObject(THUMBNAIL_SUFFIX + size); + } catch (IOException ioex) { + logger.fine("Failed attempt to delete the result of a failed thumbnail rescaling; this is most likely ok - for ex., because it was never created in the first place."); + } + } } - return true; + return thumbnailGenerated; } @@ -530,12 +568,10 @@ private static String getImageAsBase64FromInputStream(InputStream inputStream) { public static String getImageAsBase64FromFile(File imageFile) { InputStream imageInputStream = null; try { - - int imageSize = (int) imageFile.length(); - - imageInputStream = new FileInputStream(imageFile); - - return getImageAsBase64FromInputStream(imageInputStream); //, imageSize); + if (imageFile.length() > 0) { + imageInputStream = new FileInputStream(imageFile); + return getImageAsBase64FromInputStream(imageInputStream); + } } catch (IOException ex) { // too bad - but not fatal logger.warning("getImageAsBase64FromFile: Failed to read data from thumbnail file"); @@ -595,16 +631,12 @@ public static String generateImageThumbnailFromFile(String fileLocation, int siz logger.fine("image dimensions: " + width + "x" + height); - thumbFileLocation = rescaleImage(fullSizeImage, width, height, size, fileLocation); + return rescaleImage(fullSizeImage, width, height, size, fileLocation); - if (thumbFileLocation != null) { - return thumbFileLocation; - } } catch (Exception e) { logger.warning("Failed to read in an image from " + fileLocation + ": " + e.getMessage()); } return null; - } /* @@ -643,10 +675,14 @@ public static String rescaleImage(BufferedImage fullSizeImage, int width, int he try { rescaleImage(fullSizeImage, width, height, size, outputFileStream); } catch (Exception ioex) { - logger.warning("caught Exceptiopn trying to create rescaled image " + outputLocation); - return null; + logger.warning("caught Exception trying to create rescaled image " + outputLocation); + outputLocation = null; } finally { IOUtils.closeQuietly(outputFileStream); + // delete the file if the rescaleImage failed + if (outputLocation == null) { + outputFile.delete(); + } } return outputLocation; @@ -702,13 +738,19 @@ private static void rescaleImage(BufferedImage fullSizeImage, int width, int hei if (iter.hasNext()) { writer = (ImageWriter) iter.next(); } else { - throw new IOException("Failed to locatie ImageWriter plugin for image type PNG"); + throw new IOException("Failed to locate ImageWriter plugin for image type PNG"); } - BufferedImage lowRes = new BufferedImage(thumbWidth, thumbHeight, BufferedImage.TYPE_INT_ARGB); - Graphics2D g2 = lowRes.createGraphics(); - g2.drawImage(thumbImage, 0, 0, null); - g2.dispose(); + BufferedImage lowRes = null; + try { + lowRes = new BufferedImage(thumbWidth, thumbHeight, BufferedImage.TYPE_INT_ARGB); + Graphics2D g2 = lowRes.createGraphics(); + g2.drawImage(thumbImage, 0, 0, null); + g2.dispose(); + } catch (Exception ex) { + logger.warning("Failed to create LoRes Image: " + ex.getMessage()); + throw new IOException("Caught exception trying to generate thumbnail: " + ex.getMessage()); + } try (ImageOutputStream ios = ImageIO.createImageOutputStream(outputStream);) { @@ -824,6 +866,7 @@ public static String generatePDFThumbnailFromFile(String fileLocation, int size) // generate the thumbnail for the requested size, *using the already scaled-down // 400x400 png version, above*: + // (the "exists()" check below appears to be unnecessary - we've already checked early on - ?) if (!((new File(thumbFileLocation)).exists())) { thumbFileLocation = runImageMagick(imageMagickExec, previewFileLocation, thumbFileLocation, size, "png"); } diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIO.java index 1616bfabf96..bca70259cb7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIO.java @@ -124,8 +124,12 @@ public void open(DataAccessOption... options) throws IOException { logger.fine("Setting size"); this.setSize(retrieveSizeFromMedia()); } - if (dataFile.getContentType() != null && dataFile.getContentType().equals("text/tab-separated-values") - && dataFile.isTabularData() && dataFile.getDataTable() != null && (!this.noVarHeader())) { + if (dataFile.getContentType() != null + && dataFile.getContentType().equals("text/tab-separated-values") + && dataFile.isTabularData() + && dataFile.getDataTable() != null + && (!this.noVarHeader()) + && (!dataFile.getDataTable().isStoredWithVariableHeader())) { List datavariables = dataFile.getDataTable().getDataVariables(); String varHeaderLine = generateVariableHeader(datavariables); diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java index 8afc365417e..d2fdec7b323 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java @@ -40,6 +40,7 @@ import edu.harvard.iq.dataverse.Dataverse; import edu.harvard.iq.dataverse.DvObject; import edu.harvard.iq.dataverse.datavariable.DataVariable; +import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.util.FileUtil; import opennlp.tools.util.StringUtil; @@ -225,7 +226,8 @@ public void open(DataAccessOption... options) throws IOException { && dataFile.getContentType().equals("text/tab-separated-values") && dataFile.isTabularData() && dataFile.getDataTable() != null - && (!this.noVarHeader())) { + && (!this.noVarHeader()) + && (!dataFile.getDataTable().isStoredWithVariableHeader())) { List datavariables = dataFile.getDataTable().getDataVariables(); String varHeaderLine = generateVariableHeader(datavariables); @@ -990,7 +992,10 @@ private String generateTemporaryS3UploadUrl(String key, Date expiration) throws GeneratePresignedUrlRequest generatePresignedUrlRequest = new GeneratePresignedUrlRequest(bucketName, key).withMethod(HttpMethod.PUT).withExpiration(expiration); //Require user to add this header to indicate a temporary file - generatePresignedUrlRequest.putCustomRequestHeader(Headers.S3_TAGGING, "dv-state=temp"); + final boolean taggingDisabled = JvmSettings.DISABLE_S3_TAGGING.lookupOptional(Boolean.class, this.driverId).orElse(false); + if (!taggingDisabled) { + generatePresignedUrlRequest.putCustomRequestHeader(Headers.S3_TAGGING, "dv-state=temp"); + } URL presignedUrl; try { @@ -1039,7 +1044,10 @@ public JsonObjectBuilder generateTemporaryS3UploadUrls(String globalId, String s } else { JsonObjectBuilder urls = Json.createObjectBuilder(); InitiateMultipartUploadRequest initiationRequest = new InitiateMultipartUploadRequest(bucketName, key); - initiationRequest.putCustomRequestHeader(Headers.S3_TAGGING, "dv-state=temp"); + final boolean taggingDisabled = JvmSettings.DISABLE_S3_TAGGING.lookupOptional(Boolean.class, this.driverId).orElse(false); + if (!taggingDisabled) { + initiationRequest.putCustomRequestHeader(Headers.S3_TAGGING, "dv-state=temp"); + } InitiateMultipartUploadResult initiationResponse = s3.initiateMultipartUpload(initiationRequest); String uploadId = initiationResponse.getUploadId(); for (int i = 1; i <= (fileSize / minPartSize) + (fileSize % minPartSize > 0 ? 1 : 0); i++) { diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java index 51cdecf64a0..8d3efa79b51 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java @@ -57,7 +57,6 @@ public abstract class StorageIO { static final String UPLOAD_REDIRECT = "upload-redirect"; static final String UPLOAD_OUT_OF_BAND = "upload-out-of-band"; protected static final String DOWNLOAD_REDIRECT = "download-redirect"; - protected static final String DATAVERSE_INACCESSIBLE = "dataverse-inaccessible"; public StorageIO() { @@ -624,7 +623,7 @@ public static boolean isDirectUploadEnabled(String driverId) { //True by default, Stores (e.g. RemoteOverlay, Globus) can set this false to stop attempts to read bytes public static boolean isDataverseAccessible(String driverId) { - return (true && !Boolean.parseBoolean(getConfigParamForDriver(driverId, DATAVERSE_INACCESSIBLE))); + return (true && !Boolean.parseBoolean(StorageIO.getConfigParamForDriver(driverId, AbstractRemoteOverlayAccessIO.FILES_NOT_ACCESSIBLE_BY_DATAVERSE))); } // Check that storageIdentifier is consistent with store's config diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java index 105a60ab418..717f46ffd60 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java @@ -142,7 +142,8 @@ public void open(DataAccessOption... options) throws IOException { && dataFile.getContentType().equals("text/tab-separated-values") && dataFile.isTabularData() && dataFile.getDataTable() != null - && (!this.noVarHeader())) { + && (!this.noVarHeader()) + && (!dataFile.getDataTable().isStoredWithVariableHeader())) { List datavariables = dataFile.getDataTable().getDataVariables(); String varHeaderLine = generateVariableHeader(datavariables); diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/TabularSubsetGenerator.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/TabularSubsetGenerator.java index 782f7f3a52d..c369010c8cd 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/TabularSubsetGenerator.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/TabularSubsetGenerator.java @@ -60,305 +60,26 @@ public class TabularSubsetGenerator implements SubsetGenerator { - private static Logger dbgLog = Logger.getLogger(TabularSubsetGenerator.class.getPackage().getName()); + private static Logger logger = Logger.getLogger(TabularSubsetGenerator.class.getPackage().getName()); - private static int COLUMN_TYPE_STRING = 1; - private static int COLUMN_TYPE_LONG = 2; - private static int COLUMN_TYPE_DOUBLE = 3; - private static int COLUMN_TYPE_FLOAT = 4; - - private static int MAX_COLUMN_BUFFER = 8192; - - private FileChannel fileChannel = null; - - private int varcount; - private int casecount; - private int subsetcount; - - private byte[][] columnEntries = null; - - - private ByteBuffer[] columnByteBuffers; - private int[] columnBufferSizes; - private int[] columnBufferOffsets; - - private long[] columnStartOffsets; - private long[] columnTotalOffsets; - private long[] columnTotalLengths; - - public TabularSubsetGenerator() { - - } - - public TabularSubsetGenerator (DataFile datafile, List variables) throws IOException { - if (!datafile.isTabularData()) { - throw new IOException("DataFile is not tabular data."); - } - - setVarCount(datafile.getDataTable().getVarQuantity().intValue()); - setCaseCount(datafile.getDataTable().getCaseQuantity().intValue()); - - - - StorageIO dataAccess = datafile.getStorageIO(); - if (!dataAccess.isLocalFile()) { - throw new IOException("Subsetting is supported on local files only!"); - } - - //File tabfile = datafile.getFileSystemLocation().toFile(); - File tabfile = dataAccess.getFileSystemPath().toFile(); + //private static int MAX_COLUMN_BUFFER = 8192; - File rotatedImageFile = getRotatedImage(tabfile, getVarCount(), getCaseCount()); - long[] columnEndOffsets = extractColumnOffsets(rotatedImageFile, getVarCount(), getCaseCount()); - - fileChannel = (FileChannel.open(Paths.get(rotatedImageFile.getAbsolutePath()), StandardOpenOption.READ)); - - if (variables == null || variables.size() < 1 || variables.size() > getVarCount()) { - throw new IOException("Illegal number of variables in the subset request"); - } - - subsetcount = variables.size(); - columnTotalOffsets = new long[subsetcount]; - columnTotalLengths = new long[subsetcount]; - columnByteBuffers = new ByteBuffer[subsetcount]; - - + public TabularSubsetGenerator() { - if (subsetcount == 1) { - if (!datafile.getDataTable().getId().equals(variables.get(0).getDataTable().getId())) { - throw new IOException("Variable in the subset request does not belong to the datafile."); - } - dbgLog.fine("single variable subset; setting fileChannel position to "+extractColumnOffset(columnEndOffsets, variables.get(0).getFileOrder())); - fileChannel.position(extractColumnOffset(columnEndOffsets, variables.get(0).getFileOrder())); - columnTotalLengths[0] = extractColumnLength(columnEndOffsets, variables.get(0).getFileOrder()); - columnTotalOffsets[0] = 0; - } else { - columnEntries = new byte[subsetcount][]; - - columnBufferSizes = new int[subsetcount]; - columnBufferOffsets = new int[subsetcount]; - columnStartOffsets = new long[subsetcount]; - - int i = 0; - for (DataVariable var : variables) { - if (!datafile.getDataTable().getId().equals(var.getDataTable().getId())) { - throw new IOException("Variable in the subset request does not belong to the datafile."); - } - columnByteBuffers[i] = ByteBuffer.allocate(MAX_COLUMN_BUFFER); - columnTotalLengths[i] = extractColumnLength(columnEndOffsets, var.getFileOrder()); - columnStartOffsets[i] = extractColumnOffset(columnEndOffsets, var.getFileOrder()); - if (columnTotalLengths[i] < MAX_COLUMN_BUFFER) { - columnByteBuffers[i].limit((int)columnTotalLengths[i]); - } - fileChannel.position(columnStartOffsets[i]); - columnBufferSizes[i] = fileChannel.read(columnByteBuffers[i]); - columnBufferOffsets[i] = 0; - columnTotalOffsets[i] = columnBufferSizes[i]; - i++; - } - } - } - - private int getVarCount() { - return varcount; } - private void setVarCount(int varcount) { - this.varcount = varcount; - } - - private int getCaseCount() { - return casecount; - } - - private void setCaseCount(int casecount) { - this.casecount = casecount; - } - - - /* - * Note that this method operates on the *absolute* column number, i.e. - * the number of the physical column in the tabular file. This is stored - * in DataVariable.FileOrder. - * This "column number" should not be confused with the number of column - * in the subset request; a user can request any number of variable - * columns, in an order that doesn't have to follow the physical order - * of the columns in the file. - */ - private long extractColumnOffset(long[] columnEndOffsets, int column) throws IOException { - if (columnEndOffsets == null || columnEndOffsets.length <= column) { - throw new IOException("Offsets table not initialized; or column out of bounds."); - } - long columnOffset; - - if (column > 0) { - columnOffset = columnEndOffsets[column - 1]; - } else { - columnOffset = getVarCount() * 8; - } - return columnOffset; - } - - /* - * See the comment for the method above. + /** + * This class used to be much more complex. There were methods for subsetting + * from fixed-width field files; including using the optimized, "90 deg. rotated" + * versions of such files (i.e. you create a *columns-wise* copy of your data + * file in which the columns are stored sequentially, and a table of byte + * offsets of each column. You can then read individual variable columns + * for cheap; at the expense of doubling the storage size of your tabular + * data files. These methods were not used, so they were deleted (in Jan. 2024 + * prior to 6.2. + * Please consult git history if you are interested in looking at that code. */ - private long extractColumnLength(long[] columnEndOffsets, int column) throws IOException { - if (columnEndOffsets == null || columnEndOffsets.length <= column) { - throw new IOException("Offsets table not initialized; or column out of bounds."); - } - long columnLength; - - if (column > 0) { - columnLength = columnEndOffsets[column] - columnEndOffsets[column - 1]; - } else { - columnLength = columnEndOffsets[0] - varcount * 8; - } - - return columnLength; - } - - - private void bufferMoreColumnBytes(int column) throws IOException { - if (columnTotalOffsets[column] >= columnTotalLengths[column]) { - throw new IOException("attempt to buffer bytes past the column boundary"); - } - fileChannel.position(columnStartOffsets[column] + columnTotalOffsets[column]); - - columnByteBuffers[column].clear(); - if (columnTotalLengths[column] < columnTotalOffsets[column] + MAX_COLUMN_BUFFER) { - dbgLog.fine("Limiting the buffer to "+(columnTotalLengths[column] - columnTotalOffsets[column])+" bytes"); - columnByteBuffers[column].limit((int) (columnTotalLengths[column] - columnTotalOffsets[column])); - } - columnBufferSizes[column] = fileChannel.read(columnByteBuffers[column]); - dbgLog.fine("Read "+columnBufferSizes[column]+" bytes for subset column "+column); - columnBufferOffsets[column] = 0; - columnTotalOffsets[column] += columnBufferSizes[column]; - } - - public byte[] readColumnEntryBytes(int column) { - return readColumnEntryBytes(column, true); - } - - - public byte[] readColumnEntryBytes(int column, boolean addTabs) { - byte[] leftover = null; - byte[] ret = null; - - if (columnBufferOffsets[column] >= columnBufferSizes[column]) { - try { - bufferMoreColumnBytes(column); - if (columnBufferSizes[column] < 1) { - return null; - } - } catch (IOException ioe) { - return null; - } - } - - int byteindex = columnBufferOffsets[column]; - try { - while (columnByteBuffers[column].array()[byteindex] != '\n') { - byteindex++; - if (byteindex == columnBufferSizes[column]) { - // save the leftover: - if (leftover == null) { - leftover = new byte[columnBufferSizes[column] - columnBufferOffsets[column]]; - System.arraycopy(columnByteBuffers[column].array(), columnBufferOffsets[column], leftover, 0, columnBufferSizes[column] - columnBufferOffsets[column]); - } else { - byte[] merged = new byte[leftover.length + columnBufferSizes[column]]; - - System.arraycopy(leftover, 0, merged, 0, leftover.length); - System.arraycopy(columnByteBuffers[column].array(), 0, merged, leftover.length, columnBufferSizes[column]); - leftover = merged; - merged = null; - } - // read more bytes: - bufferMoreColumnBytes(column); - if (columnBufferSizes[column] < 1) { - return null; - } - byteindex = 0; - } - } - - // presumably, we have found our '\n': - if (leftover == null) { - ret = new byte[byteindex - columnBufferOffsets[column] + 1]; - System.arraycopy(columnByteBuffers[column].array(), columnBufferOffsets[column], ret, 0, byteindex - columnBufferOffsets[column] + 1); - } else { - ret = new byte[leftover.length + byteindex + 1]; - System.arraycopy(leftover, 0, ret, 0, leftover.length); - System.arraycopy(columnByteBuffers[column].array(), 0, ret, leftover.length, byteindex + 1); - } - - } catch (IOException ioe) { - return null; - } - - columnBufferOffsets[column] = (byteindex + 1); - - if (column < columnBufferOffsets.length - 1) { - ret[ret.length - 1] = '\t'; - } - return ret; - } - - public int readSingleColumnSubset(byte[] buffer) throws IOException { - if (columnTotalOffsets[0] == columnTotalLengths[0]) { - return -1; - } - - if (columnByteBuffers[0] == null) { - dbgLog.fine("allocating single column subset buffer."); - columnByteBuffers[0] = ByteBuffer.allocate(buffer.length); - } - - int bytesread = fileChannel.read(columnByteBuffers[0]); - dbgLog.fine("single column subset: read "+bytesread+" bytes."); - if (columnTotalOffsets[0] + bytesread > columnTotalLengths[0]) { - bytesread = (int)(columnTotalLengths[0] - columnTotalOffsets[0]); - } - System.arraycopy(columnByteBuffers[0].array(), 0, buffer, 0, bytesread); - - columnTotalOffsets[0] += bytesread; - columnByteBuffers[0].clear(); - return bytesread > 0 ? bytesread : -1; - } - - - public byte[] readSubsetLineBytes() throws IOException { - byte[] ret = null; - int total = 0; - for (int i = 0; i < subsetcount; i++) { - columnEntries[i] = readColumnEntryBytes(i); - if (columnEntries[i] == null) { - throw new IOException("Failed to read subset line entry"); - } - total += columnEntries[i].length; - } - - ret = new byte[total]; - int offset = 0; - for (int i = 0; i < subsetcount; i++) { - System.arraycopy(columnEntries[i], 0, ret, offset, columnEntries[i].length); - offset += columnEntries[i].length; - } - dbgLog.fine("line: "+new String(ret)); - return ret; - } - - - public void close() { - if (fileChannel != null) { - try { - fileChannel.close(); - } catch (IOException ioe) { - // don't care. - } - } - } - public void subsetFile(String infile, String outfile, List columns, Long numCases) { subsetFile(infile, outfile, columns, numCases, "\t"); } @@ -411,11 +132,15 @@ public void subsetFile(InputStream in, String outfile, List columns, Lo * files, OK to use on small files: */ - public static Double[] subsetDoubleVector(InputStream in, int column, int numCases) { + public static Double[] subsetDoubleVector(InputStream in, int column, int numCases, boolean skipHeader) { Double[] retVector = new Double[numCases]; try (Scanner scanner = new Scanner(in)) { scanner.useDelimiter("\\n"); + if (skipHeader) { + skipFirstLine(scanner); + } + for (int caseIndex = 0; caseIndex < numCases; caseIndex++) { if (scanner.hasNext()) { String[] line = (scanner.next()).split("\t", -1); @@ -463,11 +188,15 @@ public static Double[] subsetDoubleVector(InputStream in, int column, int numCas * Same deal as with the method above - straightforward, but (potentially) slow. * Not a resource hog though - will only try to store one vector in memory. */ - public static Float[] subsetFloatVector(InputStream in, int column, int numCases) { + public static Float[] subsetFloatVector(InputStream in, int column, int numCases, boolean skipHeader) { Float[] retVector = new Float[numCases]; try (Scanner scanner = new Scanner(in)) { scanner.useDelimiter("\\n"); + if (skipHeader) { + skipFirstLine(scanner); + } + for (int caseIndex = 0; caseIndex < numCases; caseIndex++) { if (scanner.hasNext()) { String[] line = (scanner.next()).split("\t", -1); @@ -513,11 +242,15 @@ public static Float[] subsetFloatVector(InputStream in, int column, int numCases * Same deal as with the method above - straightforward, but (potentially) slow. * Not a resource hog though - will only try to store one vector in memory. */ - public static Long[] subsetLongVector(InputStream in, int column, int numCases) { + public static Long[] subsetLongVector(InputStream in, int column, int numCases, boolean skipHeader) { Long[] retVector = new Long[numCases]; try (Scanner scanner = new Scanner(in)) { scanner.useDelimiter("\\n"); + if (skipHeader) { + skipFirstLine(scanner); + } + for (int caseIndex = 0; caseIndex < numCases; caseIndex++) { if (scanner.hasNext()) { String[] line = (scanner.next()).split("\t", -1); @@ -549,11 +282,15 @@ public static Long[] subsetLongVector(InputStream in, int column, int numCases) * Same deal as with the method above - straightforward, but (potentially) slow. * Not a resource hog though - will only try to store one vector in memory. */ - public static String[] subsetStringVector(InputStream in, int column, int numCases) { + public static String[] subsetStringVector(InputStream in, int column, int numCases, boolean skipHeader) { String[] retVector = new String[numCases]; try (Scanner scanner = new Scanner(in)) { scanner.useDelimiter("\\n"); + if (skipHeader) { + skipFirstLine(scanner); + } + for (int caseIndex = 0; caseIndex < numCases; caseIndex++) { if (scanner.hasNext()) { String[] line = (scanner.next()).split("\t", -1); @@ -621,819 +358,10 @@ public static String[] subsetStringVector(InputStream in, int column, int numCas } - /* - * Straightforward method for subsetting a tab-delimited data file, extracting - * all the columns representing continuous variables and returning them as - * a 2-dimensional array of Doubles; - * Inefficient on large files, OK to use on small ones. - */ - public static Double[][] subsetDoubleVectors(InputStream in, Set columns, int numCases) throws IOException { - Double[][] retVector = new Double[columns.size()][numCases]; - try (Scanner scanner = new Scanner(in)) { - scanner.useDelimiter("\\n"); - - for (int caseIndex = 0; caseIndex < numCases; caseIndex++) { - if (scanner.hasNext()) { - String[] line = (scanner.next()).split("\t", -1); - int j = 0; - for (Integer i : columns) { - try { - // TODO: verify that NaN and +-Inf are going to be - // handled correctly here! -- L.A. - // NO, "+-Inf" is not handled correctly; see the - // comment further down below. - retVector[j][caseIndex] = new Double(line[i]); - } catch (NumberFormatException ex) { - retVector[j][caseIndex] = null; // missing value - } - j++; - } - } else { - throw new IOException("Tab file has fewer rows than the stored number of cases!"); - } - } - - int tailIndex = numCases; - while (scanner.hasNext()) { - String nextLine = scanner.next(); - if (!"".equals(nextLine)) { - throw new IOException("Tab file has more nonempty rows than the stored number of cases ("+numCases+")! current index: "+tailIndex+", line: "+nextLine); - } - tailIndex++; - } - - } - return retVector; - - } - - public String[] subsetStringVector(DataFile datafile, int column) throws IOException { - return (String[])subsetObjectVector(datafile, column, COLUMN_TYPE_STRING); - } - - public Double[] subsetDoubleVector(DataFile datafile, int column) throws IOException { - return (Double[])subsetObjectVector(datafile, column, COLUMN_TYPE_DOUBLE); - } - - public Long[] subsetLongVector(DataFile datafile, int column) throws IOException { - return (Long[])subsetObjectVector(datafile, column, COLUMN_TYPE_LONG); - } - - // Float methods are temporary; - // In normal operations we'll be treating all the floating point types as - // doubles. I need to be able to handle floats for some 4.0 vs 3.* ingest - // tests. -- L.A. - - public Float[] subsetFloatVector(DataFile datafile, int column) throws IOException { - return (Float[])subsetObjectVector(datafile, column, COLUMN_TYPE_FLOAT); - } - - public String[] subsetStringVector(File tabfile, int column, int varcount, int casecount) throws IOException { - return (String[])subsetObjectVector(tabfile, column, varcount, casecount, COLUMN_TYPE_STRING); - } - - public Double[] subsetDoubleVector(File tabfile, int column, int varcount, int casecount) throws IOException { - return (Double[])subsetObjectVector(tabfile, column, varcount, casecount, COLUMN_TYPE_DOUBLE); - } - - public Long[] subsetLongVector(File tabfile, int column, int varcount, int casecount) throws IOException { - return (Long[])subsetObjectVector(tabfile, column, varcount, casecount, COLUMN_TYPE_LONG); - } - - public Float[] subsetFloatVector(File tabfile, int column, int varcount, int casecount) throws IOException { - return (Float[])subsetObjectVector(tabfile, column, varcount, casecount, COLUMN_TYPE_FLOAT); - } - - public Object[] subsetObjectVector(DataFile dataFile, int column, int columntype) throws IOException { - if (!dataFile.isTabularData()) { - throw new IOException("DataFile is not tabular data."); - } - - int varcount = dataFile.getDataTable().getVarQuantity().intValue(); - int casecount = dataFile.getDataTable().getCaseQuantity().intValue(); - - if (column >= varcount) { - throw new IOException("Column "+column+" is out of bounds."); - } - - StorageIO dataAccess = dataFile.getStorageIO(); - if (!dataAccess.isLocalFile()) { - throw new IOException("Subsetting is supported on local files only!"); - } - - //File tabfile = datafile.getFileSystemLocation().toFile(); - File tabfile = dataAccess.getFileSystemPath().toFile(); - - if (columntype == COLUMN_TYPE_STRING) { - String filename = dataFile.getFileMetadata().getLabel(); - if (filename != null) { - filename = filename.replaceFirst("^_", ""); - Integer fnumvalue = null; - try { - fnumvalue = new Integer(filename); - } catch (Exception ex){ - fnumvalue = null; - } - if (fnumvalue != null) { - //if ((fnumvalue.intValue() < 112497)) { // && (fnumvalue.intValue() > 60015)) { - if ((fnumvalue.intValue() < 111931)) { // && (fnumvalue.intValue() > 60015)) { - if (!(fnumvalue.intValue() == 60007 - || fnumvalue.intValue() == 59997 - || fnumvalue.intValue() == 60015 - || fnumvalue.intValue() == 59948 - || fnumvalue.intValue() == 60012 - || fnumvalue.intValue() == 52585 - || fnumvalue.intValue() == 60005 - || fnumvalue.intValue() == 60002 - || fnumvalue.intValue() == 59954 - || fnumvalue.intValue() == 60008 - || fnumvalue.intValue() == 54972 - || fnumvalue.intValue() == 55010 - || fnumvalue.intValue() == 54996 - || fnumvalue.intValue() == 53527 - || fnumvalue.intValue() == 53546 - || fnumvalue.intValue() == 55002 - || fnumvalue.intValue() == 55006 - || fnumvalue.intValue() == 54998 - || fnumvalue.intValue() == 52552 - // SPSS/SAV cases with similar issue - compat mode must be disabled - //|| fnumvalue.intValue() == 101826 // temporary - tricky file with accents and v. 16... - || fnumvalue.intValue() == 54618 // another SAV file, with long strings... - || fnumvalue.intValue() == 54619 // [same] - || fnumvalue.intValue() == 57983 - || fnumvalue.intValue() == 58262 - || fnumvalue.intValue() == 58288 - || fnumvalue.intValue() == 58656 - || fnumvalue.intValue() == 59144 - // || fnumvalue.intValue() == 69626 [nope!] - )) { - dbgLog.info("\"Old\" file name detected; using \"compatibility mode\" for a character vector subset;"); - return subsetObjectVector(tabfile, column, varcount, casecount, columntype, true); - } - } - } - } + private static void skipFirstLine(Scanner scanner) { + if (!scanner.hasNext()) { + throw new RuntimeException("Failed to read the variable name header line from the tab-delimited file!"); } - - return subsetObjectVector(tabfile, column, varcount, casecount, columntype); - } - - public Object[] subsetObjectVector(File tabfile, int column, int varcount, int casecount, int columntype) throws IOException { - return subsetObjectVector(tabfile, column, varcount, casecount, columntype, false); - } - - - - public Object[] subsetObjectVector(File tabfile, int column, int varcount, int casecount, int columntype, boolean compatmode) throws IOException { - - Object[] retVector = null; - - boolean isString = false; - boolean isDouble = false; - boolean isLong = false; - boolean isFloat = false; - - //Locale loc = new Locale("en", "US"); - - if (columntype == COLUMN_TYPE_STRING) { - isString = true; - retVector = new String[casecount]; - } else if (columntype == COLUMN_TYPE_DOUBLE) { - isDouble = true; - retVector = new Double[casecount]; - } else if (columntype == COLUMN_TYPE_LONG) { - isLong = true; - retVector = new Long[casecount]; - } else if (columntype == COLUMN_TYPE_FLOAT){ - isFloat = true; - retVector = new Float[casecount]; - } else { - throw new IOException("Unsupported column type: "+columntype); - } - - File rotatedImageFile = getRotatedImage(tabfile, varcount, casecount); - long[] columnEndOffsets = extractColumnOffsets(rotatedImageFile, varcount, casecount); - long columnOffset = 0; - long columnLength = 0; - - if (column > 0) { - columnOffset = columnEndOffsets[column - 1]; - columnLength = columnEndOffsets[column] - columnEndOffsets[column - 1]; - } else { - columnOffset = varcount * 8; - columnLength = columnEndOffsets[0] - varcount * 8; - } - int caseindex = 0; - - try (FileChannel fc = (FileChannel.open(Paths.get(rotatedImageFile.getAbsolutePath()), - StandardOpenOption.READ))) { - fc.position(columnOffset); - int MAX_COLUMN_BUFFER = 8192; - - ByteBuffer in = ByteBuffer.allocate(MAX_COLUMN_BUFFER); - - if (columnLength < MAX_COLUMN_BUFFER) { - in.limit((int) (columnLength)); - } - - long bytesRead = 0; - long bytesReadTotal = 0; - - int byteoffset = 0; - byte[] leftover = null; - - while (bytesReadTotal < columnLength) { - bytesRead = fc.read(in); - byte[] columnBytes = in.array(); - int bytecount = 0; - - while (bytecount < bytesRead) { - if (columnBytes[bytecount] == '\n') { - /* - String token = new String(columnBytes, byteoffset, bytecount-byteoffset, "UTF8"); - - if (leftover != null) { - String leftoverString = new String (leftover, "UTF8"); - token = leftoverString + token; - leftover = null; - } - */ - /* - * Note that the way I was doing it at first - above - - * was not quite the correct way - because I was creating UTF8 - * strings from the leftover bytes, and the bytes in the - * current buffer *separately*; which means, if a multi-byte - * UTF8 character got split in the middle between one buffer - * and the next, both chunks of it would become junk - * characters, on each side! - * The correct way of doing it, of course, is to create a - * merged byte buffer, and then turn it into a UTF8 string. - * -- L.A. 4.0 - */ - String token = null; - - if (leftover == null) { - token = new String(columnBytes, byteoffset, bytecount - byteoffset, "UTF8"); - } else { - byte[] merged = new byte[leftover.length + bytecount - byteoffset]; - - System.arraycopy(leftover, 0, merged, 0, leftover.length); - System.arraycopy(columnBytes, byteoffset, merged, leftover.length, bytecount - byteoffset); - token = new String(merged, "UTF8"); - leftover = null; - merged = null; - } - - if (isString) { - if ("".equals(token)) { - // An empty string is a string missing value! - // An empty string in quotes is an empty string! - retVector[caseindex] = null; - } else { - // Strip the outer quotes: - token = token.replaceFirst("^\\\"", ""); - token = token.replaceFirst("\\\"$", ""); - - // We need to restore the special characters that - // are stored in tab files escaped - quotes, new lines - // and tabs. Before we do that however, we need to - // take care of any escaped backslashes stored in - // the tab file. I.e., "foo\t" should be transformed - // to "foo"; but "foo\\t" should be transformed - // to "foo\t". This way new lines and tabs that were - // already escaped in the original data are not - // going to be transformed to unescaped tab and - // new line characters! - - String[] splitTokens = token.split(Matcher.quoteReplacement("\\\\"), -2); - - // (note that it's important to use the 2-argument version - // of String.split(), and set the limit argument to a - // negative value; otherwise any trailing backslashes - // are lost.) - - for (int i = 0; i < splitTokens.length; i++) { - splitTokens[i] = splitTokens[i].replaceAll(Matcher.quoteReplacement("\\\""), "\""); - splitTokens[i] = splitTokens[i].replaceAll(Matcher.quoteReplacement("\\t"), "\t"); - splitTokens[i] = splitTokens[i].replaceAll(Matcher.quoteReplacement("\\n"), "\n"); - splitTokens[i] = splitTokens[i].replaceAll(Matcher.quoteReplacement("\\r"), "\r"); - } - // TODO: - // Make (some of?) the above optional; for ex., we - // do need to restore the newlines when calculating UNFs; - // But if we are subsetting these vectors in order to - // create a new tab-delimited file, they will - // actually break things! -- L.A. Jul. 28 2014 - - token = StringUtils.join(splitTokens, '\\'); - - // "compatibility mode" - a hack, to be able to produce - // unfs identical to those produced by the "early" - // unf5 jar; will be removed in production 4.0. - // -- L.A. (TODO: ...) - if (compatmode && !"".equals(token)) { - if (token.length() > 128) { - if ("".equals(token.trim())) { - // don't ask... - token = token.substring(0, 129); - } else { - token = token.substring(0, 128); - // token = String.format(loc, "%.128s", token); - token = token.trim(); - // dbgLog.info("formatted and trimmed: "+token); - } - } else { - if ("".equals(token.trim())) { - // again, don't ask; - // - this replicates some bugginness - // that happens inside unf5; - token = "null"; - } else { - token = token.trim(); - } - } - } - - retVector[caseindex] = token; - } - } else if (isDouble) { - try { - // TODO: verify that NaN and +-Inf are - // handled correctly here! -- L.A. - // Verified: new Double("nan") works correctly, - // resulting in Double.NaN; - // Double("[+-]Inf") doesn't work however; - // (the constructor appears to be expecting it - // to be spelled as "Infinity", "-Infinity", etc. - if ("inf".equalsIgnoreCase(token) || "+inf".equalsIgnoreCase(token)) { - retVector[caseindex] = java.lang.Double.POSITIVE_INFINITY; - } else if ("-inf".equalsIgnoreCase(token)) { - retVector[caseindex] = java.lang.Double.NEGATIVE_INFINITY; - } else if (token == null || token.equals("")) { - // missing value: - retVector[caseindex] = null; - } else { - retVector[caseindex] = new Double(token); - } - } catch (NumberFormatException ex) { - dbgLog.warning("NumberFormatException thrown for " + token + " as Double"); - - retVector[caseindex] = null; // missing value - // TODO: ? - } - } else if (isLong) { - try { - retVector[caseindex] = new Long(token); - } catch (NumberFormatException ex) { - retVector[caseindex] = null; // assume missing value - } - } else if (isFloat) { - try { - if ("inf".equalsIgnoreCase(token) || "+inf".equalsIgnoreCase(token)) { - retVector[caseindex] = java.lang.Float.POSITIVE_INFINITY; - } else if ("-inf".equalsIgnoreCase(token)) { - retVector[caseindex] = java.lang.Float.NEGATIVE_INFINITY; - } else if (token == null || token.equals("")) { - // missing value: - retVector[caseindex] = null; - } else { - retVector[caseindex] = new Float(token); - } - } catch (NumberFormatException ex) { - dbgLog.warning("NumberFormatException thrown for " + token + " as Float"); - retVector[caseindex] = null; // assume missing value (TODO: ?) - } - } - caseindex++; - - if (bytecount == bytesRead - 1) { - byteoffset = 0; - } else { - byteoffset = bytecount + 1; - } - } else { - if (bytecount == bytesRead - 1) { - // We've reached the end of the buffer; - // This means we'll save whatever unused bytes left in - // it - i.e., the bytes between the last new line - // encountered and the end - in the leftover buffer. - - // *EXCEPT*, there may be a case of a very long String - // that is actually longer than MAX_COLUMN_BUFFER, in - // which case it is possible that we've read through - // an entire buffer of bytes without finding any - // new lines... in this case we may need to add this - // entire byte buffer to an already existing leftover - // buffer! - if (leftover == null) { - leftover = new byte[(int) bytesRead - byteoffset]; - System.arraycopy(columnBytes, byteoffset, leftover, 0, (int) bytesRead - byteoffset); - } else { - if (byteoffset != 0) { - throw new IOException("Reached the end of the byte buffer, with some leftover left from the last read; yet the offset is not zero!"); - } - byte[] merged = new byte[leftover.length + (int) bytesRead]; - - System.arraycopy(leftover, 0, merged, 0, leftover.length); - System.arraycopy(columnBytes, byteoffset, merged, leftover.length, (int) bytesRead); - // leftover = null; - leftover = merged; - merged = null; - } - byteoffset = 0; - - } - } - bytecount++; - } - - bytesReadTotal += bytesRead; - in.clear(); - if (columnLength - bytesReadTotal < MAX_COLUMN_BUFFER) { - in.limit((int) (columnLength - bytesReadTotal)); - } - } - - } - - if (caseindex != casecount) { - throw new IOException("Faile to read "+casecount+" tokens for column "+column); - //System.out.println("read "+caseindex+" tokens instead of expected "+casecount+"."); - } - - return retVector; - } - - private long[] extractColumnOffsets (File rotatedImageFile, int varcount, int casecount) throws IOException { - long[] byteOffsets = new long[varcount]; - - try (BufferedInputStream rotfileStream = new BufferedInputStream(new FileInputStream(rotatedImageFile))) { - - byte[] offsetHeader = new byte[varcount * 8]; - - int readlen = rotfileStream.read(offsetHeader); - - if (readlen != varcount * 8) { - throw new IOException("Could not read " + varcount * 8 + " header bytes from the rotated file."); - } - - for (int varindex = 0; varindex < varcount; varindex++) { - byte[] offsetBytes = new byte[8]; - System.arraycopy(offsetHeader, varindex * 8, offsetBytes, 0, 8); - - ByteBuffer offsetByteBuffer = ByteBuffer.wrap(offsetBytes); - byteOffsets[varindex] = offsetByteBuffer.getLong(); - - // System.out.println(byteOffsets[varindex]); - } - - } - - return byteOffsets; - } - - private File getRotatedImage(File tabfile, int varcount, int casecount) throws IOException { - String fileName = tabfile.getAbsolutePath(); - String rotatedImageFileName = fileName + ".90d"; - File rotatedImageFile = new File(rotatedImageFileName); - if (rotatedImageFile.exists()) { - //System.out.println("Image already exists!"); - return rotatedImageFile; - } - - return generateRotatedImage(tabfile, varcount, casecount); - - } - - private File generateRotatedImage (File tabfile, int varcount, int casecount) throws IOException { - // TODO: throw exceptions if bad file, zero varcount, etc. ... - - String fileName = tabfile.getAbsolutePath(); - String rotatedImageFileName = fileName + ".90d"; - - int MAX_OUTPUT_STREAMS = 32; - int MAX_BUFFERED_BYTES = 10 * 1024 * 1024; // 10 MB - for now? - int MAX_COLUMN_BUFFER = 8 * 1024; - - // offsetHeader will contain the byte offsets of the individual column - // vectors in the final rotated image file - byte[] offsetHeader = new byte[varcount * 8]; - int[] bufferedSizes = new int[varcount]; - long[] cachedfileSizes = new long[varcount]; - File[] columnTempFiles = new File[varcount]; - - for (int i = 0; i < varcount; i++) { - bufferedSizes[i] = 0; - cachedfileSizes[i] = 0; - } - - // TODO: adjust MAX_COLUMN_BUFFER here, so that the total size is - // no more than MAX_BUFFERED_BYTES (but no less than 1024 maybe?) - - byte[][] bufferedColumns = new byte [varcount][MAX_COLUMN_BUFFER]; - - // read the tab-delimited file: - - try (FileInputStream tabfileStream = new FileInputStream(tabfile); - Scanner scanner = new Scanner(tabfileStream)) { - scanner.useDelimiter("\\n"); - - for (int caseindex = 0; caseindex < casecount; caseindex++) { - if (scanner.hasNext()) { - String[] line = (scanner.next()).split("\t", -1); - // TODO: throw an exception if there are fewer tab-delimited - // tokens than the number of variables specified. - String token = ""; - int tokensize = 0; - for (int varindex = 0; varindex < varcount; varindex++) { - // TODO: figure out the safest way to convert strings to - // bytes here. Is it going to be safer to use getBytes("UTF8")? - // we are already making the assumption that the values - // in the tab file are in UTF8. -- L.A. - token = line[varindex] + "\n"; - tokensize = token.getBytes().length; - if (bufferedSizes[varindex] + tokensize > MAX_COLUMN_BUFFER) { - // fill the buffer and dump its contents into the temp file: - // (do note that there may be *several* MAX_COLUMN_BUFFERs - // worth of bytes in the token!) - - int tokenoffset = 0; - - if (bufferedSizes[varindex] != MAX_COLUMN_BUFFER) { - tokenoffset = MAX_COLUMN_BUFFER - bufferedSizes[varindex]; - System.arraycopy(token.getBytes(), 0, bufferedColumns[varindex], bufferedSizes[varindex], tokenoffset); - } // (otherwise the buffer is already full, and we should - // simply dump it into the temp file, without adding any - // extra bytes to it) - - File bufferTempFile = columnTempFiles[varindex]; - if (bufferTempFile == null) { - bufferTempFile = File.createTempFile("columnBufferFile", "bytes"); - columnTempFiles[varindex] = bufferTempFile; - } - - // *append* the contents of the buffer to the end of the - // temp file, if already exists: - try (BufferedOutputStream outputStream = new BufferedOutputStream( - new FileOutputStream(bufferTempFile, true))) { - outputStream.write(bufferedColumns[varindex], 0, MAX_COLUMN_BUFFER); - cachedfileSizes[varindex] += MAX_COLUMN_BUFFER; - - // keep writing MAX_COLUMN_BUFFER-size chunks of bytes into - // the temp file, for as long as there's more than MAX_COLUMN_BUFFER - // bytes left in the token: - - while (tokensize - tokenoffset > MAX_COLUMN_BUFFER) { - outputStream.write(token.getBytes(), tokenoffset, MAX_COLUMN_BUFFER); - cachedfileSizes[varindex] += MAX_COLUMN_BUFFER; - tokenoffset += MAX_COLUMN_BUFFER; - } - - } - - // buffer the remaining bytes and reset the buffered - // byte counter: - - System.arraycopy(token.getBytes(), - tokenoffset, - bufferedColumns[varindex], - 0, - tokensize - tokenoffset); - - bufferedSizes[varindex] = tokensize - tokenoffset; - - } else { - // continue buffering - System.arraycopy(token.getBytes(), 0, bufferedColumns[varindex], bufferedSizes[varindex], tokensize); - bufferedSizes[varindex] += tokensize; - } - } - } else { - throw new IOException("Tab file has fewer rows than the stored number of cases!"); - } - } - } - - // OK, we've created the individual byte vectors of the tab file columns; - // they may be partially saved in temp files and/or in memory. - // We now need to go through all these buffers and create the final - // rotated image file. - - try (BufferedOutputStream finalOut = new BufferedOutputStream( - new FileOutputStream(new File(rotatedImageFileName)))) { - - // but first we should create the offset header and write it out into - // the final file; because it should be at the head, doh! - - long columnOffset = varcount * 8; - // (this is the offset of the first column vector; it is equal to the - // size of the offset header, i.e. varcount * 8 bytes) - - for (int varindex = 0; varindex < varcount; varindex++) { - long totalColumnBytes = cachedfileSizes[varindex] + bufferedSizes[varindex]; - columnOffset += totalColumnBytes; - // totalColumnBytes; - byte[] columnOffsetByteArray = ByteBuffer.allocate(8).putLong(columnOffset).array(); - System.arraycopy(columnOffsetByteArray, 0, offsetHeader, varindex * 8, 8); - } - - finalOut.write(offsetHeader, 0, varcount * 8); - - for (int varindex = 0; varindex < varcount; varindex++) { - long cachedBytesRead = 0; - - // check if there is a cached temp file: - - File cachedTempFile = columnTempFiles[varindex]; - if (cachedTempFile != null) { - byte[] cachedBytes = new byte[MAX_COLUMN_BUFFER]; - try (BufferedInputStream cachedIn = new BufferedInputStream(new FileInputStream(cachedTempFile))) { - int readlen = 0; - while ((readlen = cachedIn.read(cachedBytes)) > -1) { - finalOut.write(cachedBytes, 0, readlen); - cachedBytesRead += readlen; - } - } - - // delete the temp file: - cachedTempFile.delete(); - - } - - if (cachedBytesRead != cachedfileSizes[varindex]) { - throw new IOException("Could not read the correct number of bytes cached for column "+varindex+"; "+ - cachedfileSizes[varindex] + " bytes expected, "+cachedBytesRead+" read."); - } - - // then check if there are any bytes buffered for this column: - - if (bufferedSizes[varindex] > 0) { - finalOut.write(bufferedColumns[varindex], 0, bufferedSizes[varindex]); - } - - } - } - - return new File(rotatedImageFileName); - - } - - /* - * Test method for taking a "rotated" image, and reversing it, reassembling - * all the columns in the original order. Which should result in a file - * byte-for-byte identical file to the original tab-delimited version. - * - * (do note that this method is not efficiently implemented; it's only - * being used for experiments so far, to confirm the accuracy of the - * accuracy of generateRotatedImage(). It should not be used for any - * practical means in the application!) - */ - private void reverseRotatedImage (File rotfile, int varcount, int casecount) throws IOException { - // open the file, read in the offset header: - try (BufferedInputStream rotfileStream = new BufferedInputStream(new FileInputStream(rotfile))) { - byte[] offsetHeader = new byte[varcount * 8]; - long[] byteOffsets = new long[varcount]; - - int readlen = rotfileStream.read(offsetHeader); - - if (readlen != varcount * 8) { - throw new IOException ("Could not read "+varcount*8+" header bytes from the rotated file."); - } - - for (int varindex = 0; varindex < varcount; varindex++) { - byte[] offsetBytes = new byte[8]; - System.arraycopy(offsetHeader, varindex*8, offsetBytes, 0, 8); - - ByteBuffer offsetByteBuffer = ByteBuffer.wrap(offsetBytes); - byteOffsets[varindex] = offsetByteBuffer.getLong(); - - //System.out.println(byteOffsets[varindex]); - } - - String [][] reversedMatrix = new String[casecount][varcount]; - - long offset = varcount * 8; - byte[] columnBytes; - - for (int varindex = 0; varindex < varcount; varindex++) { - long columnLength = byteOffsets[varindex] - offset; - - - - columnBytes = new byte[(int)columnLength]; - readlen = rotfileStream.read(columnBytes); - - if (readlen != columnLength) { - throw new IOException ("Could not read "+columnBytes+" bytes for column "+varindex); - } - /* - String columnString = new String(columnBytes); - //System.out.print(columnString); - String[] values = columnString.split("\n", -1); - - if (values.length < casecount) { - throw new IOException("count mismatch: "+values.length+" tokens found for column "+varindex); - } - - for (int caseindex = 0; caseindex < casecount; caseindex++) { - reversedMatrix[caseindex][varindex] = values[caseindex]; - }*/ - - int bytecount = 0; - int byteoffset = 0; - int caseindex = 0; - //System.out.println("generating value vector for column "+varindex); - while (bytecount < columnLength) { - if (columnBytes[bytecount] == '\n') { - String token = new String(columnBytes, byteoffset, bytecount-byteoffset); - reversedMatrix[caseindex++][varindex] = token; - byteoffset = bytecount + 1; - } - bytecount++; - } - - if (caseindex != casecount) { - throw new IOException("count mismatch: "+caseindex+" tokens found for column "+varindex); - } - offset = byteOffsets[varindex]; - } - - for (int caseindex = 0; caseindex < casecount; caseindex++) { - for (int varindex = 0; varindex < varcount; varindex++) { - System.out.print(reversedMatrix[caseindex][varindex]); - if (varindex < varcount-1) { - System.out.print("\t"); - } else { - System.out.print("\n"); - } - } - } - - } - - - } - - /** - * main() method, for testing - * usage: java edu.harvard.iq.dataverse.dataaccess.TabularSubsetGenerator testfile.tab varcount casecount column type - * make sure the CLASSPATH contains ... - * - */ - - public static void main(String[] args) { - - String tabFileName = args[0]; - int varcount = new Integer(args[1]).intValue(); - int casecount = new Integer(args[2]).intValue(); - int column = new Integer(args[3]).intValue(); - String type = args[4]; - - File tabFile = new File(tabFileName); - File rotatedImageFile = null; - - TabularSubsetGenerator subsetGenerator = new TabularSubsetGenerator(); - - /* - try { - rotatedImageFile = subsetGenerator.getRotatedImage(tabFile, varcount, casecount); - } catch (IOException ex) { - System.out.println(ex.getMessage()); - } - */ - - //System.out.println("\nFinished generating \"rotated\" column image file."); - - //System.out.println("\nOffsets:"); - - MathContext doubleMathContext = new MathContext(15, RoundingMode.HALF_EVEN); - String FORMAT_IEEE754 = "%+#.15e"; - - try { - //subsetGenerator.reverseRotatedImage(rotatedImageFile, varcount, casecount); - //String[] columns = subsetGenerator.subsetStringVector(tabFile, column, varcount, casecount); - if ("string".equals(type)) { - String[] columns = subsetGenerator.subsetStringVector(tabFile, column, varcount, casecount); - for (int i = 0; i < casecount; i++) { - System.out.println(columns[i]); - } - } else { - - Double[] columns = subsetGenerator.subsetDoubleVector(tabFile, column, varcount, casecount); - for (int i = 0; i < casecount; i++) { - if (columns[i] != null) { - BigDecimal outBigDecimal = new BigDecimal(columns[i], doubleMathContext); - System.out.println(String.format(FORMAT_IEEE754, outBigDecimal)); - } else { - System.out.println("NA"); - } - //System.out.println(columns[i]); - } - } - } catch (IOException ex) { - System.out.println(ex.getMessage()); - } - } -} - - + scanner.next(); + } +} \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/TabularSubsetInputStream.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/TabularSubsetInputStream.java deleted file mode 100644 index 89e033353c1..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/TabularSubsetInputStream.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ - -package edu.harvard.iq.dataverse.dataaccess; - -import edu.harvard.iq.dataverse.DataFile; -import edu.harvard.iq.dataverse.datavariable.DataVariable; -import java.io.IOException; -import java.io.InputStream; -import java.util.List; -import java.util.logging.Logger; - -/** - * - * @author Leonid Andreev - */ -public class TabularSubsetInputStream extends InputStream { - private static final Logger logger = Logger.getLogger(TabularSubsetInputStream.class.getCanonicalName()); - - private TabularSubsetGenerator subsetGenerator = null; - private int numberOfSubsetVariables; - private int numberOfObservations; - private int numberOfObservationsRead = 0; - private byte[] leftoverBytes = null; - - public TabularSubsetInputStream(DataFile datafile, List variables) throws IOException { - if (datafile == null) { - throw new IOException("Null datafile in subset request"); - } - if (!datafile.isTabularData()) { - throw new IOException("Subset requested on a non-tabular data file"); - } - numberOfObservations = datafile.getDataTable().getCaseQuantity().intValue(); - - if (variables == null || variables.size() < 1) { - throw new IOException("Null or empty list of variables in subset request."); - } - numberOfSubsetVariables = variables.size(); - subsetGenerator = new TabularSubsetGenerator(datafile, variables); - - } - - //@Override - public int read() throws IOException { - throw new IOException("read() method not implemented; do not use."); - } - - //@Override - public int read(byte[] b) throws IOException { - // TODO: - // Move this code into TabularSubsetGenerator - logger.fine("subset input stream: read request, on a "+b.length+" byte buffer;"); - - if (numberOfSubsetVariables == 1) { - logger.fine("calling the single variable subset read method"); - return subsetGenerator.readSingleColumnSubset(b); - } - - int bytesread = 0; - byte [] linebuffer; - - // do we have a leftover? - if (leftoverBytes != null) { - if (leftoverBytes.length < b.length) { - System.arraycopy(leftoverBytes, 0, b, 0, leftoverBytes.length); - bytesread = leftoverBytes.length; - leftoverBytes = null; - - } else { - // shouldn't really happen... unless it's a very large subset, - // or a very long string, etc. - System.arraycopy(leftoverBytes, 0, b, 0, b.length); - byte[] tmp = new byte[leftoverBytes.length - b.length]; - System.arraycopy(leftoverBytes, b.length, tmp, 0, leftoverBytes.length - b.length); - leftoverBytes = tmp; - tmp = null; - return b.length; - } - } - - while (bytesread < b.length && numberOfObservationsRead < numberOfObservations) { - linebuffer = subsetGenerator.readSubsetLineBytes(); - numberOfObservationsRead++; - - if (bytesread + linebuffer.length < b.length) { - // copy linebuffer into the return buffer: - System.arraycopy(linebuffer, 0, b, bytesread, linebuffer.length); - bytesread += linebuffer.length; - } else { - System.arraycopy(linebuffer, 0, b, bytesread, b.length - bytesread); - // save the leftover; - if (bytesread + linebuffer.length > b.length) { - leftoverBytes = new byte[bytesread + linebuffer.length - b.length]; - System.arraycopy(linebuffer, b.length - bytesread, leftoverBytes, 0, bytesread + linebuffer.length - b.length); - } - return b.length; - } - } - - // and this means we've reached the end of the tab file! - - return bytesread > 0 ? bytesread : -1; - } - - //@Override - public void close() { - if (subsetGenerator != null) { - subsetGenerator.close(); - } - } -} diff --git a/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleException.java b/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleException.java index 3329d92b7a9..474674bda73 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleException.java +++ b/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleException.java @@ -1,7 +1,9 @@ package edu.harvard.iq.dataverse.datacapturemodule; +@Deprecated(forRemoval = true, since = "2024-07-07") public class DataCaptureModuleException extends Exception { + @Deprecated(forRemoval = true, since = "2024-07-07") public DataCaptureModuleException(String message, Throwable cause) { super(message, cause); } diff --git a/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtil.java b/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtil.java index 460e4727afc..094d3976133 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtil.java @@ -12,10 +12,12 @@ import jakarta.json.JsonObject; import jakarta.json.JsonObjectBuilder; +@Deprecated(forRemoval = true, since = "2024-07-07") public class DataCaptureModuleUtil { private static final Logger logger = Logger.getLogger(DataCaptureModuleUtil.class.getCanonicalName()); + @Deprecated(forRemoval = true, since = "2024-07-07") public static boolean rsyncSupportEnabled(String uploadMethodsSettings) { logger.fine("uploadMethodsSettings: " + uploadMethodsSettings);; if (uploadMethodsSettings==null){ @@ -28,6 +30,7 @@ public static boolean rsyncSupportEnabled(String uploadMethodsSettings) { /** * generate JSON to send to DCM */ + @Deprecated(forRemoval = true, since = "2024-07-07") public static JsonObject generateJsonForUploadRequest(AuthenticatedUser user, Dataset dataset) { JsonObjectBuilder jab = Json.createObjectBuilder(); // The general rule should be to always pass the user id and dataset identifier to the DCM. @@ -39,6 +42,7 @@ public static JsonObject generateJsonForUploadRequest(AuthenticatedUser user, Da /** * transfer script from DCM */ + @Deprecated(forRemoval = true, since = "2024-07-07") public static ScriptRequestResponse getScriptFromRequest(HttpResponse uploadRequest) { int status = uploadRequest.getStatus(); JsonNode body = uploadRequest.getBody(); @@ -54,6 +58,7 @@ public static ScriptRequestResponse getScriptFromRequest(HttpResponse return scriptRequestResponse; } + @Deprecated(forRemoval = true, since = "2024-07-07") static UploadRequestResponse makeUploadRequest(HttpResponse uploadRequest) { int status = uploadRequest.getStatus(); String body = uploadRequest.getBody(); @@ -61,6 +66,7 @@ static UploadRequestResponse makeUploadRequest(HttpResponse uploadReques return new UploadRequestResponse(uploadRequest.getStatus(), body); } + @Deprecated(forRemoval = true, since = "2024-07-07") public static String getMessageFromException(DataCaptureModuleException ex) { if (ex == null) { return "DataCaptureModuleException was null!"; @@ -76,6 +82,7 @@ public static String getMessageFromException(DataCaptureModuleException ex) { return message + " was caused by " + cause.getMessage(); } + @Deprecated(forRemoval = true, since = "2024-07-07") public static String getScriptName(DatasetVersion datasetVersion) { return "upload-" + datasetVersion.getDataset().getIdentifier().replace("/", "_") + ".bash"; } diff --git a/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java b/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java index 03a0044a987..98bd26b51d6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java @@ -1,17 +1,14 @@ package edu.harvard.iq.dataverse.dataset; -import edu.harvard.iq.dataverse.DataFile; -import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.DatasetField; -import edu.harvard.iq.dataverse.DatasetVersion; -import edu.harvard.iq.dataverse.FileMetadata; -import edu.harvard.iq.dataverse.TermsOfUseAndAccess; +import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IpAddress; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.dataaccess.DataAccess; import static edu.harvard.iq.dataverse.api.ApiConstants.DS_VERSION_DRAFT; import static edu.harvard.iq.dataverse.dataaccess.DataAccess.getStorageIO; + +import edu.harvard.iq.dataverse.dataaccess.InputStreamIO; import edu.harvard.iq.dataverse.dataaccess.StorageIO; import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter; import edu.harvard.iq.dataverse.util.BundleUtil; @@ -31,15 +28,14 @@ import java.util.*; import java.util.logging.Logger; import javax.imageio.ImageIO; + +import jakarta.enterprise.inject.spi.CDI; import org.apache.commons.io.IOUtils; -import static edu.harvard.iq.dataverse.dataaccess.DataAccess.getStorageIO; import edu.harvard.iq.dataverse.datasetutility.FileSizeChecker; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.license.License; -import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.StringUtil; import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json; -import static edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder.jsonObjectBuilder; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.EnumUtils; @@ -218,7 +214,8 @@ public static boolean deleteDatasetLogo(Dataset dataset) { storageIO.deleteAuxObject(datasetLogoThumbnail + thumbExtension + ImageThumbConverter.DEFAULT_CARDIMAGE_SIZE); } catch (IOException ex) { - logger.info("Failed to delete dataset logo: " + ex.getMessage()); + logger.fine("Failed to delete dataset logo: " + ex.getMessage() + + " (this is most likely harmless; this method is often called without checking if the custom dataset logo was in fact present)"); return false; } return true; @@ -293,7 +290,7 @@ public static Dataset persistDatasetLogoToStorageAndCreateThumbnails(Dataset dat dataAccess = DataAccess.getStorageIO(dataset); } catch(IOException ioex){ - //TODO: Add a suitable waing message + //TODO: Add a suitable warning message logger.warning("Failed to save the file, storage id " + dataset.getStorageIdentifier() + " (" + ioex.getMessage() + ")"); } @@ -355,30 +352,44 @@ public static Dataset persistDatasetLogoToStorageAndCreateThumbnails(Dataset dat // We'll try to pre-generate the rescaled versions in both the // DEFAULT_DATASET_LOGO (currently 140) and DEFAULT_CARDIMAGE_SIZE (48) String thumbFileLocation = ImageThumbConverter.rescaleImage(fullSizeImage, width, height, ImageThumbConverter.DEFAULT_DATASETLOGO_SIZE, tmpFileForResize.toPath().toString()); - logger.fine("thumbFileLocation = " + thumbFileLocation); - logger.fine("tmpFileLocation=" + tmpFileForResize.toPath().toString()); - //now we must save the updated thumbnail - try { - dataAccess.savePathAsAux(Paths.get(thumbFileLocation), datasetLogoThumbnail+thumbExtension+ImageThumbConverter.DEFAULT_DATASETLOGO_SIZE); - } catch (IOException ex) { - logger.severe("Failed to move updated thumbnail file from " + tmpFile.getAbsolutePath() + " to its DataAccess location" + ": " + ex); + if (thumbFileLocation == null) { + logger.warning("Rescale Thumbnail Image to logo failed"); + dataset.setPreviewImageAvailable(false); + dataset.setUseGenericThumbnail(true); + } else { + logger.fine("thumbFileLocation = " + thumbFileLocation); + logger.fine("tmpFileLocation=" + tmpFileForResize.toPath().toString()); + //now we must save the updated thumbnail + try { + dataAccess.savePathAsAux(Paths.get(thumbFileLocation), datasetLogoThumbnail + thumbExtension + ImageThumbConverter.DEFAULT_DATASETLOGO_SIZE); + } catch (IOException ex) { + logger.severe("Failed to move updated thumbnail file from " + tmpFile.getAbsolutePath() + " to its DataAccess location" + ": " + ex); + } } thumbFileLocation = ImageThumbConverter.rescaleImage(fullSizeImage, width, height, ImageThumbConverter.DEFAULT_CARDIMAGE_SIZE, tmpFileForResize.toPath().toString()); - logger.fine("thumbFileLocation = " + thumbFileLocation); - logger.fine("tmpFileLocation=" + tmpFileForResize.toPath().toString()); - //now we must save the updated thumbnail - try { - dataAccess.savePathAsAux(Paths.get(thumbFileLocation), datasetLogoThumbnail+thumbExtension+ImageThumbConverter.DEFAULT_CARDIMAGE_SIZE); - } catch (IOException ex) { - logger.severe("Failed to move updated thumbnail file from " + tmpFile.getAbsolutePath() + " to its DataAccess location" + ": " + ex); + if (thumbFileLocation == null) { + logger.warning("Rescale Thumbnail Image to card failed"); + dataset.setPreviewImageAvailable(false); + dataset.setUseGenericThumbnail(true); + } else { + logger.fine("thumbFileLocation = " + thumbFileLocation); + logger.fine("tmpFileLocation=" + tmpFileForResize.toPath().toString()); + //now we must save the updated thumbnail + try { + dataAccess.savePathAsAux(Paths.get(thumbFileLocation), datasetLogoThumbnail + thumbExtension + ImageThumbConverter.DEFAULT_CARDIMAGE_SIZE); + } catch (IOException ex) { + logger.severe("Failed to move updated thumbnail file from " + tmpFile.getAbsolutePath() + " to its DataAccess location" + ": " + ex); + } } //This deletes the tempfiles created for rescaling and encoding boolean tmpFileWasDeleted = tmpFile.delete(); boolean originalTempFileWasDeleted = tmpFileForResize.delete(); try { - Files.delete(Paths.get(thumbFileLocation)); + if (thumbFileLocation != null) { + Files.delete(Paths.get(thumbFileLocation)); + } } catch (IOException ioex) { logger.fine("Failed to delete temporary thumbnail file"); } @@ -463,8 +474,19 @@ public static InputStream getLogoAsInputStream(Dataset dataset) { } try { - in = ImageThumbConverter.getImageThumbnailAsInputStream(thumbnailFile.getStorageIO(), - ImageThumbConverter.DEFAULT_DATASETLOGO_SIZE).getInputStream(); + + boolean origImageFailed = thumbnailFile.isPreviewImageFail(); + InputStreamIO isIO = ImageThumbConverter.getImageThumbnailAsInputStream(thumbnailFile.getStorageIO(), + ImageThumbConverter.DEFAULT_DATASETLOGO_SIZE); + if (!origImageFailed && thumbnailFile.isPreviewImageFail()) { + // We found an older 0 length thumbnail. Newer image uploads will not have this issue. + // Once cleaned up, this thumbnail will no longer have this issue + // ImageThumbConverter fixed the DataFile + // Now we need to update dataset since this is a bad logo + DatasetServiceBean datasetService = CDI.current().select(DatasetServiceBean.class).get(); + datasetService.clearDatasetLevelThumbnail(dataset); + } + in = isIO != null ? isIO.getInputStream() : null; } catch (IOException ioex) { logger.warning("getLogo(): Failed to get logo from DataFile for " + dataset.getStorageIdentifier() + " (" + ioex.getMessage() + ")"); diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/CommandContext.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/CommandContext.java index f74c1222bb0..96330271367 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/CommandContext.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/CommandContext.java @@ -1,9 +1,7 @@ package edu.harvard.iq.dataverse.engine.command; -import edu.harvard.iq.dataverse.DOIDataCiteServiceBean; -import edu.harvard.iq.dataverse.DOIEZIdServiceBean; -import edu.harvard.iq.dataverse.HandlenetServiceBean; import edu.harvard.iq.dataverse.DataFileServiceBean; +import edu.harvard.iq.dataverse.DatasetFieldServiceBean; import edu.harvard.iq.dataverse.DatasetLinkingServiceBean; import edu.harvard.iq.dataverse.DatasetServiceBean; import edu.harvard.iq.dataverse.DatasetVersionServiceBean; @@ -18,6 +16,7 @@ import edu.harvard.iq.dataverse.FileDownloadServiceBean; import edu.harvard.iq.dataverse.GuestbookResponseServiceBean; import edu.harvard.iq.dataverse.GuestbookServiceBean; +import edu.harvard.iq.dataverse.MetadataBlockServiceBean; import edu.harvard.iq.dataverse.search.IndexServiceBean; import edu.harvard.iq.dataverse.PermissionServiceBean; import edu.harvard.iq.dataverse.RoleAssigneeServiceBean; @@ -32,8 +31,7 @@ import edu.harvard.iq.dataverse.datacapturemodule.DataCaptureModuleServiceBean; import edu.harvard.iq.dataverse.engine.DataverseEngine; import edu.harvard.iq.dataverse.ingest.IngestServiceBean; -import edu.harvard.iq.dataverse.pidproviders.FakePidProviderServiceBean; -import edu.harvard.iq.dataverse.pidproviders.PermaLinkPidProviderServiceBean; +import edu.harvard.iq.dataverse.pidproviders.PidProviderFactoryBean; import edu.harvard.iq.dataverse.privateurl.PrivateUrlServiceBean; import edu.harvard.iq.dataverse.search.IndexBatchServiceBean; import edu.harvard.iq.dataverse.search.SolrIndexServiceBean; @@ -100,15 +98,7 @@ public interface CommandContext { public DataverseFieldTypeInputLevelServiceBean fieldTypeInputLevels(); - public DOIEZIdServiceBean doiEZId(); - - public DOIDataCiteServiceBean doiDataCite(); - - public FakePidProviderServiceBean fakePidProvider(); - - public HandlenetServiceBean handleNet(); - - public PermaLinkPidProviderServiceBean permaLinkProvider(); + public PidProviderFactoryBean pidProviderFactory(); public GuestbookServiceBean guestbooks(); @@ -145,7 +135,9 @@ public interface CommandContext { public ConfirmEmailServiceBean confirmEmail(); public ActionLogServiceBean actionLog(); - + + public MetadataBlockServiceBean metadataBlocks(); + public void beginCommandSequence(); public boolean completeCommandSequence(Command command); @@ -155,4 +147,6 @@ public interface CommandContext { public Stack getCommandsCalled(); public void addCommand(Command command); + + public DatasetFieldServiceBean dsField(); } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/exception/RateLimitCommandException.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/exception/RateLimitCommandException.java new file mode 100644 index 00000000000..99a665b31ac --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/exception/RateLimitCommandException.java @@ -0,0 +1,16 @@ +package edu.harvard.iq.dataverse.engine.command.exception; + +import edu.harvard.iq.dataverse.engine.command.Command; + +/** + * An exception raised when a command cannot be executed, due to the + * issuing user being rate limited. + * + * @author + */ +public class RateLimitCommandException extends CommandException { + + public RateLimitCommandException(String message, Command aCommand) { + super(message, aCommand); + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractCreateDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractCreateDatasetCommand.java index 303d8e1c25f..ab78a88c9a7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractCreateDatasetCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractCreateDatasetCommand.java @@ -1,24 +1,19 @@ package edu.harvard.iq.dataverse.engine.command.impl; -import edu.harvard.iq.dataverse.*; +import edu.harvard.iq.dataverse.DataFile; +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; -import edu.harvard.iq.dataverse.batch.util.LoggingUtil; import edu.harvard.iq.dataverse.dataaccess.DataAccess; -import edu.harvard.iq.dataverse.datacapturemodule.DataCaptureModuleUtil; -import edu.harvard.iq.dataverse.datacapturemodule.ScriptRequestResponse; import edu.harvard.iq.dataverse.engine.command.CommandContext; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; -import edu.harvard.iq.dataverse.engine.command.exception.CommandExecutionException; -import edu.harvard.iq.dataverse.settings.SettingsServiceBean; +import edu.harvard.iq.dataverse.pidproviders.PidProvider; import static edu.harvard.iq.dataverse.util.StringUtil.isEmpty; -import java.io.IOException; import java.util.Objects; -import java.util.logging.Level; import java.util.logging.Logger; -import org.apache.solr.client.solrj.SolrServerException; /**; * An abstract base class for commands that creates {@link Dataset}s. @@ -81,9 +76,10 @@ public Dataset execute(CommandContext ctxt) throws CommandException { additionalParameterTests(ctxt); Dataset theDataset = getDataset(); - GlobalIdServiceBean idServiceBean = GlobalIdServiceBean.getBean(ctxt); + PidProvider pidProvider = ctxt.dvObjects().getEffectivePidGenerator(theDataset); + if ( isEmpty(theDataset.getIdentifier()) ) { - theDataset.setIdentifier(idServiceBean.generateDatasetIdentifier(theDataset)); + pidProvider.generatePid(theDataset); } DatasetVersion dsv = getVersionToPersist(theDataset); @@ -94,6 +90,8 @@ public Dataset execute(CommandContext ctxt) throws CommandException { if(!harvested) { checkSystemMetadataKeyIfNeeded(dsv, null); } + + registerExternalVocabValuesIfAny(ctxt, dsv); theDataset.setCreator((AuthenticatedUser) getRequest().getUser()); @@ -105,19 +103,18 @@ public Dataset execute(CommandContext ctxt) throws CommandException { dataFile.setCreateDate(theDataset.getCreateDate()); } - String nonNullDefaultIfKeyNotFound = ""; if (theDataset.getProtocol()==null) { - theDataset.setProtocol(ctxt.settings().getValueForKey(SettingsServiceBean.Key.Protocol, nonNullDefaultIfKeyNotFound)); + theDataset.setProtocol(pidProvider.getProtocol()); } if (theDataset.getAuthority()==null) { - theDataset.setAuthority(ctxt.settings().getValueForKey(SettingsServiceBean.Key.Authority, nonNullDefaultIfKeyNotFound)); + theDataset.setAuthority(pidProvider.getAuthority()); } if (theDataset.getStorageIdentifier() == null) { String driverId = theDataset.getEffectiveStorageDriverId(); theDataset.setStorageIdentifier(driverId + DataAccess.SEPARATOR + theDataset.getAuthorityForFileStorage() + "/" + theDataset.getIdentifierForFileStorage()); } if (theDataset.getIdentifier()==null) { - theDataset.setIdentifier(idServiceBean.generateDatasetIdentifier(theDataset)); + pidProvider.generatePid(theDataset); } // Attempt the registration if importing dataset through the API, or the app (but not harvest) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractDatasetCommand.java index 6061461306d..1a1f4f9318b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractDatasetCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractDatasetCommand.java @@ -2,10 +2,13 @@ import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetField; +import edu.harvard.iq.dataverse.DatasetFieldServiceBean; import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.DatasetVersionDifference; import edu.harvard.iq.dataverse.DatasetVersionUser; import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.MetadataBlock; +import edu.harvard.iq.dataverse.TermsOfUseAndAccess; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.engine.command.AbstractCommand; import edu.harvard.iq.dataverse.engine.command.CommandContext; @@ -13,6 +16,8 @@ import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.exception.CommandExecutionException; import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; +import edu.harvard.iq.dataverse.pidproviders.PidProvider; +import edu.harvard.iq.dataverse.pidproviders.PidUtil; import edu.harvard.iq.dataverse.util.BundleUtil; import java.sql.Timestamp; @@ -22,10 +27,8 @@ import java.util.logging.Logger; import static java.util.stream.Collectors.joining; +import jakarta.ejb.EJB; import jakarta.validation.ConstraintViolation; -import edu.harvard.iq.dataverse.GlobalIdServiceBean; -import edu.harvard.iq.dataverse.MetadataBlock; -import edu.harvard.iq.dataverse.TermsOfUseAndAccess; import edu.harvard.iq.dataverse.settings.JvmSettings; /** @@ -152,18 +155,18 @@ protected void validateOrDie(DatasetVersion dsv, Boolean lenient) throws Command */ protected void registerExternalIdentifier(Dataset theDataset, CommandContext ctxt, boolean retry) throws CommandException { if (!theDataset.isIdentifierRegistered()) { - GlobalIdServiceBean globalIdServiceBean = GlobalIdServiceBean.getBean(theDataset.getProtocol(), ctxt); - if ( globalIdServiceBean != null ) { + PidProvider pidProvider = PidUtil.getPidProvider(theDataset.getGlobalId().getProviderId()); + if ( pidProvider != null ) { try { - if (globalIdServiceBean.alreadyRegistered(theDataset)) { + if (pidProvider.alreadyRegistered(theDataset)) { int attempts = 0; if(retry) { do { - theDataset.setIdentifier(globalIdServiceBean.generateDatasetIdentifier(theDataset)); + pidProvider.generatePid(theDataset); logger.log(Level.INFO, "Attempting to register external identifier for dataset {0} (trying: {1}).", new Object[]{theDataset.getId(), theDataset.getIdentifier()}); attempts++; - } while (globalIdServiceBean.alreadyRegistered(theDataset) && attempts <= FOOLPROOF_RETRIAL_ATTEMPTS_LIMIT); + } while (pidProvider.alreadyRegistered(theDataset) && attempts <= FOOLPROOF_RETRIAL_ATTEMPTS_LIMIT); } if(!retry) { logger.warning("Reserving PID for: " + getDataset().getId() + " during publication failed."); @@ -177,7 +180,7 @@ protected void registerExternalIdentifier(Dataset theDataset, CommandContext ctx } // Invariant: Dataset identifier does not exist in the remote registry try { - globalIdServiceBean.createIdentifier(theDataset); + pidProvider.createIdentifier(theDataset); theDataset.setGlobalIdCreateTime(getTimestamp()); theDataset.setIdentifierRegistered(true); } catch (Throwable ex) { @@ -185,7 +188,7 @@ protected void registerExternalIdentifier(Dataset theDataset, CommandContext ctx } } catch (Throwable e) { - throw new CommandException(BundleUtil.getStringFromBundle("dataset.publish.error", globalIdServiceBean.getProviderInformation()), this); + throw new CommandException(BundleUtil.getStringFromBundle("dataset.publish.error", pidProvider.getProviderInformation()), this); } } else { throw new IllegalCommandException("This dataset may not be published because its id registry service is not supported.", this); @@ -230,4 +233,13 @@ protected void checkSystemMetadataKeyIfNeeded(DatasetVersion newVersion, Dataset } } } + + protected void registerExternalVocabValuesIfAny(CommandContext ctxt, DatasetVersion newVersion) { + for (DatasetField df : newVersion.getFlatDatasetFields()) { + logger.fine("Found id: " + df.getDatasetFieldType().getId()); + if (ctxt.dsField().getCVocConf(true).containsKey(df.getDatasetFieldType().getId())) { + ctxt.dsField().registerExternalVocabValues(df); + } + } + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractGetPublishedFileMetadataCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractGetPublishedFileMetadataCommand.java new file mode 100644 index 00000000000..4fef2c27efb --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractGetPublishedFileMetadataCommand.java @@ -0,0 +1,39 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.DataFile; +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetVersion; +import edu.harvard.iq.dataverse.FileMetadata; +import edu.harvard.iq.dataverse.authorization.Permission; +import edu.harvard.iq.dataverse.engine.command.AbstractCommand; +import edu.harvard.iq.dataverse.engine.command.CommandContext; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; + +@RequiredPermissions({}) +abstract class AbstractGetPublishedFileMetadataCommand extends AbstractCommand { + protected final DataFile dataFile; + protected final boolean includeDeaccessioned; + + public AbstractGetPublishedFileMetadataCommand(DataverseRequest request, DataFile dataFile, boolean includeDeaccessioned) { + super(request, dataFile); + this.dataFile = dataFile; + this.includeDeaccessioned = includeDeaccessioned; + } + + protected FileMetadata getLatestPublishedFileMetadata(CommandContext ctxt) { + return dataFile.getFileMetadatas().stream().filter(fileMetadata -> { + DatasetVersion.VersionState versionState = fileMetadata.getDatasetVersion().getVersionState(); + return (!versionState.equals(DatasetVersion.VersionState.DRAFT) + && isDatasetVersionAccessible(fileMetadata.getDatasetVersion(), dataFile.getOwner(), ctxt)); + }).reduce(null, DataFile::getTheNewerFileMetadata); + } + + protected boolean isDatasetVersionAccessible(DatasetVersion datasetVersion, Dataset ownerDataset, CommandContext ctxt) { + return datasetVersion.isReleased() || isDatasetVersionDeaccessionedAndAccessible(datasetVersion, ownerDataset, ctxt); + } + + private boolean isDatasetVersionDeaccessionedAndAccessible(DatasetVersion datasetVersion, Dataset ownerDataset, CommandContext ctxt) { + return includeDeaccessioned && datasetVersion.isDeaccessioned() && ctxt.permissions().requestOn(getRequest(), ownerDataset).has(Permission.EditDataset); + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractSubmitToArchiveCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractSubmitToArchiveCommand.java index b988fd05f03..29c27d0396d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractSubmitToArchiveCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractSubmitToArchiveCommand.java @@ -1,6 +1,5 @@ package edu.harvard.iq.dataverse.engine.command.impl; -import edu.harvard.iq.dataverse.DOIDataCiteRegisterService; import edu.harvard.iq.dataverse.DataCitation; import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetVersion; @@ -14,6 +13,7 @@ import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; +import edu.harvard.iq.dataverse.pidproviders.doi.datacite.DOIDataCiteRegisterService; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.bagit.BagGenerator; import edu.harvard.iq.dataverse.util.bagit.OREMap; diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AssignRoleCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AssignRoleCommand.java index 5577d541012..121af765737 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AssignRoleCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AssignRoleCommand.java @@ -3,7 +3,6 @@ */ package edu.harvard.iq.dataverse.engine.command.impl; -import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.Dataverse; import edu.harvard.iq.dataverse.authorization.DataverseRole; @@ -18,7 +17,10 @@ import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; +import edu.harvard.iq.dataverse.util.BundleUtil; + import java.util.Collections; +import java.util.HashSet; import java.util.Map; import java.util.Set; @@ -67,17 +69,38 @@ public RoleAssignment execute(CommandContext ctxt) throws CommandException { throw new IllegalCommandException("User " + user.getUserIdentifier() + " is deactivated and cannot be given a role.", this); } } + if(isExistingRole(ctxt)){ + throw new IllegalCommandException(BundleUtil.getStringFromBundle("datasets.api.grant.role.assignee.has.role.error"), this); + } // TODO make sure the role is defined on the dataverse. RoleAssignment roleAssignment = new RoleAssignment(role, grantee, defPoint, privateUrlToken, anonymizedAccess); return ctxt.roles().save(roleAssignment); } + private boolean isExistingRole(CommandContext ctxt) { + return ctxt.roles() + .directRoleAssignments(grantee, defPoint) + .stream() + .map(RoleAssignment::getRole) + .anyMatch(it -> it.equals(role)); + } + @Override public Map> getRequiredPermissions() { // for data file check permission on owning dataset - return Collections.singletonMap("", - defPoint instanceof Dataverse ? Collections.singleton(Permission.ManageDataversePermissions) - : defPoint instanceof Dataset ? Collections.singleton(Permission.ManageDatasetPermissions) : Collections.singleton(Permission.ManageFilePermissions)); + Set requiredPermissions = new HashSet(); + + if (defPoint instanceof Dataverse) { + requiredPermissions.add(Permission.ManageDataversePermissions); + } else if (defPoint instanceof Dataset) { + requiredPermissions.add(Permission.ManageDatasetPermissions); + } else { + requiredPermissions.add(Permission.ManageFilePermissions); + } + + requiredPermissions.addAll(role.permissions()); + + return Collections.singletonMap("", requiredPermissions); } @Override diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CheckRateLimitForCollectionPageCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CheckRateLimitForCollectionPageCommand.java new file mode 100644 index 00000000000..b23e6034c9a --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CheckRateLimitForCollectionPageCommand.java @@ -0,0 +1,16 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.DvObject; +import edu.harvard.iq.dataverse.engine.command.AbstractVoidCommand; +import edu.harvard.iq.dataverse.engine.command.CommandContext; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; + +public class CheckRateLimitForCollectionPageCommand extends AbstractVoidCommand { + public CheckRateLimitForCollectionPageCommand(DataverseRequest aRequest, DvObject dvObject) { + super(aRequest, dvObject); + } + + @Override + protected void executeImpl(CommandContext ctxt) throws CommandException { } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CheckRateLimitForDatasetPageCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CheckRateLimitForDatasetPageCommand.java new file mode 100644 index 00000000000..da8c1e4d8e3 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CheckRateLimitForDatasetPageCommand.java @@ -0,0 +1,17 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.DvObject; +import edu.harvard.iq.dataverse.engine.command.AbstractVoidCommand; +import edu.harvard.iq.dataverse.engine.command.CommandContext; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; + +public class CheckRateLimitForDatasetPageCommand extends AbstractVoidCommand { + + public CheckRateLimitForDatasetPageCommand(DataverseRequest aRequest, DvObject dvObject) { + super(aRequest, dvObject); + } + + @Override + protected void executeImpl(CommandContext ctxt) throws CommandException { } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDatasetVersionCommand.java index bcaece55fed..6539ac27ea2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDatasetVersionCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDatasetVersionCommand.java @@ -59,7 +59,8 @@ public DatasetVersion execute(CommandContext ctxt) throws CommandException { //Will throw an IllegalCommandException if a system metadatablock is changed and the appropriate key is not supplied. checkSystemMetadataKeyIfNeeded(newVersion, latest); - + registerExternalVocabValuesIfAny(ctxt, newVersion); + List newVersionMetadatum = new ArrayList<>(latest.getFileMetadatas().size()); for ( FileMetadata fmd : latest.getFileMetadatas() ) { FileMetadata fmdCopy = fmd.createCopy(); diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDatasetCommand.java index c9ebe735e31..c22a2cdb4a2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDatasetCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDatasetCommand.java @@ -3,6 +3,7 @@ import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.GlobalId; import edu.harvard.iq.dataverse.RoleAssignment; import edu.harvard.iq.dataverse.Template; import edu.harvard.iq.dataverse.UserNotification; @@ -12,12 +13,13 @@ import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; +import edu.harvard.iq.dataverse.pidproviders.PidProvider; +import edu.harvard.iq.dataverse.pidproviders.PidUtil; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import static edu.harvard.iq.dataverse.util.StringUtil.nonEmpty; import java.util.logging.Logger; -import edu.harvard.iq.dataverse.GlobalIdServiceBean; import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; import java.util.List; import java.sql.Timestamp; @@ -71,13 +73,18 @@ public CreateNewDatasetCommand(Dataset theDataset, DataverseRequest aRequest, Te */ @Override protected void additionalParameterTests(CommandContext ctxt) throws CommandException { - if ( nonEmpty(getDataset().getIdentifier()) ) { - GlobalIdServiceBean idServiceBean = GlobalIdServiceBean.getBean(getDataset().getProtocol(), ctxt); - if ( !idServiceBean.isGlobalIdUnique(getDataset().getGlobalId()) ) { - throw new IllegalCommandException(String.format("Dataset with identifier '%s', protocol '%s' and authority '%s' already exists", - getDataset().getIdentifier(), getDataset().getProtocol(), getDataset().getAuthority()), - this); - } + if (nonEmpty(getDataset().getIdentifier())) { + GlobalId pid = getDataset().getGlobalId(); + if (pid != null) { + PidProvider pidProvider = PidUtil.getPidProvider(pid.getProviderId()); + + if (!pidProvider.isGlobalIdUnique(pid)) { + throw new IllegalCommandException(String.format( + "Dataset with identifier '%s', protocol '%s' and authority '%s' already exists", + getDataset().getIdentifier(), getDataset().getProtocol(), getDataset().getAuthority()), + this); + } + } } } @@ -88,11 +95,11 @@ protected DatasetVersion getVersionToPersist( Dataset theDataset ) { @Override protected void handlePid(Dataset theDataset, CommandContext ctxt) throws CommandException { - GlobalIdServiceBean idServiceBean = GlobalIdServiceBean.getBean(ctxt); - if(!idServiceBean.isConfigured()) { - throw new IllegalCommandException("PID Provider " + idServiceBean.getProviderInformation().get(0) + " is not configured.", this); + PidProvider pidProvider = PidUtil.getPidProvider(theDataset.getGlobalId().getProviderId()); + if(!pidProvider.canManagePID()) { + throw new IllegalCommandException("PID Provider " + pidProvider.getId() + " is not configured.", this); } - if ( !idServiceBean.registerWhenPublished() ) { + if ( !pidProvider.registerWhenPublished() ) { // pre-register a persistent id registerExternalIdentifier(theDataset, ctxt, true); } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDataFileCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDataFileCommand.java index e2730ec06d3..0812c52a846 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDataFileCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDataFileCommand.java @@ -1,6 +1,7 @@ package edu.harvard.iq.dataverse.engine.command.impl; import edu.harvard.iq.dataverse.DataFile; +import edu.harvard.iq.dataverse.GlobalId; import edu.harvard.iq.dataverse.search.IndexServiceBean; import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; @@ -11,6 +12,8 @@ import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.exception.CommandExecutionException; import edu.harvard.iq.dataverse.engine.command.exception.PermissionException; +import edu.harvard.iq.dataverse.pidproviders.PidProvider; +import edu.harvard.iq.dataverse.pidproviders.PidUtil; import edu.harvard.iq.dataverse.util.FileUtil; import edu.harvard.iq.dataverse.util.StringUtil; import java.io.IOException; @@ -23,7 +26,6 @@ import java.util.Collections; import java.util.logging.Level; import java.util.logging.Logger; -import edu.harvard.iq.dataverse.GlobalIdServiceBean; /** * Deletes a data file, both DB entity and filesystem object. @@ -202,15 +204,18 @@ public FileVisitResult postVisitDirectory(final Path dir, final IOException e) */ } } - GlobalIdServiceBean idServiceBean = GlobalIdServiceBean.getBean(ctxt); - try { - if (idServiceBean.alreadyRegistered(doomed)) { - idServiceBean.deleteIdentifier(doomed); + GlobalId pid = doomed.getGlobalId(); + if (pid != null) { + PidProvider pidProvider = PidUtil.getPidProvider(pid.getProviderId()); + + try { + if (pidProvider.alreadyRegistered(doomed)) { + pidProvider.deleteIdentifier(doomed); + } + } catch (Exception e) { + logger.log(Level.WARNING, "Identifier deletion was not successfull:", e.getMessage()); } - } catch (Exception e) { - logger.log(Level.WARNING, "Identifier deletion was not successfull:", e.getMessage()); } - DataFile doomedAndMerged = ctxt.em().merge(doomed); ctxt.em().remove(doomedAndMerged); /** diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeletePidCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeletePidCommand.java index 274aeb3c3fd..c4910dd10c2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeletePidCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeletePidCommand.java @@ -1,7 +1,6 @@ package edu.harvard.iq.dataverse.engine.command.impl; import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.GlobalIdServiceBean; import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.engine.command.AbstractVoidCommand; @@ -11,7 +10,8 @@ import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; import edu.harvard.iq.dataverse.engine.command.exception.PermissionException; -import edu.harvard.iq.dataverse.settings.SettingsServiceBean; +import edu.harvard.iq.dataverse.pidproviders.PidProvider; +import edu.harvard.iq.dataverse.pidproviders.PidUtil; import edu.harvard.iq.dataverse.util.BundleUtil; import org.apache.commons.httpclient.HttpException; @@ -38,25 +38,26 @@ public DeletePidCommand(DataverseRequest request, Dataset dataset) { protected void executeImpl(CommandContext ctxt) throws CommandException { if (!(getUser() instanceof AuthenticatedUser) || !getUser().isSuperuser()) { - throw new PermissionException(BundleUtil.getStringFromBundle("admin.api.auth.mustBeSuperUser"), - this, Collections.singleton(Permission.EditDataset), dataset); + throw new PermissionException(BundleUtil.getStringFromBundle("admin.api.auth.mustBeSuperUser"), this, + Collections.singleton(Permission.EditDataset), dataset); } - String nonNullDefaultIfKeyNotFound = ""; - String protocol = ctxt.settings().getValueForKey(SettingsServiceBean.Key.Protocol, nonNullDefaultIfKeyNotFound); - GlobalIdServiceBean idServiceBean = GlobalIdServiceBean.getBean(protocol, ctxt); + PidProvider pidProvider = PidUtil.getPidProvider(dataset.getGlobalId().getProviderId()); + try { - idServiceBean.deleteIdentifier(dataset); + pidProvider.deleteIdentifier(dataset); // Success! Clear the create time, etc. dataset.setGlobalIdCreateTime(null); dataset.setIdentifierRegistered(false); ctxt.datasets().merge(dataset); } catch (HttpException hex) { - String message = BundleUtil.getStringFromBundle("pids.deletePid.failureExpected", Arrays.asList(dataset.getGlobalId().asString(), Integer.toString(hex.getReasonCode()))); + String message = BundleUtil.getStringFromBundle("pids.deletePid.failureExpected", + Arrays.asList(dataset.getGlobalId().asString(), Integer.toString(hex.getReasonCode()))); logger.info(message); throw new IllegalCommandException(message, this); } catch (Exception ex) { - String message = BundleUtil.getStringFromBundle("pids.deletePid.failureOther", Arrays.asList(dataset.getGlobalId().asString(), ex.getLocalizedMessage())); + String message = BundleUtil.getStringFromBundle("pids.deletePid.failureOther", + Arrays.asList(dataset.getGlobalId().asString(), ex.getLocalizedMessage())); logger.info(message); throw new IllegalCommandException(message, this); } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DestroyDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DestroyDatasetCommand.java index 41093444360..be3e28029e4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DestroyDatasetCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DestroyDatasetCommand.java @@ -2,7 +2,9 @@ import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.GlobalId; import edu.harvard.iq.dataverse.authorization.DataverseRole; import edu.harvard.iq.dataverse.search.IndexServiceBean; import edu.harvard.iq.dataverse.RoleAssignment; @@ -15,6 +17,8 @@ import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.exception.PermissionException; +import edu.harvard.iq.dataverse.pidproviders.PidProvider; +import edu.harvard.iq.dataverse.pidproviders.PidUtil; import edu.harvard.iq.dataverse.search.IndexResponse; import java.util.ArrayList; import java.util.Collections; @@ -22,7 +26,7 @@ import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; -import edu.harvard.iq.dataverse.GlobalIdServiceBean; + import edu.harvard.iq.dataverse.batch.util.LoggingUtil; import java.io.IOException; import java.util.concurrent.Future; @@ -61,17 +65,17 @@ protected void executeImpl(CommandContext ctxt) throws CommandException { throw new PermissionException("Destroy can only be called by superusers.", this, Collections.singleton(Permission.DeleteDatasetDraft), doomed); } + Dataset managedDoomed = ctxt.em().merge(doomed); // If there is a dedicated thumbnail DataFile, it needs to be reset // explicitly, or we'll get a constraint violation when deleting: - doomed.setThumbnailFile(null); - final Dataset managedDoomed = ctxt.em().merge(doomed); - + managedDoomed.setThumbnailFile(null); + // files need to iterate through and remove 'by hand' to avoid // optimistic lock issues... (plus the physical files need to be // deleted too!) - - Iterator dfIt = doomed.getFiles().iterator(); + DatasetVersion dv = managedDoomed.getLatestVersion(); + Iterator dfIt = managedDoomed.getFiles().iterator(); while (dfIt.hasNext()){ DataFile df = dfIt.next(); // Gather potential Solr IDs of files. As of this writing deaccessioned files are never indexed. @@ -82,50 +86,52 @@ protected void executeImpl(CommandContext ctxt) throws CommandException { ctxt.engine().submit(new DeleteDataFileCommand(df, getRequest(), true)); dfIt.remove(); } - - //also, lets delete the uploaded thumbnails! - if (!doomed.isHarvested()) { - deleteDatasetLogo(doomed); - } + dv.setFileMetadatas(null); // ASSIGNMENTS - for (RoleAssignment ra : ctxt.roles().directRoleAssignments(doomed)) { + for (RoleAssignment ra : ctxt.roles().directRoleAssignments(managedDoomed)) { ctxt.em().remove(ra); } // ROLES - for (DataverseRole ra : ctxt.roles().findByOwnerId(doomed.getId())) { + for (DataverseRole ra : ctxt.roles().findByOwnerId(managedDoomed.getId())) { ctxt.em().remove(ra); } - if (!doomed.isHarvested()) { - GlobalIdServiceBean idServiceBean = GlobalIdServiceBean.getBean(ctxt); - try { - if (idServiceBean.alreadyRegistered(doomed)) { - idServiceBean.deleteIdentifier(doomed); - for (DataFile df : doomed.getFiles()) { - idServiceBean.deleteIdentifier(df); + if (!managedDoomed.isHarvested()) { + //also, lets delete the uploaded thumbnails! + deleteDatasetLogo(managedDoomed); + // and remove the PID (perhaps should be after the remove in case that causes a roll-back?) + GlobalId pid = managedDoomed.getGlobalId(); + if (pid != null) { + PidProvider pidProvider = PidUtil.getPidProvider(pid.getProviderId()); + try { + if (pidProvider.alreadyRegistered(managedDoomed)) { + pidProvider.deleteIdentifier(managedDoomed); + //Files are handled in DeleteDataFileCommand } + } catch (Exception e) { + logger.log(Level.WARNING, "Identifier deletion was not successful:", e.getMessage()); } - } catch (Exception e) { - logger.log(Level.WARNING, "Identifier deletion was not successful:", e.getMessage()); } - } + } toReIndex = managedDoomed.getOwner(); - // dataset - ctxt.em().remove(managedDoomed); - // add potential Solr IDs of datasets to list for deletion - String solrIdOfPublishedDatasetVersion = IndexServiceBean.solrDocIdentifierDataset + doomed.getId(); + String solrIdOfPublishedDatasetVersion = IndexServiceBean.solrDocIdentifierDataset + managedDoomed.getId(); datasetAndFileSolrIdsToDelete.add(solrIdOfPublishedDatasetVersion); - String solrIdOfDraftDatasetVersion = IndexServiceBean.solrDocIdentifierDataset + doomed.getId() + IndexServiceBean.draftSuffix; + String solrIdOfDraftDatasetVersion = IndexServiceBean.solrDocIdentifierDataset + managedDoomed.getId() + IndexServiceBean.draftSuffix; datasetAndFileSolrIdsToDelete.add(solrIdOfDraftDatasetVersion); String solrIdOfDraftDatasetVersionPermission = solrIdOfDraftDatasetVersion + IndexServiceBean.discoverabilityPermissionSuffix; datasetAndFileSolrIdsToDelete.add(solrIdOfDraftDatasetVersionPermission); - String solrIdOfDeaccessionedDatasetVersion = IndexServiceBean.solrDocIdentifierDataset + doomed.getId() + IndexServiceBean.deaccessionedSuffix; + String solrIdOfDeaccessionedDatasetVersion = IndexServiceBean.solrDocIdentifierDataset + managedDoomed.getId() + IndexServiceBean.deaccessionedSuffix; datasetAndFileSolrIdsToDelete.add(solrIdOfDeaccessionedDatasetVersion); + + // dataset + ctxt.em().remove(managedDoomed); + + } @Override diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java index 89cfc732455..287e877f6e0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java @@ -20,6 +20,8 @@ import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.export.ExportService; +import edu.harvard.iq.dataverse.pidproviders.PidProvider; +import edu.harvard.iq.dataverse.pidproviders.PidUtil; import edu.harvard.iq.dataverse.privateurl.PrivateUrl; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.BundleUtil; @@ -30,7 +32,7 @@ import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; -import edu.harvard.iq.dataverse.GlobalIdServiceBean; + import edu.harvard.iq.dataverse.batch.util.LoggingUtil; import edu.harvard.iq.dataverse.dataaccess.StorageIO; import edu.harvard.iq.dataverse.engine.command.Command; @@ -265,7 +267,6 @@ public boolean onSuccess(CommandContext ctxt, Object r) { } catch (Exception e) { logger.warning("Failure to send dataset published messages for : " + dataset.getId() + " : " + e.getMessage()); } - ctxt.index().asyncIndexDataset(dataset, true); //re-indexing dataverses that have additional subjects if (!dataversesToIndex.isEmpty()){ @@ -295,7 +296,8 @@ public boolean onSuccess(CommandContext ctxt, Object r) { logger.log(Level.WARNING, "Finalization: exception caught while exporting: "+ex.getMessage(), ex); // ... but it is important to only update the export time stamp if the // export was indeed successful. - } + } + ctxt.index().asyncIndexDataset(dataset, true); return retVal; } @@ -349,7 +351,7 @@ private void validateDataFiles(Dataset dataset, CommandContext ctxt) throws Comm // major release; we can revisit the decision if there's any // indication that this makes publishing take significantly longer. String driverId = FileUtil.getStorageDriver(dataFile); - if(StorageIO.isDataverseAccessible(driverId) && maxFileSize == -1 || dataFile.getFilesize() < maxFileSize) { + if(StorageIO.isDataverseAccessible(driverId) && (maxFileSize == -1 || dataFile.getFilesize() < maxFileSize)) { FileUtil.validateDataFileChecksum(dataFile); } else { @@ -384,56 +386,52 @@ private void validateDataFiles(Dataset dataset, CommandContext ctxt) throws Comm } private void publicizeExternalIdentifier(Dataset dataset, CommandContext ctxt) throws CommandException { - String protocol = getDataset().getProtocol(); - String authority = getDataset().getAuthority(); - GlobalIdServiceBean idServiceBean = GlobalIdServiceBean.getBean(protocol, ctxt); - - if (idServiceBean != null) { - - try { - String currentGlobalIdProtocol = ctxt.settings().getValueForKey(SettingsServiceBean.Key.Protocol, ""); - String currentGlobalAuthority = ctxt.settings().getValueForKey(SettingsServiceBean.Key.Authority, ""); - String dataFilePIDFormat = ctxt.settings().getValueForKey(SettingsServiceBean.Key.DataFilePIDFormat, "DEPENDENT"); - boolean isFilePIDsEnabled = ctxt.systemConfig().isFilePIDsEnabledForCollection(getDataset().getOwner()); - // We will skip trying to register the global identifiers for datafiles - // if "dependent" file-level identifiers are requested, AND the naming - // protocol, or the authority of the dataset global id is different from - // what's currently configured for the Dataverse. In other words - // we can't get "dependent" DOIs assigned to files in a dataset - // with the registered id that is a handle; or even a DOI, but in - // an authority that's different from what's currently configured. - // Additionaly in 4.9.3 we have added a system variable to disable - // registering file PIDs on the installation level. - if (((currentGlobalIdProtocol.equals(protocol) && currentGlobalAuthority.equals(authority)) - || dataFilePIDFormat.equals("INDEPENDENT")) - && isFilePIDsEnabled - && dataset.getLatestVersion().getMinorVersionNumber() != null - && dataset.getLatestVersion().getMinorVersionNumber().equals((long) 0)) { - //A false return value indicates a failure in calling the service - for (DataFile df : dataset.getFiles()) { - logger.log(Level.FINE, "registering global id for file {0}", df.getId()); - //A false return value indicates a failure in calling the service - if (!idServiceBean.publicizeIdentifier(df)) { - throw new Exception(); - } - df.setGlobalIdCreateTime(getTimestamp()); - df.setIdentifierRegistered(true); + PidProvider pidProvider = ctxt.dvObjects().getEffectivePidGenerator(dataset); + try { + // We will skip trying to register the global identifiers for datafiles + // if "dependent" file-level identifiers are requested, AND the naming + // protocol, or the authority of the dataset global id is different from + // what's currently configured for the Dataverse. In other words + // we can't get "dependent" DOIs assigned to files in a dataset + // with the registered id that is a handle; or even a DOI, but in + // an authority that's different from what's currently configured. + // Additionaly in 4.9.3 we have added a system variable to disable + // registering file PIDs on the installation level. + boolean registerGlobalIdsForFiles = ctxt.systemConfig().isFilePIDsEnabledForCollection( + getDataset().getOwner()) + && pidProvider.canCreatePidsLike(dataset.getGlobalId()); + + if (registerGlobalIdsForFiles + && dataset.getLatestVersion().getMinorVersionNumber() != null + && dataset.getLatestVersion().getMinorVersionNumber().equals((long) 0)) { + // A false return value indicates a failure in calling the service + for (DataFile df : dataset.getFiles()) { + logger.log(Level.FINE, "registering global id for file {0}", df.getId()); + // A false return value indicates a failure in calling the service + if (!pidProvider.publicizeIdentifier(df)) { + throw new Exception(); } + df.setGlobalIdCreateTime(getTimestamp()); + df.setIdentifierRegistered(true); } - if (!idServiceBean.publicizeIdentifier(dataset)) { - throw new Exception(); - } - dataset.setGlobalIdCreateTime(new Date()); // TODO these two methods should be in the responsibility of the idServiceBean. - dataset.setIdentifierRegistered(true); - } catch (Throwable e) { - logger.warning("Failed to register the identifier "+dataset.getGlobalId().asString()+", or to register a file in the dataset; notifying the user(s), unlocking the dataset"); - - // Send failure notification to the user: - notifyUsersDatasetPublishStatus(ctxt, dataset, UserNotification.Type.PUBLISHFAILED_PIDREG); - - ctxt.datasets().removeDatasetLocks(dataset, DatasetLock.Reason.finalizePublication); - throw new CommandException(BundleUtil.getStringFromBundle("dataset.publish.error", idServiceBean.getProviderInformation()), this); } + if (!pidProvider.publicizeIdentifier(dataset)) { + throw new Exception(); + } + dataset.setGlobalIdCreateTime(new Date()); // TODO these two methods should be in the responsibility of the + // pidProvider. + dataset.setIdentifierRegistered(true); + } catch (Throwable e) { + logger.warning("Failed to register the identifier " + dataset.getGlobalId().asString() + + ", or to register a file in the dataset; notifying the user(s), unlocking the dataset"); + + // Send failure notification to the user: + notifyUsersDatasetPublishStatus(ctxt, dataset, UserNotification.Type.PUBLISHFAILED_PIDREG); + + ctxt.datasets().removeDatasetLocks(dataset, DatasetLock.Reason.finalizePublication); + throw new CommandException( + BundleUtil.getStringFromBundle("dataset.publish.error", pidProvider.getProviderInformation()), + this); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDataFileCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDataFileCommand.java index fdf47bbd2dd..369f3cbfda6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDataFileCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDataFileCommand.java @@ -11,35 +11,34 @@ import edu.harvard.iq.dataverse.engine.command.CommandContext; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; + import java.util.Collections; import java.util.Map; import java.util.Set; /** - * * @author Matthew */ // no annotations here, since permissions are dynamically decided // based off GetDatasetCommand for similar permissions checking public class GetDataFileCommand extends AbstractCommand { - private final DataFile df; + private final DataFile dataFile; - public GetDataFileCommand(DataverseRequest aRequest, DataFile anAffectedDataset) { - super(aRequest, anAffectedDataset); - df = anAffectedDataset; + public GetDataFileCommand(DataverseRequest aRequest, DataFile dataFile) { + super(aRequest, dataFile); + this.dataFile = dataFile; } @Override public DataFile execute(CommandContext ctxt) throws CommandException { - return df; + return dataFile; } @Override public Map> getRequiredPermissions() { return Collections.singletonMap("", - df.isReleased() ? Collections.emptySet() - : Collections.singleton(Permission.ViewUnpublishedDataset)); + dataFile.isReleased() ? Collections.emptySet() + : Collections.singleton(Permission.ViewUnpublishedDataset)); } - } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDraftFileMetadataIfAvailableCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDraftFileMetadataIfAvailableCommand.java index 14999548b34..1d83f0dd1f4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDraftFileMetadataIfAvailableCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDraftFileMetadataIfAvailableCommand.java @@ -1,8 +1,6 @@ package edu.harvard.iq.dataverse.engine.command.impl; import edu.harvard.iq.dataverse.DataFile; -import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.engine.command.AbstractCommand; @@ -12,25 +10,19 @@ import edu.harvard.iq.dataverse.engine.command.exception.CommandException; /** - * * @author Matthew */ -@RequiredPermissions( Permission.ViewUnpublishedDataset ) -public class GetDraftFileMetadataIfAvailableCommand extends AbstractCommand{ - private final DataFile df; +@RequiredPermissions(Permission.ViewUnpublishedDataset) +public class GetDraftFileMetadataIfAvailableCommand extends AbstractCommand { + private final DataFile dataFile; - public GetDraftFileMetadataIfAvailableCommand(DataverseRequest aRequest, DataFile dataFile) { - super(aRequest, dataFile); - df = dataFile; + public GetDraftFileMetadataIfAvailableCommand(DataverseRequest request, DataFile dataFile) { + super(request, dataFile); + this.dataFile = dataFile; } @Override public FileMetadata execute(CommandContext ctxt) throws CommandException { - FileMetadata fm = df.getLatestFileMetadata(); - if(fm.getDatasetVersion().getVersionState().equals(DatasetVersion.VersionState.DRAFT)) { - return df.getLatestFileMetadata(); - } - return null; + return dataFile.getDraftFileMetadata(); } - } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLatestAccessibleDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLatestAccessibleDatasetVersionCommand.java index 7bcc851bde2..431b3ff47c6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLatestAccessibleDatasetVersionCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLatestAccessibleDatasetVersionCommand.java @@ -25,24 +25,25 @@ public class GetLatestAccessibleDatasetVersionCommand extends AbstractCommand { private final Dataset ds; private final boolean includeDeaccessioned; - private boolean checkPerms; + private boolean checkPermsWhenDeaccessioned; public GetLatestAccessibleDatasetVersionCommand(DataverseRequest aRequest, Dataset anAffectedDataset) { - this(aRequest, anAffectedDataset, false, false); + this(aRequest, anAffectedDataset,false, false); } - public GetLatestAccessibleDatasetVersionCommand(DataverseRequest aRequest, Dataset anAffectedDataset, boolean includeDeaccessioned, boolean checkPerms) { + public GetLatestAccessibleDatasetVersionCommand(DataverseRequest aRequest, Dataset anAffectedDataset, boolean includeDeaccessioned, boolean checkPermsWhenDeaccessioned) { super(aRequest, anAffectedDataset); ds = anAffectedDataset; this.includeDeaccessioned = includeDeaccessioned; - this.checkPerms = checkPerms; + this.checkPermsWhenDeaccessioned = checkPermsWhenDeaccessioned; } @Override public DatasetVersion execute(CommandContext ctxt) throws CommandException { - if (ds.getLatestVersion().isDraft() && ctxt.permissions().requestOn(getRequest(), ds).has(Permission.ViewUnpublishedDataset)) { + if (ds.getLatestVersion().isDraft() && + ctxt.permissions().requestOn(getRequest(), ds).has(Permission.ViewUnpublishedDataset)) { return ctxt.engine().submit(new GetDraftDatasetVersionCommand(getRequest(), ds)); } - return ctxt.engine().submit(new GetLatestPublishedDatasetVersionCommand(getRequest(), ds, includeDeaccessioned, checkPerms)); + return ctxt.engine().submit(new GetLatestPublishedDatasetVersionCommand(getRequest(), ds, includeDeaccessioned, checkPermsWhenDeaccessioned)); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLatestAccessibleFileMetadataCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLatestAccessibleFileMetadataCommand.java new file mode 100644 index 00000000000..05f3c73bde0 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLatestAccessibleFileMetadataCommand.java @@ -0,0 +1,30 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.DataFile; +import edu.harvard.iq.dataverse.FileMetadata; +import edu.harvard.iq.dataverse.authorization.Permission; +import edu.harvard.iq.dataverse.engine.command.CommandContext; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; + +public class GetLatestAccessibleFileMetadataCommand extends AbstractGetPublishedFileMetadataCommand { + + public GetLatestAccessibleFileMetadataCommand(DataverseRequest request, DataFile dataFile, boolean includeDeaccessioned) { + super(request, dataFile, includeDeaccessioned); + } + + @Override + public FileMetadata execute(CommandContext ctxt) throws CommandException { + FileMetadata fileMetadata = null; + + if (ctxt.permissions().requestOn(getRequest(), dataFile.getOwner()).has(Permission.ViewUnpublishedDataset)) { + fileMetadata = dataFile.getDraftFileMetadata(); + } + + if (fileMetadata == null) { + fileMetadata = getLatestPublishedFileMetadata(ctxt); + } + + return fileMetadata; + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLatestPublishedFileMetadataCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLatestPublishedFileMetadataCommand.java new file mode 100644 index 00000000000..fc13dba1a34 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLatestPublishedFileMetadataCommand.java @@ -0,0 +1,19 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.DataFile; +import edu.harvard.iq.dataverse.FileMetadata; +import edu.harvard.iq.dataverse.engine.command.CommandContext; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; + +public class GetLatestPublishedFileMetadataCommand extends AbstractGetPublishedFileMetadataCommand { + + public GetLatestPublishedFileMetadataCommand(DataverseRequest request, DataFile dataFile, boolean includeDeaccessioned) { + super(request, dataFile, includeDeaccessioned); + } + + @Override + public FileMetadata execute(CommandContext ctxt) throws CommandException { + return getLatestPublishedFileMetadata(ctxt); + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetSpecificPublishedFileMetadataByDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetSpecificPublishedFileMetadataByDatasetVersionCommand.java new file mode 100644 index 00000000000..deffbfb57ee --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetSpecificPublishedFileMetadataByDatasetVersionCommand.java @@ -0,0 +1,34 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.DataFile; +import edu.harvard.iq.dataverse.DatasetVersion; +import edu.harvard.iq.dataverse.FileMetadata; +import edu.harvard.iq.dataverse.engine.command.CommandContext; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; + +public class GetSpecificPublishedFileMetadataByDatasetVersionCommand extends AbstractGetPublishedFileMetadataCommand { + private final long majorVersion; + private final long minorVersion; + + public GetSpecificPublishedFileMetadataByDatasetVersionCommand(DataverseRequest request, DataFile dataFile, long majorVersion, long minorVersion, boolean includeDeaccessioned) { + super(request, dataFile, includeDeaccessioned); + this.majorVersion = majorVersion; + this.minorVersion = minorVersion; + } + + @Override + public FileMetadata execute(CommandContext ctxt) throws CommandException { + return dataFile.getFileMetadatas().stream() + .filter(fileMetadata -> isRequestedVersionFileMetadata(fileMetadata, ctxt)) + .findFirst() + .orElse(null); + } + + private boolean isRequestedVersionFileMetadata(FileMetadata fileMetadata, CommandContext ctxt) { + DatasetVersion datasetVersion = fileMetadata.getDatasetVersion(); + return isDatasetVersionAccessible(datasetVersion, dataFile.getOwner(), ctxt) + && datasetVersion.getVersionNumber().equals(majorVersion) + && datasetVersion.getMinorVersionNumber().equals(minorVersion); + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GrantSuperuserStatusCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GrantSuperuserStatusCommand.java index 42af43b7247..ec8c8976260 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GrantSuperuserStatusCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GrantSuperuserStatusCommand.java @@ -14,7 +14,7 @@ import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.exception.PermissionException; -import edu.harvard.iq.dataverse.GlobalIdServiceBean; +import edu.harvard.iq.dataverse.pidproviders.PidProvider; /** * diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ImportDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ImportDatasetCommand.java index 478272950bd..772c989264c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ImportDatasetCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ImportDatasetCommand.java @@ -1,14 +1,14 @@ package edu.harvard.iq.dataverse.engine.command.impl; import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.GlobalIdServiceBean; import edu.harvard.iq.dataverse.engine.command.CommandContext; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.exception.CommandExecutionException; import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; import edu.harvard.iq.dataverse.engine.command.exception.PermissionException; -import edu.harvard.iq.dataverse.pidproviders.FakePidProviderServiceBean; +import edu.harvard.iq.dataverse.pidproviders.PidProvider; +import edu.harvard.iq.dataverse.pidproviders.PidUtil; import java.io.IOException; import java.util.Collections; @@ -80,9 +80,9 @@ protected void additionalParameterTests(CommandContext ctxt) throws CommandExcep * Dataverse) but aren't findable to be used. That could be the case if, for * example, someone was importing a draft dataset from elsewhere. */ - GlobalIdServiceBean globalIdServiceBean = GlobalIdServiceBean.getBean(ds.getProtocol(), ctxt); - if (globalIdServiceBean != null) { - if (globalIdServiceBean.alreadyRegistered(ds.getGlobalId(), true)) { + PidProvider pidProvider = PidUtil.getPidProvider(ds.getGlobalId().getProviderId()); + if (pidProvider != null) { + if (pidProvider.alreadyRegistered(ds.getGlobalId(), true)) { return; } } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListFeaturedCollectionsCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListFeaturedCollectionsCommand.java new file mode 100644 index 00000000000..4dca522e499 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListFeaturedCollectionsCommand.java @@ -0,0 +1,50 @@ + +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.DataverseFeaturedDataverse; +import edu.harvard.iq.dataverse.DvObject; +import edu.harvard.iq.dataverse.authorization.Permission; +import edu.harvard.iq.dataverse.engine.command.AbstractCommand; +import edu.harvard.iq.dataverse.engine.command.CommandContext; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * + * @author stephenkraffmiller + */ +public class ListFeaturedCollectionsCommand extends AbstractCommand> { + + private final Dataverse dv; + + public ListFeaturedCollectionsCommand(DataverseRequest aRequest, Dataverse aDataverse) { + super(aRequest, aDataverse); + dv = aDataverse; + } + + @Override + public List execute(CommandContext ctxt) throws CommandException { + List featuredTarget = new ArrayList<>(); + List featuredList = ctxt.featuredDataverses().findByDataverseId(dv.getId()); + for (DataverseFeaturedDataverse dfd : featuredList) { + Dataverse fd = dfd.getFeaturedDataverse(); + featuredTarget.add(fd); + } + return featuredTarget; + + } + + @Override + public Map> getRequiredPermissions() { + return Collections.singletonMap("", + dv.isReleased() ? Collections.emptySet() + : Collections.singleton(Permission.ViewUnpublishedDataverse)); + } + +} diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListMetadataBlocksCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListMetadataBlocksCommand.java index 912318cf155..8275533ced2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListMetadataBlocksCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListMetadataBlocksCommand.java @@ -7,6 +7,7 @@ import edu.harvard.iq.dataverse.engine.command.CommandContext; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; + import java.util.Collections; import java.util.List; import java.util.Map; @@ -14,29 +15,40 @@ /** * Lists the metadata blocks of a {@link Dataverse}. - * + * * @author michael */ // no annotations here, since permissions are dynamically decided -public class ListMetadataBlocksCommand extends AbstractCommand>{ - - private final Dataverse dv; - - public ListMetadataBlocksCommand(DataverseRequest aRequest, Dataverse aDataverse) { - super(aRequest, aDataverse); - dv = aDataverse; +public class ListMetadataBlocksCommand extends AbstractCommand> { + + private final Dataverse dataverse; + private final boolean onlyDisplayedOnCreate; + + public ListMetadataBlocksCommand(DataverseRequest request, Dataverse dataverse, boolean onlyDisplayedOnCreate) { + super(request, dataverse); + this.dataverse = dataverse; + this.onlyDisplayedOnCreate = onlyDisplayedOnCreate; } @Override public List execute(CommandContext ctxt) throws CommandException { - return dv.getMetadataBlocks(); + if (onlyDisplayedOnCreate) { + return listMetadataBlocksDisplayedOnCreate(ctxt, dataverse); + } + return dataverse.getMetadataBlocks(); } - + + private List listMetadataBlocksDisplayedOnCreate(CommandContext ctxt, Dataverse dataverse) { + if (dataverse.isMetadataBlockRoot() || dataverse.getOwner() == null) { + return ctxt.metadataBlocks().listMetadataBlocksDisplayedOnCreate(dataverse); + } + return listMetadataBlocksDisplayedOnCreate(ctxt, dataverse.getOwner()); + } + @Override public Map> getRequiredPermissions() { return Collections.singletonMap("", - dv.isReleased() ? Collections.emptySet() - : Collections.singleton(Permission.ViewUnpublishedDataverse)); - } - + dataverse.isReleased() ? Collections.emptySet() + : Collections.singleton(Permission.ViewUnpublishedDataverse)); + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MergeInAccountCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MergeInAccountCommand.java index 1ec51764d73..03f4dceef88 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MergeInAccountCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MergeInAccountCommand.java @@ -14,7 +14,6 @@ import edu.harvard.iq.dataverse.UserNotification; import edu.harvard.iq.dataverse.authorization.AuthenticatedUserLookup; import edu.harvard.iq.dataverse.authorization.providers.builtin.BuiltinUser; -import edu.harvard.iq.dataverse.authorization.providers.oauth2.OAuth2TokenData; import edu.harvard.iq.dataverse.authorization.users.ApiToken; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.batch.util.LoggingUtil; @@ -25,7 +24,6 @@ import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; -import edu.harvard.iq.dataverse.passwordreset.PasswordResetData; import edu.harvard.iq.dataverse.search.IndexResponse; import edu.harvard.iq.dataverse.search.savedsearch.SavedSearch; import edu.harvard.iq.dataverse.workflows.WorkflowComment; @@ -177,6 +175,7 @@ protected void executeImpl(CommandContext ctxt) throws CommandException { ctxt.em().createNativeQuery("Delete from OAuth2TokenData where user_id ="+consumedAU.getId()).executeUpdate(); + ctxt.em().createNativeQuery("DELETE FROM explicitgroup_authenticateduser consumed USING explicitgroup_authenticateduser ongoing WHERE consumed.containedauthenticatedusers_id="+ongoingAU.getId()+" AND ongoing.containedauthenticatedusers_id="+consumedAU.getId()).executeUpdate(); ctxt.em().createNativeQuery("UPDATE explicitgroup_authenticateduser SET containedauthenticatedusers_id="+ongoingAU.getId()+" WHERE containedauthenticatedusers_id="+consumedAU.getId()).executeUpdate(); ctxt.actionLog().changeUserIdentifierInHistory(consumedAU.getIdentifier(), ongoingAU.getIdentifier()); diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java index f5ef121dee2..6b95f3b6de1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java @@ -2,7 +2,6 @@ import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetLock; -import edu.harvard.iq.dataverse.GlobalIdServiceBean; import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.engine.command.Command; @@ -11,6 +10,7 @@ import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; +import edu.harvard.iq.dataverse.pidproviders.PidProvider; import edu.harvard.iq.dataverse.privateurl.PrivateUrl; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.BundleUtil; @@ -130,24 +130,15 @@ public PublishDatasetResult execute(CommandContext ctxt) throws CommandException // ... // Additionaly in 4.9.3 we have added a system variable to disable // registering file PIDs on the installation level. - String currentGlobalIdProtocol = ctxt.settings().getValueForKey(SettingsServiceBean.Key.Protocol, ""); - String currentGlobalAuthority= ctxt.settings().getValueForKey(SettingsServiceBean.Key.Authority, ""); - String dataFilePIDFormat = ctxt.settings().getValueForKey(SettingsServiceBean.Key.DataFilePIDFormat, "DEPENDENT"); boolean registerGlobalIdsForFiles = - (currentGlobalIdProtocol.equals(theDataset.getProtocol()) || dataFilePIDFormat.equals("INDEPENDENT")) - && ctxt.systemConfig().isFilePIDsEnabledForCollection(theDataset.getOwner()); - - if ( registerGlobalIdsForFiles ){ - registerGlobalIdsForFiles = currentGlobalAuthority.equals( theDataset.getAuthority() ); - } + ctxt.systemConfig().isFilePIDsEnabledForCollection(getDataset().getOwner()) && + ctxt.dvObjects().getEffectivePidGenerator(getDataset()).canCreatePidsLike(getDataset().getGlobalId()); boolean validatePhysicalFiles = ctxt.systemConfig().isDatafileValidationOnPublishEnabled(); // As of v5.0, publishing a dataset is always done asynchronously, // with the dataset locked for the duration of the operation. - //if ((registerGlobalIdsForFiles || validatePhysicalFiles) - // && theDataset.getFiles().size() > ctxt.systemConfig().getPIDAsynchRegFileCount()) { String info = "Publishing the dataset; "; info += registerGlobalIdsForFiles ? "Registering PIDs for Datafiles; " : ""; @@ -178,15 +169,6 @@ public PublishDatasetResult execute(CommandContext ctxt) throws CommandException // method: //ctxt.datasets().callFinalizePublishCommandAsynchronously(theDataset.getId(), ctxt, request, datasetExternallyReleased); return new PublishDatasetResult(theDataset, Status.Inprogress); - - /** - * Code for for "synchronous" (while-you-wait) publishing - * is preserved below, commented out: - } else { - // Synchronous publishing (no workflow involved) - theDataset = ctxt.engine().submit(new FinalizeDatasetPublicationCommand(theDataset, getRequest(),datasetExternallyReleased)); - return new PublishDatasetResult(theDataset, Status.Completed); - } */ } } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RegisterDvObjectCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RegisterDvObjectCommand.java index 779bc7fb7fe..7b80871a1e0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RegisterDvObjectCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RegisterDvObjectCommand.java @@ -4,20 +4,17 @@ import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DvObject; -import edu.harvard.iq.dataverse.GlobalId; +import edu.harvard.iq.dataverse.DvObjectContainer; import edu.harvard.iq.dataverse.engine.command.AbstractVoidCommand; import edu.harvard.iq.dataverse.engine.command.CommandContext; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; -import edu.harvard.iq.dataverse.settings.SettingsServiceBean; +import edu.harvard.iq.dataverse.pidproviders.PidProvider; +import edu.harvard.iq.dataverse.pidproviders.handle.HandlePidProvider; + import java.sql.Timestamp; import java.util.Date; -import edu.harvard.iq.dataverse.GlobalIdServiceBean; -import edu.harvard.iq.dataverse.HandlenetServiceBean; -import edu.harvard.iq.dataverse.batch.util.LoggingUtil; -import java.io.IOException; -import org.apache.solr.client.solrj.SolrServerException; /** * @@ -44,48 +41,37 @@ public RegisterDvObjectCommand(DataverseRequest aRequest, DvObject target, Boole @Override protected void executeImpl(CommandContext ctxt) throws CommandException { + DvObjectContainer container = (target instanceof DvObjectContainer) ? (DvObjectContainer) target : target.getOwner(); + // Get the pidProvider that is configured to mint new IDs + PidProvider pidProvider = ctxt.dvObjects().getEffectivePidGenerator(container); if(this.migrateHandle){ //Only continue if you can successfully migrate the handle - if (!processMigrateHandle(ctxt)) return; + if (HandlePidProvider.HDL_PROTOCOL.equals(pidProvider.getProtocol()) || !processMigrateHandle(ctxt)) return; } - String nonNullDefaultIfKeyNotFound = ""; - String protocol = ctxt.settings().getValueForKey(SettingsServiceBean.Key.Protocol, nonNullDefaultIfKeyNotFound); - String authority = ctxt.settings().getValueForKey(SettingsServiceBean.Key.Authority, nonNullDefaultIfKeyNotFound); - // Get the idServiceBean that is configured to mint new IDs - GlobalIdServiceBean idServiceBean = GlobalIdServiceBean.getBean(protocol, ctxt); + try { //Test to see if identifier already present //if so, leave. if (target.getIdentifier() == null || target.getIdentifier().isEmpty()) { - if (target.isInstanceofDataset()) { - target.setIdentifier(idServiceBean.generateDatasetIdentifier((Dataset) target)); - - } else { - target.setIdentifier(idServiceBean.generateDataFileIdentifier((DataFile) target)); - } - if (target.getProtocol() == null) { - target.setProtocol(protocol); - } - if (target.getAuthority() == null) { - target.setAuthority(authority); - } + pidProvider.generatePid(target); } - if (idServiceBean.alreadyRegistered(target)) { + + if (pidProvider.alreadyRegistered(target)) { return; } - String doiRetString = idServiceBean.createIdentifier(target); + String doiRetString = pidProvider.createIdentifier(target); if (doiRetString != null && doiRetString.contains(target.getIdentifier())) { - if (!idServiceBean.registerWhenPublished()) { + if (!pidProvider.registerWhenPublished()) { // Should register ID before publicize() is called - // For example, DOIEZIdServiceBean tries to recreate the id if the identifier isn't registered before + // For example, DOIEZIdProvider tries to recreate the id if the identifier isn't registered before // publicizeIdentifier is called target.setIdentifierRegistered(true); target.setGlobalIdCreateTime(new Timestamp(new Date().getTime())); } if (target.isReleased()) { - idServiceBean.publicizeIdentifier(target); + pidProvider.publicizeIdentifier(target); } - if (idServiceBean.registerWhenPublished() && target.isReleased()) { + if (pidProvider.registerWhenPublished() && target.isReleased()) { target.setGlobalIdCreateTime(new Timestamp(new Date().getTime())); target.setIdentifierRegistered(true); } @@ -95,27 +81,21 @@ protected void executeImpl(CommandContext ctxt) throws CommandException { Dataset dataset = (Dataset) target; for (DataFile df : dataset.getFiles()) { if (df.getIdentifier() == null || df.getIdentifier().isEmpty()) { - df.setIdentifier(idServiceBean.generateDataFileIdentifier(df)); - if (df.getProtocol() == null || df.getProtocol().isEmpty()) { - df.setProtocol(protocol); - } - if (df.getAuthority() == null || df.getAuthority().isEmpty()) { - df.setAuthority(authority); - } + pidProvider.generatePid(df); } - doiRetString = idServiceBean.createIdentifier(df); + doiRetString = pidProvider.createIdentifier(df); if (doiRetString != null && doiRetString.contains(df.getIdentifier())) { - if (!idServiceBean.registerWhenPublished()) { + if (!pidProvider.registerWhenPublished()) { // Should register ID before publicize() is called - // For example, DOIEZIdServiceBean tries to recreate the id if the identifier isn't registered before + // For example, DOIEZIdProvider tries to recreate the id if the identifier isn't registered before // publicizeIdentifier is called df.setIdentifierRegistered(true); df.setGlobalIdCreateTime(new Timestamp(new Date().getTime())); } if (df.isReleased()) { - idServiceBean.publicizeIdentifier(df); + pidProvider.publicizeIdentifier(df); } - if (idServiceBean.registerWhenPublished() && df.isReleased()) { + if (pidProvider.registerWhenPublished() && df.isReleased()) { df.setGlobalIdCreateTime(new Timestamp(new Date().getTime())); df.setIdentifierRegistered(true); } @@ -145,7 +125,7 @@ protected void executeImpl(CommandContext ctxt) throws CommandException { private Boolean processMigrateHandle (CommandContext ctxt){ boolean retval = true; if(!target.isInstanceofDataset()) return false; - if(!target.getProtocol().equals(HandlenetServiceBean.HDL_PROTOCOL)) return false; + if(!target.getProtocol().equals(HandlePidProvider.HDL_PROTOCOL)) return false; AlternativePersistentIdentifier api = new AlternativePersistentIdentifier(); api.setProtocol(target.getProtocol()); diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommand.java index a29e7fdd59c..6b7baa7d01b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommand.java @@ -27,6 +27,7 @@ * "actiontype" in the actionlogrecord rather than "InternalError" if you throw * a CommandExecutionException. */ +@Deprecated(forRemoval = true, since = "2024-07-07") @RequiredPermissions(Permission.EditDataset) public class RequestRsyncScriptCommand extends AbstractCommand { diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ReservePidCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ReservePidCommand.java index 6b2872f3397..b7e3ddd8ce6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ReservePidCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ReservePidCommand.java @@ -1,7 +1,6 @@ package edu.harvard.iq.dataverse.engine.command.impl; import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.GlobalIdServiceBean; import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.engine.command.AbstractVoidCommand; @@ -11,6 +10,8 @@ import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; import edu.harvard.iq.dataverse.engine.command.exception.PermissionException; +import edu.harvard.iq.dataverse.pidproviders.PidProvider; +import edu.harvard.iq.dataverse.pidproviders.PidUtil; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.BundleUtil; import java.util.Arrays; @@ -41,11 +42,10 @@ protected void executeImpl(CommandContext ctxt) throws CommandException { this, Collections.singleton(Permission.EditDataset), dataset); } - String nonNullDefaultIfKeyNotFound = ""; - String protocol = ctxt.settings().getValueForKey(SettingsServiceBean.Key.Protocol, nonNullDefaultIfKeyNotFound); - GlobalIdServiceBean idServiceBean = GlobalIdServiceBean.getBean(protocol, ctxt); + PidProvider pidProvider = ctxt.dvObjects().getEffectivePidGenerator(dataset); + try { - String returnString = idServiceBean.createIdentifier(dataset); + String returnString = pidProvider.createIdentifier(dataset); logger.fine(returnString); // No errors caught, so mark PID as reserved. dataset.setGlobalIdCreateTime(new Date()); diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ReturnDatasetToAuthorCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ReturnDatasetToAuthorCommand.java index caf37ad4de1..f3b33f82524 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ReturnDatasetToAuthorCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ReturnDatasetToAuthorCommand.java @@ -25,6 +25,11 @@ public class ReturnDatasetToAuthorCommand extends AbstractDatasetCommand uploadLogoSizeLimit) { - throw new IllegalCommandException("File is larger than maximum size: " + uploadLogoSizeLimit + ".", this); + throw new IllegalCommandException(BundleUtil.getStringFromBundle("datasets.api.thumbnail.fileToLarge", List.of(String.valueOf(uploadLogoSizeLimit))), this); } FileInputStream fileAsStream = null; try { @@ -107,23 +111,25 @@ public DatasetThumbnail execute(CommandContext ctxt) throws CommandException { Logger.getLogger(UpdateDatasetThumbnailCommand.class.getName()).log(Level.SEVERE, null, ex); } Dataset datasetWithNewThumbnail = ctxt.datasets().setNonDatasetFileAsThumbnail(dataset, fileAsStream); - IOUtils.closeQuietly(fileAsStream); + IOUtils.closeQuietly(fileAsStream); if (datasetWithNewThumbnail != null) { - return datasetWithNewThumbnail.getDatasetThumbnail(ImageThumbConverter.DEFAULT_CARDIMAGE_SIZE); - } else { - return null; + DatasetThumbnail thumbnail = datasetWithNewThumbnail.getDatasetThumbnail(ImageThumbConverter.DEFAULT_CARDIMAGE_SIZE); + if (thumbnail != null) { + return thumbnail; + } } + throw new IllegalCommandException(BundleUtil.getStringFromBundle("datasets.api.thumbnail.nonDatasetFailed"), this); case removeThumbnail: - Dataset ds2 = ctxt.datasets().removeDatasetThumbnail(dataset); + Dataset ds2 = ctxt.datasets().clearDatasetLevelThumbnail(dataset); DatasetThumbnail datasetThumbnail2 = ds2.getDatasetThumbnail(ImageThumbConverter.DEFAULT_CARDIMAGE_SIZE); if (datasetThumbnail2 == null) { return null; } else { - throw new CommandException("User wanted to remove the thumbnail it still has one!", this); + throw new CommandException(BundleUtil.getStringFromBundle("datasets.api.thumbnail.notDeleted"), this); } default: - throw new IllegalCommandException("Whatever you are trying to do to the dataset thumbnail is not supported.", this); + throw new IllegalCommandException(BundleUtil.getStringFromBundle("datasets.api.thumbnail.actionNotSupported"), this); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java index 7591bebe796..994f4c7dfb6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java @@ -1,6 +1,12 @@ package edu.harvard.iq.dataverse.engine.command.impl; -import edu.harvard.iq.dataverse.*; +import edu.harvard.iq.dataverse.DataFile; +import edu.harvard.iq.dataverse.DataFileCategory; +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetLock; +import edu.harvard.iq.dataverse.DatasetVersion; +import edu.harvard.iq.dataverse.DatasetVersionDifference; +import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.engine.command.CommandContext; @@ -8,7 +14,6 @@ import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; -import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.DatasetFieldUtil; import edu.harvard.iq.dataverse.util.FileMetadataUtil; @@ -115,8 +120,11 @@ public Dataset execute(CommandContext ctxt) throws CommandException { //Will throw an IllegalCommandException if a system metadatablock is changed and the appropriate key is not supplied. checkSystemMetadataKeyIfNeeded(getDataset().getOrCreateEditVersion(fmVarMet), persistedVersion); - - + + getDataset().getOrCreateEditVersion().setLastUpdateTime(getTimestamp()); + + registerExternalVocabValuesIfAny(ctxt, getDataset().getOrCreateEditVersion(fmVarMet)); + try { // Invariant: Dataset has no locks preventing the update String lockInfoMessage = "saving current edits"; @@ -256,7 +264,6 @@ public Dataset execute(CommandContext ctxt) throws CommandException { ctxt.ingest().recalculateDatasetVersionUNF(theDataset.getOrCreateEditVersion()); } - theDataset.getOrCreateEditVersion().setLastUpdateTime(getTimestamp()); theDataset.setModificationTime(getTimestamp()); savedDataset = ctxt.em().merge(theDataset); diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java index fe9415f39f9..bdb69dc918f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java @@ -18,7 +18,6 @@ import java.util.ArrayList; import java.util.List; import java.util.logging.Logger; -import jakarta.persistence.TypedQuery; /** * Update an existing dataverse. @@ -30,10 +29,10 @@ public class UpdateDataverseCommand extends AbstractCommand { private final Dataverse editedDv; private final List facetList; - private final List featuredDataverseList; - private final List inputLevelList; - - private boolean datasetsReindexRequired = false; + private final List featuredDataverseList; + private final List inputLevelList; + + private boolean datasetsReindexRequired = false; public UpdateDataverseCommand(Dataverse editedDv, List facetList, List featuredDataverseList, DataverseRequest aRequest, List inputLevelList ) { diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseInputLevelsCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseInputLevelsCommand.java new file mode 100644 index 00000000000..cf7b4a6f69c --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseInputLevelsCommand.java @@ -0,0 +1,51 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.DataverseFieldTypeInputLevel; +import edu.harvard.iq.dataverse.MetadataBlock; +import edu.harvard.iq.dataverse.authorization.Permission; +import edu.harvard.iq.dataverse.engine.command.AbstractCommand; +import edu.harvard.iq.dataverse.engine.command.CommandContext; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; + +import java.util.ArrayList; +import java.util.List; + +@RequiredPermissions(Permission.EditDataverse) +public class UpdateDataverseInputLevelsCommand extends AbstractCommand { + private final Dataverse dataverse; + private final List inputLevelList; + + public UpdateDataverseInputLevelsCommand(Dataverse dataverse, DataverseRequest request, List inputLevelList) { + super(request, dataverse); + this.dataverse = dataverse; + this.inputLevelList = new ArrayList<>(inputLevelList); + } + + @Override + public Dataverse execute(CommandContext ctxt) throws CommandException { + if (inputLevelList == null || inputLevelList.isEmpty()) { + throw new CommandException("Error while updating dataverse input levels: Input level list cannot be null or empty", this); + } + addInputLevelMetadataBlocks(); + dataverse.setMetadataBlockRoot(true); + return ctxt.engine().submit(new UpdateDataverseCommand(dataverse, null, null, getRequest(), inputLevelList)); + } + + private void addInputLevelMetadataBlocks() { + List dataverseMetadataBlocks = dataverse.getMetadataBlocks(); + for (DataverseFieldTypeInputLevel inputLevel : inputLevelList) { + MetadataBlock inputLevelMetadataBlock = inputLevel.getDatasetFieldType().getMetadataBlock(); + if (!dataverseHasMetadataBlock(dataverseMetadataBlocks, inputLevelMetadataBlock)) { + dataverseMetadataBlocks.add(inputLevelMetadataBlock); + } + } + dataverse.setMetadataBlocks(dataverseMetadataBlocks); + } + + private boolean dataverseHasMetadataBlock(List dataverseMetadataBlocks, MetadataBlock metadataBlock) { + return dataverseMetadataBlocks.stream().anyMatch(block -> block.getId().equals(metadataBlock.getId())); + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDvObjectPIDMetadataCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDvObjectPIDMetadataCommand.java index 7230f9f9c0a..0c463cddec1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDvObjectPIDMetadataCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDvObjectPIDMetadataCommand.java @@ -2,7 +2,6 @@ import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.GlobalIdServiceBean; import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.engine.command.AbstractVoidCommand; @@ -11,6 +10,8 @@ import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.exception.PermissionException; +import edu.harvard.iq.dataverse.pidproviders.PidProvider; +import edu.harvard.iq.dataverse.pidproviders.PidUtil; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.BundleUtil; import java.sql.Timestamp; @@ -46,31 +47,31 @@ protected void executeImpl(CommandContext ctxt) throws CommandException { //the single dataset update api checks for drafts before calling the command return; } - GlobalIdServiceBean idServiceBean = GlobalIdServiceBean.getBean(target.getProtocol(), ctxt); + PidProvider pidProvider = PidUtil.getPidProvider(target.getGlobalId().getProviderId()); + try { - Boolean doiRetString = idServiceBean.publicizeIdentifier(target); + Boolean doiRetString = pidProvider.publicizeIdentifier(target); if (doiRetString) { target.setGlobalIdCreateTime(new Timestamp(new Date().getTime())); ctxt.em().merge(target); ctxt.em().flush(); // When updating, we want to traverse through files even if the dataset itself // didn't need updating. - String currentGlobalIdProtocol = ctxt.settings().getValueForKey(SettingsServiceBean.Key.Protocol, ""); - String dataFilePIDFormat = ctxt.settings().getValueForKey(SettingsServiceBean.Key.DataFilePIDFormat, "DEPENDENT"); boolean isFilePIDsEnabled = ctxt.systemConfig().isFilePIDsEnabledForCollection(target.getOwner()); // We will skip trying to update the global identifiers for datafiles if they // aren't being used. // If they are, we need to assure that there's an existing PID or, as when - // creating PIDs, that the protocol matches that of the dataset DOI if - // we're going to create a DEPENDENT file PID. - String protocol = target.getProtocol(); + // creating PIDs, that it's possible. + + boolean canCreatePidsForFiles = + isFilePIDsEnabled && ctxt.dvObjects().getEffectivePidGenerator(target).canCreatePidsLike(target.getGlobalId()); + for (DataFile df : target.getFiles()) { if (isFilePIDsEnabled && // using file PIDs and (!(df.getIdentifier() == null || df.getIdentifier().isEmpty()) || // identifier exists, or - currentGlobalIdProtocol.equals(protocol) || // right protocol to create dependent DOIs, or - dataFilePIDFormat.equals("INDEPENDENT"))// or independent. TODO(pm) - check authority too + canCreatePidsForFiles) // we can create PIDs for files ) { - doiRetString = idServiceBean.publicizeIdentifier(df); + doiRetString = pidProvider.publicizeIdentifier(df); if (doiRetString) { df.setGlobalIdCreateTime(new Timestamp(new Date().getTime())); ctxt.em().merge(df); diff --git a/src/main/java/edu/harvard/iq/dataverse/export/DDIExportServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/export/DDIExportServiceBean.java index 5119b4b96c7..edd01ae98a3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/DDIExportServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/DDIExportServiceBean.java @@ -545,6 +545,16 @@ private void createDataFileDDI(XMLStreamWriter xmlw, Set excludedFieldSe List vars = variableService.findByDataTableId(dt.getId()); if (checkField("catgry", excludedFieldSet, includedFieldSet)) { if (checkIsWithoutFrequencies(vars)) { + // @todo: the method called here to calculate frequencies + // when they are missing from the database (for whatever + // reasons) subsets the physical tab-delimited file and + // calculates them in real time. this is very expensive operation + // potentially. let's make sure that, when we do this, we + // save the resulting frequencies in the database, so that + // we don't have to do this again. Also, let's double check + // whether the "checkIsWithoutFrequencies()" method is doing + // the right thing - as it appears to return true when there + // are no categorical variables in the DataTable (?) calculateFrequencies(df, vars); } } @@ -580,6 +590,7 @@ private boolean checkIsWithoutFrequencies(List vars) { private void calculateFrequencies(DataFile df, List vars) { + // @todo: see the comment in the part of the code that calls this method try { DataConverter dc = new DataConverter(); File tabFile = dc.downloadFromStorageIO(df.getStorageIO()); diff --git a/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java b/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java index a7967f6ccb6..f0d77eb8b52 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/InternalExportDataProvider.java @@ -8,12 +8,11 @@ import jakarta.json.JsonArrayBuilder; import jakarta.json.JsonObject; import jakarta.json.JsonObjectBuilder; - -import edu.harvard.iq.dataverse.DOIDataCiteRegisterService; import edu.harvard.iq.dataverse.DataCitation; import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.FileMetadata; +import edu.harvard.iq.dataverse.pidproviders.doi.datacite.DOIDataCiteRegisterService; import io.gdcc.spi.export.ExportDataProvider; import edu.harvard.iq.dataverse.util.bagit.OREMap; import edu.harvard.iq.dataverse.util.json.JsonPrinter; diff --git a/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java index 7b0a92a4372..4b8822e8b66 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java @@ -13,16 +13,16 @@ import com.google.gson.Gson; -import edu.harvard.iq.dataverse.DOIServiceBean; import edu.harvard.iq.dataverse.DatasetFieldConstant; import edu.harvard.iq.dataverse.GlobalId; -import edu.harvard.iq.dataverse.HandlenetServiceBean; import edu.harvard.iq.dataverse.api.dto.DatasetDTO; import edu.harvard.iq.dataverse.api.dto.DatasetVersionDTO; import edu.harvard.iq.dataverse.api.dto.FieldDTO; import edu.harvard.iq.dataverse.api.dto.MetadataBlockDTO; import edu.harvard.iq.dataverse.util.PersonOrOrgUtil; import edu.harvard.iq.dataverse.pidproviders.PidUtil; +import edu.harvard.iq.dataverse.pidproviders.doi.AbstractDOIProvider; +import edu.harvard.iq.dataverse.pidproviders.handle.HandlePidProvider; import edu.harvard.iq.dataverse.util.json.JsonUtil; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -189,10 +189,10 @@ public static void writeIdentifierElement(XMLStreamWriter xmlw, String identifie if (StringUtils.isNotBlank(identifier)) { Map identifier_map = new HashMap(); - if (StringUtils.containsIgnoreCase(identifier, DOIServiceBean.DOI_RESOLVER_URL)) { + if (StringUtils.containsIgnoreCase(identifier, AbstractDOIProvider.DOI_RESOLVER_URL)) { identifier_map.put("identifierType", "DOI"); identifier = StringUtils.substring(identifier, identifier.indexOf("10.")); - } else if (StringUtils.containsIgnoreCase(identifier, HandlenetServiceBean.HDL_RESOLVER_URL)) { + } else if (StringUtils.containsIgnoreCase(identifier, HandlePidProvider.HDL_RESOLVER_URL)) { identifier_map.put("identifierType", "Handle"); if (StringUtils.contains(identifier, "http")) { identifier = identifier.replace(identifier.substring(0, identifier.indexOf("/") + 2), ""); @@ -437,7 +437,7 @@ public static void writeSubjectsElement(XMLStreamWriter xmlw, DatasetVersionDTO for (String subject : fieldDTO.getMultipleVocab()) { if (StringUtils.isNotBlank(subject)) { subject_check = writeOpenTag(xmlw, "subjects", subject_check); - writeSubjectElement(xmlw, null, null, subject, language); + writeSubjectElement(xmlw, null, null, null, subject, language); } } } @@ -446,7 +446,8 @@ public static void writeSubjectsElement(XMLStreamWriter xmlw, DatasetVersionDTO for (HashSet fieldDTOs : fieldDTO.getMultipleCompound()) { String subject = null; String subjectScheme = null; - String schemeURI = null; + String keywordTermURI = null; + String keywordVocabURI = null; for (Iterator iterator = fieldDTOs.iterator(); iterator.hasNext();) { FieldDTO next = iterator.next(); @@ -454,18 +455,22 @@ public static void writeSubjectsElement(XMLStreamWriter xmlw, DatasetVersionDTO subject = next.getSinglePrimitive(); } + if (DatasetFieldConstant.keywordTermURI.equals(next.getTypeName())) { + keywordTermURI = next.getSinglePrimitive(); + } + if (DatasetFieldConstant.keywordVocab.equals(next.getTypeName())) { subjectScheme = next.getSinglePrimitive(); } - + if (DatasetFieldConstant.keywordVocabURI.equals(next.getTypeName())) { - schemeURI = next.getSinglePrimitive(); + keywordVocabURI = next.getSinglePrimitive(); } } if (StringUtils.isNotBlank(subject)) { subject_check = writeOpenTag(xmlw, "subjects", subject_check); - writeSubjectElement(xmlw, subjectScheme, schemeURI, subject, language); + writeSubjectElement(xmlw, subjectScheme, keywordTermURI, keywordVocabURI, subject, language); } } } @@ -493,7 +498,7 @@ public static void writeSubjectsElement(XMLStreamWriter xmlw, DatasetVersionDTO if (StringUtils.isNotBlank(subject)) { subject_check = writeOpenTag(xmlw, "subjects", subject_check); - writeSubjectElement(xmlw, subjectScheme, schemeURI, subject, language); + writeSubjectElement(xmlw, subjectScheme, null, schemeURI, subject, language); } } } @@ -513,7 +518,7 @@ public static void writeSubjectsElement(XMLStreamWriter xmlw, DatasetVersionDTO * @param language * @throws XMLStreamException */ - private static void writeSubjectElement(XMLStreamWriter xmlw, String subjectScheme, String schemeURI, String value, String language) throws XMLStreamException { + private static void writeSubjectElement(XMLStreamWriter xmlw, String subjectScheme, String valueURI, String schemeURI, String value, String language) throws XMLStreamException { // write a subject Map subject_map = new HashMap(); @@ -524,6 +529,9 @@ private static void writeSubjectElement(XMLStreamWriter xmlw, String subjectSche if (StringUtils.isNotBlank(subjectScheme)) { subject_map.put("subjectScheme", subjectScheme); } + if (StringUtils.isNotBlank(valueURI)) { + subject_map.put("valueURI", valueURI); + } if (StringUtils.isNotBlank(schemeURI)) { subject_map.put("schemeURI", schemeURI); } diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java index 7f1f46c06cb..c103f6981e1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java @@ -40,6 +40,7 @@ public class ExternalTool implements Serializable { public static final String TOOL_NAME = "toolName"; public static final String ALLOWED_API_CALLS = "allowedApiCalls"; public static final String REQUIREMENTS = "requirements"; + public static final String AUX_FILES_EXIST = "auxFilesExist"; @Id @GeneratedValue(strategy = GenerationType.IDENTITY) @@ -362,4 +363,9 @@ public void setRequirements(String requirements) { this.requirements = requirements; } + public boolean accessesAuxFiles() { + String reqs = getRequirements(); + return reqs!=null && reqs.contains(AUX_FILES_EXIST); + } + } diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java index e13843eadfa..5ee0bf1355d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java @@ -5,6 +5,8 @@ import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.DataFileServiceBean; import edu.harvard.iq.dataverse.authorization.users.ApiToken; +import edu.harvard.iq.dataverse.dataaccess.DataAccess; +import edu.harvard.iq.dataverse.dataaccess.StorageIO; import edu.harvard.iq.dataverse.externaltools.ExternalTool.Type; import edu.harvard.iq.dataverse.util.URLTokenUtil.ReservedWord; import edu.harvard.iq.dataverse.util.json.JsonUtil; @@ -141,9 +143,10 @@ public List findExternalToolsByFile(List allExternal List externalTools = new ArrayList<>(); //Map tabular data to it's mimetype (the isTabularData() check assures that this code works the same as before, but it may need to change if tabular data is split into subtypes with differing mimetypes) final String contentType = file.isTabularData() ? DataFileServiceBean.MIME_TYPE_TSV_ALT : file.getContentType(); + boolean isAccessible = StorageIO.isDataverseAccessible(DataAccess.getStorageDriverFromIdentifier(file.getStorageIdentifier())); allExternalTools.forEach((externalTool) -> { //Match tool and file type, then check requirements - if (contentType.equals(externalTool.getContentType()) && meetsRequirements(externalTool, file)) { + if (contentType.equals(externalTool.getContentType()) && meetsRequirements(externalTool, file) && (isAccessible || externalTool.accessesAuxFiles())) { externalTools.add(externalTool); } }); @@ -159,7 +162,7 @@ public boolean meetsRequirements(ExternalTool externalTool, DataFile dataFile) { } boolean meetsRequirements = true; JsonObject requirementsObj = JsonUtil.getJsonObject(requirements); - JsonArray auxFilesExist = requirementsObj.getJsonArray("auxFilesExist"); + JsonArray auxFilesExist = requirementsObj.getJsonArray(ExternalTool.AUX_FILES_EXIST); for (JsonValue jsonValue : auxFilesExist) { String formatTag = jsonValue.asJsonObject().getString("formatTag"); String formatVersion = jsonValue.asJsonObject().getString("formatVersion"); diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java index 3e60441850b..fb50214c259 100644 --- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java @@ -642,7 +642,7 @@ public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, S String logTimestamp = logFormatter.format(new Date()); Logger globusLogger = Logger.getLogger( "edu.harvard.iq.dataverse.upload.client.DatasetServiceBean." + "GlobusUpload" + logTimestamp); - String logFileName = "../logs" + File.separator + "globusUpload_id_" + dataset.getId() + "_" + logTimestamp + String logFileName = System.getProperty("com.sun.aas.instanceRoot") + File.separator + "logs" + File.separator + "globusUpload_id_" + dataset.getId() + "_" + logTimestamp + ".log"; FileHandler fileHandler; boolean fileHandlerSuceeded; @@ -920,7 +920,7 @@ public void globusDownload(String jsonData, Dataset dataset, User authUser) thro Logger globusLogger = Logger.getLogger( "edu.harvard.iq.dataverse.upload.client.DatasetServiceBean." + "GlobusDownload" + logTimestamp); - String logFileName = "../logs" + File.separator + "globusDownload_id_" + dataset.getId() + "_" + logTimestamp + String logFileName = System.getProperty("com.sun.aas.instanceRoot") + File.separator + "logs" + File.separator + "globusDownload_id_" + dataset.getId() + "_" + logTimestamp + ".log"; FileHandler fileHandler; boolean fileHandlerSuceeded; @@ -985,10 +985,14 @@ public void globusDownload(String jsonData, Dataset dataset, User authUser) thro if (taskStatus.startsWith("FAILED") || taskStatus.startsWith("INACTIVE")) { String comment = "Reason : " + taskStatus.split("#")[1] + "
Short Description : " + taskStatus.split("#")[2]; - userNotificationService.sendNotification((AuthenticatedUser) authUser, new Timestamp(new Date().getTime()), - UserNotification.Type.GLOBUSDOWNLOADCOMPLETEDWITHERRORS, dataset.getId(), comment, true); - globusLogger.info("Globus task failed during download process"); - } else { + if (authUser != null && authUser instanceof AuthenticatedUser) { + userNotificationService.sendNotification((AuthenticatedUser) authUser, new Timestamp(new Date().getTime()), + UserNotification.Type.GLOBUSDOWNLOADCOMPLETEDWITHERRORS, dataset.getId(), comment, true); + } + + globusLogger.info("Globus task failed during download process: "+comment); + } else if (authUser != null && authUser instanceof AuthenticatedUser) { + boolean taskSkippedFiles = (task.getSkip_source_errors() == null) ? false : task.getSkip_source_errors(); if (!taskSkippedFiles) { userNotificationService.sendNotification((AuthenticatedUser) authUser, @@ -1257,11 +1261,11 @@ public void writeGuestbookAndStartTransfer(GuestbookResponse guestbookResponse, Long fileId = Long.parseLong(idAsString); // If we need to create a GuestBookResponse record, we have to // look up the DataFile object for this file: + df = dataFileService.findCheapAndEasy(fileId); + selectedFiles.add(df); if (!doNotSaveGuestbookResponse) { - df = dataFileService.findCheapAndEasy(fileId); guestbookResponse.setDataFile(df); fileDownloadService.writeGuestbookResponseRecord(guestbookResponse); - selectedFiles.add(df); } } catch (NumberFormatException nfe) { logger.warning( diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java index 20884e3360c..e0b5c2dfbfb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java @@ -88,7 +88,6 @@ public class HarvesterServiceBean { public static final String HARVEST_RESULT_FAILED="failed"; public static final String DATAVERSE_PROPRIETARY_METADATA_FORMAT="dataverse_json"; public static final String DATAVERSE_PROPRIETARY_METADATA_API="/api/datasets/export?exporter="+DATAVERSE_PROPRIETARY_METADATA_FORMAT+"&persistentId="; - public static final String DATAVERSE_HARVEST_STOP_FILE="../logs/stopharvest_"; public HarvesterServiceBean() { @@ -399,7 +398,7 @@ private void deleteHarvestedDatasetIfExists(String persistentIdentifier, Dataver private boolean checkIfStoppingJob(HarvestingClient harvestingClient) { Long pid = ProcessHandle.current().pid(); - String stopFileName = DATAVERSE_HARVEST_STOP_FILE + harvestingClient.getName() + "." + pid; + String stopFileName = System.getProperty("com.sun.aas.instanceRoot") + File.separator + "logs" + File.separator + "stopharvest_" + harvestingClient.getName() + "." + pid; Path stopFilePath = Paths.get(stopFileName); if (Files.exists(stopFilePath)) { diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClient.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClient.java index 40db55f2a0c..0667f5594ce 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClient.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClient.java @@ -243,6 +243,14 @@ public String getCustomHttpHeaders() { public void setCustomHttpHeaders(String customHttpHeaders) { this.customHttpHeaders = customHttpHeaders; } + + private boolean allowHarvestingMissingCVV; + public boolean getAllowHarvestingMissingCVV() { + return allowHarvestingMissingCVV; + } + public void setAllowHarvestingMissingCVV(boolean allowHarvestingMissingCVV) { + this.allowHarvestingMissingCVV = allowHarvestingMissingCVV; + } // TODO: do we need "orphanRemoval=true"? -- L.A. 4.4 // TODO: should it be @OrderBy("startTime")? -- L.A. 4.4 diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAIRecordServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAIRecordServiceBean.java index 1b4a7bc7db0..cc15d4c978b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAIRecordServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAIRecordServiceBean.java @@ -40,10 +40,6 @@ @Stateless @Named public class OAIRecordServiceBean implements java.io.Serializable { - @EJB - OAISetServiceBean oaiSetService; - @EJB - IndexServiceBean indexService; @EJB DatasetServiceBean datasetService; @EJB @@ -55,13 +51,24 @@ public class OAIRecordServiceBean implements java.io.Serializable { EntityManager em; private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.harvest.server.OAIRecordServiceBean"); - - public void updateOaiRecords(String setName, List datasetIds, Date updateTime, boolean doExport) { - updateOaiRecords(setName, datasetIds, updateTime, doExport, logger); - } - public void updateOaiRecords(String setName, List datasetIds, Date updateTime, boolean doExport, Logger setUpdateLogger) { - + /** + * Updates the OAI records for the set specified + * @param setName name of the OAI set + * @param datasetIds ids of the datasets that are candidates for this OAI set + * @param updateTime time stamp + * @param doExport attempt to export datasets that haven't been exported yet + * @param confirmed true if the datasetIds above were looked up in the database + * - as opposed to in the search engine. Meaning, that it is + * confirmed that any dataset not on this list that's currently + * in the set is no longer in the database and should be + * marked as deleted without any further checks. Otherwise + * we'll want to double-check if the dataset still exists + * as published. This is to prevent marking existing datasets + * as deleted during a full reindex etc. + * @param setUpdateLogger dedicated Logger + */ + public void updateOaiRecords(String setName, List datasetIds, Date updateTime, boolean doExport, boolean confirmed, Logger setUpdateLogger) { // create Map of OaiRecords List oaiRecords = findOaiRecordsBySetName(setName); Map recordMap = new HashMap<>(); @@ -101,9 +108,6 @@ public void updateOaiRecords(String setName, List datasetIds, Date updateT DatasetVersion releasedVersion = dataset.getReleasedVersion(); Date publicationDate = releasedVersion == null ? null : releasedVersion.getReleaseTime(); - //if (dataset.getPublicationDate() != null - // && (dataset.getLastExportTime() == null - // || dataset.getLastExportTime().before(dataset.getPublicationDate()))) { if (publicationDate != null && (dataset.getLastExportTime() == null || dataset.getLastExportTime().before(publicationDate))) { @@ -125,7 +129,9 @@ public void updateOaiRecords(String setName, List datasetIds, Date updateT } // anything left in the map should be marked as removed! - markOaiRecordsAsRemoved( recordMap.values(), updateTime, setUpdateLogger); + markOaiRecordsAsRemoved(recordMap.values(), updateTime, confirmed, setUpdateLogger); + + } @@ -162,7 +168,7 @@ record = new OAIRecord(setName, dataset.getGlobalId().asString(), new Date()); } } - + /* // Updates any existing OAI records for this dataset // Should be called whenever there's a change in the release status of the Dataset // (i.e., when it's published or deaccessioned), so that the timestamps and @@ -201,13 +207,31 @@ public void updateOaiRecordsForDataset(Dataset dataset) { logger.fine("Null returned - no records found."); } } +*/ - public void markOaiRecordsAsRemoved(Collection records, Date updateTime, Logger setUpdateLogger) { + public void markOaiRecordsAsRemoved(Collection records, Date updateTime, boolean confirmed, Logger setUpdateLogger) { for (OAIRecord oaiRecord : records) { if ( !oaiRecord.isRemoved() ) { - setUpdateLogger.fine("marking OAI record "+oaiRecord.getGlobalId()+" as removed"); - oaiRecord.setRemoved(true); - oaiRecord.setLastUpdateTime(updateTime); + boolean confirmedRemoved = confirmed; + if (!confirmedRemoved) { + Dataset lookedUp = datasetService.findByGlobalId(oaiRecord.getGlobalId()); + if (lookedUp == null) { + confirmedRemoved = true; + } else if (lookedUp.getLastExportTime() == null) { + confirmedRemoved = true; + } else { + boolean isReleased = lookedUp.getReleasedVersion() != null; + if (!isReleased) { + confirmedRemoved = true; + } + } + } + + if (confirmedRemoved) { + setUpdateLogger.fine("marking OAI record "+oaiRecord.getGlobalId()+" as removed"); + oaiRecord.setRemoved(true); + oaiRecord.setLastUpdateTime(updateTime); + } } else { setUpdateLogger.fine("OAI record "+oaiRecord.getGlobalId()+" is already marked as removed."); } diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAISetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAISetServiceBean.java index d5c78c36b98..242187db7f1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAISetServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAISetServiceBean.java @@ -171,6 +171,8 @@ public void exportOaiSet(OAISet oaiSet, Logger exportLogger) { String query = managedSet.getDefinition(); List datasetIds; + boolean databaseLookup = false; // As opposed to a search engine lookup + try { if (!oaiSet.isDefaultSet()) { datasetIds = expandSetQuery(query); @@ -181,6 +183,7 @@ public void exportOaiSet(OAISet oaiSet, Logger exportLogger) { // including the unpublished drafts and deaccessioned ones. // Those will be filtered out further down the line. datasetIds = datasetService.findAllLocalDatasetIds(); + databaseLookup = true; } } catch (OaiSetException ose) { datasetIds = null; @@ -191,7 +194,7 @@ public void exportOaiSet(OAISet oaiSet, Logger exportLogger) { // they will be properly marked as "deleted"! -- L.A. 4.5 //if (datasetIds != null && !datasetIds.isEmpty()) { exportLogger.info("Calling OAI Record Service to re-export " + datasetIds.size() + " datasets."); - oaiRecordService.updateOaiRecords(managedSet.getSpec(), datasetIds, new Date(), true, exportLogger); + oaiRecordService.updateOaiRecords(managedSet.getSpec(), datasetIds, new Date(), true, databaseLookup, exportLogger); //} managedSet.setUpdateInProgress(false); @@ -200,7 +203,7 @@ public void exportOaiSet(OAISet oaiSet, Logger exportLogger) { public void exportAllSets() { String logTimestamp = logFormatter.format(new Date()); Logger exportLogger = Logger.getLogger("edu.harvard.iq.dataverse.harvest.client.OAISetServiceBean." + "UpdateAllSets." + logTimestamp); - String logFileName = "../logs" + File.separator + "oaiSetsUpdate_" + logTimestamp + ".log"; + String logFileName = System.getProperty("com.sun.aas.instanceRoot") + File.separator + "logs" + File.separator + "oaiSetsUpdate_" + logTimestamp + ".log"; FileHandler fileHandler = null; boolean fileHandlerSuceeded = false; try { diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java index 233ca94f5fc..f9047e3ee5f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java @@ -5,6 +5,7 @@ */ package edu.harvard.iq.dataverse.harvest.server.web.servlet; +import edu.harvard.iq.dataverse.MailServiceBean; import io.gdcc.xoai.dataprovider.DataProvider; import io.gdcc.xoai.dataprovider.repository.Repository; import io.gdcc.xoai.dataprovider.repository.RepositoryConfiguration; @@ -39,6 +40,7 @@ import java.io.IOException; +import java.util.Optional; import java.util.logging.Logger; import jakarta.ejb.EJB; import jakarta.inject.Inject; @@ -67,14 +69,14 @@ public class OAIServlet extends HttpServlet { @EJB OAIRecordServiceBean recordService; @EJB - SettingsServiceBean settingsService; - @EJB DataverseServiceBean dataverseService; @EJB DatasetServiceBean datasetService; @EJB SystemConfig systemConfig; + @EJB + MailServiceBean mailServiceBean; @Inject @ConfigProperty(name = "dataverse.oai.server.maxidentifiers", defaultValue="100") @@ -194,9 +196,13 @@ private RepositoryConfiguration createRepositoryConfiguration() { // (Note: if the setting does not exist, we are going to assume that they // have a reason not to want to configure their email address, if it is // a developer's instance, for example; or a reason not to want to - // advertise it to the world.) - InternetAddress systemEmailAddress = MailUtil.parseSystemAddress(settingsService.getValueForKey(SettingsServiceBean.Key.SystemEmail)); - String systemEmailLabel = systemEmailAddress != null ? systemEmailAddress.getAddress() : "donotreply@localhost"; + // advertise it to the world.) + String systemEmailLabel = "donotreply@localhost"; + // TODO: should we expose the support team's address if configured? + Optional systemAddress = mailServiceBean.getSystemAddress(); + if (systemAddress.isPresent()) { + systemEmailLabel = systemAddress.get().getAddress(); + } RepositoryConfiguration configuration = new RepositoryConfiguration.RepositoryConfigurationBuilder() .withAdminEmail(systemEmailLabel) diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java index 233f746fb17..9bacafd173f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java @@ -726,27 +726,17 @@ public void produceSummaryStatistics(DataFile dataFile, File generatedTabularFil } public void produceContinuousSummaryStatistics(DataFile dataFile, File generatedTabularFile) throws IOException { - - /* - // quick, but memory-inefficient way: - // - this method just loads the entire file-worth of continuous vectors - // into a Double[][] matrix. - //Double[][] variableVectors = subsetContinuousVectors(dataFile); - //calculateContinuousSummaryStatistics(dataFile, variableVectors); - - // A more sophisticated way: this subsets one column at a time, using - // the new optimized subsetting that does not have to read any extra - // bytes from the file to extract the column: - - TabularSubsetGenerator subsetGenerator = new TabularSubsetGenerator(); - */ for (int i = 0; i < dataFile.getDataTable().getVarQuantity(); i++) { if (dataFile.getDataTable().getDataVariables().get(i).isIntervalContinuous()) { logger.fine("subsetting continuous vector"); if ("float".equals(dataFile.getDataTable().getDataVariables().get(i).getFormat())) { - Float[] variableVector = TabularSubsetGenerator.subsetFloatVector(new FileInputStream(generatedTabularFile), i, dataFile.getDataTable().getCaseQuantity().intValue()); + Float[] variableVector = TabularSubsetGenerator.subsetFloatVector( + new FileInputStream(generatedTabularFile), + i, + dataFile.getDataTable().getCaseQuantity().intValue(), + dataFile.getDataTable().isStoredWithVariableHeader()); logger.fine("Calculating summary statistics on a Float vector;"); calculateContinuousSummaryStatistics(dataFile, i, variableVector); // calculate the UNF while we are at it: @@ -754,7 +744,11 @@ public void produceContinuousSummaryStatistics(DataFile dataFile, File generated calculateUNF(dataFile, i, variableVector); variableVector = null; } else { - Double[] variableVector = TabularSubsetGenerator.subsetDoubleVector(new FileInputStream(generatedTabularFile), i, dataFile.getDataTable().getCaseQuantity().intValue()); + Double[] variableVector = TabularSubsetGenerator.subsetDoubleVector( + new FileInputStream(generatedTabularFile), + i, + dataFile.getDataTable().getCaseQuantity().intValue(), + dataFile.getDataTable().isStoredWithVariableHeader()); logger.fine("Calculating summary statistics on a Double vector;"); calculateContinuousSummaryStatistics(dataFile, i, variableVector); // calculate the UNF while we are at it: @@ -776,7 +770,11 @@ public void produceDiscreteNumericSummaryStatistics(DataFile dataFile, File gene && dataFile.getDataTable().getDataVariables().get(i).isTypeNumeric()) { logger.fine("subsetting discrete-numeric vector"); - Long[] variableVector = TabularSubsetGenerator.subsetLongVector(new FileInputStream(generatedTabularFile), i, dataFile.getDataTable().getCaseQuantity().intValue()); + Long[] variableVector = TabularSubsetGenerator.subsetLongVector( + new FileInputStream(generatedTabularFile), + i, + dataFile.getDataTable().getCaseQuantity().intValue(), + dataFile.getDataTable().isStoredWithVariableHeader()); // We are discussing calculating the same summary stats for // all numerics (the same kind of sumstats that we've been calculating // for numeric continuous type) -- L.A. Jul. 2014 @@ -810,7 +808,11 @@ public void produceCharacterSummaryStatistics(DataFile dataFile, File generatedT if (dataFile.getDataTable().getDataVariables().get(i).isTypeCharacter()) { logger.fine("subsetting character vector"); - String[] variableVector = TabularSubsetGenerator.subsetStringVector(new FileInputStream(generatedTabularFile), i, dataFile.getDataTable().getCaseQuantity().intValue()); + String[] variableVector = TabularSubsetGenerator.subsetStringVector( + new FileInputStream(generatedTabularFile), + i, + dataFile.getDataTable().getCaseQuantity().intValue(), + dataFile.getDataTable().isStoredWithVariableHeader()); //calculateCharacterSummaryStatistics(dataFile, i, variableVector); // calculate the UNF while we are at it: logger.fine("Calculating UNF on a String vector"); @@ -828,20 +830,29 @@ public static void produceFrequencyStatistics(DataFile dataFile, File generatedT produceFrequencies(generatedTabularFile, vars); } - public static void produceFrequencies( File generatedTabularFile, List vars) throws IOException { + public static void produceFrequencies(File generatedTabularFile, List vars) throws IOException { for (int i = 0; i < vars.size(); i++) { Collection cats = vars.get(i).getCategories(); int caseQuantity = vars.get(i).getDataTable().getCaseQuantity().intValue(); boolean isNumeric = vars.get(i).isTypeNumeric(); + boolean skipVariableHeaderLine = vars.get(i).getDataTable().isStoredWithVariableHeader(); Object[] variableVector = null; if (cats.size() > 0) { if (isNumeric) { - variableVector = TabularSubsetGenerator.subsetFloatVector(new FileInputStream(generatedTabularFile), i, caseQuantity); + variableVector = TabularSubsetGenerator.subsetFloatVector( + new FileInputStream(generatedTabularFile), + i, + caseQuantity, + skipVariableHeaderLine); } else { - variableVector = TabularSubsetGenerator.subsetStringVector(new FileInputStream(generatedTabularFile), i, caseQuantity); + variableVector = TabularSubsetGenerator.subsetStringVector( + new FileInputStream(generatedTabularFile), + i, + caseQuantity, + skipVariableHeaderLine); } if (variableVector != null) { Hashtable freq = calculateFrequency(variableVector); @@ -923,6 +934,7 @@ public boolean ingestAsTabular(Long datafile_id) { DataFile dataFile = fileService.find(datafile_id); boolean ingestSuccessful = false; boolean forceTypeCheck = false; + boolean storingWithVariableHeader = systemConfig.isStoringIngestedFilesWithHeaders(); // Never attempt to ingest a file that's already ingested! if (dataFile.isTabularData()) { @@ -1024,11 +1036,7 @@ public boolean ingestAsTabular(Long datafile_id) { TabularDataIngest tabDataIngest = null; try { - if (additionalData != null) { - tabDataIngest = ingestPlugin.read(inputStream, additionalData); - } else { - tabDataIngest = ingestPlugin.read(inputStream, null); - } + tabDataIngest = ingestPlugin.read(inputStream, storingWithVariableHeader, additionalData); } catch (IOException ingestEx) { dataFile.SetIngestProblem(); FileUtil.createIngestFailureReport(dataFile, ingestEx.getMessage()); @@ -1081,6 +1089,7 @@ public boolean ingestAsTabular(Long datafile_id) { dataFile.setDataTable(tabDataIngest.getDataTable()); tabDataIngest.getDataTable().setDataFile(dataFile); tabDataIngest.getDataTable().setOriginalFileName(originalFileName); + dataFile.getDataTable().setStoredWithVariableHeader(storingWithVariableHeader); try { produceSummaryStatistics(dataFile, tabFile); @@ -1172,6 +1181,7 @@ public boolean ingestAsTabular(Long datafile_id) { // Replace contents of the file with the tab-delimited data produced: dataAccess.savePath(Paths.get(tabFile.getAbsolutePath())); + // Reset the file size: dataFile.setFilesize(dataAccess.getSize()); @@ -2297,7 +2307,7 @@ public static void main(String[] args) { TabularDataIngest tabDataIngest = null; try { - tabDataIngest = ingestPlugin.read(fileInputStream, null); + tabDataIngest = ingestPlugin.read(fileInputStream, false, null); } catch (IOException ingestEx) { System.err.println("Caught an exception trying to ingest file "+file+"."); System.exit(1); diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/TabularDataFileReader.java b/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/TabularDataFileReader.java index 223b171dfb5..0f23a3d9781 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/TabularDataFileReader.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/TabularDataFileReader.java @@ -20,10 +20,13 @@ package edu.harvard.iq.dataverse.ingest.tabulardata; +import edu.harvard.iq.dataverse.datavariable.DataVariable; import edu.harvard.iq.dataverse.ingest.tabulardata.spi.*; //import edu.harvard.iq.dataverse.ingest.plugin.metadata.*; import java.io.*; import static java.lang.System.*; +import java.util.Iterator; +import java.util.List; import java.util.regex.Matcher; /** @@ -98,7 +101,7 @@ public void setDataLanguageEncoding(String dataLanguageEncoding) { * * @throws java.io.IOException if a reading error occurs. */ - public abstract TabularDataIngest read(BufferedInputStream stream, File dataFile) + public abstract TabularDataIngest read(BufferedInputStream stream, boolean storeWithVariableHeader, File dataFile) throws IOException; @@ -176,5 +179,26 @@ protected String escapeCharacterString(String rawString) { return escapedString; } + + protected String generateVariableHeader(List dvs) { + String varHeader = null; + + if (dvs != null) { + Iterator iter = dvs.iterator(); + DataVariable dv; + + if (iter.hasNext()) { + dv = iter.next(); + varHeader = dv.getName(); + } + + while (iter.hasNext()) { + dv = iter.next(); + varHeader = varHeader + "\t" + dv.getName(); + } + } + + return varHeader; + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/csv/CSVFileReader.java b/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/csv/CSVFileReader.java index 57f76df3802..f8816ababb4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/csv/CSVFileReader.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/csv/CSVFileReader.java @@ -110,7 +110,7 @@ private void init() throws IOException { * @throws java.io.IOException if a reading error occurs. */ @Override - public TabularDataIngest read(BufferedInputStream stream, File dataFile) throws IOException { + public TabularDataIngest read(BufferedInputStream stream, boolean saveWithVariableHeader, File dataFile) throws IOException { init(); if (stream == null) { @@ -124,7 +124,7 @@ public TabularDataIngest read(BufferedInputStream stream, File dataFile) throws File tabFileDestination = File.createTempFile("data-", ".tab"); PrintWriter tabFileWriter = new PrintWriter(tabFileDestination.getAbsolutePath()); - int lineCount = readFile(localBufferedReader, dataTable, tabFileWriter); + int lineCount = readFile(localBufferedReader, dataTable, saveWithVariableHeader, tabFileWriter); logger.fine("Tab file produced: " + tabFileDestination.getAbsolutePath()); @@ -136,14 +136,17 @@ public TabularDataIngest read(BufferedInputStream stream, File dataFile) throws } - public int readFile(BufferedReader csvReader, DataTable dataTable, PrintWriter finalOut) throws IOException { + public int readFile(BufferedReader csvReader, DataTable dataTable, boolean saveWithVariableHeader, PrintWriter finalOut) throws IOException { List variableList = new ArrayList<>(); CSVParser parser = new CSVParser(csvReader, inFormat.withHeader()); Map headers = parser.getHeaderMap(); int i = 0; + String variableNameHeader = null; + for (String varName : headers.keySet()) { + // @todo: is .keySet() guaranteed to return the names in the right order? if (varName == null || varName.isEmpty()) { // TODO: // Add a sensible variable name validation algorithm. @@ -158,6 +161,13 @@ public int readFile(BufferedReader csvReader, DataTable dataTable, PrintWriter f dv.setTypeCharacter(); dv.setIntervalDiscrete(); + + if (saveWithVariableHeader) { + variableNameHeader = variableNameHeader == null + ? varName + : variableNameHeader.concat("\t" + varName); + } + i++; } @@ -342,6 +352,14 @@ public int readFile(BufferedReader csvReader, DataTable dataTable, PrintWriter f try (BufferedReader secondPassReader = new BufferedReader(new FileReader(firstPassTempFile))) { parser = new CSVParser(secondPassReader, inFormat.withHeader()); String[] caseRow = new String[headers.size()]; + + // Save the variable name header, if requested + if (saveWithVariableHeader) { + if (variableNameHeader == null) { + throw new IOException("failed to generate the Variable Names header"); + } + finalOut.println(variableNameHeader); + } for (CSVRecord record : parser) { if (!record.isConsistent()) { diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/dta/DTAFileReader.java b/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/dta/DTAFileReader.java index 2dec701592e..73818f8fb62 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/dta/DTAFileReader.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/dta/DTAFileReader.java @@ -505,7 +505,7 @@ private void init() throws IOException { } @Override - public TabularDataIngest read(BufferedInputStream stream, File dataFile) throws IOException { + public TabularDataIngest read(BufferedInputStream stream, boolean storeWithVariableHeader, File dataFile) throws IOException { dbgLog.info("***** DTAFileReader: read() start *****"); if (dataFile != null) { @@ -519,7 +519,7 @@ public TabularDataIngest read(BufferedInputStream stream, File dataFile) throws if (releaseNumber!=104) { decodeExpansionFields(stream); } - decodeData(stream); + decodeData(stream, storeWithVariableHeader); decodeValueLabels(stream); ingesteddata.setDataTable(dataTable); @@ -1665,7 +1665,7 @@ private void parseValueLabelsReleasel108(BufferedInputStream stream) throws IOEx dbgLog.fine("parseValueLabelsRelease108(): end"); } - private void decodeData(BufferedInputStream stream) throws IOException { + private void decodeData(BufferedInputStream stream, boolean saveWithVariableHeader) throws IOException { dbgLog.fine("\n***** decodeData(): start *****"); @@ -1719,6 +1719,11 @@ private void decodeData(BufferedInputStream stream) throws IOException { BUT, this needs to be reviewed/confirmed etc! */ //String[][] dateFormat = new String[nvar][nobs]; + + // add the variable header here, if needed + if (saveWithVariableHeader) { + pwout.println(generateVariableHeader(dataTable.getDataVariables())); + } for (int i = 0; i < nobs; i++) { byte[] dataRowBytes = new byte[bytes_per_row]; diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/dta/NewDTAFileReader.java b/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/dta/NewDTAFileReader.java index 22581834676..53607d541de 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/dta/NewDTAFileReader.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/dta/NewDTAFileReader.java @@ -339,7 +339,7 @@ private void init() throws IOException { } @Override - public TabularDataIngest read(BufferedInputStream stream, File dataFile) throws IOException { + public TabularDataIngest read(BufferedInputStream stream, boolean storeWithVariableHeader, File dataFile) throws IOException { logger.fine("NewDTAFileReader: read() start"); // shit ton of diagnostics (still) needed here!! -- L.A. @@ -363,7 +363,13 @@ public TabularDataIngest read(BufferedInputStream stream, File dataFile) throws // "characteristics" - STATA-proprietary information // (we are skipping it) readCharacteristics(dataReader); - readData(dataReader); + + String variableHeaderLine = null; + + if (storeWithVariableHeader) { + variableHeaderLine = generateVariableHeader(dataTable.getDataVariables()); + } + readData(dataReader, variableHeaderLine); // (potentially) large, (potentially) non-ASCII character strings // saved outside the section, and referenced @@ -707,7 +713,7 @@ private void readCharacteristics(DataReader reader) throws IOException { } - private void readData(DataReader reader) throws IOException { + private void readData(DataReader reader, String variableHeaderLine) throws IOException { logger.fine("Data section; at offset " + reader.getByteOffset() + "; dta map offset: " + dtaMap.getOffset_data()); logger.fine("readData(): start"); reader.readOpeningTag(TAG_DATA); @@ -731,6 +737,11 @@ private void readData(DataReader reader) throws IOException { FileOutputStream fileOutTab = new FileOutputStream(tabDelimitedDataFile); PrintWriter pwout = new PrintWriter(new OutputStreamWriter(fileOutTab, "utf8"), true); + // add the variable header here, if needed + if (variableHeaderLine != null) { + pwout.println(variableHeaderLine); + } + logger.fine("Beginning to read data stream."); for (int i = 0; i < nobs; i++) { @@ -999,6 +1010,8 @@ private void readSTRLs(DataReader reader) throws IOException { int nobs = dataTable.getCaseQuantity().intValue(); String[] line; + + //@todo: adjust for the case of storing the file with the variable header for (int obsindex = 0; obsindex < nobs; obsindex++) { if (scanner.hasNext()) { diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/por/PORFileReader.java b/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/por/PORFileReader.java index c90b0ea6950..2ee966c3e31 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/por/PORFileReader.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/por/PORFileReader.java @@ -180,7 +180,7 @@ private void init() throws IOException { } @Override - public TabularDataIngest read(BufferedInputStream stream, File additionalData) throws IOException{ + public TabularDataIngest read(BufferedInputStream stream, boolean storeWithVariableHeader, File additionalData) throws IOException{ dbgLog.fine("PORFileReader: read() start"); if (additionalData != null) { @@ -226,7 +226,7 @@ public TabularDataIngest read(BufferedInputStream stream, File additionalData) t headerId = "8S"; } - decode(headerId, bfReader); + decode(headerId, bfReader, storeWithVariableHeader); // for last iteration @@ -382,7 +382,7 @@ public TabularDataIngest read(BufferedInputStream stream, File additionalData) t return ingesteddata; } - private void decode(String headerId, BufferedReader reader) throws IOException{ + private void decode(String headerId, BufferedReader reader, boolean storeWithVariableHeader) throws IOException{ if (headerId.equals("1")) decodeProductName(reader); else if (headerId.equals("2")) decodeLicensee(reader); else if (headerId.equals("3")) decodeFileLabel(reader); @@ -398,7 +398,7 @@ private void decode(String headerId, BufferedReader reader) throws IOException{ else if (headerId.equals("C")) decodeVariableLabel(reader); else if (headerId.equals("D")) decodeValueLabel(reader); else if (headerId.equals("E")) decodeDocument(reader); - else if (headerId.equals("F")) decodeData(reader); + else if (headerId.equals("F")) decodeData(reader, storeWithVariableHeader); } @@ -1099,7 +1099,7 @@ private void decodeDocument(BufferedReader reader) throws IOException { } - private void decodeData(BufferedReader reader) throws IOException { + private void decodeData(BufferedReader reader, boolean storeWithVariableHeader) throws IOException { dbgLog.fine("decodeData(): start"); // TODO: get rid of this "variableTypeFinal"; -- L.A. 4.0 beta int[] variableTypeFinal= new int[varQnty]; @@ -1126,6 +1126,9 @@ private void decodeData(BufferedReader reader) throws IOException { // contents (variable) checker concering decimals Arrays.fill(variableTypeFinal, 0); + if (storeWithVariableHeader) { + pwout.println(StringUtils.join(variableNameList, "\t")); + } // raw-case counter int j = 0; // case diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/rdata/RDATAFileReader.java b/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/rdata/RDATAFileReader.java index eb1353fd792..50f2f89e354 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/rdata/RDATAFileReader.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/rdata/RDATAFileReader.java @@ -473,7 +473,7 @@ private void init() throws IOException { * @throws java.io.IOException if a reading error occurs. */ @Override - public TabularDataIngest read(BufferedInputStream stream, File dataFile) throws IOException { + public TabularDataIngest read(BufferedInputStream stream, boolean saveWithVariableHeader, File dataFile) throws IOException { init(); @@ -509,7 +509,7 @@ public TabularDataIngest read(BufferedInputStream stream, File dataFile) throws File tabFileDestination = File.createTempFile("data-", ".tab"); PrintWriter tabFileWriter = new PrintWriter(tabFileDestination.getAbsolutePath(), "UTF-8"); - int lineCount = csvFileReader.read(localBufferedReader, dataTable, tabFileWriter); + int lineCount = csvFileReader.read(localBufferedReader, dataTable, saveWithVariableHeader, tabFileWriter); LOG.fine("RDATAFileReader: successfully read "+lineCount+" lines of tab-delimited data."); diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/rdata/RTabFileParser.java b/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/rdata/RTabFileParser.java index f60b7733463..fbe7e401b57 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/rdata/RTabFileParser.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/rdata/RTabFileParser.java @@ -61,8 +61,8 @@ public RTabFileParser (char delimiterChar) { // should be used. - public int read(BufferedReader csvReader, DataTable dataTable, PrintWriter pwout) throws IOException { - dbgLog.warning("RTabFileParser: Inside R Tab file parser"); + public int read(BufferedReader csvReader, DataTable dataTable, boolean saveWithVariableHeader, PrintWriter pwout) throws IOException { + dbgLog.fine("RTabFileParser: Inside R Tab file parser"); int varQnty = 0; @@ -94,14 +94,17 @@ public int read(BufferedReader csvReader, DataTable dataTable, PrintWriter pwout boolean[] isTimeVariable = new boolean[varQnty]; boolean[] isBooleanVariable = new boolean[varQnty]; + String variableNameHeader = null; + if (dataTable.getDataVariables() != null) { for (int i = 0; i < varQnty; i++) { DataVariable var = dataTable.getDataVariables().get(i); if (var == null) { - // throw exception! + throw new IOException ("null dataVariable passed to the parser"); + } if (var.getType() == null) { - // throw exception! + throw new IOException ("null dataVariable type passed to the parser"); } if (var.isTypeCharacter()) { isCharacterVariable[i] = true; @@ -128,13 +131,24 @@ public int read(BufferedReader csvReader, DataTable dataTable, PrintWriter pwout } } } else { - // throw excepion "unknown variable format type" - ? + throw new IOException ("unknown dataVariable format passed to the parser"); } - + if (saveWithVariableHeader) { + variableNameHeader = variableNameHeader == null + ? var.getName() + : variableNameHeader.concat("\t" + var.getName()); + } } } else { - // throw exception! + throw new IOException ("null dataVariables list passed to the parser"); + } + + if (saveWithVariableHeader) { + if (variableNameHeader == null) { + throw new IOException ("failed to generate the Variable Names header"); + } + pwout.println(variableNameHeader); } while ((line = csvReader.readLine()) != null) { diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/sav/SAVFileReader.java b/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/sav/SAVFileReader.java index 682b8f1166c..5eecbdfb666 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/sav/SAVFileReader.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/sav/SAVFileReader.java @@ -338,7 +338,7 @@ private void init() throws IOException { } } - public TabularDataIngest read(BufferedInputStream stream, File dataFile) throws IOException{ + public TabularDataIngest read(BufferedInputStream stream, boolean storeWithVariableHeader, File dataFile) throws IOException{ dbgLog.info("SAVFileReader: read() start"); if (dataFile != null) { @@ -422,7 +422,7 @@ public TabularDataIngest read(BufferedInputStream stream, File dataFile) throws methodCurrentlyExecuted = "decodeRecordTypeData"; dbgLog.fine("***** SAVFileReader: executing method decodeRecordTypeData"); - decodeRecordTypeData(stream); + decodeRecordTypeData(stream, storeWithVariableHeader); } catch (IllegalArgumentException e) { @@ -2308,7 +2308,7 @@ void decodeRecordType999(BufferedInputStream stream) throws IOException { - void decodeRecordTypeData(BufferedInputStream stream) throws IOException { + void decodeRecordTypeData(BufferedInputStream stream, boolean storeWithVariableHeader) throws IOException { dbgLog.fine("decodeRecordTypeData(): start"); ///String fileUnfValue = null; @@ -2320,9 +2320,9 @@ void decodeRecordTypeData(BufferedInputStream stream) throws IOException { throw new IllegalArgumentException("stream == null!"); } if (isDataSectionCompressed){ - decodeRecordTypeDataCompressed(stream); + decodeRecordTypeDataCompressed(stream, storeWithVariableHeader); } else { - decodeRecordTypeDataUnCompressed(stream); + decodeRecordTypeDataUnCompressed(stream, storeWithVariableHeader); } /* UNF calculation was here... */ @@ -2362,7 +2362,7 @@ PrintWriter createOutputWriter (BufferedInputStream stream) throws IOException { } - void decodeRecordTypeDataCompressed(BufferedInputStream stream) throws IOException { + void decodeRecordTypeDataCompressed(BufferedInputStream stream, boolean storeWithVariableHeader) throws IOException { dbgLog.fine("***** decodeRecordTypeDataCompressed(): start *****"); @@ -2395,7 +2395,10 @@ void decodeRecordTypeDataCompressed(BufferedInputStream stream) throws IOExcepti dbgLog.fine("printFormatTable:\n" + printFormatTable); variableFormatTypeList = new String[varQnty]; - + // write the variable header out, if instructed to do so + if (storeWithVariableHeader) { + pwout.println(generateVariableHeader(dataTable.getDataVariables())); + } for (int i = 0; i < varQnty; i++) { variableFormatTypeList[i] = SPSSConstants.FORMAT_CATEGORY_TABLE.get( @@ -2947,7 +2950,7 @@ void decodeRecordTypeDataCompressed(BufferedInputStream stream) throws IOExcepti } - void decodeRecordTypeDataUnCompressed(BufferedInputStream stream) throws IOException { + void decodeRecordTypeDataUnCompressed(BufferedInputStream stream, boolean storeWithVariableHeader) throws IOException { dbgLog.fine("***** decodeRecordTypeDataUnCompressed(): start *****"); if (stream ==null){ @@ -3013,6 +3016,11 @@ void decodeRecordTypeDataUnCompressed(BufferedInputStream stream) throws IOExcep ///dataTable2 = new Object[varQnty][caseQnty]; // storage of date formats to pass to UNF ///dateFormats = new String[varQnty][caseQnty]; + + // write the variable header out, if instructed to do so + if (storeWithVariableHeader) { + pwout.println(generateVariableHeader(dataTable.getDataVariables())); + } try { for (int i = 0; ; i++){ // case-wise loop diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/xlsx/XLSXFileReader.java b/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/xlsx/XLSXFileReader.java index ea3f3868f24..ef91793690e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/xlsx/XLSXFileReader.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/xlsx/XLSXFileReader.java @@ -36,7 +36,6 @@ import org.apache.commons.lang3.StringUtils; import org.apache.poi.xssf.eventusermodel.XSSFReader; -import org.apache.poi.xssf.usermodel.XSSFRichTextString; import org.apache.poi.xssf.model.SharedStrings; import org.apache.poi.openxml4j.opc.OPCPackage; import org.xml.sax.Attributes; @@ -81,7 +80,9 @@ private void init() throws IOException { * @throws java.io.IOException if a reading error occurs. */ @Override - public TabularDataIngest read(BufferedInputStream stream, File dataFile) throws IOException { + public TabularDataIngest read(BufferedInputStream stream, boolean storeWithVariableHeader, File dataFile) throws IOException { + // @todo: implement handling of "saveWithVariableHeader" option + init(); TabularDataIngest ingesteddata = new TabularDataIngest(); @@ -118,6 +119,10 @@ public TabularDataIngest read(BufferedInputStream stream, File dataFile) throws String[] caseRow = new String[varQnty]; String[] valueTokens; + // add the variable header here, if needed + if (storeWithVariableHeader) { + finalWriter.println(generateVariableHeader(dataTable.getDataVariables())); + } while ((line = secondPassReader.readLine()) != null) { // chop the line: @@ -549,7 +554,7 @@ public static void main(String[] args) throws Exception { BufferedInputStream xlsxInputStream = new BufferedInputStream(new FileInputStream(new File(args[0]))); - TabularDataIngest dataIngest = testReader.read(xlsxInputStream, null); + TabularDataIngest dataIngest = testReader.read(xlsxInputStream, false, null); dataTable = dataIngest.getDataTable(); diff --git a/src/main/java/edu/harvard/iq/dataverse/makedatacount/MakeDataCountLoggingServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/makedatacount/MakeDataCountLoggingServiceBean.java index 5edf2fde0c3..c3bf85e699a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/makedatacount/MakeDataCountLoggingServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/makedatacount/MakeDataCountLoggingServiceBean.java @@ -46,6 +46,12 @@ public void logEntry(MakeDataCountEntry entry) { public String getLogFileName() { return "counter_"+new SimpleDateFormat("yyyy-MM-dd").format(new Timestamp(new Date().getTime()))+".log"; } + + // Sanitize the values to a safe string for the log file + static String sanitize(String in) { + // Log lines are tab delimited so tabs must be replaced. Replacing escape sequences with a space. + return in != null ? in.replaceAll("\\s+", " ") : null; + } public static class MakeDataCountEntry { @@ -367,7 +373,7 @@ public String getTitle() { * @param title the title to set */ public final void setTitle(String title) { - this.title = title; + this.title = sanitize(title); } /** @@ -384,7 +390,7 @@ public String getPublisher() { * @param publisher the publisher to set */ public final void setPublisher(String publisher) { - this.publisher = publisher; + this.publisher = sanitize(publisher); } /** @@ -401,7 +407,7 @@ public String getPublisherId() { * @param publisherId the publisherId to set */ public final void setPublisherId(String publisherId) { - this.publisherId = publisherId; + this.publisherId = sanitize(publisherId); } /** @@ -418,7 +424,7 @@ public String getAuthors() { * @param authors the authors to set */ public final void setAuthors(String authors) { - this.authors = authors; + this.authors = sanitize(authors); } /** @@ -452,7 +458,7 @@ public String getVersion() { * @param version the version to set */ public final void setVersion(String version) { - this.version = version; + this.version = sanitize(version); } /** @@ -469,7 +475,7 @@ public String getOtherId() { * @param otherId the otherId to set */ public void setOtherId(String otherId) { - this.otherId = otherId; + this.otherId = sanitize(otherId); } /** diff --git a/src/main/java/edu/harvard/iq/dataverse/makedatacount/MakeDataCountProcessState.java b/src/main/java/edu/harvard/iq/dataverse/makedatacount/MakeDataCountProcessState.java new file mode 100644 index 00000000000..2241a2c4ca8 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/makedatacount/MakeDataCountProcessState.java @@ -0,0 +1,75 @@ +package edu.harvard.iq.dataverse.makedatacount; + +import jakarta.persistence.*; + +import java.io.Serializable; +import java.sql.Timestamp; +import java.time.Instant; +import java.util.Arrays; + +@Entity +@Table(indexes = {@Index(columnList="yearMonth")}) +public class MakeDataCountProcessState implements Serializable { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(nullable = false) + private Long id; + + public enum MDCProcessState { + NEW("new"), DONE("done"), SKIP("skip"), PROCESSING("processing"), FAILED("failed"); + private final String text; + private MDCProcessState(final String text) { + this.text = text; + } + public static MDCProcessState fromString(String text) { + if (text != null) { + for (MDCProcessState state : MDCProcessState.values()) { + if (text.equals(state.text)) { + return state; + } + } + } + throw new IllegalArgumentException("State must be one of these values: " + Arrays.asList(MDCProcessState.values()) + "."); + } + @Override + public String toString() { + return text; + } + } + @Column(nullable = false) + private String yearMonth; + @Column(nullable = false) + private MDCProcessState state; + @Column(nullable = true) + private Timestamp stateChangeTimestamp; + + public MakeDataCountProcessState() { } + public MakeDataCountProcessState (String yearMonth, String state) { + this.setYearMonth(yearMonth); + this.setState(state); + } + + public void setYearMonth(String yearMonth) throws IllegalArgumentException { + // Todo: add constraint + if (yearMonth == null || (!yearMonth.matches("\\d{4}-\\d{2}") && !yearMonth.matches("\\d{4}-\\d{2}-\\d{2}"))) { + throw new IllegalArgumentException("YEAR-MONTH date format must be either yyyy-mm or yyyy-mm-dd"); + } + this.yearMonth = yearMonth; + } + public String getYearMonth() { + return this.yearMonth; + } + public void setState(MDCProcessState state) { + this.state = state; + this.stateChangeTimestamp = Timestamp.from(Instant.now()); + } + public void setState(String state) throws IllegalArgumentException { + setState(MDCProcessState.fromString(state)); + } + public MDCProcessState getState() { + return this.state; + } + public Timestamp getStateChangeTime() { + return stateChangeTimestamp; + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/makedatacount/MakeDataCountProcessStateServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/makedatacount/MakeDataCountProcessStateServiceBean.java new file mode 100644 index 00000000000..5d7ec8ff047 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/makedatacount/MakeDataCountProcessStateServiceBean.java @@ -0,0 +1,61 @@ +package edu.harvard.iq.dataverse.makedatacount; + +import jakarta.ejb.EJBException; +import jakarta.ejb.Stateless; +import jakarta.inject.Named; +import jakarta.persistence.EntityManager; +import jakarta.persistence.PersistenceContext; +import jakarta.persistence.Query; + +import java.util.List; + +@Named +@Stateless +public class MakeDataCountProcessStateServiceBean { + + @PersistenceContext(unitName = "VDCNet-ejbPU") + protected EntityManager em; + + public MakeDataCountProcessState getMakeDataCountProcessState(String yearMonth) { + validateYearMonth(yearMonth); + MakeDataCountProcessState mdcps = null; + String queryStr = "SELECT d FROM MakeDataCountProcessState d WHERE d.yearMonth = '" + yearMonth + "' "; + Query query = em.createQuery(queryStr); + List resultList = query.getResultList(); + if (resultList.size() > 1) { + throw new EJBException("More than one MakeDataCount Process State record found for YearMonth " + yearMonth + "."); + } + if (resultList.size() == 1) { + mdcps = (MakeDataCountProcessState) resultList.get(0); + } + return mdcps; + } + + public MakeDataCountProcessState setMakeDataCountProcessState(String yearMonth, String state) { + MakeDataCountProcessState mdcps = getMakeDataCountProcessState(yearMonth); + if (mdcps == null) { + mdcps = new MakeDataCountProcessState(yearMonth, state); + } else { + mdcps.setState(state); + } + return em.merge(mdcps); + } + + public boolean deleteMakeDataCountProcessState(String yearMonth) { + MakeDataCountProcessState mdcps = getMakeDataCountProcessState(yearMonth); + if (mdcps == null) { + return false; + } else { + em.remove(mdcps); + em.flush(); + return true; + } + } + + private void validateYearMonth(String yearMonth) { + // Check yearMonth format. either yyyy-mm or yyyy-mm-dd + if (yearMonth == null || (!yearMonth.matches("\\d{4}-\\d{2}") && !yearMonth.matches("\\d{4}-\\d{2}-\\d{2}"))) { + throw new IllegalArgumentException("YEAR-MONTH date format must be either yyyy-mm or yyyy-mm-dd"); + } + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java index 1b5619c53e0..a74474efa15 100644 --- a/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java @@ -572,6 +572,54 @@ public JsonArray uniqueDatasetDownloads(String yyyymm, Dataverse d) { } + //Accounts + + /* + * + * @param yyyymm Month in YYYY-MM format. + */ + public long accountsToMonth(String yyyymm) throws ParseException { + Query query = em.createNativeQuery("" + + "select count(authenticateduser.id)\n" + + "from authenticateduser\n" + + "where authenticateduser.createdtime is not null\n" + + "and date_trunc('month', createdtime) <= to_date('" + yyyymm + "','YYYY-MM');" + ); + logger.log(Level.FINE, "Metric query: {0}", query); + + return (long) query.getSingleResult(); + } + + /* + * + * @param days interval since the current date to list + * the number of user accounts created + */ + public long accountsPastDays(int days) { + Query query = em.createNativeQuery("" + + "select count(id)\n" + + "from authenticateduser\n" + + "where authenticateduser.createdtime is not null\n" + + "and authenticateduser.createdtime > current_date - interval '" + days + "' day;" + ); + logger.log(Level.FINE, "Metric query: {0}", query); + + return (long) query.getSingleResult(); + } + + public JsonArray accountsTimeSeries() { + Query query = em.createNativeQuery("" + + "select distinct to_char(au.createdtime, 'YYYY-MM'), count(id)\n" + + "from authenticateduser as au\n" + + "where au.createdtime is not null\n" + + "group by to_char(au.createdtime, 'YYYY-MM')\n" + + "order by to_char(au.createdtime, 'YYYY-MM');"); + + logger.log(Level.FINE, "Metric query: {0}", query); + List results = query.getResultList(); + return MetricsUtil.timeSeriesToJson(results); + } + //MDC diff --git a/src/main/java/edu/harvard/iq/dataverse/mydata/DataRetrieverAPI.java b/src/main/java/edu/harvard/iq/dataverse/mydata/DataRetrieverAPI.java index 0a64f42d840..6c99155d8a4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/mydata/DataRetrieverAPI.java +++ b/src/main/java/edu/harvard/iq/dataverse/mydata/DataRetrieverAPI.java @@ -3,6 +3,7 @@ */ package edu.harvard.iq.dataverse.mydata; +import edu.harvard.iq.dataverse.DatasetServiceBean; import edu.harvard.iq.dataverse.DataverseRoleServiceBean; import edu.harvard.iq.dataverse.DataverseServiceBean; import edu.harvard.iq.dataverse.DataverseSession; @@ -26,10 +27,10 @@ import edu.harvard.iq.dataverse.search.SearchFields; import edu.harvard.iq.dataverse.search.SortBy; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.logging.Logger; -import java.util.Locale; import jakarta.ejb.EJB; import jakarta.inject.Inject; import jakarta.json.Json; @@ -63,7 +64,7 @@ public class DataRetrieverAPI extends AbstractApiBean { private static final String retrieveDataPartialAPIPath = "retrieve"; @Inject - DataverseSession session; + DataverseSession session; @EJB DataverseRoleServiceBean dataverseRoleService; @@ -81,6 +82,8 @@ public class DataRetrieverAPI extends AbstractApiBean { //MyDataQueryHelperServiceBean myDataQueryHelperServiceBean; @EJB GroupServiceBean groupService; + @EJB + DatasetServiceBean datasetService; private List roleList; private DataverseRolePermissionHelper rolePermissionHelper; @@ -274,9 +277,7 @@ public String retrieveMyDataAsJsonString( @QueryParam("dataset_valid") List datasetValidities) { boolean OTHER_USER = false; - String localeCode = session.getLocaleCode(); - String noMsgResultsFound = BundleUtil.getStringFromPropertyFile("dataretrieverAPI.noMsgResultsFound", - "Bundle", new Locale(localeCode)); + String noMsgResultsFound = BundleUtil.getStringFromBundle("dataretrieverAPI.noMsgResultsFound"); if ((session.getUser() != null) && (session.getUser().isAuthenticated())) { authUser = (AuthenticatedUser) session.getUser(); @@ -284,7 +285,10 @@ public String retrieveMyDataAsJsonString( try { authUser = getRequestAuthenticatedUserOrDie(crc); } catch (WrappedResponse e) { - return this.getJSONErrorString("Requires authentication. Please login.", "retrieveMyDataAsJsonString. User not found! Shouldn't be using this anyway"); + return this.getJSONErrorString( + BundleUtil.getStringFromBundle("dataretrieverAPI.authentication.required"), + BundleUtil.getStringFromBundle("dataretrieverAPI.authentication.required.opt") + ); } } @@ -297,7 +301,9 @@ public String retrieveMyDataAsJsonString( authUser = searchUser; OTHER_USER = true; } else { - return this.getJSONErrorString("No user found for: \"" + userIdentifier + "\"", null); + return this.getJSONErrorString( + BundleUtil.getStringFromBundle("dataretrieverAPI.user.not.found", Arrays.asList(userIdentifier)), + null); } } @@ -337,8 +343,7 @@ public String retrieveMyDataAsJsonString( myDataFinder = new MyDataFinder(rolePermissionHelper, roleAssigneeService, dvObjectServiceBean, - groupService, - noMsgResultsFound); + groupService); this.myDataFinder.runFindDataSteps(filterParams); if (myDataFinder.hasError()){ return this.getJSONErrorString(myDataFinder.getErrorMessage(), myDataFinder.getErrorMessage()); @@ -393,11 +398,14 @@ public String retrieveMyDataAsJsonString( } catch (SearchException ex) { solrQueryResponse = null; - this.logger.severe("Solr SearchException: " + ex.getMessage()); + logger.severe("Solr SearchException: " + ex.getMessage()); } - if (solrQueryResponse==null){ - return this.getJSONErrorString("Sorry! There was an error with the search service.", "Sorry! There was a SOLR Error"); + if (solrQueryResponse == null) { + return this.getJSONErrorString( + BundleUtil.getStringFromBundle("dataretrieverAPI.solr.error"), + BundleUtil.getStringFromBundle("dataretrieverAPI.solr.error.opt") + ); } // --------------------------------- @@ -491,9 +499,10 @@ private JsonArrayBuilder formatSolrDocs(SolrQueryResponse solrResponse, RoleTagR // ------------------------------------------- // (a) Get core card data from solr // ------------------------------------------- - myDataCardInfo = doc.getJsonForMyData(); - if (!doc.getEntity().isInstanceofDataFile()){ + myDataCardInfo = doc.getJsonForMyData(isValid(doc)); + + if (doc.getEntity() != null && !doc.getEntity().isInstanceofDataFile()){ String parentAlias = dataverseService.getParentAliasString(doc); myDataCardInfo.add("parent_alias",parentAlias); } @@ -514,4 +523,8 @@ private JsonArrayBuilder formatSolrDocs(SolrQueryResponse solrResponse, RoleTagR return jsonSolrDocsArrayBuilder; } + + private boolean isValid(SolrSearchResult result) { + return result.isValid(x -> true); + } } \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/mydata/MyDataFilterParams.java b/src/main/java/edu/harvard/iq/dataverse/mydata/MyDataFilterParams.java index 2ab248fcc0b..2acb93d37f5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/mydata/MyDataFilterParams.java +++ b/src/main/java/edu/harvard/iq/dataverse/mydata/MyDataFilterParams.java @@ -12,6 +12,7 @@ import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.search.SearchConstants; import edu.harvard.iq.dataverse.search.SearchFields; +import edu.harvard.iq.dataverse.util.BundleUtil; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -178,26 +179,25 @@ public List getRoleIds(){ } - - private void checkParams(){ - - if ((this.userIdentifier == null)||(this.userIdentifier.isEmpty())){ - this.addError("Sorry! No user was found!"); + private void checkParams() { + if ((this.userIdentifier == null) || (this.userIdentifier.isEmpty())) { + this.addError(BundleUtil.getStringFromBundle("myDataFilterParams.error.no.user")); return; } - if ((this.roleIds == null)||(this.roleIds.isEmpty())){ - this.addError("No results. Please select at least one Role."); + if ((this.roleIds == null) || (this.roleIds.isEmpty())) { + this.addError(BundleUtil.getStringFromBundle("myDataFilterParams.error.result.no.role")); return; } - if ((this.dvObjectTypes == null)||(this.dvObjectTypes.isEmpty())){ - this.addError("No results. Please select one of Dataverses, Datasets, Files."); + if ((this.dvObjectTypes == null) || (this.dvObjectTypes.isEmpty())) { + this.addError(BundleUtil.getStringFromBundle("myDataFilterParams.error.result.no.dvobject")); return; } - - if ((this.publicationStatuses == null)||(this.publicationStatuses.isEmpty())){ - this.addError("No results. Please select one of " + StringUtils.join(MyDataFilterParams.defaultPublishedStates, ", ") + "."); + + if ((this.publicationStatuses == null) || (this.publicationStatuses.isEmpty())) { + this.addError(BundleUtil.getStringFromBundle("dataretrieverAPI.user.not.found", + Arrays.asList(StringUtils.join(MyDataFilterParams.defaultPublishedStates, ", ")))); return; } } @@ -292,7 +292,7 @@ public String getSolrFragmentForPublicationStatus(){ } public String getSolrFragmentForDatasetValidity(){ - if ((this.datasetValidities == null) || (this.datasetValidities.isEmpty())){ + if ((this.datasetValidities == null) || (this.datasetValidities.isEmpty()) || (this.datasetValidities.size() > 1)){ return ""; } diff --git a/src/main/java/edu/harvard/iq/dataverse/mydata/MyDataFinder.java b/src/main/java/edu/harvard/iq/dataverse/mydata/MyDataFinder.java index 917884f3549..5626a442762 100644 --- a/src/main/java/edu/harvard/iq/dataverse/mydata/MyDataFinder.java +++ b/src/main/java/edu/harvard/iq/dataverse/mydata/MyDataFinder.java @@ -11,7 +11,9 @@ import edu.harvard.iq.dataverse.authorization.DataverseRolePermissionHelper; import edu.harvard.iq.dataverse.authorization.groups.GroupServiceBean; import edu.harvard.iq.dataverse.search.SearchFields; +import edu.harvard.iq.dataverse.util.BundleUtil; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -47,7 +49,6 @@ public class MyDataFinder { private RoleAssigneeServiceBean roleAssigneeService; private DvObjectServiceBean dvObjectServiceBean; private GroupServiceBean groupService; - private String noMsgResultsFound; //private RoleAssigneeServiceBean roleService = new RoleAssigneeServiceBean(); //private MyDataQueryHelperServiceBean myDataQueryHelperService; // -------------------- @@ -86,12 +87,11 @@ public class MyDataFinder { private List fileGrandparentFileIds = new ArrayList<>(); // dataverse has file permissions - public MyDataFinder(DataverseRolePermissionHelper rolePermissionHelper, RoleAssigneeServiceBean roleAssigneeService, DvObjectServiceBean dvObjectServiceBean, GroupServiceBean groupService, String _noMsgResultsFound) { + public MyDataFinder(DataverseRolePermissionHelper rolePermissionHelper, RoleAssigneeServiceBean roleAssigneeService, DvObjectServiceBean dvObjectServiceBean, GroupServiceBean groupService) { this.rolePermissionHelper = rolePermissionHelper; this.roleAssigneeService = roleAssigneeService; this.dvObjectServiceBean = dvObjectServiceBean; this.groupService = groupService; - this.noMsgResultsFound = _noMsgResultsFound; this.loadHarvestedDataverseIds(); } @@ -213,7 +213,7 @@ private List getSolrFilterQueries(boolean totalCountsOnly){ // ----------------------------------------------------------------- String dvObjectFQ = this.getSolrDvObjectFilterQuery(); if (dvObjectFQ ==null){ - this.addErrorMessage(noMsgResultsFound); + this.addErrorMessage(BundleUtil.getStringFromBundle("myDataFinder.error.result.empty")); return null; } filterQueries.add(dvObjectFQ); @@ -239,7 +239,7 @@ private List getSolrFilterQueries(boolean totalCountsOnly){ //fq=publicationStatus:"Unpublished"&fq=publicationStatus:"Draft" // ----------------------------------------------------------------- - // (4) FQ by dataset metadata vlidity + // (4) FQ by dataset metadata validity // ----------------------------------------------------------------- filterQueries.add(this.filterParams.getSolrFragmentForDatasetValidity()); //fq=datasetValid:(true OR false) @@ -286,7 +286,7 @@ public String getSolrDvObjectFilterQuery(){ if ((distinctEntityIds.isEmpty()) && (distinctParentIds.isEmpty())) { - this.addErrorMessage(noMsgResultsFound); + this.addErrorMessage(BundleUtil.getStringFromBundle("myDataFinder.error.result.empty")); return null; } @@ -430,24 +430,25 @@ public JsonArrayBuilder getListofSelectedRoles(){ } - private boolean runStep1RoleAssignments(){ + private boolean runStep1RoleAssignments() { List results = this.roleAssigneeService.getAssigneeAndRoleIdListFor(filterParams); //logger.info("runStep1RoleAssignments results: " + results.toString()); - if (results == null){ - this.addErrorMessage("Sorry, the EntityManager isn't working (still)."); + if (results == null) { + this.addErrorMessage(BundleUtil.getStringFromBundle("myDataFinder.error.result.null")); return false; - }else if (results.isEmpty()){ + } else if (results.isEmpty()) { List roleNames = this.rolePermissionHelper.getRoleNamesByIdList(this.filterParams.getRoleIds()); - if ((roleNames == null)||(roleNames.isEmpty())){ - this.addErrorMessage("Sorry, you have no assigned roles."); - }else{ - if (roleNames.size()==1){ - this.addErrorMessage("Sorry, nothing was found for this role: " + StringUtils.join(roleNames, ", ")); - }else{ - this.addErrorMessage("Sorry, nothing was found for these roles: " + StringUtils.join(roleNames, ", ")); + if ((roleNames == null) || (roleNames.isEmpty())) { + this.addErrorMessage(BundleUtil.getStringFromBundle("myDataFinder.error.result.no.role")); + } else { + final List args = Arrays.asList(StringUtils.join(roleNames, ", ")); + if (roleNames.size() == 1) { + this.addErrorMessage(BundleUtil.getStringFromBundle("myDataFinder.error.result.role.empty", args)); + } else { + this.addErrorMessage(BundleUtil.getStringFromBundle("myDataFinder.error.result.roles.empty", args)); } } return false; @@ -497,7 +498,7 @@ private boolean runStep2DirectAssignments(){ List results = this.dvObjectServiceBean.getDvObjectInfoForMyData(directDvObjectIds); //List results = this.roleAssigneeService.getAssignmentsFor(this.userIdentifier); if (results.isEmpty()){ - this.addErrorMessage("Sorry, you have no assigned Dataverses, Datasets, or Files."); + this.addErrorMessage(BundleUtil.getStringFromBundle("myDataFinder.error.result.no.dvobject")); return false; } diff --git a/src/main/java/edu/harvard/iq/dataverse/openapi/OpenApi.java b/src/main/java/edu/harvard/iq/dataverse/openapi/OpenApi.java new file mode 100644 index 00000000000..6bd54916e0d --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/openapi/OpenApi.java @@ -0,0 +1,101 @@ +package edu.harvard.iq.dataverse.openapi; + +import java.io.*; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.logging.*; + +import jakarta.json.Json; +import jakarta.json.JsonObject; +import jakarta.servlet.ServletException; +import jakarta.servlet.annotation.WebServlet; +import jakarta.servlet.http.*; +import jakarta.ws.rs.core.*; +import org.apache.commons.io.IOUtils; +import edu.harvard.iq.dataverse.api.Info; +import edu.harvard.iq.dataverse.util.BundleUtil; + +@WebServlet("/openapi") +public class OpenApi extends HttpServlet { + + private static final Logger logger = Logger.getLogger(Info.class.getCanonicalName()); + + private static final String YAML_FORMAT = "yaml"; + private static final String JSON_FORMAT = "json"; + + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + + + String format = req.getParameter("format"); + String accept = req.getHeader("Accept"); + + /* + * We first check for the headers, if the request accepts application/json + * have to check for the format parameter, if it is different from json + * return BAD_REQUEST (400) + */ + if (MediaType.APPLICATION_JSON.equals(accept)){ + if (format != null && !JSON_FORMAT.equals(format)){ + List args = Arrays.asList(accept, format); + String bundleResponse = BundleUtil.getStringFromBundle("openapi.exception.unaligned", args); + resp.sendError(Response.Status.BAD_REQUEST.getStatusCode(), + bundleResponse); + return; + } else { + format = JSON_FORMAT; + } + } + + /* + * We currently support only JSON or YAML being the second the default + * if no format is specified, if a different format is specified we return + * UNSUPPORTED_MEDIA_TYPE (415) specifying that the format is not supported + */ + + format = format == null ? YAML_FORMAT : format.toLowerCase(); + + if (JSON_FORMAT.equals(format)) { + resp.setContentType(MediaType.APPLICATION_JSON_TYPE.toString()); + } else if (YAML_FORMAT.equals(format)){ + resp.setContentType(MediaType.TEXT_PLAIN_TYPE.toString()); + } else { + + List args = Arrays.asList(format); + String bundleResponse = BundleUtil.getStringFromBundle("openapi.exception.invalid.format", args); + + JsonObject errorResponse = Json.createObjectBuilder() + .add("status", "ERROR") + .add("code", HttpServletResponse.SC_UNSUPPORTED_MEDIA_TYPE) + .add("message", bundleResponse) + .build(); + + resp.setContentType(MediaType.APPLICATION_JSON_TYPE.toString()); + resp.setStatus(HttpServletResponse.SC_UNSUPPORTED_MEDIA_TYPE); + + PrintWriter responseWriter = resp.getWriter(); + responseWriter.println(errorResponse.toString()); + responseWriter.flush(); + return; + } + + try { + String baseFileName = "/META-INF/openapi." + format; + ClassLoader classLoader = this.getClass().getClassLoader(); + URL aliasesResource = classLoader.getResource(baseFileName); + InputStream openapiDefinitionStream = aliasesResource.openStream(); + String content = IOUtils.toString(openapiDefinitionStream, StandardCharsets.UTF_8); + resp.getWriter().write(content); + } catch (Exception e) { + logger.log(Level.SEVERE, "OpenAPI Definition format not found " + format + ":" + e.getMessage(), e); + String bundleResponse = BundleUtil.getStringFromBundle("openapi.exception"); + resp.sendError(Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), + bundleResponse); + } + + + } + +} diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/AbstractPidProvider.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/AbstractPidProvider.java new file mode 100644 index 00000000000..a3dcf6cbb3b --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/AbstractPidProvider.java @@ -0,0 +1,550 @@ +package edu.harvard.iq.dataverse.pidproviders; + +import edu.harvard.iq.dataverse.DataFile; +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetField; +import edu.harvard.iq.dataverse.DvObject; +import edu.harvard.iq.dataverse.GlobalId; +import edu.harvard.iq.dataverse.util.SystemConfig; +import jakarta.json.Json; +import jakarta.json.JsonObject; +import jakarta.json.JsonObjectBuilder; + +import java.util.*; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.commons.lang3.RandomStringUtils; +import com.beust.jcommander.Strings; + +public abstract class AbstractPidProvider implements PidProvider { + + private static final Logger logger = Logger.getLogger(AbstractPidProvider.class.getCanonicalName()); + + public static String UNAVAILABLE = ":unav"; + public static final String SEPARATOR = "/"; + + protected PidProviderFactoryBean pidProviderService; + + private String protocol; + + private String authority = null; + + private String shoulder = null; + + private String identifierGenerationStyle = null; + + private String datafilePidFormat = null; + + private HashSet managedSet; + + private HashSet excludedSet; + + private String id; + private String label; + + protected AbstractPidProvider(String id, String label, String protocol) { + this.id = id; + this.label = label; + this.protocol = protocol; + this.managedSet = new HashSet(); + this.excludedSet = new HashSet(); + } + + protected AbstractPidProvider(String id, String label, String protocol, String authority, String shoulder, + String identifierGenerationStyle, String datafilePidFormat, String managedList, String excludedList) { + this.id = id; + this.label = label; + this.protocol = protocol; + this.authority = authority; + this.shoulder = shoulder; + this.identifierGenerationStyle = identifierGenerationStyle; + this.datafilePidFormat = datafilePidFormat; + this.managedSet = new HashSet(Arrays.asList(managedList.split(",\\s"))); + this.excludedSet = new HashSet(Arrays.asList(excludedList.split(",\\s"))); + if (logger.isLoggable(Level.FINE)) { + Iterator iter = managedSet.iterator(); + while (iter.hasNext()) { + logger.fine("managedSet in " + getId() + ": " + iter.next()); + } + iter = excludedSet.iterator(); + while (iter.hasNext()) { + logger.fine("excludedSet in " + getId() + ": " + iter.next()); + } + } + } + + @Override + public Map getMetadataForCreateIndicator(DvObject dvObjectIn) { + logger.log(Level.FINE, "getMetadataForCreateIndicator(DvObject)"); + Map metadata = new HashMap<>(); + metadata = addBasicMetadata(dvObjectIn, metadata); + metadata.put("datacite.publicationyear", generateYear(dvObjectIn)); + metadata.put("_target", getTargetUrl(dvObjectIn)); + return metadata; + } + + protected Map getUpdateMetadata(DvObject dvObjectIn) { + logger.log(Level.FINE, "getUpdateMetadataFromDataset"); + Map metadata = new HashMap<>(); + metadata = addBasicMetadata(dvObjectIn, metadata); + return metadata; + } + + protected Map addBasicMetadata(DvObject dvObjectIn, Map metadata) { + + String authorString = dvObjectIn.getAuthorString(); + if (authorString.isEmpty() || authorString.contains(DatasetField.NA_VALUE)) { + authorString = UNAVAILABLE; + } + + String producerString = pidProviderService.getProducer(); + + if (producerString.isEmpty() || producerString.equals(DatasetField.NA_VALUE)) { + producerString = UNAVAILABLE; + } + + String titleString = dvObjectIn.getCurrentName(); + + if (titleString.isEmpty() || titleString.equals(DatasetField.NA_VALUE)) { + titleString = UNAVAILABLE; + } + + metadata.put("datacite.creator", authorString); + metadata.put("datacite.title", titleString); + metadata.put("datacite.publisher", producerString); + metadata.put("datacite.publicationyear", generateYear(dvObjectIn)); + return metadata; + } + + protected Map addDOIMetadataForDestroyedDataset(DvObject dvObjectIn) { + Map metadata = new HashMap<>(); + String authorString = UNAVAILABLE; + String producerString = UNAVAILABLE; + String titleString = "This item has been removed from publication"; + + metadata.put("datacite.creator", authorString); + metadata.put("datacite.title", titleString); + metadata.put("datacite.publisher", producerString); + metadata.put("datacite.publicationyear", "9999"); + return metadata; + } + + protected String getTargetUrl(DvObject dvObjectIn) { + logger.log(Level.FINE, "getTargetUrl"); + return SystemConfig.getDataverseSiteUrlStatic() + dvObjectIn.getTargetUrl() + + dvObjectIn.getGlobalId().asString(); + } + + @Override + public String getIdentifier(DvObject dvObject) { + GlobalId gid = dvObject.getGlobalId(); + return gid != null ? gid.asString() : null; + } + + protected String generateYear(DvObject dvObjectIn) { + return dvObjectIn.getYearPublishedCreated(); + } + + public Map getMetadataForTargetURL(DvObject dvObject) { + logger.log(Level.FINE, "getMetadataForTargetURL"); + HashMap metadata = new HashMap<>(); + metadata.put("_target", getTargetUrl(dvObject)); + return metadata; + } + + @Override + public boolean alreadyRegistered(DvObject dvo) throws Exception { + if (dvo == null) { + logger.severe("Null DvObject sent to alreadyRegistered()."); + return false; + } + GlobalId globalId = dvo.getGlobalId(); + if (globalId == null) { + return false; + } + return alreadyRegistered(globalId, false); + } + + public abstract boolean alreadyRegistered(GlobalId globalId, boolean noProviderDefault) throws Exception; + + /* + * ToDo: the DvObject being sent in provides partial support for the case where + * it has a different authority/protocol than what is configured (i.e. a legacy + * Pid that can actually be updated by the Pid account being used.) Removing + * this now would potentially break/make it harder to handle that case prior to + * support for configuring multiple Pid providers. Once that exists, it would be + * cleaner to always find the PidProvider associated with the + * protocol/authority/shoulder of the current dataset and then not pass the + * DvObject as a param. (This would also remove calls to get the settings since + * that would be done at construction.) + */ + @Override + public DvObject generatePid(DvObject dvObject) { + + if (dvObject.getProtocol() == null) { + dvObject.setProtocol(getProtocol()); + } else { + if (!dvObject.getProtocol().equals(getProtocol())) { + logger.warning("The protocol of the DvObject (" + dvObject.getProtocol() + + ") does not match the configured protocol (" + getProtocol() + ")"); + throw new IllegalArgumentException("The protocol of the DvObject (" + dvObject.getProtocol() + + ") doesn't match that of the provider, id: " + getId()); + } + } + if (dvObject.getAuthority() == null) { + dvObject.setAuthority(getAuthority()); + } else { + if (!dvObject.getAuthority().equals(getAuthority())) { + logger.warning("The authority of the DvObject (" + dvObject.getAuthority() + + ") does not match the configured authority (" + getAuthority() + ")"); + throw new IllegalArgumentException("The authority of the DvObject (" + dvObject.getAuthority() + + ") doesn't match that of the provider, id: " + getId()); + } + } + if (dvObject.isInstanceofDataset()) { + dvObject.setIdentifier(generateDatasetIdentifier((Dataset) dvObject)); + } else { + dvObject.setIdentifier(generateDataFileIdentifier((DataFile) dvObject)); + } + return dvObject; + } + + private String generateDatasetIdentifier(Dataset dataset) { + String shoulder = getShoulder(); + + switch (getIdentifierGenerationStyle()) { + case "randomString": + return generateIdentifierAsRandomString(dataset, shoulder); + case "storedProcGenerated": + return generateIdentifierFromStoredProcedureIndependent(dataset, shoulder); + default: + /* Should we throw an exception instead?? -- L.A. 4.6.2 */ + return generateIdentifierAsRandomString(dataset, shoulder); + } + } + + /** + * Check that a identifier entered by the user is unique (not currently used for + * any other study in this Dataverse Network) also check for duplicate in EZID + * if needed + * + * @param userIdentifier + * @param dataset + * @return {@code true} if the identifier is unique, {@code false} otherwise. + */ + public boolean isGlobalIdUnique(GlobalId globalId) { + if (!pidProviderService.isGlobalIdLocallyUnique(globalId)) { + return false; // duplication found in local database + } + + // not in local DB, look in the persistent identifier service + try { + return !alreadyRegistered(globalId, false); + } catch (Exception e) { + // we can live with failure - means identifier not found remotely + } + + return true; + } + + /** + * Parse a Persistent Id and set the protocol, authority, and identifier + * + * Example 1: doi:10.5072/FK2/BYM3IW protocol: doi authority: 10.5072 + * identifier: FK2/BYM3IW + * + * Example 2: hdl:1902.1/111012 protocol: hdl authority: 1902.1 identifier: + * 111012 + * + * @param identifierString + * @param separator the string that separates the authority from the + * identifier. + * @param destination the global id that will contain the parsed data. + * @return {@code destination}, after its fields have been updated, or + * {@code null} if parsing failed. + */ + @Override + public GlobalId parsePersistentId(String fullIdentifierString) { + // Occasionally, the protocol separator character ':' comes in still + // URL-encoded as %3A (usually as a result of the URL having been + // encoded twice): + fullIdentifierString = fullIdentifierString.replace("%3A", ":"); + + int index1 = fullIdentifierString.indexOf(':'); + if (index1 > 0) { // ':' found with one or more characters before it + String protocol = fullIdentifierString.substring(0, index1); + GlobalId globalId = parsePersistentId(protocol, fullIdentifierString.substring(index1 + 1)); + return globalId; + } + logger.log(Level.INFO, "Error parsing identifier: {0}: '':'' not found in string", + fullIdentifierString); + return null; + } + + protected GlobalId parsePersistentId(String protocol, String identifierString) { + String authority; + String identifier; + if (identifierString == null) { + return null; + } + int index = identifierString.indexOf(getSeparator()); + if (index > 0 && (index + 1) < identifierString.length()) { + // '/' found with one or more characters + // before and after it + // Strip any whitespace, ; and ' from authority (should finding them cause a + // failure instead?) + authority = PidProvider.formatIdentifierString(identifierString.substring(0, index)); + + if (PidProvider.testforNullTerminator(authority)) { + return null; + } + identifier = PidProvider.formatIdentifierString(identifierString.substring(index + 1)); + if (PidProvider.testforNullTerminator(identifier)) { + return null; + } + + } else { + logger.log(Level.INFO, "Error parsing identifier: {0}: '':/'' not found in string", + identifierString); + return null; + } + return parsePersistentId(protocol, authority, identifier); + } + + public GlobalId parsePersistentId(String protocol, String authority, String identifier) { + logger.fine("Parsing: " + protocol + ":" + authority + getSeparator() + identifier + " in " + getId()); + if (!PidProvider.isValidGlobalId(protocol, authority, identifier)) { + return null; + } + // Check authority/identifier if this is a provider that manages specific + // identifiers + // /is not one of the unmanaged providers that has null authority + if (getAuthority() != null) { + + String cleanIdentifier = protocol + ":" + authority + getSeparator() + identifier; + /* + * Test if this provider manages this identifier - return null if it does not. + * It does match if ((the identifier's authority and shoulder match the + * provider's), or the identifier is in the managed set), and, in either case, + * the identifier is not in the excluded set. + */ + logger.fine("clean pid in " + getId() + ": " + cleanIdentifier); + logger.fine("managed in " + getId() + ": " + getManagedSet().contains(cleanIdentifier)); + logger.fine("excluded from " + getId() + ": " + getExcludedSet().contains(cleanIdentifier)); + + if (!(((authority.equals(getAuthority()) && identifier.startsWith(getShoulder())) + || getManagedSet().contains(cleanIdentifier)) && !getExcludedSet().contains(cleanIdentifier))) { + return null; + } + } + return new GlobalId(protocol, authority, identifier, getSeparator(), getUrlPrefix(), getId()); + } + + public String getSeparator() { + // The standard default + return SEPARATOR; + } + + private String generateDataFileIdentifier(DataFile datafile) { + String doiDataFileFormat = getDatafilePidFormat(); + + String prepend = ""; + if (doiDataFileFormat.equals(SystemConfig.DataFilePIDFormat.DEPENDENT.toString())) { + // If format is dependent then pre-pend the dataset identifier + prepend = datafile.getOwner().getIdentifier() + SEPARATOR; + datafile.setProtocol(datafile.getOwner().getProtocol()); + datafile.setAuthority(datafile.getOwner().getAuthority()); + } else { + // If there's a shoulder prepend independent identifiers with it + prepend = getShoulder(); + datafile.setProtocol(getProtocol()); + datafile.setAuthority(getAuthority()); + } + + switch (getIdentifierGenerationStyle()) { + case "randomString": + return generateIdentifierAsRandomString(datafile, prepend); + case "storedProcGenerated": + if (doiDataFileFormat.equals(SystemConfig.DataFilePIDFormat.INDEPENDENT.toString())) { + return generateIdentifierFromStoredProcedureIndependent(datafile, prepend); + } else { + return generateIdentifierFromStoredProcedureDependent(datafile, prepend); + } + default: + /* Should we throw an exception instead?? -- L.A. 4.6.2 */ + return generateIdentifierAsRandomString(datafile, prepend); + } + } + + /* + * This method checks locally for a DvObject with the same PID and if that is + * OK, checks with the PID service. + * + * @param dvo - the object to check (ToDo - get protocol/authority from this + * PidProvider object) + * + * @param prepend - for Datasets, this is always the shoulder, for DataFiles, it + * could be the shoulder or the parent Dataset identifier + */ + private String generateIdentifierAsRandomString(DvObject dvo, String prepend) { + String identifier = null; + do { + identifier = prepend + RandomStringUtils.randomAlphanumeric(6).toUpperCase(); + } while (!isGlobalIdUnique(new GlobalId(dvo.getProtocol(), dvo.getAuthority(), identifier, this.getSeparator(), + this.getUrlPrefix(), this.getId()))); + + return identifier; + } + + /* + * This method checks locally for a DvObject with the same PID and if that is + * OK, checks with the PID service. + * + * @param dvo - the object to check (ToDo - get protocol/authority from this + * PidProvider object) + * + * @param prepend - for Datasets, this is always the shoulder, for DataFiles, it + * could be the shoulder or the parent Dataset identifier + */ + + private String generateIdentifierFromStoredProcedureIndependent(DvObject dvo, String prepend) { + String identifier; + do { + String identifierFromStoredProcedure = pidProviderService.generateNewIdentifierByStoredProcedure(); + // some diagnostics here maybe - is it possible to determine that it's failing + // because the stored procedure hasn't been created in the database? + if (identifierFromStoredProcedure == null) { + return null; + } + identifier = prepend + identifierFromStoredProcedure; + } while (!isGlobalIdUnique(new GlobalId(dvo.getProtocol(), dvo.getAuthority(), identifier, this.getSeparator(), + this.getUrlPrefix(), this.getId()))); + + return identifier; + } + + /* + * This method is only used for DataFiles with DEPENDENT Pids. It is not for + * Datasets + * + */ + private String generateIdentifierFromStoredProcedureDependent(DataFile datafile, String prepend) { + String identifier; + Long retVal; + retVal = Long.valueOf(0L); + // ToDo - replace loops with one lookup for largest entry? (the do loop runs + // ~n**2/2 calls). The check for existingIdentifiers means this is mostly a + // local loop now, versus involving db or PidProvider calls, but still...) + + // This will catch identifiers already assigned in the current transaction (e.g. + // in FinalizeDatasetPublicationCommand) that haven't been committed to the db + // without having to make a call to the PIDProvider + Set existingIdentifiers = new HashSet(); + List files = datafile.getOwner().getFiles(); + for (DataFile f : files) { + existingIdentifiers.add(f.getIdentifier()); + } + + do { + retVal++; + identifier = prepend + retVal.toString(); + + } while (existingIdentifiers.contains(identifier) || !isGlobalIdUnique(new GlobalId(datafile.getProtocol(), + datafile.getAuthority(), identifier, this.getSeparator(), this.getUrlPrefix(), this.getId()))); + + return identifier; + } + + + @Override + public boolean canManagePID() { + // The default expectation is that PID providers are configured to manage some + // set (i.e. based on protocol/authority/shoulder) of PIDs + return true; + } + + @Override + public void setPidProviderServiceBean(PidProviderFactoryBean pidProviderServiceBean) { + this.pidProviderService = pidProviderServiceBean; + } + + @Override + public String getProtocol() { + return protocol; + } + + @Override + public String getAuthority() { + return authority; + } + + @Override + public String getShoulder() { + return shoulder; + } + + @Override + public String getIdentifierGenerationStyle() { + return identifierGenerationStyle; + } + + @Override + public String getDatafilePidFormat() { + return datafilePidFormat; + } + + @Override + public Set getManagedSet() { + return managedSet; + } + + @Override + public Set getExcludedSet() { + return excludedSet; + } + + @Override + public String getId() { + return id; + } + + @Override + public String getLabel() { + return label; + } + + @Override + /** + * True if this provider can manage PIDs in general, this pid is not in the + * managedSet (meaning it is managed but the provider does not generally manage + * it's protocol/authority/separator/shoulder) and either this provider is the + * same as the pid's or we're allowed to create INDEPENDENT pids. The latter + * clause covers the potential case where the effective pid provider/generator + * for the dataset is set to a different one that handles the dataset's pid + * itself. In this case, we can create file PIDs if they are independent. + * + * @param pid - the related pid to check + * @return true if this provider can manage PIDs like the one supplied + */ + public boolean canCreatePidsLike(GlobalId pid) { + return canManagePID() && !managedSet.contains(pid.asString()) + && (getIdentifierGenerationStyle().equals("INDEPENDENT") || getId().equals(pid.getProviderId())); + } + + @Override + public JsonObject getProviderSpecification() { + JsonObjectBuilder providerSpecification = Json.createObjectBuilder(); + providerSpecification.add("id", id); + providerSpecification.add("label", label); + providerSpecification.add("protocol", protocol); + providerSpecification.add("authority", authority); + providerSpecification.add("separator", getSeparator()); + providerSpecification.add("shoulder", shoulder); + providerSpecification.add("identifierGenerationStyle", identifierGenerationStyle); + providerSpecification.add("datafilePidFormat", datafilePidFormat); + providerSpecification.add("managedSet", Strings.join(",", managedSet.toArray())); + providerSpecification.add("excludedSet", Strings.join(",", excludedSet.toArray())); + return providerSpecification.build(); + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/PermaLinkPidProviderServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/PermaLinkPidProviderServiceBean.java deleted file mode 100644 index d145a7ec106..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/PermaLinkPidProviderServiceBean.java +++ /dev/null @@ -1,160 +0,0 @@ -package edu.harvard.iq.dataverse.pidproviders; - -import edu.harvard.iq.dataverse.AbstractGlobalIdServiceBean; -import edu.harvard.iq.dataverse.DvObject; -import edu.harvard.iq.dataverse.GlobalId; -import edu.harvard.iq.dataverse.GlobalIdServiceBean; -import edu.harvard.iq.dataverse.settings.JvmSettings; -import edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key; -import edu.harvard.iq.dataverse.util.SystemConfig; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.logging.Logger; - -import jakarta.annotation.PostConstruct; -import jakarta.ejb.Stateless; - -/** - * PermaLink provider - * This is a minimalist permanent ID provider intended for use with 'real' datasets/files where the use case none-the-less doesn't lend itself to the use of DOIs or Handles, e.g. - * * due to cost - * * for a catalog/archive where Dataverse has a dataset representing a dataset with DOI/handle stored elsewhere - * - * The initial implementation will mint identifiers locally and will provide the existing page URLs (using the ?persistentID= format). - * This will be overridable by a configurable parameter to support use of an external resolver. - * - */ -@Stateless -public class PermaLinkPidProviderServiceBean extends AbstractGlobalIdServiceBean { - - private static final Logger logger = Logger.getLogger(PermaLinkPidProviderServiceBean.class.getCanonicalName()); - - public static final String PERMA_PROTOCOL = "perma"; - public static final String PERMA_PROVIDER_NAME = "PERMA"; - - //ToDo - handle dataset/file defaults for local system - public static final String PERMA_RESOLVER_URL = JvmSettings.PERMALINK_BASEURL - .lookupOptional() - .orElse(SystemConfig.getDataverseSiteUrlStatic()); - - String authority = null; - private String separator = ""; - - @PostConstruct - private void init() { - if(PERMA_PROTOCOL.equals(settingsService.getValueForKey(Key.Protocol))){ - authority = settingsService.getValueForKey(Key.Authority); - configured=true; - }; - - } - - - //Only used in PidUtilTest - haven't figured out how to mock a PostConstruct call directly - // ToDo - remove after work to allow more than one Pid Provider which is expected to not use stateless beans - public void reInit() { - init(); - } - - @Override - public String getSeparator() { - //The perma default - return separator; - } - - @Override - public boolean alreadyRegistered(GlobalId globalId, boolean noProviderDefault) { - // Perma doesn't manage registration, so we assume all local PIDs can be treated - // as registered - boolean existsLocally = !dvObjectService.isGlobalIdLocallyUnique(globalId); - return existsLocally ? existsLocally : noProviderDefault; - } - - @Override - public boolean registerWhenPublished() { - return false; - } - - @Override - public List getProviderInformation() { - return List.of(PERMA_PROVIDER_NAME, PERMA_RESOLVER_URL); - } - - @Override - public String createIdentifier(DvObject dvo) throws Throwable { - //Call external resolver and send landing URL? - //FWIW: Return value appears to only be used in RegisterDvObjectCommand where success requires finding the dvo identifier in this string. (Also logged a couple places). - return(dvo.getGlobalId().asString()); - } - - @Override - public Map getIdentifierMetadata(DvObject dvo) { - Map map = new HashMap<>(); - return map; - } - - @Override - public String modifyIdentifierTargetURL(DvObject dvo) throws Exception { - return getTargetUrl(dvo); - } - - @Override - public void deleteIdentifier(DvObject dvo) throws Exception { - // no-op - } - - @Override - public boolean publicizeIdentifier(DvObject dvObject) { - //Generate if needed (i.e. datafile case where we don't create/register early (even with reigsterWhenPublished == false)) - if(dvObject.getIdentifier() == null || dvObject.getIdentifier().isEmpty() ){ - dvObject = generateIdentifier(dvObject); - } - //Call external resolver and send landing URL? - return true; - } - - @Override - public GlobalId parsePersistentId(String pidString) { - //ToDo - handle local PID resolver for dataset/file - if (pidString.startsWith(getUrlPrefix())) { - pidString = pidString.replace(getUrlPrefix(), - (PERMA_PROTOCOL + ":")); - } - return super.parsePersistentId(pidString); - } - - @Override - public GlobalId parsePersistentId(String protocol, String identifierString) { - logger.fine("Checking Perma: " + identifierString); - if (!PERMA_PROTOCOL.equals(protocol)) { - return null; - } - String identifier = null; - if (authority != null) { - if (identifierString.startsWith(authority)) { - identifier = identifierString.substring(authority.length()); - } - } - identifier = GlobalIdServiceBean.formatIdentifierString(identifier); - if (GlobalIdServiceBean.testforNullTerminator(identifier)) { - return null; - } - return new GlobalId(PERMA_PROTOCOL, authority, identifier, separator, getUrlPrefix(), PERMA_PROVIDER_NAME); - } - - @Override - public GlobalId parsePersistentId(String protocol, String authority, String identifier) { - if (!PERMA_PROTOCOL.equals(protocol)) { - return null; - } - return super.parsePersistentId(protocol, authority, identifier); - } - - @Override - public String getUrlPrefix() { - - return PERMA_RESOLVER_URL + "/citation?persistentId=" + PERMA_PROTOCOL + ":"; - } -} diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidHelper.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidHelper.java deleted file mode 100644 index 5bc855a9593..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidHelper.java +++ /dev/null @@ -1,43 +0,0 @@ -package edu.harvard.iq.dataverse.pidproviders; - -import java.util.Arrays; -import jakarta.annotation.PostConstruct; -import jakarta.ejb.EJB; -import jakarta.ejb.Singleton; -import jakarta.ejb.Startup; - -import edu.harvard.iq.dataverse.DOIDataCiteServiceBean; -import edu.harvard.iq.dataverse.DOIEZIdServiceBean; -import edu.harvard.iq.dataverse.HandlenetServiceBean; - - /** - * This is a small helper bean - * As it is a singleton and built at application start (=deployment), it will inject the (stateless) - * dataverse service into the BrandingUtil once it's ready. - */ - @Startup - @Singleton - public class PidHelper { - - @EJB - DOIDataCiteServiceBean datacitePidSvc; - @EJB - DOIEZIdServiceBean ezidPidSvc; - @EJB - HandlenetServiceBean handlePidSvc; - @EJB - FakePidProviderServiceBean fakePidSvc; - @EJB - PermaLinkPidProviderServiceBean permaPidSvc; - @EJB - UnmanagedDOIServiceBean unmanagedDOISvc; - @EJB - UnmanagedHandlenetServiceBean unmanagedHandleSvc; - - @PostConstruct - public void listServices() { - PidUtil.addAllToProviderList(Arrays.asList(datacitePidSvc, ezidPidSvc, handlePidSvc, permaPidSvc, fakePidSvc)); - PidUtil.addAllToUnmanagedProviderList(Arrays.asList(unmanagedDOISvc, unmanagedHandleSvc)); - } - - } \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/GlobalIdServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidProvider.java similarity index 56% rename from src/main/java/edu/harvard/iq/dataverse/GlobalIdServiceBean.java rename to src/main/java/edu/harvard/iq/dataverse/pidproviders/PidProvider.java index aebf13778c3..ea3a243f25c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/GlobalIdServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidProvider.java @@ -1,19 +1,16 @@ -package edu.harvard.iq.dataverse; +package edu.harvard.iq.dataverse.pidproviders; -import static edu.harvard.iq.dataverse.GlobalIdServiceBean.logger; -import edu.harvard.iq.dataverse.engine.command.CommandContext; -import edu.harvard.iq.dataverse.pidproviders.PermaLinkPidProviderServiceBean; -import edu.harvard.iq.dataverse.pidproviders.PidUtil; -import edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key; +import edu.harvard.iq.dataverse.DvObject; +import edu.harvard.iq.dataverse.GlobalId; +import jakarta.json.JsonObject; +import jakarta.json.JsonValue; import java.util.*; -import java.util.function.Function; -import java.util.logging.Level; import java.util.logging.Logger; -public interface GlobalIdServiceBean { +public interface PidProvider { - static final Logger logger = Logger.getLogger(GlobalIdServiceBean.class.getCanonicalName()); + static final Logger logger = Logger.getLogger(PidProvider.class.getCanonicalName()); boolean alreadyRegistered(DvObject dvo) throws Exception; @@ -36,7 +33,6 @@ public interface GlobalIdServiceBean { boolean registerWhenPublished(); boolean canManagePID(); - boolean isConfigured(); List getProviderInformation(); @@ -52,36 +48,24 @@ public interface GlobalIdServiceBean { Map getMetadataForTargetURL(DvObject dvObject); - DvObject generateIdentifier(DvObject dvObject); + DvObject generatePid(DvObject dvObject); String getIdentifier(DvObject dvObject); boolean publicizeIdentifier(DvObject studyIn); - String generateDatasetIdentifier(Dataset dataset); - String generateDataFileIdentifier(DataFile datafile); boolean isGlobalIdUnique(GlobalId globalId); String getUrlPrefix(); String getSeparator(); - static GlobalIdServiceBean getBean(String protocol, CommandContext ctxt) { - final Function protocolHandler = BeanDispatcher.DISPATCHER.get(protocol); - if ( protocolHandler != null ) { - GlobalIdServiceBean theBean = protocolHandler.apply(ctxt); - if(theBean != null && theBean.isConfigured()) { - logger.fine("getBean returns " + theBean.getProviderInformation().get(0) + " for protocol " + protocol); - } - return theBean; - } else { - logger.log(Level.SEVERE, "Unknown protocol: {0}", protocol); - return null; - } - } - - static GlobalIdServiceBean getBean(CommandContext ctxt) { - return getBean(ctxt.settings().getValueForKey(Key.Protocol, ""), ctxt); - } + String getProtocol(); + String getProviderType(); + String getId(); + String getLabel(); + String getAuthority(); + String getShoulder(); + String getIdentifierGenerationStyle(); public static Optional parse(String identifierString) { try { @@ -111,6 +95,7 @@ public static Optional parse(String identifierString) { * {@code null} if parsing failed. */ public GlobalId parsePersistentId(String identifierString); + public GlobalId parsePersistentId(String protocol, String authority, String identifier); @@ -119,16 +104,16 @@ public static boolean isValidGlobalId(String protocol, String authority, String if (protocol == null || authority == null || identifier == null) { return false; } - if(!authority.equals(GlobalIdServiceBean.formatIdentifierString(authority))) { + if(!authority.equals(PidProvider.formatIdentifierString(authority))) { return false; } - if (GlobalIdServiceBean.testforNullTerminator(authority)) { + if (PidProvider.testforNullTerminator(authority)) { return false; } - if(!identifier.equals(GlobalIdServiceBean.formatIdentifierString(identifier))) { + if(!identifier.equals(PidProvider.formatIdentifierString(identifier))) { return false; } - if (GlobalIdServiceBean.testforNullTerminator(identifier)) { + if (PidProvider.testforNullTerminator(identifier)) { return false; } return true; @@ -177,40 +162,28 @@ static boolean checkDOIAuthority(String doiAuthority){ return true; } -} - - -/* - * ToDo - replace this with a mechanism like BrandingUtilHelper that would read - * the config and create PidProviders, one per set of config values and serve - * those as needed. The help has to be a bean to autostart and to hand the - * required service beans to the PidProviders. That may boil down to just the - * dvObjectService (to check for local identifier conflicts) since it will be - * the helper that has to read settings/get systewmConfig values. - * - */ - -/** - * Static utility class for dispatching implementing beans, based on protocol and providers. - * @author michael - */ -class BeanDispatcher { - static final Map> DISPATCHER = new HashMap<>(); - - static { - DISPATCHER.put("hdl", ctxt->ctxt.handleNet() ); - DISPATCHER.put("doi", ctxt->{ - String doiProvider = ctxt.settings().getValueForKey(Key.DoiProvider, ""); - switch ( doiProvider ) { - case "EZID": return ctxt.doiEZId(); - case "DataCite": return ctxt.doiDataCite(); - case "FAKE": return ctxt.fakePidProvider(); - default: - logger.log(Level.SEVERE, "Unknown doiProvider: {0}", doiProvider); - return null; - } - }); - - DISPATCHER.put(PermaLinkPidProviderServiceBean.PERMA_PROTOCOL, ctxt->ctxt.permaLinkProvider() ); - } + + public void setPidProviderServiceBean(PidProviderFactoryBean pidProviderFactoryBean); + + String getDatafilePidFormat(); + + Set getManagedSet(); + + Set getExcludedSet(); + + /** + * Whether related pids can be created by this pid provider + * @see edu.harvard.iq.dataverse.pidproviders.AbstractPidProvider#canCreatePidsLike(GlobalId) more details in the abstract implementation + * + * @param pid + * @return - whether related pids can be created by this pid provider. + */ + boolean canCreatePidsLike(GlobalId pid); + + /** + * Returns a JSON representation of this pid provider including it's id, label, protocol, authority, separator, and identifier. + * @return + */ + public JsonObject getProviderSpecification(); + } \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidProviderFactory.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidProviderFactory.java new file mode 100644 index 00000000000..f7c1a4b9174 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidProviderFactory.java @@ -0,0 +1,8 @@ +package edu.harvard.iq.dataverse.pidproviders; + +public interface PidProviderFactory { + + String getType(); + + PidProvider createPidProvider(String id); +} diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidProviderFactoryBean.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidProviderFactoryBean.java new file mode 100644 index 00000000000..40044408c63 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidProviderFactoryBean.java @@ -0,0 +1,250 @@ +package edu.harvard.iq.dataverse.pidproviders; + +import java.io.IOException; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Optional; +import java.util.ServiceLoader; +import java.util.logging.Level; +import java.util.logging.Logger; + +import jakarta.annotation.PostConstruct; +import jakarta.ejb.EJB; +import jakarta.ejb.Singleton; +import jakarta.ejb.Startup; +import jakarta.inject.Inject; +import edu.harvard.iq.dataverse.settings.JvmSettings; +import edu.harvard.iq.dataverse.settings.SettingsServiceBean; +import edu.harvard.iq.dataverse.util.SystemConfig; +import edu.harvard.iq.dataverse.DataverseServiceBean; +import edu.harvard.iq.dataverse.DvObjectServiceBean; +import edu.harvard.iq.dataverse.GlobalId; +import edu.harvard.iq.dataverse.pidproviders.doi.UnmanagedDOIProvider; +import edu.harvard.iq.dataverse.pidproviders.doi.datacite.DataCiteDOIProvider; +import edu.harvard.iq.dataverse.pidproviders.doi.ezid.EZIdDOIProvider; +import edu.harvard.iq.dataverse.pidproviders.doi.fake.FakeDOIProvider; +import edu.harvard.iq.dataverse.pidproviders.handle.HandlePidProvider; +import edu.harvard.iq.dataverse.pidproviders.handle.UnmanagedHandlePidProvider; +import edu.harvard.iq.dataverse.pidproviders.perma.PermaLinkPidProvider; +import edu.harvard.iq.dataverse.pidproviders.perma.UnmanagedPermaLinkPidProvider; + +/** + * This Bean loads all of the PidProviderFactory types available (e.g. EZID, + * DataCite, Handle, PermaLink) and then reads the configuration to load + * particular PidProviders (e.g. a DataCite provider with a specific + * authority/shoulder, username/password, etc.) + */ +@Startup +@Singleton +public class PidProviderFactoryBean { + + private static final Logger logger = Logger.getLogger(PidProviderFactoryBean.class.getCanonicalName()); + + @Inject + DataverseServiceBean dataverseService; + @EJB + protected SettingsServiceBean settingsService; + @Inject + protected DvObjectServiceBean dvObjectService; + @Inject + SystemConfig systemConfig; + + private ServiceLoader loader; + private Map pidProviderFactoryMap = new HashMap<>(); + + @PostConstruct + public void init() { + loadProviderFactories(); + loadProviders(); + } + + private void loadProviderFactories() { + /* + * Step 1 - find the PROVIDERS dir and add all jar files there to a class loader + */ + List jarUrls = new ArrayList<>(); + Optional providerPathSetting = JvmSettings.PIDPROVIDERS_DIRECTORY.lookupOptional(String.class); + if (providerPathSetting.isPresent()) { + Path providersDir = Paths.get(providerPathSetting.get()); + // Get all JAR files from the configured directory + try (DirectoryStream stream = Files.newDirectoryStream(providersDir, "*.jar")) { + // Using the foreach loop here to enable catching the URI/URL exceptions + for (Path path : stream) { + logger.log(Level.FINE, "Adding {0}", path.toUri().toURL()); + // This is the syntax required to indicate a jar file from which classes should + // be loaded (versus a class file). + jarUrls.add(new URL("jar:" + path.toUri().toURL() + "!/")); + } + } catch (IOException e) { + logger.warning("Problem accessing external Providers: " + e.getLocalizedMessage()); + } + } + URLClassLoader cl = URLClassLoader.newInstance(jarUrls.toArray(new URL[0]), this.getClass().getClassLoader()); + + /* + * Step 2 - load all PidProcviderFactories that can be found, using the jars as + * additional sources + */ + loader = ServiceLoader.load(PidProviderFactory.class, cl); + /* + * Step 3 - Fill pidProviderFactoryMap with type as the key, allow external + * factories to replace internal ones for the same type. FWIW: From the logging + * it appears that ServiceLoader returns classes in ~ alphabetical order rather + * than by class loader, so internal classes handling a given providerName may + * be processed before or after external ones. + */ + loader.forEach(providerFactory -> { + String type = providerFactory.getType(); + logger.fine("Loaded PidProviderFactory of type: " + type); + // If no entry for this providerName yet or if it is an external provider + if (!pidProviderFactoryMap.containsKey(type) || providerFactory.getClass().getClassLoader().equals(cl)) { + logger.fine("Adding PidProviderFactory of type: " + type + " to the map"); + pidProviderFactoryMap.put(type, providerFactory); + } + logger.log(Level.FINE, + "Loaded PidProviderFactory of type: " + type + " from " + + providerFactory.getClass().getCanonicalName() + " and classloader: " + + providerFactory.getClass().getClassLoader().getClass().getCanonicalName()); + }); + } + + private void loadProviders() { + Optional providers = JvmSettings.PID_PROVIDERS.lookupOptional(String[].class); + if (!providers.isPresent()) { + logger.warning( + "No PidProviders configured via dataverse.pid.providers. Please consider updating as older PIDProvider configuration mechanisms will be removed in a future version of Dataverse."); + } else { + for (String id : providers.get()) { + //Allows spaces in PID_PROVIDERS setting + id=id.trim(); + Optional type = JvmSettings.PID_PROVIDER_TYPE.lookupOptional(id); + if (!type.isPresent()) { + logger.warning("PidProvider " + id + + " listed in dataverse.pid.providers is not properly configured and will not be used."); + } else { + String typeString = type.get(); + if (pidProviderFactoryMap.containsKey(typeString)) { + PidProvider provider = pidProviderFactoryMap.get(typeString).createPidProvider(id); + provider.setPidProviderServiceBean(this); + PidUtil.addToProviderList(provider); + } + } + } + } + String protocol = settingsService.getValueForKey(SettingsServiceBean.Key.Protocol); + String authority = settingsService.getValueForKey(SettingsServiceBean.Key.Authority); + String shoulder = settingsService.getValueForKey(SettingsServiceBean.Key.Shoulder); + String provider = settingsService.getValueForKey(SettingsServiceBean.Key.DoiProvider); + + if (protocol != null && authority != null && shoulder != null && provider != null) { + logger.warning("Found legacy settings: " + protocol + " " + authority + " " + shoulder + " " + provider + + "Please consider updating as this PIDProvider configuration mechanism will be removed in a future version of Dataverse"); + if (PidUtil.getPidProvider(protocol, authority, shoulder) != null) { + logger.warning( + "Legacy PID provider settings found - ignored since a provider for the same protocol, authority, shoulder has been registered"); + } else { + PidProvider legacy = null; + // Try to add a legacy provider + String identifierGenerationStyle = settingsService + .getValueForKey(SettingsServiceBean.Key.IdentifierGenerationStyle, "random"); + String dataFilePidFormat = settingsService.getValueForKey(SettingsServiceBean.Key.DataFilePIDFormat, + "DEPENDENT"); + switch (protocol) { + case "doi": + switch (provider) { + case "EZID": + + String baseUrl = JvmSettings.LEGACY_EZID_API_URL.lookup(); + String username = JvmSettings.LEGACY_EZID_USERNAME.lookup(); + String password = JvmSettings.LEGACY_EZID_PASSWORD.lookup(); + PidUtil.addToProviderList(new EZIdDOIProvider("legacy", "legacy", authority, shoulder, + identifierGenerationStyle, dataFilePidFormat, "", "", baseUrl, username, password)); + + break; + case "DataCite": + String mdsUrl = JvmSettings.LEGACY_DATACITE_MDS_API_URL.lookup(); + String restUrl = JvmSettings.LEGACY_DATACITE_REST_API_URL.lookup(); + // Defaults for testing where no account is set up + String dcUsername = JvmSettings.LEGACY_DATACITE_USERNAME.lookup(); + String dcPassword = JvmSettings.LEGACY_DATACITE_PASSWORD.lookup(); + if (mdsUrl != null && restUrl != null && dcUsername != null && dcPassword != null) { + legacy = new DataCiteDOIProvider("legacy", "legacy", authority, shoulder, + identifierGenerationStyle, dataFilePidFormat, "", "", mdsUrl, restUrl, dcUsername, + dcPassword); + } + break; + case "FAKE": + logger.warning("Adding FAKE provider"); + legacy = new FakeDOIProvider("legacy", "legacy", authority, shoulder, identifierGenerationStyle, + dataFilePidFormat, "", ""); + break; + } + break; + case "hdl": + int index = JvmSettings.LEGACY_HANDLENET_INDEX.lookup(Integer.class); + String path = JvmSettings.LEGACY_HANDLENET_KEY_PATH.lookup(); + String passphrase = JvmSettings.LEGACY_HANDLENET_KEY_PASSPHRASE.lookup(); + boolean independentHandleService = settingsService + .isTrueForKey(SettingsServiceBean.Key.IndependentHandleService, false); + String handleAuthHandle = settingsService.getValueForKey(SettingsServiceBean.Key.HandleAuthHandle); + + legacy = new HandlePidProvider("legacy", "legacy", authority, shoulder, identifierGenerationStyle, + dataFilePidFormat, "", "", index, independentHandleService, handleAuthHandle, path, + passphrase); + break; + case "perma": + String baseUrl = JvmSettings.LEGACY_PERMALINK_BASEURL.lookup(); + legacy = new PermaLinkPidProvider("legacy", "legacy", authority, shoulder, + identifierGenerationStyle, dataFilePidFormat, "", "", baseUrl, + PermaLinkPidProvider.SEPARATOR); + } + if (legacy != null) { + legacy.setPidProviderServiceBean(this); + PidUtil.addToProviderList(legacy); + } + } + logger.info("Have " + PidUtil.getManagedProviderIds().size() + " managed PID providers"); + } + PidUtil.addAllToUnmanagedProviderList(Arrays.asList(new UnmanagedDOIProvider(), + new UnmanagedHandlePidProvider(), new UnmanagedPermaLinkPidProvider())); + } + + public String getProducer() { + return dataverseService.getRootDataverseName(); + } + + public boolean isGlobalIdLocallyUnique(GlobalId globalId) { + return dvObjectService.isGlobalIdLocallyUnique(globalId); + } + + String generateNewIdentifierByStoredProcedure() { + return dvObjectService.generateNewIdentifierByStoredProcedure(); + } + + public PidProvider getDefaultPidGenerator() { + Optional pidProviderDefaultId = JvmSettings.PID_DEFAULT_PROVIDER.lookupOptional(String.class); + if (pidProviderDefaultId.isPresent()) { + return PidUtil.getPidProvider(pidProviderDefaultId.get()); + } else { + String nonNullDefaultIfKeyNotFound = ""; + String protocol = settingsService.getValueForKey(SettingsServiceBean.Key.Protocol, + nonNullDefaultIfKeyNotFound); + String authority = settingsService.getValueForKey(SettingsServiceBean.Key.Authority, + nonNullDefaultIfKeyNotFound); + String shoulder = settingsService.getValueForKey(SettingsServiceBean.Key.Shoulder, + nonNullDefaultIfKeyNotFound); + + return PidUtil.getPidProvider(protocol, authority, shoulder); + } + } +} \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidUtil.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidUtil.java index 78305648f67..279f18dcd0e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidUtil.java @@ -1,9 +1,8 @@ package edu.harvard.iq.dataverse.pidproviders; -import edu.harvard.iq.dataverse.DOIServiceBean; import edu.harvard.iq.dataverse.GlobalId; -import edu.harvard.iq.dataverse.GlobalIdServiceBean; -import edu.harvard.iq.dataverse.HandlenetServiceBean; +import edu.harvard.iq.dataverse.pidproviders.doi.AbstractDOIProvider; +import edu.harvard.iq.dataverse.pidproviders.handle.HandlePidProvider; import edu.harvard.iq.dataverse.util.BundleUtil; import java.io.IOException; import java.io.InputStream; @@ -14,6 +13,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.logging.Logger; import jakarta.json.Json; @@ -113,24 +113,22 @@ public static JsonObjectBuilder queryDoi(GlobalId globalId, String baseUrl, Stri * @return DOI in the form 10.7910/DVN/TJCLKP (no "doi:") */ private static String acceptOnlyDoi(GlobalId globalId) { - if (!DOIServiceBean.DOI_PROTOCOL.equals(globalId.getProtocol())) { + if (!AbstractDOIProvider.DOI_PROTOCOL.equals(globalId.getProtocol())) { throw new IllegalArgumentException(BundleUtil.getStringFromBundle("pids.datacite.errors.DoiOnly")); } return globalId.getAuthority() + "/" + globalId.getIdentifier(); } - static Map providerMap = new HashMap(); - static Map unmanagedProviderMap = new HashMap(); + static Map providerMap = new HashMap(); + static Map unmanagedProviderMap = new HashMap(); - public static void addAllToProviderList(List list) { - for (GlobalIdServiceBean pidProvider : list) { - providerMap.put(pidProvider.getProviderInformation().get(0), pidProvider); - } + public static void addToProviderList(PidProvider pidProvider) { + providerMap.put(pidProvider.getId(), pidProvider); } - public static void addAllToUnmanagedProviderList(List list) { - for (GlobalIdServiceBean pidProvider : list) { - unmanagedProviderMap.put(pidProvider.getProviderInformation().get(0), pidProvider); + public static void addAllToUnmanagedProviderList(List list) { + for (PidProvider pidProvider : list) { + unmanagedProviderMap.put(pidProvider.getId(), pidProvider); } } @@ -141,7 +139,7 @@ public static void addAllToUnmanagedProviderList(List list) */ public static GlobalId parseAsGlobalID(String identifier) { logger.fine("In parseAsGlobalId: " + providerMap.size()); - for (GlobalIdServiceBean pidProvider : providerMap.values()) { + for (PidProvider pidProvider : providerMap.values()) { logger.fine(" Checking " + String.join(",", pidProvider.getProviderInformation())); GlobalId globalId = pidProvider.parsePersistentId(identifier); if (globalId != null) { @@ -149,7 +147,7 @@ public static GlobalId parseAsGlobalID(String identifier) { } } // If no providers can managed this PID, at least allow it to be recognized - for (GlobalIdServiceBean pidProvider : unmanagedProviderMap.values()) { + for (PidProvider pidProvider : unmanagedProviderMap.values()) { logger.fine(" Checking " + String.join(",", pidProvider.getProviderInformation())); GlobalId globalId = pidProvider.parsePersistentId(identifier); if (globalId != null) { @@ -167,14 +165,14 @@ public static GlobalId parseAsGlobalID(String identifier) { public static GlobalId parseAsGlobalID(String protocol, String authority, String identifier) { logger.fine("Looking for " + protocol + " " + authority + " " + identifier); logger.fine("In parseAsGlobalId: " + providerMap.size()); - for (GlobalIdServiceBean pidProvider : providerMap.values()) { + for (PidProvider pidProvider : providerMap.values()) { logger.fine(" Checking " + String.join(",", pidProvider.getProviderInformation())); GlobalId globalId = pidProvider.parsePersistentId(protocol, authority, identifier); if (globalId != null) { return globalId; } } - for (GlobalIdServiceBean pidProvider : unmanagedProviderMap.values()) { + for (PidProvider pidProvider : unmanagedProviderMap.values()) { logger.fine(" Checking " + String.join(",", pidProvider.getProviderInformation())); GlobalId globalId = pidProvider.parsePersistentId(protocol, authority, identifier); if (globalId != null) { @@ -191,28 +189,96 @@ public static GlobalId parseAsGlobalID(String protocol, String authority, String * This method should be deprecated/removed when further refactoring to support * multiple PID providers is done. At that point, when the providers aren't * beans, this code can be moved into other classes that go in the providerMap. - * If this method is not kept in sync with the DOIServiceBean and - * HandlenetServiceBean implementations, the tests using it won't be valid tests - * of the production code. + * If this method is not kept in sync with the AbstractDOIProvider and HandlePidProvider + * implementations, the tests using it won't be valid tests of the production + * code. */ private static GlobalId parseUnmanagedDoiOrHandle(String protocol, String authority, String identifier) { // Default recognition - could be moved to new classes in the future. - if (!GlobalIdServiceBean.isValidGlobalId(protocol, authority, identifier)) { + if (!PidProvider.isValidGlobalId(protocol, authority, identifier)) { return null; } String urlPrefix = null; switch (protocol) { - case DOIServiceBean.DOI_PROTOCOL: - if (!GlobalIdServiceBean.checkDOIAuthority(authority)) { + case AbstractDOIProvider.DOI_PROTOCOL: + if (!PidProvider.checkDOIAuthority(authority)) { return null; } - urlPrefix = DOIServiceBean.DOI_RESOLVER_URL; + urlPrefix = AbstractDOIProvider.DOI_RESOLVER_URL; break; - case HandlenetServiceBean.HDL_PROTOCOL: - urlPrefix = HandlenetServiceBean.HDL_RESOLVER_URL; + case HandlePidProvider.HDL_PROTOCOL: + urlPrefix = HandlePidProvider.HDL_RESOLVER_URL; break; } return new GlobalId(protocol, authority, identifier, "/", urlPrefix, null); } + + /** + * Get a PidProvider by name. GlobalIds have a getProviderName() method so this + * method is often used as + * getPidProvider(dvObject.getGlobalId().getProviderName(); (which will fail if + * the GlobalId is null - use PidProviderFactoryBean.getPidProvider(DvObject) if + * you aren't sure. + * + */ + + public static PidProvider getPidProvider(String name) { + for (PidProvider pidProvider : providerMap.values()) { + if (name.equals(pidProvider.getId())) { + return pidProvider; + } + } + for (PidProvider pidProvider : unmanagedProviderMap.values()) { + if (name.equals(pidProvider.getId())) { + return pidProvider; + } + } + return null; + } + + + + /** + * Method to clear all managed/unmanaged PidProviders. Only for testing as these + * lists are only loaded once by the @Stateless PidProviderFactoryBean in Dataverse. + */ + public static void clearPidProviders() { + providerMap.clear(); + unmanagedProviderMap.clear(); + } + + /** + * Get a PidProvider by protocol/authority/shoulder. + */ + public static PidProvider getPidProvider(String protocol, String authority, String shoulder) { + return getPidProvider(protocol, authority, shoulder, AbstractPidProvider.SEPARATOR); + } + + public static PidProvider getPidProvider(String protocol, String authority, String shoulder, String separator) { + for (PidProvider pidProvider : providerMap.values()) { + if (protocol.equals(pidProvider.getProtocol()) && authority.equals(pidProvider.getAuthority()) + && shoulder.equals(pidProvider.getShoulder()) && separator.equals(pidProvider.getSeparator())) { + return pidProvider; + } + } + for (PidProvider pidProvider : unmanagedProviderMap.values()) { + if (protocol.equals(pidProvider.getProtocol())) { + return pidProvider; + } + } + return null; + } + + public static Set getManagedProviderIds() { + return providerMap.keySet(); + } + + public static JsonObject getProviders() { + JsonObjectBuilder builder = Json.createObjectBuilder(); + for (PidProvider pidProvider : providerMap.values()) { + builder.add(pidProvider.getId(), pidProvider.getProviderSpecification()); + } + return builder.build(); + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/AbstractDOIProvider.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/AbstractDOIProvider.java new file mode 100644 index 00000000000..43e34e74c59 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/AbstractDOIProvider.java @@ -0,0 +1,123 @@ +package edu.harvard.iq.dataverse.pidproviders.doi; + +import java.util.Arrays; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; + +import edu.harvard.iq.dataverse.DataFile; +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetField; +import edu.harvard.iq.dataverse.DvObject; +import edu.harvard.iq.dataverse.GlobalId; +import edu.harvard.iq.dataverse.pidproviders.AbstractPidProvider; +import edu.harvard.iq.dataverse.pidproviders.PidProvider; + + +public abstract class AbstractDOIProvider extends AbstractPidProvider { + + private static final Logger logger = Logger.getLogger(AbstractDOIProvider.class.getCanonicalName()); + + public static final String DOI_PROTOCOL = "doi"; + public static final String DOI_RESOLVER_URL = "https://doi.org/"; + public static final String HTTP_DOI_RESOLVER_URL = "http://doi.org/"; + public static final String DXDOI_RESOLVER_URL = "https://dx.doi.org/"; + public static final String HTTP_DXDOI_RESOLVER_URL = "http://dx.doi.org/"; + + public AbstractDOIProvider(String id, String label, String providerAuthority, String providerShoulder, String identifierGenerationStyle, String datafilePidFormat, String managedList, String excludedList) { + super(id, label, DOI_PROTOCOL, providerAuthority, providerShoulder, identifierGenerationStyle, datafilePidFormat, managedList, excludedList); + } + + //For Unmanged provider + public AbstractDOIProvider(String name, String label) { + super(name, label, DOI_PROTOCOL); + } + + @Override + public GlobalId parsePersistentId(String pidString) { + if (pidString.startsWith(DOI_RESOLVER_URL)) { + pidString = pidString.replace(DOI_RESOLVER_URL, + (DOI_PROTOCOL + ":")); + } else if (pidString.startsWith(HTTP_DOI_RESOLVER_URL)) { + pidString = pidString.replace(HTTP_DOI_RESOLVER_URL, + (DOI_PROTOCOL + ":")); + } else if (pidString.startsWith(DXDOI_RESOLVER_URL)) { + pidString = pidString.replace(DXDOI_RESOLVER_URL, + (DOI_PROTOCOL + ":")); + } + return super.parsePersistentId(pidString); + } + + @Override + public GlobalId parsePersistentId(String protocol, String identifierString) { + + if (!DOI_PROTOCOL.equals(protocol)) { + return null; + } + GlobalId globalId = super.parsePersistentId(protocol, identifierString); + if (globalId!=null && !PidProvider.checkDOIAuthority(globalId.getAuthority())) { + return null; + } + return globalId; + } + + @Override + public GlobalId parsePersistentId(String protocol, String authority, String identifier) { + + if (!DOI_PROTOCOL.equals(protocol)) { + return null; + } + return super.parsePersistentId(protocol, authority, identifier); + } + + public String getUrlPrefix() { + return DOI_RESOLVER_URL; + } + + protected String getProviderKeyName() { + return null; + } + + public String getProtocol() { + return DOI_PROTOCOL; + } + + public String getMetadataFromDvObject(String identifier, Map metadata, DvObject dvObject) { + + Dataset dataset = null; + + if (dvObject instanceof Dataset) { + dataset = (Dataset) dvObject; + } else { + dataset = (Dataset) dvObject.getOwner(); + } + + XmlMetadataTemplate metadataTemplate = new XmlMetadataTemplate(); + metadataTemplate.setIdentifier(identifier.substring(identifier.indexOf(':') + 1)); + metadataTemplate.setCreators(Arrays.asList(metadata.get("datacite.creator").split("; "))); + metadataTemplate.setAuthors(dataset.getLatestVersion().getDatasetAuthors()); + if (dvObject.isInstanceofDataset()) { + metadataTemplate.setDescription(dataset.getLatestVersion().getDescriptionPlainText()); + } + if (dvObject.isInstanceofDataFile()) { + DataFile df = (DataFile) dvObject; + String fileDescription = df.getDescription(); + metadataTemplate.setDescription(fileDescription == null ? "" : fileDescription); + } + + metadataTemplate.setContacts(dataset.getLatestVersion().getDatasetContacts()); + metadataTemplate.setProducers(dataset.getLatestVersion().getDatasetProducers()); + metadataTemplate.setTitle(dvObject.getCurrentName()); + String producerString = pidProviderService.getProducer(); + if (producerString.isEmpty() || producerString.equals(DatasetField.NA_VALUE)) { + producerString = UNAVAILABLE; + } + metadataTemplate.setPublisher(producerString); + metadataTemplate.setPublisherYear(metadata.get("datacite.publicationyear")); + + String xmlMetadata = metadataTemplate.generateXML(dvObject); + logger.log(Level.FINE, "XML to send to DataCite: {0}", xmlMetadata); + return xmlMetadata; + } + +} \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/UnmanagedDOIServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/UnmanagedDOIProvider.java similarity index 73% rename from src/main/java/edu/harvard/iq/dataverse/pidproviders/UnmanagedDOIServiceBean.java rename to src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/UnmanagedDOIProvider.java index f7e9372cc9b..d4e674f8396 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/UnmanagedDOIServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/UnmanagedDOIProvider.java @@ -1,17 +1,11 @@ -package edu.harvard.iq.dataverse.pidproviders; +package edu.harvard.iq.dataverse.pidproviders.doi; import java.io.IOException; import java.util.List; import java.util.Map; -import java.util.logging.Logger; - -import jakarta.annotation.PostConstruct; -import jakarta.ejb.Stateless; - import org.apache.commons.httpclient.HttpException; import org.apache.commons.lang3.NotImplementedException; -import edu.harvard.iq.dataverse.DOIServiceBean; import edu.harvard.iq.dataverse.DvObject; import edu.harvard.iq.dataverse.GlobalId; @@ -20,15 +14,13 @@ * */ -@Stateless -public class UnmanagedDOIServiceBean extends DOIServiceBean { +public class UnmanagedDOIProvider extends AbstractDOIProvider { - private static final Logger logger = Logger.getLogger(UnmanagedDOIServiceBean.class.getCanonicalName()); + public static final String ID = "UnmanagedDOIProvider"; - @PostConstruct - private void init() { - // Always on - configured = true; + public UnmanagedDOIProvider() { + //Also using ID as label + super(ID, ID); } @Override @@ -73,11 +65,15 @@ public boolean publicizeIdentifier(DvObject dvObject) { @Override public List getProviderInformation() { - return List.of("UnmanagedDOIProvider", ""); + return List.of(getId(), ""); } + @Override + public String getProviderType() { + return "unamagedDOI"; + } // PID recognition - // Done by DOIServiceBean + // Done by AbstractDOIProvider } diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java new file mode 100644 index 00000000000..30e4dfd79cc --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -0,0 +1,314 @@ +package edu.harvard.iq.dataverse.pidproviders.doi; + +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + +import edu.harvard.iq.dataverse.DataFile; +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetAuthor; +import edu.harvard.iq.dataverse.DvObject; +import edu.harvard.iq.dataverse.pidproviders.AbstractPidProvider; + +public class XmlMetadataTemplate { + + private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.DataCiteMetadataTemplate"); + private static String template; + + static { + try (InputStream in = XmlMetadataTemplate.class.getResourceAsStream("datacite_metadata_template.xml")) { + template = new String(in.readAllBytes(), StandardCharsets.UTF_8); + } catch (Exception e) { + logger.log(Level.SEVERE, "datacite metadata template load error"); + logger.log(Level.SEVERE, "String " + e.toString()); + logger.log(Level.SEVERE, "localized message " + e.getLocalizedMessage()); + logger.log(Level.SEVERE, "cause " + e.getCause()); + logger.log(Level.SEVERE, "message " + e.getMessage()); + } + } + + private String xmlMetadata; + private String identifier; + private List datafileIdentifiers; + private List creators; + private String title; + private String publisher; + private String publisherYear; + private List authors; + private String description; + private List contacts; + private List producers; + + public List getProducers() { + return producers; + } + + public void setProducers(List producers) { + this.producers = producers; + } + + public List getContacts() { + return contacts; + } + + public void setContacts(List contacts) { + this.contacts = contacts; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public List getAuthors() { + return authors; + } + + public void setAuthors(List authors) { + this.authors = authors; + } + + public XmlMetadataTemplate() { + } + + public List getDatafileIdentifiers() { + return datafileIdentifiers; + } + + public void setDatafileIdentifiers(List datafileIdentifiers) { + this.datafileIdentifiers = datafileIdentifiers; + } + + public XmlMetadataTemplate(String xmlMetaData) { + this.xmlMetadata = xmlMetaData; + Document doc = Jsoup.parseBodyFragment(xmlMetaData); + Elements identifierElements = doc.select("identifier"); + if (identifierElements.size() > 0) { + identifier = identifierElements.get(0).html(); + } + Elements creatorElements = doc.select("creatorName"); + creators = new ArrayList<>(); + for (Element creatorElement : creatorElements) { + creators.add(creatorElement.html()); + } + Elements titleElements = doc.select("title"); + if (titleElements.size() > 0) { + title = titleElements.get(0).html(); + } + Elements publisherElements = doc.select("publisher"); + if (publisherElements.size() > 0) { + publisher = publisherElements.get(0).html(); + } + Elements publisherYearElements = doc.select("publicationYear"); + if (publisherYearElements.size() > 0) { + publisherYear = publisherYearElements.get(0).html(); + } + } + + public String generateXML(DvObject dvObject) { + // Can't use "UNKNOWN" here because DataCite will respond with "[facet + // 'pattern'] the value 'unknown' is not accepted by the pattern '[\d]{4}'" + String publisherYearFinal = "9999"; + // FIXME: Investigate why this.publisherYear is sometimes null now that pull + // request #4606 has been merged. + if (this.publisherYear != null) { + // Added to prevent a NullPointerException when trying to destroy datasets when + // using DataCite rather than EZID. + publisherYearFinal = this.publisherYear; + } + xmlMetadata = template.replace("${identifier}", getIdentifier().trim()).replace("${title}", this.title) + .replace("${publisher}", this.publisher).replace("${publisherYear}", publisherYearFinal) + .replace("${description}", this.description); + + StringBuilder creatorsElement = new StringBuilder(); + if (authors != null && !authors.isEmpty()) { + for (DatasetAuthor author : authors) { + creatorsElement.append(""); + creatorsElement.append(author.getName().getDisplayValue()); + creatorsElement.append(""); + + if (author.getIdType() != null && author.getIdValue() != null && !author.getIdType().isEmpty() + && !author.getIdValue().isEmpty() && author.getAffiliation() != null + && !author.getAffiliation().getDisplayValue().isEmpty()) { + + if (author.getIdType().equals("ORCID")) { + creatorsElement.append( + "" + + author.getIdValue() + ""); + } + if (author.getIdType().equals("ISNI")) { + creatorsElement.append( + "" + + author.getIdValue() + ""); + } + if (author.getIdType().equals("LCNA")) { + creatorsElement.append( + "" + + author.getIdValue() + ""); + } + } + if (author.getAffiliation() != null && !author.getAffiliation().getDisplayValue().isEmpty()) { + creatorsElement + .append("" + author.getAffiliation().getDisplayValue() + ""); + } + creatorsElement.append(""); + } + + } else { + creatorsElement.append("").append(AbstractPidProvider.UNAVAILABLE) + .append(""); + } + + xmlMetadata = xmlMetadata.replace("${creators}", creatorsElement.toString()); + + StringBuilder contributorsElement = new StringBuilder(); + if (this.getContacts() != null) { + for (String[] contact : this.getContacts()) { + if (!contact[0].isEmpty()) { + contributorsElement.append("" + + contact[0] + ""); + if (!contact[1].isEmpty()) { + contributorsElement.append("" + contact[1] + ""); + } + contributorsElement.append(""); + } + } + } + + if (this.getProducers() != null) { + for (String[] producer : this.getProducers()) { + contributorsElement.append("" + producer[0] + + ""); + if (!producer[1].isEmpty()) { + contributorsElement.append("" + producer[1] + ""); + } + contributorsElement.append(""); + } + } + + String relIdentifiers = generateRelatedIdentifiers(dvObject); + + xmlMetadata = xmlMetadata.replace("${relatedIdentifiers}", relIdentifiers); + + xmlMetadata = xmlMetadata.replace("{$contributors}", contributorsElement.toString()); + return xmlMetadata; + } + + private String generateRelatedIdentifiers(DvObject dvObject) { + + StringBuilder sb = new StringBuilder(); + if (dvObject.isInstanceofDataset()) { + Dataset dataset = (Dataset) dvObject; + if (!dataset.getFiles().isEmpty() && !(dataset.getFiles().get(0).getIdentifier() == null)) { + + datafileIdentifiers = new ArrayList<>(); + for (DataFile dataFile : dataset.getFiles()) { + if (dataFile.getGlobalId() != null) { + if (sb.toString().isEmpty()) { + sb.append(""); + } + sb.append("" + + dataFile.getGlobalId() + ""); + } + } + + if (!sb.toString().isEmpty()) { + sb.append(""); + } + } + } else if (dvObject.isInstanceofDataFile()) { + DataFile df = (DataFile) dvObject; + sb.append(""); + sb.append("" + + df.getOwner().getGlobalId() + ""); + sb.append(""); + } + return sb.toString(); + } + + public void generateFileIdentifiers(DvObject dvObject) { + + if (dvObject.isInstanceofDataset()) { + Dataset dataset = (Dataset) dvObject; + + if (!dataset.getFiles().isEmpty() && !(dataset.getFiles().get(0).getIdentifier() == null)) { + + datafileIdentifiers = new ArrayList<>(); + for (DataFile dataFile : dataset.getFiles()) { + datafileIdentifiers.add(dataFile.getIdentifier()); + int x = xmlMetadata.indexOf("") - 1; + xmlMetadata = xmlMetadata.replace("{relatedIdentifier}", dataFile.getIdentifier()); + xmlMetadata = xmlMetadata.substring(0, x) + "${relatedIdentifier}" + + template.substring(x, template.length() - 1); + + } + + } else { + xmlMetadata = xmlMetadata.replace( + "${relatedIdentifier}", + ""); + } + } + } + + public static String getTemplate() { + return template; + } + + public static void setTemplate(String template) { + XmlMetadataTemplate.template = template; + } + + public String getIdentifier() { + return identifier; + } + + public void setIdentifier(String identifier) { + this.identifier = identifier; + } + + public List getCreators() { + return creators; + } + + public void setCreators(List creators) { + this.creators = creators; + } + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + public String getPublisher() { + return publisher; + } + + public void setPublisher(String publisher) { + this.publisher = publisher; + } + + public String getPublisherYear() { + return publisherYear; + } + + public void setPublisherYear(String publisherYear) { + this.publisherYear = publisherYear; + } + +} \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DOIDataCiteRegisterService.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DOIDataCiteRegisterService.java new file mode 100644 index 00000000000..0e322eace05 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DOIDataCiteRegisterService.java @@ -0,0 +1,222 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package edu.harvard.iq.dataverse.pidproviders.doi.datacite; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.commons.text.StringEscapeUtils; + +import edu.harvard.iq.dataverse.DataFile; +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetField; +import edu.harvard.iq.dataverse.DvObject; +import edu.harvard.iq.dataverse.branding.BrandingUtil; +import edu.harvard.iq.dataverse.pidproviders.AbstractPidProvider; +import edu.harvard.iq.dataverse.pidproviders.doi.XmlMetadataTemplate; + +/** + * + * @author luopc + */ +public class DOIDataCiteRegisterService { + + private static final Logger logger = Logger.getLogger(DOIDataCiteRegisterService.class.getCanonicalName()); + + + //A singleton since it, and the httpClient in it can be reused. + private DataCiteRESTfullClient client=null; + + public DOIDataCiteRegisterService(String url, String username, String password) { + client = new DataCiteRESTfullClient(url, username, password); + } + + /** + * This "reserveIdentifier" method is heavily based on the + * "registerIdentifier" method below but doesn't, this one doesn't doesn't + * register a URL, which causes the "state" of DOI to transition from + * "draft" to "findable". Here are some DataCite docs on the matter: + * + * "DOIs can exist in three states: draft, registered, and findable. DOIs + * are in the draft state when metadata have been registered, and will + * transition to the findable state when registering a URL." -- + * https://support.datacite.org/docs/mds-api-guide#doi-states + */ + public String reserveIdentifier(String identifier, Map metadata, DvObject dvObject) throws IOException { + String retString = ""; + String xmlMetadata = getMetadataFromDvObject(identifier, metadata, dvObject); + + retString = client.postMetadata(xmlMetadata); + + return retString; + } + + public String registerIdentifier(String identifier, Map metadata, DvObject dvObject) throws IOException { + String retString = ""; + String xmlMetadata = getMetadataFromDvObject(identifier, metadata, dvObject); + String target = metadata.get("_target"); + + retString = client.postMetadata(xmlMetadata); + client.postUrl(identifier.substring(identifier.indexOf(":") + 1), target); + + return retString; + } + + public String deactivateIdentifier(String identifier, Map metadata, DvObject dvObject) throws IOException { + String retString = ""; + + String metadataString = getMetadataForDeactivateIdentifier(identifier, metadata, dvObject); + retString = client.postMetadata(metadataString); + retString = client.inactiveDataset(identifier.substring(identifier.indexOf(":") + 1)); + + return retString; + } + + public static String getMetadataFromDvObject(String identifier, Map metadata, DvObject dvObject) { + + Dataset dataset = null; + + if (dvObject instanceof Dataset) { + dataset = (Dataset) dvObject; + } else { + dataset = (Dataset) dvObject.getOwner(); + } + + XmlMetadataTemplate metadataTemplate = new XmlMetadataTemplate(); + metadataTemplate.setIdentifier(identifier.substring(identifier.indexOf(':') + 1)); + metadataTemplate.setCreators(Arrays.asList(metadata.get("datacite.creator").split("; "))); + metadataTemplate.setAuthors(dataset.getLatestVersion().getDatasetAuthors()); + if (dvObject.isInstanceofDataset()) { + //While getDescriptionPlainText strips < and > from HTML, it leaves '&' (at least so we need to xml escape as well + String description = StringEscapeUtils.escapeXml10(dataset.getLatestVersion().getDescriptionPlainText()); + if (description.isEmpty() || description.equals(DatasetField.NA_VALUE)) { + description = AbstractPidProvider.UNAVAILABLE; + } + metadataTemplate.setDescription(description); + } + if (dvObject.isInstanceofDataFile()) { + DataFile df = (DataFile) dvObject; + //Note: File metadata is not escaped like dataset metadata is, so adding an xml escape here. + //This could/should be removed if the datafile methods add escaping + String fileDescription = StringEscapeUtils.escapeXml10(df.getDescription()); + metadataTemplate.setDescription(fileDescription == null ? AbstractPidProvider.UNAVAILABLE : fileDescription); + } + + metadataTemplate.setContacts(dataset.getLatestVersion().getDatasetContacts()); + metadataTemplate.setProducers(dataset.getLatestVersion().getDatasetProducers()); + String title = dvObject.getCurrentName(); + if(dvObject.isInstanceofDataFile()) { + //Note file title is not currently escaped the way the dataset title is, so adding it here. + title = StringEscapeUtils.escapeXml10(title); + } + + if (title.isEmpty() || title.equals(DatasetField.NA_VALUE)) { + title = AbstractPidProvider.UNAVAILABLE; + } + + metadataTemplate.setTitle(title); + String producerString = BrandingUtil.getRootDataverseCollectionName(); + if (producerString.isEmpty() || producerString.equals(DatasetField.NA_VALUE)) { + producerString = AbstractPidProvider.UNAVAILABLE; + } + metadataTemplate.setPublisher(producerString); + metadataTemplate.setPublisherYear(metadata.get("datacite.publicationyear")); + + String xmlMetadata = metadataTemplate.generateXML(dvObject); + logger.log(Level.FINE, "XML to send to DataCite: {0}", xmlMetadata); + return xmlMetadata; + } + + public static String getMetadataForDeactivateIdentifier(String identifier, Map metadata, DvObject dvObject) { + + XmlMetadataTemplate metadataTemplate = new XmlMetadataTemplate(); + metadataTemplate.setIdentifier(identifier.substring(identifier.indexOf(':') + 1)); + metadataTemplate.setCreators(Arrays.asList(metadata.get("datacite.creator").split("; "))); + + metadataTemplate.setDescription(AbstractPidProvider.UNAVAILABLE); + + String title =metadata.get("datacite.title"); + + System.out.print("Map metadata title: "+ metadata.get("datacite.title")); + + metadataTemplate.setAuthors(null); + + metadataTemplate.setTitle(title); + String producerString = AbstractPidProvider.UNAVAILABLE; + + metadataTemplate.setPublisher(producerString); + metadataTemplate.setPublisherYear(metadata.get("datacite.publicationyear")); + + String xmlMetadata = metadataTemplate.generateXML(dvObject); + logger.log(Level.FINE, "XML to send to DataCite: {0}", xmlMetadata); + return xmlMetadata; + } + + public String modifyIdentifier(String identifier, Map metadata, DvObject dvObject) + throws IOException { + + String xmlMetadata = getMetadataFromDvObject(identifier, metadata, dvObject); + + logger.fine("XML to send to DataCite: " + xmlMetadata); + + String status = metadata.get("_status").trim(); + String target = metadata.get("_target"); + String retString = ""; + switch (status) { + case DataCiteDOIProvider.DRAFT: + // draft DOIs aren't currently being updated after every edit - ToDo - should + // this be changed or made optional? + retString = "success to reserved " + identifier; + break; + case DataCiteDOIProvider.FINDABLE: + try { + retString = client.postMetadata(xmlMetadata); + client.postUrl(identifier.substring(identifier.indexOf(":") + 1), target); + } catch (UnsupportedEncodingException ex) { + logger.log(Level.SEVERE, null, ex); + } catch (RuntimeException rte) { + logger.log(Level.SEVERE, "Error creating DOI at DataCite: {0}", rte.getMessage()); + logger.log(Level.SEVERE, "Exception", rte); + } + break; + case DataCiteDOIProvider.REGISTERED: + retString = client.inactiveDataset(identifier.substring(identifier.indexOf(":") + 1)); + break; + } + return retString; + } + + public boolean testDOIExists(String identifier) { + boolean doiExists; + try { + doiExists = client.testDOIExists(identifier.substring(identifier.indexOf(":") + 1)); + } catch (Exception e) { + logger.log(Level.INFO, identifier, e); + return false; + } + return doiExists; + } + + Map getMetadata(String identifier) throws IOException { + Map metadata = new HashMap<>(); + try { + String xmlMetadata = client.getMetadata(identifier.substring(identifier.indexOf(":") + 1)); + XmlMetadataTemplate template = new XmlMetadataTemplate(xmlMetadata); + metadata.put("datacite.creator", String.join("; ", template.getCreators())); + metadata.put("datacite.title", template.getTitle()); + metadata.put("datacite.publisher", template.getPublisher()); + metadata.put("datacite.publicationyear", template.getPublisherYear()); + } catch (RuntimeException e) { + logger.log(Level.INFO, identifier, e); + } + return metadata; + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteDOIProvider.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteDOIProvider.java new file mode 100644 index 00000000000..23078e2719b --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteDOIProvider.java @@ -0,0 +1,319 @@ +package edu.harvard.iq.dataverse.pidproviders.doi.datacite; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.Base64; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; + +import edu.harvard.iq.dataverse.DataFile; +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetVersion; +import edu.harvard.iq.dataverse.DvObject; +import edu.harvard.iq.dataverse.FileMetadata; +import edu.harvard.iq.dataverse.GlobalId; +import edu.harvard.iq.dataverse.pidproviders.doi.AbstractDOIProvider; +import org.apache.commons.httpclient.HttpException; +import org.apache.commons.httpclient.HttpStatus; + +/** + * + * @author luopc + */ +public class DataCiteDOIProvider extends AbstractDOIProvider { + + private static final Logger logger = Logger.getLogger(DataCiteDOIProvider.class.getCanonicalName()); + + static final String FINDABLE = "findable"; //public - published dataset versions + static final String DRAFT = "draft"; //reserved but not findable yet - draft/unpublished datasets + static final String REGISTERED = "registered"; //was findable once, not anymore - deaccessioned datasets + static final String NONE = "none"; //no record - draft/unpublished datasets where the initial request to reserve has failed + + public static final String TYPE = "datacite"; + + + private String mdsUrl; + private String apiUrl; + private String username; + private String password; + + private DOIDataCiteRegisterService doiDataCiteRegisterService; + + public DataCiteDOIProvider(String id, String label, String providerAuthority, String providerShoulder, + String identifierGenerationStyle, String datafilePidFormat, String managedList, String excludedList, + String mdsUrl, String apiUrl, String username, String password) { + super(id, label, providerAuthority, providerShoulder, identifierGenerationStyle, datafilePidFormat, managedList, + excludedList); + this.mdsUrl = mdsUrl; + this.apiUrl = apiUrl; + this.username = username; + this.password = password; + doiDataCiteRegisterService = new DOIDataCiteRegisterService(mdsUrl, username, password); + } + + @Override + public boolean registerWhenPublished() { + return false; + } + + @Override + public boolean alreadyRegistered(GlobalId pid, boolean noProviderDefault) { + logger.log(Level.FINE, "alreadyRegistered"); + if (pid == null || pid.asString().isEmpty()) { + logger.fine("No identifier sent."); + return false; + } + boolean alreadyRegistered; + String identifier = pid.asString(); + try { + alreadyRegistered = doiDataCiteRegisterService.testDOIExists(identifier); + } catch (Exception e) { + logger.log(Level.WARNING, "alreadyRegistered failed"); + return false; + } + return alreadyRegistered; + } + + @Override + public String createIdentifier(DvObject dvObject) throws Exception { + logger.log(Level.FINE, "createIdentifier"); + if (dvObject.getIdentifier() == null || dvObject.getIdentifier().isEmpty()) { + dvObject = generatePid(dvObject); + } + String identifier = getIdentifier(dvObject); + Map metadata = getMetadataForCreateIndicator(dvObject); + metadata.put("_status", DRAFT); + try { + String retString = doiDataCiteRegisterService.reserveIdentifier(identifier, metadata, dvObject); + logger.log(Level.FINE, "create DOI identifier retString : " + retString); + return retString; + } catch (Exception e) { + logger.log(Level.WARNING, "Identifier not created: create failed", e); + throw e; + } + } + + @Override + public Map getIdentifierMetadata(DvObject dvObject) { + logger.log(Level.FINE, "getIdentifierMetadata"); + String identifier = getIdentifier(dvObject); + Map metadata = new HashMap<>(); + try { + metadata = doiDataCiteRegisterService.getMetadata(identifier); + metadata.put("_status", getPidStatus(dvObject)); + } catch (Exception e) { + logger.log(Level.WARNING, "getIdentifierMetadata failed", e); + } + return metadata; + } + + /** + * Modifies the DOI metadata for a Dataset + * + * @param dvObject the dvObject whose metadata needs to be modified + * @return the Dataset identifier, or null if the modification failed + * @throws java.lang.Exception + */ + @Override + public String modifyIdentifierTargetURL(DvObject dvObject) throws Exception { + logger.log(Level.FINE, "modifyIdentifier"); + String identifier = getIdentifier(dvObject); + try { + Map metadata = getIdentifierMetadata(dvObject); + doiDataCiteRegisterService.modifyIdentifier(identifier, metadata, dvObject); + } catch (Exception e) { + logger.log(Level.WARNING, "modifyMetadata failed", e); + throw e; + } + return identifier; + } + + /* + * Deletes a DOI if it is in DRAFT/DRAFT state or removes metadata and + * changes it from PUBLIC/FINDABLE to REGISTERED. + */ + @Override + public void deleteIdentifier(DvObject dvObject) throws IOException, HttpException { + logger.log(Level.FINE, "deleteIdentifier"); + String identifier = getIdentifier(dvObject); + String idStatus = getPidStatus(dvObject); + switch (idStatus) { + case DRAFT: + logger.log(Level.FINE, "Delete status is reserved.."); + deleteDraftIdentifier(dvObject); + break; + case FINDABLE: + // if public then it has been released set to REGISTERED/unavailable and reset + // target to n2t url + Map metadata = addDOIMetadataForDestroyedDataset(dvObject); + metadata.put("_status", "registered"); + metadata.put("_target", getTargetUrl(dvObject)); + doiDataCiteRegisterService.deactivateIdentifier(identifier, metadata, dvObject); + break; + + case REGISTERED: + case NONE: + // Nothing to do + } + } + + /** + * Deletes DOI from the DataCite side, if possible. Only "draft" DOIs can be + * deleted. + */ + private void deleteDraftIdentifier(DvObject dvObject) throws IOException { + + GlobalId doi = dvObject.getGlobalId(); + /** + * Deletes the DOI from DataCite if it can. Returns 204 if PID was deleted (only + * possible for "draft" DOIs), 405 (method not allowed) if the DOI wasn't + * deleted (because it's in "findable" state, for example, 404 if the DOI wasn't + * found, and possibly other status codes such as 500 if DataCite is down. + */ + + URL url = new URL(getApiUrl() + "/dois/" + doi.getAuthority() + "/" + doi.getIdentifier()); + HttpURLConnection connection = null; + connection = (HttpURLConnection) url.openConnection(); + connection.setRequestMethod("DELETE"); + String userpass = getUsername() + ":" + getPassword(); + String basicAuth = "Basic " + new String(Base64.getEncoder().encode(userpass.getBytes())); + connection.setRequestProperty("Authorization", basicAuth); + int status = connection.getResponseCode(); + if (status != HttpStatus.SC_NO_CONTENT) { + logger.warning( + "Incorrect Response Status from DataCite: " + status + " : " + connection.getResponseMessage()); + throw new HttpException("Status: " + status); + } + logger.fine("deleteDoi status for " + doi.asString() + ": " + status); + } + + @Override + public boolean publicizeIdentifier(DvObject dvObject) { + logger.log(Level.FINE, "updateIdentifierStatus"); + if (dvObject.getIdentifier() == null || dvObject.getIdentifier().isEmpty()) { + dvObject = generatePid(dvObject); + } + String identifier = getIdentifier(dvObject); + Map metadata = getUpdateMetadata(dvObject); + metadata.put("_status", FINDABLE); + metadata.put("datacite.publicationyear", generateYear(dvObject)); + metadata.put("_target", getTargetUrl(dvObject)); + try { + doiDataCiteRegisterService.registerIdentifier(identifier, metadata, dvObject); + return true; + } catch (Exception e) { + logger.log(Level.WARNING, "modifyMetadata failed: " + e.getMessage(), e); + return false; + } + } + + @Override + public List getProviderInformation() { + return List.of(getId(), "https://status.datacite.org"); + } + + @Override + protected String getProviderKeyName() { + return "DataCite"; + } + + @Override + public String getProviderType() { + // TODO Auto-generated method stub + return null; + } + + public String getMdsUrl() { + return mdsUrl; + } + + public String getApiUrl() { + return apiUrl; + } + + public String getUsername() { + return username; + } + + public String getPassword() { + return password; + } + + /** + * Method to determine the status of a dvObject's PID. It replaces keeping a + * separate DOIDataCiteRegisterCache. We could also try to get this info from + * DataCite directly, but it appears to not be in the xml metadata return, so it + * would require another/different api call (possible ToDo). + * + * @param dvObject - Dataset or DataFile + * @return PID status - NONE, DRAFT, FINDABLE, or REGISTERED + */ + String getPidStatus(DvObject dvObject) { + String status = NONE; + if (dvObject instanceof Dataset) { + Dataset dataset = (Dataset) dvObject; + // return true, if all published versions were deaccessioned + boolean hasDeaccessionedVersions = false; + for (DatasetVersion testDsv : dataset.getVersions()) { + if (testDsv.isReleased()) { + // With any released version, we're done + return FINDABLE; + } + // Also check for draft version + if (testDsv.isDraft()) { + if (dataset.isIdentifierRegistered()) { + status = DRAFT; + // Keep interating to see if there's a released version + } + } + if (testDsv.isDeaccessioned()) { + hasDeaccessionedVersions = true; + // Keep interating to see if there's a released version + } + } + if (hasDeaccessionedVersions) { + if (dataset.isIdentifierRegistered()) { + return REGISTERED; + } + } + return status; + } else if (dvObject instanceof DataFile) { + DataFile df = (DataFile) dvObject; + // return true, if all published versions were deaccessioned + boolean isInDeaccessionedVersions = false; + for (FileMetadata fmd : df.getFileMetadatas()) { + DatasetVersion testDsv = fmd.getDatasetVersion(); + if (testDsv.isReleased()) { + // With any released version, we're done + return FINDABLE; + } + // Also check for draft version + if (testDsv.isDraft()) { + if (df.isIdentifierRegistered()) { + status = DRAFT; + // Keep interating to see if there's a released/deaccessioned version + } + } + if (testDsv.isDeaccessioned()) { + isInDeaccessionedVersions = true; + // Keep interating to see if there's a released version + } + } + if (isInDeaccessionedVersions) { + if (df.isIdentifierRegistered()) { + return REGISTERED; + } + } + + } + return status; + } + + + + +} diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteProviderFactory.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteProviderFactory.java new file mode 100644 index 00000000000..99d13b2647c --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteProviderFactory.java @@ -0,0 +1,43 @@ +package edu.harvard.iq.dataverse.pidproviders.doi.datacite; + +import com.google.auto.service.AutoService; + +import edu.harvard.iq.dataverse.pidproviders.PidProvider; +import edu.harvard.iq.dataverse.pidproviders.PidProviderFactory; +import edu.harvard.iq.dataverse.settings.JvmSettings; +import edu.harvard.iq.dataverse.util.SystemConfig; + +@AutoService(PidProviderFactory.class) +public class DataCiteProviderFactory implements PidProviderFactory { + + @Override + public PidProvider createPidProvider(String providerId) { + String providerType = JvmSettings.PID_PROVIDER_TYPE.lookup(providerId); + if (!providerType.equals(DataCiteDOIProvider.TYPE)) { + // Being asked to create a non-DataCite provider + return null; + } + String providerLabel = JvmSettings.PID_PROVIDER_LABEL.lookup(providerId); + String providerAuthority = JvmSettings.PID_PROVIDER_AUTHORITY.lookup(providerId); + String providerShoulder = JvmSettings.PID_PROVIDER_SHOULDER.lookupOptional(providerId).orElse(""); + String identifierGenerationStyle = JvmSettings.PID_PROVIDER_IDENTIFIER_GENERATION_STYLE + .lookupOptional(providerId).orElse("randomString"); + String datafilePidFormat = JvmSettings.PID_PROVIDER_DATAFILE_PID_FORMAT.lookupOptional(providerId) + .orElse(SystemConfig.DataFilePIDFormat.DEPENDENT.toString()); + String managedList = JvmSettings.PID_PROVIDER_MANAGED_LIST.lookupOptional(providerId).orElse(""); + String excludedList = JvmSettings.PID_PROVIDER_EXCLUDED_LIST.lookupOptional(providerId).orElse(""); + + String mdsUrl = JvmSettings.DATACITE_MDS_API_URL.lookupOptional(providerId).orElse("https://mds.test.datacite.org"); + String apiUrl = JvmSettings.DATACITE_REST_API_URL.lookupOptional(providerId).orElse("https://api.test.datacite.org"); + String username = JvmSettings.DATACITE_USERNAME.lookup(providerId); + String password = JvmSettings.DATACITE_PASSWORD.lookup(providerId); + + return new DataCiteDOIProvider(providerId, providerLabel, providerAuthority, providerShoulder, identifierGenerationStyle, + datafilePidFormat, managedList, excludedList, mdsUrl, apiUrl, username, password); + } + + public String getType() { + return DataCiteDOIProvider.TYPE; + } + +} diff --git a/src/main/java/edu/harvard/iq/dataverse/DataCiteRESTfullClient.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteRESTfullClient.java similarity index 92% rename from src/main/java/edu/harvard/iq/dataverse/DataCiteRESTfullClient.java rename to src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteRESTfullClient.java index 491f19ab36c..d185b0249b9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataCiteRESTfullClient.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteRESTfullClient.java @@ -3,7 +3,7 @@ * To change this template file, choose Tools | Templates * and open the template in the editor. */ -package edu.harvard.iq.dataverse; +package edu.harvard.iq.dataverse.pidproviders.doi.datacite; import java.io.Closeable; @@ -45,21 +45,14 @@ public class DataCiteRESTfullClient implements Closeable { private HttpClientContext context; private String encoding = "utf-8"; - public DataCiteRESTfullClient(String url, String username, String password) throws IOException { + public DataCiteRESTfullClient(String url, String username, String password) { this.url = url; - try { - context = HttpClientContext.create(); - CredentialsProvider credsProvider = new BasicCredentialsProvider(); - credsProvider.setCredentials(new AuthScope(null, -1), - new UsernamePasswordCredentials(username, password)); - context.setCredentialsProvider(credsProvider); - - httpClient = HttpClients.createDefault(); - } catch (Exception ioe) { - close(); - logger.log(Level.SEVERE,"Fail to init Client",ioe); - throw new RuntimeException("Fail to init Client", ioe); - } + context = HttpClientContext.create(); + CredentialsProvider credsProvider = new BasicCredentialsProvider(); + credsProvider.setCredentials(new AuthScope(null, -1), new UsernamePasswordCredentials(username, password)); + context.setCredentialsProvider(credsProvider); + + httpClient = HttpClients.createDefault(); } public void close() { diff --git a/src/main/java/edu/harvard/iq/dataverse/DOIEZIdServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/ezid/EZIdDOIProvider.java similarity index 89% rename from src/main/java/edu/harvard/iq/dataverse/DOIEZIdServiceBean.java rename to src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/ezid/EZIdDOIProvider.java index 86b74b72f30..fe8f1ec9c70 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DOIEZIdServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/ezid/EZIdDOIProvider.java @@ -1,5 +1,10 @@ -package edu.harvard.iq.dataverse; +package edu.harvard.iq.dataverse.pidproviders.doi.ezid; +import edu.harvard.iq.dataverse.DataFile; +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DvObject; +import edu.harvard.iq.dataverse.GlobalId; +import edu.harvard.iq.dataverse.pidproviders.doi.AbstractDOIProvider; import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.ucsb.nceas.ezid.EZIDException; import edu.ucsb.nceas.ezid.EZIDService; @@ -13,40 +18,40 @@ * * @author skraffmiller */ -@Stateless -public class DOIEZIdServiceBean extends DOIServiceBean { +public class EZIdDOIProvider extends AbstractDOIProvider { - private static final Logger logger = Logger.getLogger(DOIEZIdServiceBean.class.getCanonicalName()); + + + private static final Logger logger = Logger.getLogger(EZIdDOIProvider.class.getCanonicalName()); EZIDService ezidService; - // This has a sane default in microprofile-config.properties - private final String baseUrl = JvmSettings.EZID_API_URL.lookup(); + public static final String TYPE = "ezid"; + + private String baseUrl; - public DOIEZIdServiceBean() { + + public EZIdDOIProvider(String id, String label, String providerAuthority, String providerShoulder, String identifierGenerationStyle, + String datafilePidFormat, String managedList, String excludedList, String baseUrl, String username, String password) { + super(id, label, providerAuthority, providerShoulder, identifierGenerationStyle, datafilePidFormat, managedList, excludedList); // Creating the service doesn't do any harm, just initializing some object data here. // Makes sure we don't run into NPEs from the other methods, but will obviously fail if the // login below does not work. - this.ezidService = new EZIDService(this.baseUrl); + this.baseUrl = baseUrl; + this.ezidService = new EZIDService(baseUrl); try { - // These have (obviously) no default, but still are optional to make the provider optional - String username = JvmSettings.EZID_USERNAME.lookupOptional().orElse(null); - String password = JvmSettings.EZID_PASSWORD.lookupOptional().orElse(null); - - if (username != null ^ password != null) { - logger.log(Level.WARNING, "You must give both username and password. Will not try to login."); - } - + if (username != null && password != null) { this.ezidService.login(username, password); - this.configured = true; + } else { + logger.log(Level.WARNING, "You must give both username and password. Will not try to login."); } } catch (EZIDException e) { // We only do the warnings here, but the object still needs to be created. // The EJB stateless thing expects this to go through, and it is requested on any // global id parsing. - logger.log(Level.WARNING, "Login failed to {0}", this.baseUrl); + logger.log(Level.WARNING, "Login failed to {0}", baseUrl); logger.log(Level.WARNING, "Exception String: {0}", e.toString()); logger.log(Level.WARNING, "Localized message: {0}", e.getLocalizedMessage()); logger.log(Level.WARNING, "Cause:", e.getCause()); @@ -227,14 +232,14 @@ private boolean updateIdentifierStatus(DvObject dvObject, String statusIn) { @Override public List getProviderInformation(){ - return List.of("EZID", this.baseUrl); + return List.of(getId(), this.baseUrl); } @Override public String createIdentifier(DvObject dvObject) throws Throwable { logger.log(Level.FINE, "createIdentifier"); if(dvObject.getIdentifier() == null || dvObject.getIdentifier().isEmpty() ){ - dvObject = generateIdentifier(dvObject); + dvObject = generatePid(dvObject); } String identifier = getIdentifier(dvObject); Map metadata = getMetadataForCreateIndicator(dvObject); @@ -271,7 +276,7 @@ public String createIdentifier(DvObject dvObject) throws Throwable { * @return A HashMap with the same values as {@code map} */ private HashMap asHashMap(Map map) { - return (map instanceof HashMap) ? (HashMap)map : new HashMap<>(map); + return (map instanceof HashMap) ? (HashMap)map : new HashMap<>(map); } @Override @@ -279,5 +284,9 @@ protected String getProviderKeyName() { return "EZID"; } + @Override + public String getProviderType() { + return TYPE; + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/ezid/EZIdProviderFactory.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/ezid/EZIdProviderFactory.java new file mode 100644 index 00000000000..95ad9bdeff0 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/ezid/EZIdProviderFactory.java @@ -0,0 +1,42 @@ +package edu.harvard.iq.dataverse.pidproviders.doi.ezid; + +import com.google.auto.service.AutoService; + +import edu.harvard.iq.dataverse.pidproviders.PidProvider; +import edu.harvard.iq.dataverse.pidproviders.PidProviderFactory; +import edu.harvard.iq.dataverse.settings.JvmSettings; +import edu.harvard.iq.dataverse.util.SystemConfig; + +@AutoService(PidProviderFactory.class) +public class EZIdProviderFactory implements PidProviderFactory { + + @Override + public PidProvider createPidProvider(String providerId) { + String providerType = JvmSettings.PID_PROVIDER_TYPE.lookup(providerId); + if (!providerType.equals(EZIdDOIProvider.TYPE)) { + // Being asked to create a non-EZId provider + return null; + } + String providerLabel = JvmSettings.PID_PROVIDER_LABEL.lookup(providerId); + String providerAuthority = JvmSettings.PID_PROVIDER_AUTHORITY.lookup(providerId); + String providerShoulder = JvmSettings.PID_PROVIDER_SHOULDER.lookupOptional(providerId).orElse(""); + String identifierGenerationStyle = JvmSettings.PID_PROVIDER_IDENTIFIER_GENERATION_STYLE + .lookupOptional(providerId).orElse("randomString"); + String datafilePidFormat = JvmSettings.PID_PROVIDER_DATAFILE_PID_FORMAT.lookupOptional(providerId) + .orElse(SystemConfig.DataFilePIDFormat.DEPENDENT.toString()); + String managedList = JvmSettings.PID_PROVIDER_MANAGED_LIST.lookupOptional(providerId).orElse(""); + String excludedList = JvmSettings.PID_PROVIDER_EXCLUDED_LIST.lookupOptional(providerId).orElse(""); + + String baseUrl = JvmSettings.EZID_API_URL.lookupOptional(providerId).orElse("https://ezid.cdlib.org"); + String username = JvmSettings.EZID_USERNAME.lookup(providerId); + String password = JvmSettings.EZID_PASSWORD.lookup(providerId); + + return new EZIdDOIProvider(providerId, providerLabel, providerAuthority, providerShoulder, identifierGenerationStyle, datafilePidFormat, + managedList, excludedList, baseUrl, username, password); + } + + public String getType() { + return EZIdDOIProvider.TYPE; + } + +} diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/FakePidProviderServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/fake/FakeDOIProvider.java similarity index 59% rename from src/main/java/edu/harvard/iq/dataverse/pidproviders/FakePidProviderServiceBean.java rename to src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/fake/FakeDOIProvider.java index 3bd9d9dd022..a967fb40620 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/FakePidProviderServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/fake/FakeDOIProvider.java @@ -1,22 +1,22 @@ -package edu.harvard.iq.dataverse.pidproviders; +package edu.harvard.iq.dataverse.pidproviders.doi.fake; -import edu.harvard.iq.dataverse.DOIServiceBean; import edu.harvard.iq.dataverse.DvObject; import edu.harvard.iq.dataverse.GlobalId; +import edu.harvard.iq.dataverse.pidproviders.doi.AbstractDOIProvider; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.logging.Logger; -import jakarta.ejb.Stateless; +public class FakeDOIProvider extends AbstractDOIProvider { -@Stateless -public class FakePidProviderServiceBean extends DOIServiceBean { + public static final String TYPE = "FAKE"; - private static final Logger logger = Logger.getLogger(FakePidProviderServiceBean.class.getCanonicalName()); + public FakeDOIProvider(String id, String label, String providerAuthority, String providerShoulder, String identifierGenerationStyle, + String datafilePidFormat, String managedList, String excludedList) { + super(id, label, providerAuthority, providerShoulder, identifierGenerationStyle, datafilePidFormat, managedList, excludedList); + } - //Only need to check locally public boolean isGlobalIdUnique(GlobalId globalId) { try { @@ -29,7 +29,7 @@ public boolean isGlobalIdUnique(GlobalId globalId) { @Override public boolean alreadyRegistered(GlobalId globalId, boolean noProviderDefault) { - boolean existsLocally = !dvObjectService.isGlobalIdLocallyUnique(globalId); + boolean existsLocally = !pidProviderService.isGlobalIdLocallyUnique(globalId); return existsLocally ? existsLocally : noProviderDefault; } @@ -40,7 +40,7 @@ public boolean registerWhenPublished() { @Override public List getProviderInformation() { - return List.of("FAKE", "https://dataverse.org"); + return List.of(getId(), "https://dataverse.org"); } @Override @@ -65,7 +65,10 @@ public void deleteIdentifier(DvObject dvo) throws Exception { } @Override - public boolean publicizeIdentifier(DvObject studyIn) { + public boolean publicizeIdentifier(DvObject dvObject) { + if(dvObject.isInstanceofDataFile() && dvObject.getGlobalId()==null) { + generatePid(dvObject); + } return true; } @@ -74,4 +77,9 @@ protected String getProviderKeyName() { return "FAKE"; } + @Override + public String getProviderType() { + return TYPE; + } + } diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/fake/FakeProviderFactory.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/fake/FakeProviderFactory.java new file mode 100644 index 00000000000..292c39d4383 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/fake/FakeProviderFactory.java @@ -0,0 +1,38 @@ +package edu.harvard.iq.dataverse.pidproviders.doi.fake; + +import com.google.auto.service.AutoService; + +import edu.harvard.iq.dataverse.pidproviders.PidProvider; +import edu.harvard.iq.dataverse.pidproviders.PidProviderFactory; +import edu.harvard.iq.dataverse.settings.JvmSettings; +import edu.harvard.iq.dataverse.util.SystemConfig; + +@AutoService(PidProviderFactory.class) +public class FakeProviderFactory implements PidProviderFactory { + + @Override + public PidProvider createPidProvider(String providerId) { + String providerType = JvmSettings.PID_PROVIDER_TYPE.lookup(providerId); + if (!providerType.equals(FakeDOIProvider.TYPE)) { + // Being asked to create a non-EZId provider + return null; + } + String providerLabel = JvmSettings.PID_PROVIDER_LABEL.lookup(providerId); + String providerAuthority = JvmSettings.PID_PROVIDER_AUTHORITY.lookup(providerId); + String providerShoulder = JvmSettings.PID_PROVIDER_SHOULDER.lookupOptional(providerId).orElse(""); + String identifierGenerationStyle = JvmSettings.PID_PROVIDER_IDENTIFIER_GENERATION_STYLE + .lookupOptional(providerId).orElse("randomString"); + String datafilePidFormat = JvmSettings.PID_PROVIDER_DATAFILE_PID_FORMAT.lookupOptional(providerId) + .orElse(SystemConfig.DataFilePIDFormat.DEPENDENT.toString()); + String managedList = JvmSettings.PID_PROVIDER_MANAGED_LIST.lookupOptional(providerId).orElse("");; + String excludedList = JvmSettings.PID_PROVIDER_EXCLUDED_LIST.lookupOptional(providerId).orElse("");; + + return new FakeDOIProvider(providerId, providerLabel, providerAuthority, providerShoulder, identifierGenerationStyle, + datafilePidFormat, managedList, excludedList); + } + + public String getType() { + return FakeDOIProvider.TYPE; + } + +} diff --git a/src/main/java/edu/harvard/iq/dataverse/HandlenetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/handle/HandlePidProvider.java similarity index 87% rename from src/main/java/edu/harvard/iq/dataverse/HandlenetServiceBean.java rename to src/main/java/edu/harvard/iq/dataverse/pidproviders/handle/HandlePidProvider.java index 4942db9e7ec..2627bc76fd9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/HandlenetServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/handle/HandlePidProvider.java @@ -18,10 +18,12 @@ Version 3.0. */ -package edu.harvard.iq.dataverse; +package edu.harvard.iq.dataverse.pidproviders.handle; -import edu.harvard.iq.dataverse.settings.JvmSettings; -import edu.harvard.iq.dataverse.settings.SettingsServiceBean; +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DvObject; +import edu.harvard.iq.dataverse.GlobalId; +import edu.harvard.iq.dataverse.pidproviders.AbstractPidProvider; import java.io.File; import java.io.FileInputStream; @@ -29,8 +31,6 @@ import java.util.*; import java.util.logging.Level; import java.util.logging.Logger; -import jakarta.ejb.EJB; -import jakarta.ejb.Stateless; import java.security.PrivateKey; /* Handlenet imports: */ @@ -60,23 +60,32 @@ * As of now, it only does the registration updates, to accommodate * the modifyRegistration datasets API sub-command. */ -@Stateless -public class HandlenetServiceBean extends AbstractGlobalIdServiceBean { - - @EJB - DataverseServiceBean dataverseService; - @EJB - SettingsServiceBean settingsService; - private static final Logger logger = Logger.getLogger(HandlenetServiceBean.class.getCanonicalName()); +public class HandlePidProvider extends AbstractPidProvider { + + private static final Logger logger = Logger.getLogger(HandlePidProvider.class.getCanonicalName()); public static final String HDL_PROTOCOL = "hdl"; - int handlenetIndex = JvmSettings.HANDLENET_INDEX.lookup(Integer.class); + public static final String TYPE = "hdl"; public static final String HTTP_HDL_RESOLVER_URL = "http://hdl.handle.net/"; public static final String HDL_RESOLVER_URL = "https://hdl.handle.net/"; + + - public HandlenetServiceBean() { - logger.log(Level.FINE,"Constructor"); - configured = true; + int handlenetIndex; + private boolean isIndependentHandleService; + private String authHandle; + private String keyPath; + private String keyPassphrase; + + public HandlePidProvider(String id, String label, String authority, String shoulder, String identifierGenerationStyle, + String datafilePidFormat, String managedList, String excludedList, int index, boolean isIndependentService, String authHandle, String path, String passphrase) { + super(id, label, HDL_PROTOCOL, authority, shoulder, identifierGenerationStyle, datafilePidFormat, managedList, excludedList); + this.handlenetIndex = index; + this.isIndependentHandleService = isIndependentService; + this.authHandle = authHandle; + this.keyPath = path; + this.keyPassphrase = passphrase; + } @Override @@ -231,10 +240,9 @@ private ResolutionRequest buildResolutionRequest(final String handle) { private PublicKeyAuthenticationInfo getAuthInfo(String handlePrefix) { logger.log(Level.FINE,"getAuthInfo"); byte[] key = null; - String adminCredFile = JvmSettings.HANDLENET_KEY_PATH.lookup(); - int handlenetIndex = JvmSettings.HANDLENET_INDEX.lookup(Integer.class); + String adminCredFile = getKeyPath(); - key = readKey(adminCredFile); + key = readKey(adminCredFile); PrivateKey privkey = null; privkey = readPrivKey(key, adminCredFile); String authHandle = getAuthenticationHandle(handlePrefix); @@ -244,8 +252,8 @@ private PublicKeyAuthenticationInfo getAuthInfo(String handlePrefix) { } private String getRegistrationUrl(DvObject dvObject) { logger.log(Level.FINE,"getRegistrationUrl"); - String siteUrl = systemConfig.getDataverseSiteUrl(); - String targetUrl = siteUrl + dvObject.getTargetUrl() + "hdl:" + dvObject.getAuthority() + String siteUrl = SystemConfig.getDataverseSiteUrlStatic(); + String targetUrl = siteUrl + dvObject.getTargetUrl() + "hdl:" + dvObject.getAuthority() + "/" + dvObject.getIdentifier(); return targetUrl; } @@ -278,8 +286,7 @@ private PrivateKey readPrivKey(byte[] key, final String file) { try { byte[] secKey = null; if ( Util.requiresSecretKey(key) ) { - String secret = JvmSettings.HANDLENET_KEY_PASSPHRASE.lookup(); - secKey = secret.getBytes(StandardCharsets.UTF_8); + secKey = getKeyPassphrase().getBytes(StandardCharsets.UTF_8); } key = Util.decrypt(key, secKey); privkey = Util.getPrivateKeyFromBytes(key, 0); @@ -304,9 +311,9 @@ private String getAuthenticationHandle(DvObject dvObject){ private String getAuthenticationHandle(String handlePrefix) { logger.log(Level.FINE,"getAuthenticationHandle"); - if (systemConfig.getHandleAuthHandle()!=null) { - return systemConfig.getHandleAuthHandle(); - } else if (systemConfig.isIndependentHandleService()) { + if (getHandleAuthHandle()!=null) { + return getHandleAuthHandle(); + } else if (isIndependentHandleService()) { return handlePrefix + "/ADMIN"; } else { return "0.NA/" + handlePrefix; @@ -348,9 +355,8 @@ public void deleteIdentifier(DvObject dvObject) throws Exception { String handle = getDvObjectHandle(dvObject); String authHandle = getAuthenticationHandle(dvObject); - String adminCredFile = JvmSettings.HANDLENET_KEY_PATH.lookup(); - int handlenetIndex = JvmSettings.HANDLENET_INDEX.lookup(Integer.class); - + String adminCredFile = getKeyPath(); + byte[] key = readKey(adminCredFile); PrivateKey privkey = readPrivKey(key, adminCredFile); @@ -383,7 +389,7 @@ private boolean updateIdentifierStatus(DvObject dvObject, String statusIn) { @Override public List getProviderInformation(){ - return List.of("Handle", "https://hdl.handle.net"); + return List.of(getId(), HDL_RESOLVER_URL); } @@ -401,7 +407,7 @@ public String createIdentifier(DvObject dvObject) throws Throwable { @Override public boolean publicizeIdentifier(DvObject dvObject) { if (dvObject.getIdentifier() == null || dvObject.getIdentifier().isEmpty()){ - generateIdentifier(dvObject); + generatePid(dvObject); } return updateIdentifierStatus(dvObject, "public"); @@ -438,6 +444,32 @@ public GlobalId parsePersistentId(String protocol, String authority, String iden public String getUrlPrefix() { return HDL_RESOLVER_URL; } + + @Override + public String getProtocol() { + return HDL_PROTOCOL; + } + + @Override + public String getProviderType() { + return TYPE; + } + + public String getKeyPath() { + return keyPath; + } + + public String getKeyPassphrase() { + return keyPassphrase; + } + + public boolean isIndependentHandleService() { + return isIndependentHandleService; + } + + public String getHandleAuthHandle() { + return authHandle; + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/handle/HandleProviderFactory.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/handle/HandleProviderFactory.java new file mode 100644 index 00000000000..eef5bed8432 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/handle/HandleProviderFactory.java @@ -0,0 +1,45 @@ +package edu.harvard.iq.dataverse.pidproviders.handle; + +import com.google.auto.service.AutoService; + +import edu.harvard.iq.dataverse.pidproviders.PidProvider; +import edu.harvard.iq.dataverse.pidproviders.PidProviderFactory; +import edu.harvard.iq.dataverse.settings.JvmSettings; +import edu.harvard.iq.dataverse.util.SystemConfig; + +@AutoService(PidProviderFactory.class) +public class HandleProviderFactory implements PidProviderFactory { + + @Override + public PidProvider createPidProvider(String providerId) { + String providerType = JvmSettings.PID_PROVIDER_TYPE.lookup(providerId); + if (!providerType.equals(HandlePidProvider.TYPE)) { + // Being asked to create a non-EZId provider + return null; + } + String providerLabel = JvmSettings.PID_PROVIDER_LABEL.lookup(providerId); + String providerAuthority = JvmSettings.PID_PROVIDER_AUTHORITY.lookup(providerId); + String providerShoulder = JvmSettings.PID_PROVIDER_SHOULDER.lookupOptional(providerId).orElse(""); + String identifierGenerationStyle = JvmSettings.PID_PROVIDER_IDENTIFIER_GENERATION_STYLE + .lookupOptional(providerId).orElse("randomString"); + String datafilePidFormat = JvmSettings.PID_PROVIDER_DATAFILE_PID_FORMAT.lookupOptional(providerId) + .orElse(SystemConfig.DataFilePIDFormat.DEPENDENT.toString()); + String managedList = JvmSettings.PID_PROVIDER_MANAGED_LIST.lookupOptional(providerId).orElse(""); + String excludedList = JvmSettings.PID_PROVIDER_EXCLUDED_LIST.lookupOptional(providerId).orElse(""); + + int index = JvmSettings.HANDLENET_INDEX.lookupOptional(Integer.class, providerId).orElse(300); + boolean independentHandleService = JvmSettings.HANDLENET_INDEPENDENT_SERVICE + .lookupOptional(Boolean.class, providerId).orElse(false); + String handleAuthHandle = JvmSettings.HANDLENET_AUTH_HANDLE.lookup(providerId); + String path = JvmSettings.HANDLENET_KEY_PATH.lookup(providerId); + String passphrase = JvmSettings.HANDLENET_KEY_PASSPHRASE.lookup(providerId); + return new HandlePidProvider(providerId, providerLabel, providerAuthority, providerShoulder, identifierGenerationStyle, + datafilePidFormat, managedList, excludedList, index, independentHandleService, handleAuthHandle, path, + passphrase); + } + + public String getType() { + return HandlePidProvider.TYPE; + } + +} diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/UnmanagedHandlenetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/handle/UnmanagedHandlePidProvider.java similarity index 61% rename from src/main/java/edu/harvard/iq/dataverse/pidproviders/UnmanagedHandlenetServiceBean.java rename to src/main/java/edu/harvard/iq/dataverse/pidproviders/handle/UnmanagedHandlePidProvider.java index c856c5363e0..075e10d8164 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/UnmanagedHandlenetServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/handle/UnmanagedHandlePidProvider.java @@ -1,27 +1,32 @@ -package edu.harvard.iq.dataverse.pidproviders; +package edu.harvard.iq.dataverse.pidproviders.handle; -import edu.harvard.iq.dataverse.AbstractGlobalIdServiceBean; import edu.harvard.iq.dataverse.DvObject; import edu.harvard.iq.dataverse.GlobalId; -import edu.harvard.iq.dataverse.HandlenetServiceBean; +import edu.harvard.iq.dataverse.pidproviders.AbstractPidProvider; + import java.util.*; import java.util.logging.Level; import java.util.logging.Logger; import jakarta.ejb.Stateless; import org.apache.commons.lang3.NotImplementedException; -/** This class is just used to parse Handles that are not managed by any account configured in Dataverse - * It does not implement any of the methods related to PID CRUD +/** + * This class is just used to parse Handles that are not managed by any account + * configured in Dataverse It does not implement any of the methods related to + * PID CRUD * */ -@Stateless -public class UnmanagedHandlenetServiceBean extends AbstractGlobalIdServiceBean { - private static final Logger logger = Logger.getLogger(UnmanagedHandlenetServiceBean.class.getCanonicalName()); +public class UnmanagedHandlePidProvider extends AbstractPidProvider { + + private static final Logger logger = Logger.getLogger(UnmanagedHandlePidProvider.class.getCanonicalName()); + public static final String ID = "UnmanagedHandleProvider"; - public UnmanagedHandlenetServiceBean() { + public UnmanagedHandlePidProvider() { + // Also using ID as label + super(ID, ID, HandlePidProvider.HDL_PROTOCOL); logger.log(Level.FINE, "Constructor"); - configured = true; + } @Override @@ -56,7 +61,7 @@ public void deleteIdentifier(DvObject dvObject) throws Exception { @Override public List getProviderInformation() { - return List.of("UnmanagedHandle", ""); + return List.of(getId(), ""); } @Override @@ -71,19 +76,18 @@ public boolean publicizeIdentifier(DvObject dvObject) { @Override public GlobalId parsePersistentId(String pidString) { - if (pidString.startsWith(HandlenetServiceBean.HDL_RESOLVER_URL)) { - pidString = pidString.replace(HandlenetServiceBean.HDL_RESOLVER_URL, - (HandlenetServiceBean.HDL_PROTOCOL + ":")); - } else if (pidString.startsWith(HandlenetServiceBean.HTTP_HDL_RESOLVER_URL)) { - pidString = pidString.replace(HandlenetServiceBean.HTTP_HDL_RESOLVER_URL, - (HandlenetServiceBean.HDL_PROTOCOL + ":")); + if (pidString.startsWith(HandlePidProvider.HDL_RESOLVER_URL)) { + pidString = pidString.replace(HandlePidProvider.HDL_RESOLVER_URL, (HandlePidProvider.HDL_PROTOCOL + ":")); + } else if (pidString.startsWith(HandlePidProvider.HTTP_HDL_RESOLVER_URL)) { + pidString = pidString.replace(HandlePidProvider.HTTP_HDL_RESOLVER_URL, + (HandlePidProvider.HDL_PROTOCOL + ":")); } return super.parsePersistentId(pidString); } @Override public GlobalId parsePersistentId(String protocol, String identifierString) { - if (!HandlenetServiceBean.HDL_PROTOCOL.equals(protocol)) { + if (!HandlePidProvider.HDL_PROTOCOL.equals(protocol)) { return null; } GlobalId globalId = super.parsePersistentId(protocol, identifierString); @@ -92,7 +96,7 @@ public GlobalId parsePersistentId(String protocol, String identifierString) { @Override public GlobalId parsePersistentId(String protocol, String authority, String identifier) { - if (!HandlenetServiceBean.HDL_PROTOCOL.equals(protocol)) { + if (!HandlePidProvider.HDL_PROTOCOL.equals(protocol)) { return null; } return super.parsePersistentId(protocol, authority, identifier); @@ -100,6 +104,11 @@ public GlobalId parsePersistentId(String protocol, String authority, String iden @Override public String getUrlPrefix() { - return HandlenetServiceBean.HDL_RESOLVER_URL; + return HandlePidProvider.HDL_RESOLVER_URL; + } + + @Override + public String getProviderType() { + return "unamagedHandle"; } } diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/perma/PermaLinkPidProvider.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/perma/PermaLinkPidProvider.java new file mode 100644 index 00000000000..91c7f527c88 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/perma/PermaLinkPidProvider.java @@ -0,0 +1,201 @@ +package edu.harvard.iq.dataverse.pidproviders.perma; + +import edu.harvard.iq.dataverse.DvObject; +import edu.harvard.iq.dataverse.GlobalId; +import edu.harvard.iq.dataverse.pidproviders.AbstractPidProvider; +import edu.harvard.iq.dataverse.pidproviders.PidProvider; +import edu.harvard.iq.dataverse.settings.JvmSettings; +import edu.harvard.iq.dataverse.util.SystemConfig; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Logger; + +/** + * PermaLink provider This is a minimalist permanent ID provider intended for + * use with 'real' datasets/files where the use case none-the-less doesn't lend + * itself to the use of DOIs or Handles, e.g. * due to cost * for a + * catalog/archive where Dataverse has a dataset representing a dataset with + * DOI/handle stored elsewhere + * + * The initial implementation will mint identifiers locally and will provide the + * existing page URLs (using the ?persistentID= format). This will be + * overridable by a configurable parameter to support use of an external + * resolver. + * + */ +public class PermaLinkPidProvider extends AbstractPidProvider { + + private static final Logger logger = Logger.getLogger(PermaLinkPidProvider.class.getCanonicalName()); + + public static final String PERMA_PROTOCOL = "perma"; + public static final String TYPE = "perma"; + public static final String SEPARATOR = ""; + + // ToDo - remove + @Deprecated + public static final String PERMA_RESOLVER_URL = JvmSettings.PERMALINK_BASE_URL.lookupOptional("permalink") + .orElse(SystemConfig.getDataverseSiteUrlStatic()); + + + private String separator = SEPARATOR; + + private String baseUrl; + + public PermaLinkPidProvider(String id, String label, String providerAuthority, String providerShoulder, String identifierGenerationStyle, + String datafilePidFormat, String managedList, String excludedList, String baseUrl, String separator) { + super(id, label, PERMA_PROTOCOL, providerAuthority, providerShoulder, identifierGenerationStyle, datafilePidFormat, + managedList, excludedList); + this.baseUrl = baseUrl; + this.separator = separator; + } + + @Override + public String getSeparator() { + return separator; + } + + @Override + public boolean alreadyRegistered(GlobalId globalId, boolean noProviderDefault) { + // Perma doesn't manage registration, so we assume all local PIDs can be treated + // as registered + boolean existsLocally = !pidProviderService.isGlobalIdLocallyUnique(globalId); + return existsLocally ? existsLocally : noProviderDefault; + } + + @Override + public boolean registerWhenPublished() { + return false; + } + + @Override + public List getProviderInformation() { + return List.of(getId(), getBaseUrl()); + } + + @Override + public String createIdentifier(DvObject dvo) throws Throwable { + // Call external resolver and send landing URL? + // FWIW: Return value appears to only be used in RegisterDvObjectCommand where + // success requires finding the dvo identifier in this string. (Also logged a + // couple places). + return (dvo.getGlobalId().asString()); + } + + @Override + public Map getIdentifierMetadata(DvObject dvo) { + Map map = new HashMap<>(); + return map; + } + + @Override + public String modifyIdentifierTargetURL(DvObject dvo) throws Exception { + return getTargetUrl(dvo); + } + + @Override + public void deleteIdentifier(DvObject dvo) throws Exception { + // no-op + } + + @Override + public boolean publicizeIdentifier(DvObject dvObject) { + // Generate if needed (i.e. datafile case where we don't create/register early + // (even with registerWhenPublished == false)) + if (dvObject.getIdentifier() == null || dvObject.getIdentifier().isEmpty()) { + dvObject = generatePid(dvObject); + } + // Call external resolver and send landing URL? + return true; + } + + @Override + public GlobalId parsePersistentId(String pidString) { + // ToDo - handle local PID resolver for dataset/file + logger.info("Parsing in Perma: " + pidString); + if (pidString.startsWith(getUrlPrefix())) { + pidString = pidString.replace(getUrlPrefix(), (PERMA_PROTOCOL + ":")); + } + return super.parsePersistentId(pidString); + } + + @Override + public GlobalId parsePersistentId(String protocol, String identifierString) { + logger.info("Checking Perma: " + identifierString); + if (!PERMA_PROTOCOL.equals(protocol)) { + return null; + } + String cleanIdentifier = PERMA_PROTOCOL + ":" + identifierString; + // With permalinks, we have to check the sets before parsing since the permalinks in these sets could have different authority, spearator, and shoulders + if (getExcludedSet().contains(cleanIdentifier)) { + return null; + } + if(getManagedSet().contains(cleanIdentifier)) { + /** With a variable separator that could also be empty, there is no way to determine the authority and shoulder for an unknown permalink. + * Since knowing this split isn't relevant for permalinks except for minting, the code below just assumes the authority + * is the first 4 characters and that the separator and the shoulder are empty. + * If this is found to cause issues, users should be able to use a managed permalink provider as a work-around. The code here could + * be changed to allow default lengths for the authority, separator, and shoulder and/or to add a list of known (but unmanaged) authority, separator, shoulder combos. + */ + if(identifierString.length() < 4) { + return new GlobalId(protocol, "", identifierString, SEPARATOR, getUrlPrefix(), + getId()); + } + return new GlobalId(protocol, identifierString.substring(0,4), identifierString.substring(4), SEPARATOR, getUrlPrefix(), + getId()); + } + String identifier = null; + if (getAuthority() != null) { + if (identifierString.startsWith(getAuthority())) { + identifier = identifierString.substring(getAuthority().length()); + } else { + //Doesn't match authority + return null; + } + if (identifier.startsWith(separator)) { + identifier = identifier.substring(separator.length()); + } else { + //Doesn't match separator + return null; + } + } + identifier = PidProvider.formatIdentifierString(identifier); + if (PidProvider.testforNullTerminator(identifier)) { + return null; + } + if(!identifier.startsWith(getShoulder())) { + //Doesn't match shoulder + return null; + } + return new GlobalId(PERMA_PROTOCOL, getAuthority(), identifier, separator, getUrlPrefix(), getId()); + } + + @Override + public GlobalId parsePersistentId(String protocol, String authority, String identifier) { + if (!PERMA_PROTOCOL.equals(protocol)) { + return null; + } + return super.parsePersistentId(protocol, authority, identifier); + } + + @Override + public String getUrlPrefix() { + + return getBaseUrl() + "/citation?persistentId=" + PERMA_PROTOCOL + ":"; + } + + @Override + public String getProtocol() { + return PERMA_PROTOCOL; + } + + @Override + public String getProviderType() { + return PERMA_PROTOCOL; + } + + public String getBaseUrl() { + return baseUrl; + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/perma/PermaLinkProviderFactory.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/perma/PermaLinkProviderFactory.java new file mode 100644 index 00000000000..32b89223062 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/perma/PermaLinkProviderFactory.java @@ -0,0 +1,43 @@ +package edu.harvard.iq.dataverse.pidproviders.perma; + +import com.google.auto.service.AutoService; + +import edu.harvard.iq.dataverse.pidproviders.PidProvider; +import edu.harvard.iq.dataverse.pidproviders.PidProviderFactory; +import edu.harvard.iq.dataverse.settings.JvmSettings; +import edu.harvard.iq.dataverse.util.SystemConfig; + +@AutoService(PidProviderFactory.class) +public class PermaLinkProviderFactory implements PidProviderFactory { + + @Override + public PidProvider createPidProvider(String providerId) { + String providerType = JvmSettings.PID_PROVIDER_TYPE.lookup(providerId); + if (!providerType.equals(PermaLinkPidProvider.TYPE)) { + // Being asked to create a non-EZId provider + return null; + } + String providerLabel = JvmSettings.PID_PROVIDER_LABEL.lookup(providerId); + String providerAuthority = JvmSettings.PID_PROVIDER_AUTHORITY.lookup(providerId); + String providerShoulder = JvmSettings.PID_PROVIDER_SHOULDER.lookupOptional(providerId).orElse(""); + String identifierGenerationStyle = JvmSettings.PID_PROVIDER_IDENTIFIER_GENERATION_STYLE + .lookupOptional(providerId).orElse("randomString"); + String datafilePidFormat = JvmSettings.PID_PROVIDER_DATAFILE_PID_FORMAT.lookupOptional(providerId) + .orElse(SystemConfig.DataFilePIDFormat.DEPENDENT.toString()); + String managedList = JvmSettings.PID_PROVIDER_MANAGED_LIST.lookupOptional(providerId).orElse(""); + String excludedList = JvmSettings.PID_PROVIDER_EXCLUDED_LIST.lookupOptional(providerId).orElse(""); + + String baseUrl = JvmSettings.PERMALINK_BASE_URL.lookupOptional(providerId) + .orElse(SystemConfig.getDataverseSiteUrlStatic()); + ; + String separator = JvmSettings.PERMALINK_SEPARATOR.lookupOptional(providerId).orElse(""); + + return new PermaLinkPidProvider(providerId, providerLabel, providerAuthority, providerShoulder, identifierGenerationStyle, + datafilePidFormat, managedList, excludedList, baseUrl, separator); + } + + public String getType() { + return PermaLinkPidProvider.TYPE; + } + +} diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/perma/UnmanagedPermaLinkPidProvider.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/perma/UnmanagedPermaLinkPidProvider.java new file mode 100644 index 00000000000..b7961a41c50 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/perma/UnmanagedPermaLinkPidProvider.java @@ -0,0 +1,114 @@ +package edu.harvard.iq.dataverse.pidproviders.perma; + +import edu.harvard.iq.dataverse.DvObject; +import edu.harvard.iq.dataverse.GlobalId; +import edu.harvard.iq.dataverse.pidproviders.AbstractPidProvider; +import edu.harvard.iq.dataverse.util.SystemConfig; + +import java.util.*; +import java.util.logging.Level; +import java.util.logging.Logger; +import jakarta.ejb.Stateless; +import org.apache.commons.lang3.NotImplementedException; + +/** This class is just used to parse Handles that are not managed by any account configured in Dataverse + * It does not implement any of the methods related to PID CRUD + * + */ +public class UnmanagedPermaLinkPidProvider extends AbstractPidProvider { + + private static final Logger logger = Logger.getLogger(UnmanagedPermaLinkPidProvider.class.getCanonicalName()); + public static final String ID = "UnmanagedPermaLinkProvider"; + + public UnmanagedPermaLinkPidProvider() { + // Also using ID as label + super(ID, ID, PermaLinkPidProvider.PERMA_PROTOCOL); + logger.log(Level.FINE, "Constructor"); + } + + @Override + public boolean canManagePID() { + return false; + } + + @Override + public boolean registerWhenPublished() { + throw new NotImplementedException(); + } + + @Override + public boolean alreadyRegistered(GlobalId pid, boolean noProviderDefault) throws Exception { + throw new NotImplementedException(); + } + + @Override + public Map getIdentifierMetadata(DvObject dvObject) { + throw new NotImplementedException(); + } + + @Override + public String modifyIdentifierTargetURL(DvObject dvObject) throws Exception { + throw new NotImplementedException(); + } + + @Override + public void deleteIdentifier(DvObject dvObject) throws Exception { + throw new NotImplementedException(); + } + + @Override + public List getProviderInformation() { + return List.of(getId(), ""); + } + + @Override + public String createIdentifier(DvObject dvObject) throws Throwable { + throw new NotImplementedException(); + } + + @Override + public boolean publicizeIdentifier(DvObject dvObject) { + throw new NotImplementedException(); + } + + @Override + public GlobalId parsePersistentId(String protocol, String identifierString) { + if (!PermaLinkPidProvider.PERMA_PROTOCOL.equals(protocol)) { + return null; + } + /** With a variable separator that could also be empty, there is no way to determine the authority and shoulder for an unknown/unmanaged permalink. + * Since knowing this split isn't relevant for unmanaged permalinks, the code below just assumes the authority + * is the first 4 characters and that the separator and the shoulder are empty. + * If this is found to cause issues, users should be able to use a managed permalink provider as a work-around. The code here could + * be changed to allow default lengths for the authority, separator, and shoulder and/or to add a list of known (but unmanaged) authority, separator, shoulder combos. + */ + if(identifierString.length() < 4) { + logger.warning("A short unmanaged permalink was found - assuming the authority is empty: " + identifierString); + return super.parsePersistentId(protocol, "", identifierString); + } + return super.parsePersistentId(protocol, identifierString.substring(0, 4), identifierString.substring(4)); + } + + @Override + public GlobalId parsePersistentId(String protocol, String authority, String identifier) { + if (!PermaLinkPidProvider.PERMA_PROTOCOL.equals(protocol)) { + return null; + } + return super.parsePersistentId(protocol, authority, identifier); + } + + @Override + public String getUrlPrefix() { + return SystemConfig.getDataverseSiteUrlStatic()+ "/citation?persistentId=" + PermaLinkPidProvider.PERMA_PROTOCOL + ":"; + } + + @Override + public String getProviderType() { + return PermaLinkPidProvider.TYPE; + } + + @Override + public String getSeparator() { + return PermaLinkPidProvider.SEPARATOR; + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/repositorystorageabstractionlayer/RepositoryStorageAbstractionLayerPage.java b/src/main/java/edu/harvard/iq/dataverse/repositorystorageabstractionlayer/RepositoryStorageAbstractionLayerPage.java index c252d2e3330..9edb536eda2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/repositorystorageabstractionlayer/RepositoryStorageAbstractionLayerPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/repositorystorageabstractionlayer/RepositoryStorageAbstractionLayerPage.java @@ -11,6 +11,7 @@ import jakarta.inject.Named; import jakarta.json.JsonArray; +@Deprecated(forRemoval = true, since = "2024-07-07") @Stateless @Named public class RepositoryStorageAbstractionLayerPage { @@ -22,17 +23,20 @@ public class RepositoryStorageAbstractionLayerPage { @EJB StorageSiteServiceBean storageSiteServiceBean; + @Deprecated(forRemoval = true, since = "2024-07-07") public String getLocalDataAccessDirectory(DatasetVersion datasetVersion) { String localDataAccessParentDir = settingsService.getValueForKey(SettingsServiceBean.Key.LocalDataAccessPath); return RepositoryStorageAbstractionLayerUtil.getLocalDataAccessDirectory(localDataAccessParentDir, datasetVersion.getDataset()); } + @Deprecated(forRemoval = true, since = "2024-07-07") public List getRsyncSites(DatasetVersion datasetVersion) { List storageSites = storageSiteServiceBean.findAll(); JsonArray storageSitesAsJson = RepositoryStorageAbstractionLayerUtil.getStorageSitesAsJson(storageSites); return RepositoryStorageAbstractionLayerUtil.getRsyncSites(datasetVersion.getDataset(), storageSitesAsJson); } + @Deprecated(forRemoval = true, since = "2024-07-07") public String getVerifyDataCommand(DatasetVersion datasetVersion) { return RepositoryStorageAbstractionLayerUtil.getVerifyDataCommand(datasetVersion.getDataset()); } diff --git a/src/main/java/edu/harvard/iq/dataverse/repositorystorageabstractionlayer/RepositoryStorageAbstractionLayerUtil.java b/src/main/java/edu/harvard/iq/dataverse/repositorystorageabstractionlayer/RepositoryStorageAbstractionLayerUtil.java index 8501fba3ce0..0d547402676 100644 --- a/src/main/java/edu/harvard/iq/dataverse/repositorystorageabstractionlayer/RepositoryStorageAbstractionLayerUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/repositorystorageabstractionlayer/RepositoryStorageAbstractionLayerUtil.java @@ -13,10 +13,12 @@ import jakarta.json.JsonArrayBuilder; import jakarta.json.JsonObject; +@Deprecated(forRemoval = true, since = "2024-07-07") public class RepositoryStorageAbstractionLayerUtil { private static final Logger logger = Logger.getLogger(RepositoryStorageAbstractionLayerUtil.class.getCanonicalName()); + @Deprecated(forRemoval = true, since = "2024-07-07") public static List getRsyncSites(Dataset dataset, JsonArray rsalSitesAsJson) { List rsalSites = new ArrayList<>(); boolean leafDirectoryOnly = false; @@ -30,6 +32,7 @@ public static List getRsyncSites(Dataset dataset, JsonArray rsalSites return rsalSites; } + @Deprecated(forRemoval = true, since = "2024-07-07") static String getLocalDataAccessDirectory(String localDataAccessParentDir, Dataset dataset) { if (localDataAccessParentDir == null) { localDataAccessParentDir = File.separator + "UNCONFIGURED ( " + SettingsServiceBean.Key.LocalDataAccessPath + " )"; @@ -38,6 +41,7 @@ static String getLocalDataAccessDirectory(String localDataAccessParentDir, Datas return localDataAccessParentDir + File.separator + getDirectoryContainingTheData(dataset, leafDirectoryOnly); } + @Deprecated(forRemoval = true, since = "2024-07-07") static String getVerifyDataCommand(Dataset dataset) { boolean leafDirectoryOnly = true; // TODO: if "files.sha" is defined somewhere, use it. @@ -51,6 +55,7 @@ static String getVerifyDataCommand(Dataset dataset) { * leafDirectoryOnly. See also * http://www.gnu.org/software/coreutils/manual/html_node/basename-invocation.html */ + @Deprecated(forRemoval = true, since = "2024-07-07") public static String getDirectoryContainingTheData(Dataset dataset, boolean leafDirectoryOnly) { /** * FIXME: What if there is more than one package in the dataset? @@ -81,6 +86,7 @@ public static String getDirectoryContainingTheData(Dataset dataset, boolean leaf * RSAL or some other "big data" component live for a list of remotes sites * to which a particular dataset is replicated to. */ + @Deprecated(forRemoval = true, since = "2024-07-07") static JsonArray getStorageSitesAsJson(List storageSites) { JsonArrayBuilder arraybuilder = Json.createArrayBuilder(); if (storageSites == null || storageSites.isEmpty()) { diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index d631e2829c1..0102459ab9f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -12,9 +12,9 @@ import edu.harvard.iq.dataverse.datavariable.VariableMetadataUtil; import edu.harvard.iq.dataverse.datavariable.VariableServiceBean; import edu.harvard.iq.dataverse.harvest.client.HarvestingClient; +import edu.harvard.iq.dataverse.settings.FeatureFlags; import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; -import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.FileUtil; import edu.harvard.iq.dataverse.util.StringUtil; import edu.harvard.iq.dataverse.util.SystemConfig; @@ -36,6 +36,7 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Future; +import java.util.concurrent.Semaphore; import java.util.function.Function; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -48,6 +49,8 @@ import jakarta.ejb.Stateless; import jakarta.ejb.TransactionAttribute; import static jakarta.ejb.TransactionAttributeType.REQUIRES_NEW; + +import jakarta.inject.Inject; import jakarta.inject.Named; import jakarta.json.JsonObject; import jakarta.persistence.EntityManager; @@ -72,6 +75,9 @@ import org.apache.tika.sax.BodyContentHandler; import org.eclipse.microprofile.config.Config; import org.eclipse.microprofile.config.ConfigProvider; +import org.eclipse.microprofile.metrics.MetricUnits; +import org.eclipse.microprofile.metrics.Timer; +import org.eclipse.microprofile.metrics.annotation.Metric; import org.xml.sax.ContentHandler; @Stateless @@ -209,6 +215,9 @@ public Future indexDataverse(Dataverse dataverse, boolean processPaths) solrInputDocument.addField(SearchFields.DATAVERSE_CATEGORY, dataverse.getIndexableCategoryName()); if (dataverse.isReleased()) { solrInputDocument.addField(SearchFields.PUBLICATION_STATUS, PUBLISHED_STRING); + if (FeatureFlags.ADD_PUBLICOBJECT_SOLR_FIELD.enabled()) { + solrInputDocument.addField(SearchFields.PUBLIC_OBJECT, true); + } solrInputDocument.addField(SearchFields.RELEASE_OR_CREATE_DATE, dataverse.getPublicationDate()); } else { solrInputDocument.addField(SearchFields.PUBLICATION_STATUS, UNPUBLISHED_STRING); @@ -303,7 +312,7 @@ public Future indexDataverse(Dataverse dataverse, boolean processPaths) String status; try { if (dataverse.getId() != null) { - solrClientService.getSolrClient().add(docs); + solrClientService.getSolrClient().add(docs, COMMIT_WITHIN); } else { logger.info("WARNING: indexing of a dataverse with no id attempted"); } @@ -312,14 +321,6 @@ public Future indexDataverse(Dataverse dataverse, boolean processPaths) logger.info(status); return new AsyncResult<>(status); } - try { - solrClientService.getSolrClient().commit(); - } catch (SolrServerException | IOException ex) { - status = ex.toString(); - logger.info(status); - return new AsyncResult<>(status); - } - dvObjectService.updateContentIndexTime(dataverse); IndexResponse indexResponse = solrIndexService.indexPermissionsForOneDvObject(dataverse); String msg = "indexed dataverse " + dataverse.getId() + ":" + dataverse.getAlias() + ". Response from permission indexing: " + indexResponse.getMessage(); @@ -342,6 +343,30 @@ public void indexDatasetInNewTransaction(Long datasetId) { //Dataset dataset) { private static final Map NEXT_TO_INDEX = new ConcurrentHashMap<>(); // indexingNow is a set of dataset ids of datasets being indexed asynchronously right now private static final Map INDEXING_NOW = new ConcurrentHashMap<>(); + // semaphore for async indexing + private static final Semaphore ASYNC_INDEX_SEMAPHORE = new Semaphore(JvmSettings.MAX_ASYNC_INDEXES.lookupOptional(Integer.class).orElse(4), true); + static final int COMMIT_WITHIN = 30000; //Same as current autoHardIndex time + + @Inject + @Metric(name = "index_permit_wait_time", absolute = true, unit = MetricUnits.NANOSECONDS, + description = "Displays how long does it take to receive a permit to index a dataset") + Timer indexPermitWaitTimer; + + @Inject + @Metric(name = "index_time", absolute = true, unit = MetricUnits.NANOSECONDS, + description = "Displays how long does it take to index a dataset") + Timer indexTimer; + + /** + * Try to acquire a permit from the semaphore avoiding too many parallel indexes, potentially overwhelming Solr. + * This method will time the duration waiting for the permit, allowing indexing performance to be measured. + * @throws InterruptedException + */ + private void acquirePermitFromSemaphore() throws InterruptedException { + try (var timeContext = indexPermitWaitTimer.time()) { + ASYNC_INDEX_SEMAPHORE.acquire(); + } + } // When you pass null as Dataset parameter to this method, it indicates that the indexing of the dataset with "id" has finished // Pass non-null Dataset to schedule it for indexing @@ -386,10 +411,24 @@ synchronized private static Dataset getNextToIndex(Long id, Dataset d) { */ @Asynchronous public void asyncIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) { + try { + acquirePermitFromSemaphore(); + doAyncIndexDataset(dataset, doNormalSolrDocCleanUp); + } catch (InterruptedException e) { + String failureLogText = "Indexing failed: interrupted. You can kickoff a re-index of this dataset with: \r\n curl http://localhost:8080/api/admin/index/datasets/" + dataset.getId().toString(); + failureLogText += "\r\n" + e.getLocalizedMessage(); + LoggingUtil.writeOnSuccessFailureLog(null, failureLogText, dataset); + } finally { + ASYNC_INDEX_SEMAPHORE.release(); + } + } + + private void doAyncIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) { Long id = dataset.getId(); Dataset next = getNextToIndex(id, dataset); // if there is an ongoing index job for this dataset, next is null (ongoing index job will reindex the newest version after current indexing finishes) while (next != null) { - try { + // Time context will automatically start on creation and stop when leaving the try block + try (var timeContext = indexTimer.time()) { indexDataset(next, doNormalSolrDocCleanUp); } catch (Exception e) { // catch all possible exceptions; otherwise when something unexpected happes the dataset wold remain locked and impossible to reindex String failureLogText = "Indexing failed. You can kickoff a re-index of this dataset with: \r\n curl http://localhost:8080/api/admin/index/datasets/" + dataset.getId().toString(); @@ -403,7 +442,16 @@ public void asyncIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) { @Asynchronous public void asyncIndexDatasetList(List datasets, boolean doNormalSolrDocCleanUp) { for(Dataset dataset : datasets) { - asyncIndexDataset(dataset, true); + try { + acquirePermitFromSemaphore(); + doAyncIndexDataset(dataset, true); + } catch (InterruptedException e) { + String failureLogText = "Indexing failed: interrupted. You can kickoff a re-index of this dataset with: \r\n curl http://localhost:8080/api/admin/index/datasets/" + dataset.getId().toString(); + failureLogText += "\r\n" + e.getLocalizedMessage(); + LoggingUtil.writeOnSuccessFailureLog(null, failureLogText, dataset); + } finally { + ASYNC_INDEX_SEMAPHORE.release(); + } } } @@ -415,7 +463,7 @@ public void indexDvObject(DvObject objectIn) throws SolrServerException, IOExce } } - private void indexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) throws SolrServerException, IOException { + public void indexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) throws SolrServerException, IOException { doIndexDataset(dataset, doNormalSolrDocCleanUp); updateLastIndexedTime(dataset.getId()); } @@ -784,15 +832,7 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set langs = settingsService.getConfiguredLanguages(); Map cvocMap = datasetFieldService.getCVocConf(true); + Map> cvocManagedFieldMap = new HashMap<>(); + for (Map.Entry cvocEntry : cvocMap.entrySet()) { + if(cvocEntry.getValue().containsKey("managed-fields")) { + JsonObject managedFields = cvocEntry.getValue().getJsonObject("managed-fields"); + Set managedFieldValues = new HashSet<>(); + for (String s : managedFields.keySet()) { + managedFieldValues.add(managedFields.getString(s)); + } + cvocManagedFieldMap.put(cvocEntry.getKey(), managedFieldValues); + } + } + + + Set metadataBlocksWithValue = new HashSet<>(); for (DatasetField dsf : datasetVersion.getFlatDatasetFields()) { @@ -942,36 +1002,73 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set vals = dsf.getValues_nondisplay(); - Set searchStrings = new HashSet(); + Set searchStrings = new HashSet<>(); for (String val: vals) { searchStrings.add(val); - searchStrings.addAll(datasetFieldService.getStringsFor(val)); + // Try to get string values from externalvocabularyvalue using val as termUri + searchStrings.addAll(datasetFieldService.getIndexableStringsByTermUri(val, cvocMap.get(dsfType.getId()), dsfType.getName())); + + if(dsfType.getParentDatasetFieldType()!=null) { + List childDatasetFields = dsf.getParentDatasetFieldCompoundValue().getChildDatasetFields(); + for (DatasetField df : childDatasetFields) { + if(cvocManagedFieldMap.get(dsfType.getId()).contains(df.getDatasetFieldType().getName())) { + String solrManagedFieldSearchable = df.getDatasetFieldType().getSolrField().getNameSearchable(); + // Try to get string values from externalvocabularyvalue but for a managed fields of the CVOCConf + Set stringsForManagedField = datasetFieldService.getIndexableStringsByTermUri(val, cvocMap.get(dsfType.getId()), df.getDatasetFieldType().getName()); + logger.fine(solrManagedFieldSearchable + " filled with externalvocabularyvalue : " + stringsForManagedField); + //.addField works as addition of value not a replace of value + // it allows to add mapped values by CVOCConf before or after indexing real DatasetField value(s) of solrManagedFieldSearchable + solrInputDocument.addField(solrManagedFieldSearchable, stringsForManagedField); + } + } + } } + logger.fine(solrFieldSearchable + " filled with externalvocabularyvalue : " + searchStrings); solrInputDocument.addField(solrFieldSearchable, searchStrings); if (dsfType.getSolrField().isFacetable()) { + logger.fine(solrFieldFacetable + " gets " + vals); solrInputDocument.addField(solrFieldFacetable, vals); } } + if (dsfType.isControlledVocabulary()) { - for (ControlledVocabularyValue controlledVocabularyValue : dsf.getControlledVocabularyValues()) { - if (controlledVocabularyValue.getStrValue().equals(DatasetField.NA_VALUE)) { - continue; - } + /** If the cvv list is empty but the dfv list is not then it is assumed this was harvested + * from an installation that had controlled vocabulary entries that don't exist in our this db + * @see Feature Request/Idea: Harvest metadata values that aren't from a list of controlled values #9992 + */ + if (dsf.getControlledVocabularyValues().isEmpty()) { + for (DatasetFieldValue dfv : dsf.getDatasetFieldValues()) { + if (dfv.getValue() == null || dfv.getValue().equals(DatasetField.NA_VALUE)) { + continue; + } + solrInputDocument.addField(solrFieldSearchable, dfv.getValue()); - // Index in all used languages (display and metadata languages - if (!dsfType.isAllowMultiples() || langs.isEmpty()) { - solrInputDocument.addField(solrFieldSearchable, controlledVocabularyValue.getStrValue()); - } else { - for(String locale: langs) { - solrInputDocument.addField(solrFieldSearchable, controlledVocabularyValue.getLocaleStrValue(locale)); + if (dsfType.getSolrField().isFacetable()) { + solrInputDocument.addField(solrFieldFacetable, dfv.getValue()); } } + } else { + for (ControlledVocabularyValue controlledVocabularyValue : dsf.getControlledVocabularyValues()) { + if (controlledVocabularyValue.getStrValue().equals(DatasetField.NA_VALUE)) { + continue; + } - if (dsfType.getSolrField().isFacetable()) { - solrInputDocument.addField(solrFieldFacetable, controlledVocabularyValue.getStrValue()); + // Index in all used languages (display and metadata languages + if (!dsfType.isAllowMultiples() || langs.isEmpty()) { + solrInputDocument.addField(solrFieldSearchable, controlledVocabularyValue.getStrValue()); + } else { + for(String locale: langs) { + solrInputDocument.addField(solrFieldSearchable, controlledVocabularyValue.getLocaleStrValue(locale)); + } + } + + if (dsfType.getSolrField().isFacetable()) { + solrInputDocument.addField(solrFieldFacetable, controlledVocabularyValue.getStrValue()); + } } } } else if (dsfType.getFieldType().equals(DatasetFieldType.FieldType.TEXTBOX)) { @@ -1112,12 +1209,13 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set accessObject = null; InputStream instream = null; ContentHandler textHandler = null; @@ -1242,6 +1350,7 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set d final SolrInputDocuments docs = toSolrDocs(indexableDataset, datafilesInDraftVersion); try { - solrClientService.getSolrClient().add(docs.getDocuments()); - solrClientService.getSolrClient().commit(); + solrClientService.getSolrClient().add(docs.getDocuments(), COMMIT_WITHIN); } catch (SolrServerException | IOException ex) { if (ex.getCause() instanceof SolrServerException) { throw new SolrServerException(ex); @@ -1475,6 +1591,7 @@ private void updateLastIndexedTimeInNewTransaction(Long id) { DvObject dvObjectToModify = em.find(DvObject.class, id); dvObjectToModify.setIndexTime(new Timestamp(new Date().getTime())); dvObjectToModify = em.merge(dvObjectToModify); + em.flush(); } /** @@ -1611,6 +1728,16 @@ private List getDataversePathsFromSegments(List dataversePathSeg return subtrees; } + private void addLicenseToSolrDoc(SolrInputDocument solrInputDocument, DatasetVersion datasetVersion) { + if (datasetVersion != null && datasetVersion.getTermsOfUseAndAccess() != null) { + String licenseName = "Custom Terms"; + if(datasetVersion.getTermsOfUseAndAccess().getLicense() != null) { + licenseName = datasetVersion.getTermsOfUseAndAccess().getLicense().getName(); + } + solrInputDocument.addField(SearchFields.DATASET_LICENSE, licenseName); + } + } + private void addDataverseReleaseDateToSolrDoc(SolrInputDocument solrInputDocument, Dataverse dataverse) { if (dataverse.getPublicationDate() != null) { Calendar calendar = Calendar.getInstance(); @@ -1687,8 +1814,7 @@ private void updatePathForExistingSolrDocs(DvObject object) throws SolrServerExc sid.removeField(SearchFields.SUBTREE); sid.addField(SearchFields.SUBTREE, paths); - UpdateResponse addResponse = solrClientService.getSolrClient().add(sid); - UpdateResponse commitResponse = solrClientService.getSolrClient().commit(); + UpdateResponse addResponse = solrClientService.getSolrClient().add(sid, COMMIT_WITHIN); if (object.isInstanceofDataset()) { for (DataFile df : dataset.getFiles()) { solrQuery.setQuery(SearchUtil.constructQuery(SearchFields.ENTITY_ID, df.getId().toString())); @@ -1701,8 +1827,7 @@ private void updatePathForExistingSolrDocs(DvObject object) throws SolrServerExc } sid.removeField(SearchFields.SUBTREE); sid.addField(SearchFields.SUBTREE, paths); - addResponse = solrClientService.getSolrClient().add(sid); - commitResponse = solrClientService.getSolrClient().commit(); + addResponse = solrClientService.getSolrClient().add(sid, COMMIT_WITHIN); } } } @@ -1744,12 +1869,7 @@ public String delete(Dataverse doomed) { logger.fine("deleting Solr document for dataverse " + doomed.getId()); UpdateResponse updateResponse; try { - updateResponse = solrClientService.getSolrClient().deleteById(solrDocIdentifierDataverse + doomed.getId()); - } catch (SolrServerException | IOException ex) { - return ex.toString(); - } - try { - solrClientService.getSolrClient().commit(); + updateResponse = solrClientService.getSolrClient().deleteById(solrDocIdentifierDataverse + doomed.getId(), COMMIT_WITHIN); } catch (SolrServerException | IOException ex) { return ex.toString(); } @@ -1769,12 +1889,7 @@ public String removeSolrDocFromIndex(String doomed) { logger.fine("deleting Solr document: " + doomed); UpdateResponse updateResponse; try { - updateResponse = solrClientService.getSolrClient().deleteById(doomed); - } catch (SolrServerException | IOException ex) { - return ex.toString(); - } - try { - solrClientService.getSolrClient().commit(); + updateResponse = solrClientService.getSolrClient().deleteById(doomed, COMMIT_WITHIN); } catch (SolrServerException | IOException ex) { return ex.toString(); } diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchConstants.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchConstants.java index 73b39332013..2d6632760fb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SearchConstants.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchConstants.java @@ -40,5 +40,5 @@ public class SearchConstants { public static final String RESTRICTED = "Restricted"; public static final String EMBARGOEDTHENRESTRICTED = "EmbargoedThenRestricted"; public static final String EMBARGOEDTHENPUBLIC = "EmbargoedThenPublic"; - + public static final String RETENTIONEXPIRED = "RetentionPeriodExpired"; } diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchFields.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchFields.java index 8fb7c161517..02649cec68c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SearchFields.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchFields.java @@ -217,6 +217,15 @@ public class SearchFields { public static final String DEFINITION_POINT_DVOBJECT_ID = "definitionPointDvObjectId"; public static final String DISCOVERABLE_BY = "discoverableBy"; + /** + * publicObject_b is an experimental field tied to the + * avoid-expensive-solr-join feature flag. Rather than discoverableBy which + * is a field on permission documents, publicObject_b is a field on content + * documents (dvObjects). By indexing publicObject_b=true, we can let guests + * search on it, avoiding an expensive join for those (common) users. + */ + public static final String PUBLIC_OBJECT = "publicObject_b"; + /** * i.e. "Unpublished", "Draft" (multivalued) */ @@ -267,7 +276,7 @@ more targeted results for just datasets. The format is YYYY (i.e. public static final String FULL_TEXT = "_text_"; public static final String EMBARGO_END_DATE = "embargoEndDate"; - + public static final String RETENTION_END_DATE = "retentionEndDate"; // SpatialRecursivePrefixTreeFieldType: https://solr.apache.org/guide/8_11/spatial-search.html#rpt public static final String GEOLOCATION = "geolocation"; @@ -276,4 +285,6 @@ more targeted results for just datasets. The format is YYYY (i.e. public static final String DATASET_VALID = "datasetValid"; + public static final String DATASET_LICENSE = "license"; + } diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java index 5a5d8781726..ee0f74564f4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java @@ -355,8 +355,7 @@ The real issue here (https://github.com/IQSS/dataverse/issues/7304) is caused * https://github.com/IQSS/dataverse/issues/84 */ int numRows = 10; - HttpServletRequest httpServletRequest = (HttpServletRequest) FacesContext.getCurrentInstance().getExternalContext().getRequest(); - DataverseRequest dataverseRequest = new DataverseRequest(session.getUser(), httpServletRequest); + DataverseRequest dataverseRequest = getDataverseRequest(); List dataverses = new ArrayList<>(); dataverses.add(dataverse); solrQueryResponse = searchService.search(dataverseRequest, dataverses, queryToPassToSolr, filterQueriesFinal, sortField, sortOrder.toString(), paginationStart, onlyDataRelatedToMe, numRows, false, null, null, !isFacetsDisabled(), true); @@ -1243,6 +1242,12 @@ public List getFriendlyNamesFromFilterQuery(String filterQuery) { friendlyNames.add(friendlyName.get()); return friendlyNames; } + } else if (key.equals(SearchFields.DATASET_LICENSE)) { + try { + friendlyNames.add(BundleUtil.getStringFromPropertyFile("license." + valueWithoutQuotes.toLowerCase().replace(" ","_") + ".name", "License")); + } catch (Exception e) { + logger.fine(String.format("action=getFriendlyNamesFromFilterQuery cannot find friendlyName for key=%s value=%s", key, value)); + } } friendlyNames.add(valueWithoutQuotes); @@ -1474,9 +1479,31 @@ public boolean isActivelyEmbargoed(SolrSearchResult result) { return false; } } + + public boolean isRetentionExpired(SolrSearchResult result) { + Long retentionEndDate = result.getRetentionEndDate(); + if(retentionEndDate != null) { + return LocalDate.now().toEpochDay() > retentionEndDate; + } else { + return false; + } + } + private DataverseRequest getDataverseRequest() { + final HttpServletRequest httpServletRequest = (HttpServletRequest) FacesContext.getCurrentInstance().getExternalContext().getRequest(); + return new DataverseRequest(session.getUser(), httpServletRequest); + } + public boolean isValid(SolrSearchResult result) { - return result.isValid(); + return result.isValid(x -> { + Long id = x.getEntityId(); + DvObject obj = dvObjectService.findDvObject(id); + if(obj != null && obj instanceof Dataset) { + return permissionsWrapper.canUpdateDataset(getDataverseRequest(), (Dataset) obj); + } + logger.fine("isValid called for dvObject that is null (or not a dataset), id: " + id + "This can occur if a dataset is deleted while a search is in progress"); + return true; + }); } public enum SortOrder { diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java index 51bf3bee30b..3d6a563da58 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java @@ -16,6 +16,7 @@ import edu.harvard.iq.dataverse.authorization.users.PrivateUrlUser; import edu.harvard.iq.dataverse.authorization.users.User; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.settings.FeatureFlags; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.SystemConfig; import java.io.IOException; @@ -187,19 +188,11 @@ public SolrQueryResponse search( SolrQuery solrQuery = new SolrQuery(); query = SearchUtil.sanitizeQuery(query); solrQuery.setQuery(query); -// SortClause foo = new SortClause("name", SolrQuery.ORDER.desc); -// if (query.equals("*") || query.equals("*:*")) { -// solrQuery.setSort(new SortClause(SearchFields.NAME_SORT, SolrQuery.ORDER.asc)); if (sortField != null) { // is it ok not to specify any sort? - there are cases where we // don't care, and it must cost some extra cycles -- L.A. solrQuery.setSort(new SortClause(sortField, sortOrder)); } -// } else { -// solrQuery.setSort(sortClause); -// } -// solrQuery.setSort(sortClause); - solrQuery.setParam("fl", "*,score"); solrQuery.setParam("qt", "/select"); @@ -224,6 +217,9 @@ public SolrQueryResponse search( List metadataBlockFacets = new LinkedList<>(); if (addFacets) { + + + // ----------------------------------- // Facets to Retrieve // ----------------------------------- @@ -231,6 +227,14 @@ public SolrQueryResponse search( solrQuery.addFacetField(SearchFields.DATAVERSE_CATEGORY); solrQuery.addFacetField(SearchFields.METADATA_SOURCE); solrQuery.addFacetField(SearchFields.PUBLICATION_YEAR); + /* + * We talked about this in slack on 2021-09-14, Users can see objects on draft/unpublished + * if the owner gives permissions to all users so it makes sense to expose this facet + * to all users. The request of this change started because the order of the facets were + * changed with the PR #9635 and this was unintended. + */ + solrQuery.addFacetField(SearchFields.PUBLICATION_STATUS); + solrQuery.addFacetField(SearchFields.DATASET_LICENSE); /** * @todo when a new method on datasetFieldService is available * (retrieveFacetsByDataverse?) only show the facets that the @@ -250,6 +254,7 @@ public SolrQueryResponse search( DatasetFieldType datasetField = dataverseFacet.getDatasetFieldType(); solrQuery.addFacetField(datasetField.getSolrField().getNameFacetable()); } + // Get all metadata block facets configured to be displayed metadataBlockFacets.addAll(dataverse.getMetadataBlockFacets()); } @@ -495,6 +500,8 @@ public SolrQueryResponse search( String identifierOfDataverse = (String) solrDocument.getFieldValue(SearchFields.IDENTIFIER_OF_DATAVERSE); String nameOfDataverse = (String) solrDocument.getFieldValue(SearchFields.DATAVERSE_NAME); Long embargoEndDate = (Long) solrDocument.getFieldValue(SearchFields.EMBARGO_END_DATE); + Long retentionEndDate = (Long) solrDocument.getFieldValue(SearchFields.RETENTION_END_DATE); + // Boolean datasetValid = (Boolean) solrDocument.getFieldValue(SearchFields.DATASET_VALID); List matchedFields = new ArrayList<>(); @@ -576,7 +583,8 @@ public SolrQueryResponse search( } solrSearchResult.setEmbargoEndDate(embargoEndDate); - + solrSearchResult.setRetentionEndDate(retentionEndDate); + /** * @todo start using SearchConstants class here */ @@ -712,10 +720,12 @@ public SolrQueryResponse search( boolean unpublishedAvailable = false; boolean deaccessionedAvailable = false; boolean hideMetadataSourceFacet = true; + boolean hideLicenseFacet = true; for (FacetField facetField : queryResponse.getFacetFields()) { FacetCategory facetCategory = new FacetCategory(); List facetLabelList = new ArrayList<>(); int numMetadataSources = 0; + int numLicenses = 0; String metadataBlockName = ""; String datasetFieldName = ""; /** @@ -741,23 +751,29 @@ public SolrQueryResponse search( // logger.info("field: " + facetField.getName() + " " + facetFieldCount.getName() + " (" + facetFieldCount.getCount() + ")"); String localefriendlyName = null; if (facetFieldCount.getCount() > 0) { - if(metadataBlockName.length() > 0 ) { - localefriendlyName = getLocaleTitle(datasetFieldName,facetFieldCount.getName(), metadataBlockName); + if(metadataBlockName.length() > 0 ) { + localefriendlyName = getLocaleTitle(datasetFieldName,facetFieldCount.getName(), metadataBlockName); } else if (facetField.getName().equals(SearchFields.METADATA_TYPES)) { - Optional metadataBlockFacet = metadataBlockFacets.stream().filter(blockFacet -> blockFacet.getMetadataBlock().getName().equals(facetFieldCount.getName())).findFirst(); - if (metadataBlockFacet.isEmpty()) { + Optional metadataBlockFacet = metadataBlockFacets.stream().filter(blockFacet -> blockFacet.getMetadataBlock().getName().equals(facetFieldCount.getName())).findFirst(); + if (metadataBlockFacet.isEmpty()) { // metadata block facet is not configured to be displayed => ignore continue; - } + } - localefriendlyName = metadataBlockFacet.get().getMetadataBlock().getLocaleDisplayFacet(); - } else { - try { + localefriendlyName = metadataBlockFacet.get().getMetadataBlock().getLocaleDisplayFacet(); + } else if (facetField.getName().equals(SearchFields.DATASET_LICENSE)) { + try { + localefriendlyName = BundleUtil.getStringFromPropertyFile("license." + facetFieldCount.getName().toLowerCase().replace(" ","_") + ".name", "License"); + } catch (Exception e) { + localefriendlyName = facetFieldCount.getName(); + } + } else { + try { localefriendlyName = BundleUtil.getStringFromPropertyFile(facetFieldCount.getName(), "Bundle"); - } catch (Exception e) { + } catch (Exception e) { localefriendlyName = facetFieldCount.getName(); - } - } + } + } FacetLabel facetLabel = new FacetLabel(localefriendlyName, facetFieldCount.getCount()); // quote field facets facetLabel.setFilterQuery(facetField.getName() + ":\"" + facetFieldCount.getName() + "\""); @@ -770,15 +786,19 @@ public SolrQueryResponse search( } else if (facetFieldCount.getName().equals(IndexServiceBean.getDEACCESSIONED_STRING())) { deaccessionedAvailable = true; } - } - if (facetField.getName().equals(SearchFields.METADATA_SOURCE)) { + } else if (facetField.getName().equals(SearchFields.METADATA_SOURCE)) { numMetadataSources++; + } else if (facetField.getName().equals(SearchFields.DATASET_LICENSE)) { + numLicenses++; } } } if (numMetadataSources > 1) { hideMetadataSourceFacet = false; } + if (numLicenses > 1) { + hideLicenseFacet = false; + } facetCategory.setName(facetField.getName()); // hopefully people will never see the raw facetField.getName() because it may well have an _s at the end facetCategory.setFriendlyName(facetField.getName()); @@ -855,6 +875,10 @@ public SolrQueryResponse search( if (!hideMetadataSourceFacet) { facetCategoryList.add(facetCategory); } + } else if (facetCategory.getName().equals(SearchFields.DATASET_LICENSE)) { + if (!hideLicenseFacet) { + facetCategoryList.add(facetCategory); + } } else { facetCategoryList.add(facetCategory); } @@ -978,14 +1002,132 @@ private String getPermissionFilterQuery(DataverseRequest dataverseRequest, SolrQ user = GuestUser.get(); } + AuthenticatedUser au = null; + Set groups; + + if (user instanceof GuestUser) { + // Yes, GuestUser may be part of one or more groups; such as IP Groups. + groups = groupService.collectAncestors(groupService.groupsFor(dataverseRequest)); + } else { + if (!(user instanceof AuthenticatedUser)) { + logger.severe("Should never reach here. A User must be an AuthenticatedUser or a Guest"); + throw new IllegalStateException("A User must be an AuthenticatedUser or a Guest"); + } + + au = (AuthenticatedUser) user; + + // ---------------------------------------------------- + // (3) Is this a Super User? + // If so, they can see everything + // ---------------------------------------------------- + if (au.isSuperuser()) { + // Somewhat dangerous because this user (a superuser) will be able + // to see everything in Solr with no regard to permissions. But it's + // been this way since Dataverse 4.0. So relax. :) + + return dangerZoneNoSolrJoin; + } + + // ---------------------------------------------------- + // (4) User is logged in AND onlyDatatRelatedToMe == true + // Yes, give back everything -> the settings will be in + // the filterqueries given to search + // ---------------------------------------------------- + if (onlyDatatRelatedToMe == true) { + if (systemConfig.myDataDoesNotUsePermissionDocs()) { + logger.fine("old 4.2 behavior: MyData is not using Solr permission docs"); + return dangerZoneNoSolrJoin; + } else { + // fall-through + logger.fine("new post-4.2 behavior: MyData is using Solr permission docs"); + } + } + + // ---------------------------------------------------- + // (5) Work with Authenticated User who is not a Superuser + // ---------------------------------------------------- + + groups = groupService.collectAncestors(groupService.groupsFor(dataverseRequest)); + } + + if (FeatureFlags.AVOID_EXPENSIVE_SOLR_JOIN.enabled()) { + /** + * Instead of doing a super expensive join, we will rely on the + * new boolean field PublicObject:true for public objects. This field + * is indexed on the content document itself, rather than a permission + * document. An additional join will be added only for any extra, + * more restricted groups that the user may be part of. + * **Note the experimental nature of this optimization**. + */ + StringBuilder sb = new StringBuilder(); + StringBuilder sbgroups = new StringBuilder(); + + // All users, guests and authenticated, should see all the + // documents marked as publicObject_b:true, at least: + sb.append(SearchFields.PUBLIC_OBJECT + ":" + true); + + // One or more groups *may* also be available for this user. Once again, + // do note that Guest users may be part of some groups, such as + // IP groups. + + int groupCounter = 0; + + // An AuthenticatedUser should also be able to see all the content + // on which they have direct permissions: + if (au != null) { + groupCounter++; + sbgroups.append(IndexServiceBean.getGroupPerUserPrefix() + au.getId()); + } + + // In addition to the user referenced directly, we will also + // add joins on all the non-public groups that may exist for the + // user: + for (Group group : groups) { + String groupAlias = group.getAlias(); + if (groupAlias != null && !groupAlias.isEmpty() && !groupAlias.startsWith("builtIn")) { + groupCounter++; + if (groupCounter > 1) { + sbgroups.append(" OR "); + } + sbgroups.append(IndexServiceBean.getGroupPrefix() + groupAlias); + } + } + + if (groupCounter > 1) { + // If there is more than one group, the parentheses must be added: + sbgroups.insert(0, "("); + sbgroups.append(")"); + } + + if (groupCounter > 0) { + // If there are any groups for this user, an extra join must be + // added to the query, and the extra sub-query must be added to + // the combined Solr query: + sb.append(" OR {!join from=" + SearchFields.DEFINITION_POINT + " to=id v=$q1}"); + // Add the subquery to the combined Solr query: + solrQuery.setParam("q1", SearchFields.DISCOVERABLE_BY + ":" + sbgroups.toString()); + logger.info("The sub-query q1 set to " + SearchFields.DISCOVERABLE_BY + ":" + sbgroups.toString()); + } + + String ret = sb.toString(); + logger.info("Returning experimental query: " + ret); + return ret; + } + + // END OF EXPERIMENTAL OPTIMIZATION + + // Old, un-optimized way of handling permissions. + // Largely left intact, minus the lookups that have already been performed + // above. + // ---------------------------------------------------- // (1) Is this a GuestUser? - // Yes, see if GuestUser is part of any groups such as IP Groups. // ---------------------------------------------------- if (user instanceof GuestUser) { - String groupsFromProviders = ""; - Set groups = groupService.collectAncestors(groupService.groupsFor(dataverseRequest)); + StringBuilder sb = new StringBuilder(); + + String groupsFromProviders = ""; for (Group group : groups) { logger.fine("found group " + group.getIdentifier() + " with alias " + group.getAlias()); String groupAlias = group.getAlias(); @@ -1002,51 +1144,11 @@ private String getPermissionFilterQuery(DataverseRequest dataverseRequest, SolrQ return guestWithGroups; } - // ---------------------------------------------------- - // (2) Retrieve Authenticated User - // ---------------------------------------------------- - if (!(user instanceof AuthenticatedUser)) { - logger.severe("Should never reach here. A User must be an AuthenticatedUser or a Guest"); - throw new IllegalStateException("A User must be an AuthenticatedUser or a Guest"); - } - - AuthenticatedUser au = (AuthenticatedUser) user; - - if (addFacets) { - // Logged in user, has publication status facet - // - solrQuery.addFacetField(SearchFields.PUBLICATION_STATUS); - } - - // ---------------------------------------------------- - // (3) Is this a Super User? - // Yes, give back everything - // ---------------------------------------------------- - if (au.isSuperuser()) { - // Somewhat dangerous because this user (a superuser) will be able - // to see everything in Solr with no regard to permissions. But it's - // been this way since Dataverse 4.0. So relax. :) - - return dangerZoneNoSolrJoin; - } - - // ---------------------------------------------------- - // (4) User is logged in AND onlyDatatRelatedToMe == true - // Yes, give back everything -> the settings will be in - // the filterqueries given to search - // ---------------------------------------------------- - if (onlyDatatRelatedToMe == true) { - if (systemConfig.myDataDoesNotUsePermissionDocs()) { - logger.fine("old 4.2 behavior: MyData is not using Solr permission docs"); - return dangerZoneNoSolrJoin; - } else { - logger.fine("new post-4.2 behavior: MyData is using Solr permission docs"); - } - } - // ---------------------------------------------------- // (5) Work with Authenticated User who is not a Superuser - // ---------------------------------------------------- + // ---------------------------------------------------- + // It was already confirmed, that if the user is not GuestUser, we + // have an AuthenticatedUser au which is not null. /** * @todo all this code needs cleanup and clarification. */ @@ -1077,7 +1179,6 @@ private String getPermissionFilterQuery(DataverseRequest dataverseRequest, SolrQ * a given "content document" (dataset version, etc) in Solr. */ String groupsFromProviders = ""; - Set groups = groupService.collectAncestors(groupService.groupsFor(dataverseRequest)); StringBuilder sb = new StringBuilder(); for (Group group : groups) { logger.fine("found group " + group.getIdentifier() + " with alias " + group.getAlias()); diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java index 04021eb75b6..19235bb5a14 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java @@ -356,8 +356,7 @@ private void persistToSolr(Collection docs) throws SolrServer /** * @todo Do something with these responses from Solr. */ - UpdateResponse addResponse = solrClientService.getSolrClient().add(docs); - UpdateResponse commitResponse = solrClientService.getSolrClient().commit(); + UpdateResponse addResponse = solrClientService.getSolrClient().add(docs, IndexServiceBean.COMMIT_WITHIN); } public IndexResponse indexPermissionsOnSelfAndChildren(long definitionPointId) { @@ -497,26 +496,20 @@ public IndexResponse deleteMultipleSolrIds(List solrIdsToDelete) { return new IndexResponse("nothing to delete"); } try { - solrClientService.getSolrClient().deleteById(solrIdsToDelete); + solrClientService.getSolrClient().deleteById(solrIdsToDelete, IndexServiceBean.COMMIT_WITHIN); } catch (SolrServerException | IOException ex) { /** * @todo mark these for re-deletion */ return new IndexResponse("problem deleting the following documents from Solr: " + solrIdsToDelete); } - try { - solrClientService.getSolrClient().commit(); - } catch (SolrServerException | IOException ex) { - return new IndexResponse("problem committing deletion of the following documents from Solr: " + solrIdsToDelete); - } return new IndexResponse("no known problem deleting the following documents from Solr:" + solrIdsToDelete); } public JsonObjectBuilder deleteAllFromSolrAndResetIndexTimes() throws SolrServerException, IOException { JsonObjectBuilder response = Json.createObjectBuilder(); logger.info("attempting to delete all Solr documents before a complete re-index"); - solrClientService.getSolrClient().deleteByQuery("*:*"); - solrClientService.getSolrClient().commit(); + solrClientService.getSolrClient().deleteByQuery("*:*", IndexServiceBean.COMMIT_WITHIN); int numRowsAffected = dvObjectService.clearAllIndexTimes(); response.add(numRowsClearedByClearAllIndexTimes, numRowsAffected); response.add(messageString, "Solr index and database index timestamps cleared."); diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java index 6ad7f9dbbf6..e84c8f133da 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java @@ -7,8 +7,10 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Predicate; import java.util.logging.Logger; +import edu.harvard.iq.dataverse.*; import jakarta.json.Json; import jakarta.json.JsonArrayBuilder; import jakarta.json.JsonObject; @@ -16,22 +18,14 @@ import org.apache.commons.collections4.CollectionUtils; -import edu.harvard.iq.dataverse.DataFile; -import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.DatasetField; -import edu.harvard.iq.dataverse.DatasetRelPublication; -import edu.harvard.iq.dataverse.DatasetVersion; -import edu.harvard.iq.dataverse.DvObject; -import edu.harvard.iq.dataverse.GlobalId; -import edu.harvard.iq.dataverse.MetadataBlock; import edu.harvard.iq.dataverse.api.Util; import edu.harvard.iq.dataverse.dataset.DatasetThumbnail; +import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.util.DateUtil; import edu.harvard.iq.dataverse.util.json.JsonPrinter; import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder; public class SolrSearchResult { - private static final Logger logger = Logger.getLogger(SolrSearchResult.class.getCanonicalName()); private String id; @@ -123,6 +117,8 @@ public class SolrSearchResult { private Long embargoEndDate; + private Long retentionEndDate; + private boolean datasetValid; public String getDvTree() { @@ -403,21 +399,12 @@ public JsonArrayBuilder getRelevance() { return matchedFieldsArray; } - public JsonObject toJsonObject(boolean showRelevance, boolean showEntityIds, boolean showApiUrls) { - return toJsonObject(showRelevance, showEntityIds, showApiUrls, null); - } - - public JsonObject toJsonObject(boolean showRelevance, boolean showEntityIds, boolean showApiUrls, - List metadataFields) { - return json(showRelevance, showEntityIds, showApiUrls, metadataFields).build(); - } - /** * Add additional fields for the MyData page * * @return */ - public JsonObjectBuilder getJsonForMyData() { + public JsonObjectBuilder getJsonForMyData(boolean isValid) { JsonObjectBuilder myDataJson = json(true, true, true);// boolean showRelevance, boolean showEntityIds, boolean showApiUrls) @@ -425,7 +412,7 @@ public JsonObjectBuilder getJsonForMyData() { .add("is_draft_state", this.isDraftState()).add("is_in_review_state", this.isInReviewState()) .add("is_unpublished_state", this.isUnpublishedState()).add("is_published", this.isPublishedState()) .add("is_deaccesioned", this.isDeaccessionedState()) - .add("is_valid", this.isValid()) + .add("is_valid", isValid) .add("date_to_display_on_card", getDateToDisplayOnCard()); // Add is_deaccessioned attribute, even though MyData currently screens any deaccessioned info out @@ -436,7 +423,7 @@ public JsonObjectBuilder getJsonForMyData() { if ((this.getParent() != null) && (!this.getParent().isEmpty())) { // System.out.println("keys:" + parent.keySet().toString()); - if (this.entity.isInstanceofDataFile()) { + if (this.entity != null && this.entity.isInstanceofDataFile()) { myDataJson.add("parentIdentifier", this.getParent().get(SolrSearchResult.PARENT_IDENTIFIER)) .add("parentName", this.getParent().get("name")); @@ -450,12 +437,10 @@ public JsonObjectBuilder getJsonForMyData() { } // getJsonForMydata public JsonObjectBuilder json(boolean showRelevance, boolean showEntityIds, boolean showApiUrls) { - return json(showRelevance, showEntityIds, showApiUrls, null); + return json(showRelevance, showEntityIds, showApiUrls, null, null); } - public JsonObjectBuilder json(boolean showRelevance, boolean showEntityIds, boolean showApiUrls, - List metadataFields) { - + public JsonObjectBuilder json(boolean showRelevance, boolean showEntityIds, boolean showApiUrls, List metadataFields, Long datasetFileCount) { if (this.type == null) { return jsonObjectBuilder(); } @@ -571,7 +556,7 @@ public JsonObjectBuilder json(boolean showRelevance, boolean showEntityIds, bool subjects.add(subject); } nullSafeJsonBuilder.add("subjects", subjects); - nullSafeJsonBuilder.add("fileCount", dv.getFileMetadatas().size()); + nullSafeJsonBuilder.add("fileCount", datasetFileCount); nullSafeJsonBuilder.add("versionId", dv.getId()); nullSafeJsonBuilder.add("versionState", dv.getVersionState().toString()); if (this.isPublishedState()) { @@ -1261,11 +1246,31 @@ public void setEmbargoEndDate(Long embargoEndDate) { this.embargoEndDate = embargoEndDate; } + public Long getRetentionEndDate() { + return retentionEndDate; + } + + public void setRetentionEndDate(Long retentionEndDate) { + this.retentionEndDate = retentionEndDate; + } + public void setDatasetValid(Boolean datasetValid) { this.datasetValid = datasetValid == null || Boolean.valueOf(datasetValid); } - public boolean isValid() { - return datasetValid; + public boolean isValid(Predicate canUpdateDataset) { + if (this.datasetValid) { + return true; + } + if (!this.getType().equals("datasets")) { + return true; + } + if (this.isDraftState()) { + return false; + } + if (!JvmSettings.UI_SHOW_VALIDITY_LABEL_WHEN_PUBLISHED.lookupOptional(Boolean.class).orElse(true)) { + return true; + } + return !canUpdateDataset.test(this); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/ConfigCheckService.java b/src/main/java/edu/harvard/iq/dataverse/settings/ConfigCheckService.java index a2c3f53d59d..29a9d8956a3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/ConfigCheckService.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/ConfigCheckService.java @@ -1,25 +1,37 @@ package edu.harvard.iq.dataverse.settings; +import edu.harvard.iq.dataverse.MailServiceBean; +import edu.harvard.iq.dataverse.pidproviders.PidUtil; +import edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key; import edu.harvard.iq.dataverse.util.FileUtil; - +import edu.harvard.iq.dataverse.util.MailSessionProducer; import jakarta.annotation.PostConstruct; import jakarta.ejb.DependsOn; import jakarta.ejb.Singleton; import jakarta.ejb.Startup; +import jakarta.inject.Inject; +import jakarta.mail.internet.InternetAddress; + import java.io.IOException; import java.nio.file.FileSystemException; import java.nio.file.Files; import java.nio.file.Path; import java.util.Map; +import java.util.Optional; import java.util.logging.Level; import java.util.logging.Logger; @Startup @Singleton -@DependsOn("StartupFlywayMigrator") +@DependsOn({"StartupFlywayMigrator", "PidProviderFactoryBean"}) public class ConfigCheckService { private static final Logger logger = Logger.getLogger(ConfigCheckService.class.getCanonicalName()); + + @Inject + MailSessionProducer mailSessionProducer; + @Inject + MailServiceBean mailService; public static class ConfigurationError extends RuntimeException { public ConfigurationError(String message) { @@ -29,11 +41,14 @@ public ConfigurationError(String message) { @PostConstruct public void startup() { - if (!checkSystemDirectories()) { + if (!checkSystemDirectories() || !checkPidProviders()) { throw new ConfigurationError("Not all configuration checks passed successfully. See logs above."); } + + // Only checks resulting in warnings, nothing critical that needs to stop deployment + checkSystemMailSetup(); } - + /** * In this method, we check the existence and write-ability of all important directories we use during * normal operations. It does not include checks for the storage system. If directories are not available, @@ -77,5 +92,46 @@ public boolean checkSystemDirectories() { } return success; } + + /** + * This method is not expected to make a deployment fail, but send out clear warning messages about missing or + * wrong configuration settings. + */ + public void checkSystemMailSetup() { + // Check if a system mail setting has been provided or issue warning about disabled mail notifications + Optional mailAddress = mailService.getSystemAddress(); + + // Not present -> warning + if (mailAddress.isEmpty()) { + logger.warning("Could not find a system mail setting in database (key :" + Key.SystemEmail + ", deprecated) or JVM option '" + JvmSettings.SYSTEM_EMAIL.getScopedKey() + "'"); + logger.warning("Mail notifications and system messages are deactivated until you provide a configuration"); + } + + // If there is an app server provided mail config, let's determine if the setup is matching + // TODO: when support for appserver provided mail session goes away, this code can be deleted + if (mailSessionProducer.hasSessionFromAppServer()) { + if (mailAddress.isEmpty()) { + logger.warning("Found a mail session provided by app server, but no system mail address (see logs above)"); + // Check if the "from" in the session is the same as the system mail address (see issue 4210) + } else { + String sessionFrom = mailSessionProducer.getSession().getProperty("mail.from"); + if (! mailAddress.get().toString().equals(sessionFrom)) { + logger.warning(() -> String.format( + "Found app server mail session provided 'from' (%s) does not match system mail setting (%s)", + sessionFrom, mailAddress.get())); + } + } + } + } + /** + * Verifies that at least one PidProvider capable of editing/minting PIDs is + * configured. Requires the @DependsOn("PidProviderFactoryBean") annotation above + * since it is the @PostCOnstruct init() method of that class that loads the PidProviders + * + * @return True if all checks successful, false otherwise. + */ + private boolean checkPidProviders() { + return PidUtil.getManagedProviderIds().size() > 0; + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java b/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java index afa5a1c986a..14a7ab86f22 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java @@ -36,6 +36,28 @@ public enum FeatureFlags { * @since Dataverse @TODO: */ API_BEARER_AUTH("api-bearer-auth"), + /** + * For published (public) objects, don't use a join when searching Solr. + * Experimental! Requires a reindex with the following feature flag enabled, + * in order to add the boolean publicObject_b:true field to all the public + * Solr documents. + * + * @apiNote Raise flag by setting + * "dataverse.feature.avoid-expensive-solr-join" + * @since Dataverse 6.3 + */ + AVOID_EXPENSIVE_SOLR_JOIN("avoid-expensive-solr-join"), + /** + * With this flag enabled, the boolean field publicObject_b:true will be + * added to all the indexed Solr documents for publicly-available collections, + * datasets and files. This flag makes it possible to rely on it in searches, + * instead of the very expensive join (the feature flag above). + * + * @apiNote Raise flag by setting + * "dataverse.feature.add-publicobject-solr-field" + * @since Dataverse 6.3 + */ + ADD_PUBLICOBJECT_SOLR_FIELD("add-publicobject-solr-field"), ; final String flag; diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java index 3bc06738a7e..9d13be005c9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java @@ -51,6 +51,10 @@ public enum JvmSettings { DOCROOT_DIRECTORY(SCOPE_FILES, "docroot"), GUESTBOOK_AT_REQUEST(SCOPE_FILES, "guestbook-at-request"), GLOBUS_CACHE_MAXAGE(SCOPE_FILES, "globus-cache-maxage"), + + //STORAGE DRIVER SETTINGS + SCOPE_DRIVER(SCOPE_FILES), + DISABLE_S3_TAGGING(SCOPE_DRIVER, "disable-tagging"), // SOLR INDEX SETTINGS SCOPE_SOLR(PREFIX, "solr"), @@ -60,6 +64,10 @@ public enum JvmSettings { SOLR_CORE(SCOPE_SOLR, "core"), SOLR_PATH(SCOPE_SOLR, "path"), + // INDEX CONCURENCY + SCOPE_SOLR_CONCURENCY(SCOPE_SOLR, "concurrency"), + MAX_ASYNC_INDEXES(SCOPE_SOLR_CONCURENCY, "max-async-indexes"), + // RSERVE CONNECTION SCOPE_RSERVE(PREFIX, "rserve"), RSERVE_HOST(SCOPE_RSERVE, "host"), @@ -91,40 +99,116 @@ public enum JvmSettings { // PERSISTENT IDENTIFIER SETTINGS SCOPE_PID(PREFIX, "pid"), - - // PROVIDER EZID (legacy) - these settings were formerly kept together with DataCite ones - SCOPE_PID_EZID(SCOPE_PID, "ezid"), - EZID_API_URL(SCOPE_PID_EZID, "api-url", "doi.baseurlstring"), - EZID_USERNAME(SCOPE_PID_EZID, "username", "doi.username"), - EZID_PASSWORD(SCOPE_PID_EZID, "password", "doi.password"), + PID_PROVIDERS(SCOPE_PID, "providers"), + PID_DEFAULT_PROVIDER(SCOPE_PID, "default-provider"), + SCOPE_PID_PROVIDER(SCOPE_PID), + PID_PROVIDER_TYPE(SCOPE_PID_PROVIDER, "type"), + PID_PROVIDER_LABEL(SCOPE_PID_PROVIDER, "label"), + PID_PROVIDER_AUTHORITY(SCOPE_PID_PROVIDER, "authority"), + PID_PROVIDER_SHOULDER(SCOPE_PID_PROVIDER, "shoulder"), + PID_PROVIDER_IDENTIFIER_GENERATION_STYLE(SCOPE_PID_PROVIDER, "identifier-generation-style"), + PID_PROVIDER_DATAFILE_PID_FORMAT(SCOPE_PID_PROVIDER, "datafile-pid-format"), + PID_PROVIDER_MANAGED_LIST(SCOPE_PID_PROVIDER, "managed-list"), + PID_PROVIDER_EXCLUDED_LIST(SCOPE_PID_PROVIDER, "excluded-list"), + + + // PROVIDER EZID - these settings were formerly kept together with DataCite ones + SCOPE_PID_EZID(SCOPE_PID_PROVIDER, "ezid"), + EZID_API_URL(SCOPE_PID_EZID, "api-url"), + EZID_USERNAME(SCOPE_PID_EZID, "username"), + EZID_PASSWORD(SCOPE_PID_EZID, "password"), // PROVIDER DATACITE - SCOPE_PID_DATACITE(SCOPE_PID, "datacite"), - DATACITE_MDS_API_URL(SCOPE_PID_DATACITE, "mds-api-url", "doi.baseurlstring"), - DATACITE_REST_API_URL(SCOPE_PID_DATACITE, "rest-api-url", "doi.dataciterestapiurlstring", "doi.mdcbaseurlstring"), - DATACITE_USERNAME(SCOPE_PID_DATACITE, "username", "doi.username"), - DATACITE_PASSWORD(SCOPE_PID_DATACITE, "password", "doi.password"), + SCOPE_PID_DATACITE(SCOPE_PID_PROVIDER, "datacite"), + DATACITE_MDS_API_URL(SCOPE_PID_DATACITE, "mds-api-url"), + DATACITE_REST_API_URL(SCOPE_PID_DATACITE, "rest-api-url"), + DATACITE_USERNAME(SCOPE_PID_DATACITE, "username"), + DATACITE_PASSWORD(SCOPE_PID_DATACITE, "password"), // PROVIDER PERMALINK - SCOPE_PID_PERMALINK(SCOPE_PID, "permalink"), - PERMALINK_BASEURL(SCOPE_PID_PERMALINK, "base-url", "perma.baseurlstring"), + SCOPE_PID_PERMALINK(SCOPE_PID_PROVIDER, "permalink"), + PERMALINK_BASE_URL(SCOPE_PID_PERMALINK, "base-url"), + PERMALINK_SEPARATOR(SCOPE_PID_PERMALINK, "separator"), // PROVIDER HANDLE - SCOPE_PID_HANDLENET(SCOPE_PID, "handlenet"), - HANDLENET_INDEX(SCOPE_PID_HANDLENET, "index", "dataverse.handlenet.index"), + SCOPE_PID_HANDLENET(SCOPE_PID_PROVIDER, "handlenet"), + HANDLENET_INDEX(SCOPE_PID_HANDLENET, "index"), + HANDLENET_INDEPENDENT_SERVICE(SCOPE_PID_HANDLENET, "independent-service"), + HANDLENET_AUTH_HANDLE(SCOPE_PID_HANDLENET, "auth-handle"), SCOPE_PID_HANDLENET_KEY(SCOPE_PID_HANDLENET, "key"), - HANDLENET_KEY_PATH(SCOPE_PID_HANDLENET_KEY, "path", "dataverse.handlenet.admcredfile"), - HANDLENET_KEY_PASSPHRASE(SCOPE_PID_HANDLENET_KEY, "passphrase", "dataverse.handlenet.admprivphrase"), + HANDLENET_KEY_PATH(SCOPE_PID_HANDLENET_KEY, "path"), + HANDLENET_KEY_PASSPHRASE(SCOPE_PID_HANDLENET_KEY, "passphrase"), + + /* + * The deprecated legacy settings below are from when you could only have a + * single PIDProvider. They mirror the settings above, but are global,not within + * the SCOPE_PID_PROVIDER of an individual provider. + */ + /** + * DEPRECATED PROVIDER DATACITE + * + * @deprecated - legacy single provider setting providing backward compatibility + */ + @Deprecated(forRemoval = true, since = "2024-02-13") + SCOPE_LEGACY_PID_DATACITE(SCOPE_PID, "datacite"), + LEGACY_DATACITE_MDS_API_URL(SCOPE_LEGACY_PID_DATACITE, "mds-api-url", "doi.baseurlstring"), + LEGACY_DATACITE_REST_API_URL(SCOPE_LEGACY_PID_DATACITE, "rest-api-url", "doi.dataciterestapiurlstring", + "doi.mdcbaseurlstring"), + LEGACY_DATACITE_USERNAME(SCOPE_LEGACY_PID_DATACITE, "username", "doi.username"), + LEGACY_DATACITE_PASSWORD(SCOPE_LEGACY_PID_DATACITE, "password", "doi.password"), + + /** + * DEPRECATED PROVIDER EZID + * + * @deprecated - legacy single provider setting providing backward compatibility + */ + @Deprecated(forRemoval = true, since = "2024-02-13") + SCOPE_LEGACY_PID_EZID(SCOPE_PID, "ezid"), LEGACY_EZID_API_URL(SCOPE_LEGACY_PID_EZID, "api-url"), + LEGACY_EZID_USERNAME(SCOPE_LEGACY_PID_EZID, "username"), LEGACY_EZID_PASSWORD(SCOPE_LEGACY_PID_EZID, "password"), + + /** + * DEPRECATED PROVIDER PERMALINK + * + * @deprecated - legacy single provider setting providing backward compatibility + */ + @Deprecated(forRemoval = true, since = "2024-02-13") + SCOPE_LEGACY_PID_PERMALINK(SCOPE_PID, "permalink"), + LEGACY_PERMALINK_BASEURL(SCOPE_LEGACY_PID_PERMALINK, "base-url", "perma.baseurlstring"), + + /** + * DEPRECATED PROVIDER HANDLE + * + * @deprecated - legacy single provider setting providing backward compatibility + */ + @Deprecated(forRemoval = true, since = "2024-02-13") + SCOPE_LEGACY_PID_HANDLENET(SCOPE_PID, "handlenet"), + LEGACY_HANDLENET_INDEX(SCOPE_LEGACY_PID_HANDLENET, "index", "dataverse.handlenet.index"), + @Deprecated(forRemoval = true, since = "2024-02-13") + SCOPE_LEGACY_PID_HANDLENET_KEY(SCOPE_LEGACY_PID_HANDLENET, "key"), + LEGACY_HANDLENET_KEY_PATH(SCOPE_LEGACY_PID_HANDLENET_KEY, "path", "dataverse.handlenet.admcredfile"), + LEGACY_HANDLENET_KEY_PASSPHRASE(SCOPE_LEGACY_PID_HANDLENET_KEY, "passphrase", "dataverse.handlenet.admprivphrase"), // SPI SETTINGS SCOPE_SPI(PREFIX, "spi"), SCOPE_EXPORTERS(SCOPE_SPI, "exporters"), EXPORTERS_DIRECTORY(SCOPE_EXPORTERS, "directory"), + SCOPE_PIDPROVIDERS(SCOPE_SPI, "pidproviders"), + PIDPROVIDERS_DIRECTORY(SCOPE_PIDPROVIDERS, "directory"), // MAIL SETTINGS SCOPE_MAIL(PREFIX, "mail"), + SYSTEM_EMAIL(SCOPE_MAIL, "system-email"), SUPPORT_EMAIL(SCOPE_MAIL, "support-email"), CC_SUPPORT_ON_CONTACT_EMAIL(SCOPE_MAIL, "cc-support-on-contact-email"), + MAIL_DEBUG(SCOPE_MAIL, "debug"), + // Mail Transfer Agent settings + SCOPE_MAIL_MTA(SCOPE_MAIL, "mta"), + MAIL_MTA_AUTH(SCOPE_MAIL_MTA, "auth"), + MAIL_MTA_USER(SCOPE_MAIL_MTA, "user"), + MAIL_MTA_PASSWORD(SCOPE_MAIL_MTA, "password"), + MAIL_MTA_SUPPORT_UTF8(SCOPE_MAIL_MTA, "allow-utf8-addresses"), + // Placeholder setting for a large list of extra settings + MAIL_MTA_SETTING(SCOPE_MAIL_MTA), // AUTH SETTINGS SCOPE_AUTH(PREFIX, "auth"), @@ -146,6 +230,7 @@ public enum JvmSettings { SCOPE_UI(PREFIX, "ui"), UI_ALLOW_REVIEW_INCOMPLETE(SCOPE_UI, "allow-review-for-incomplete"), UI_SHOW_VALIDITY_FILTER(SCOPE_UI, "show-validity-filter"), + UI_SHOW_VALIDITY_LABEL_WHEN_PUBLISHED(SCOPE_UI, "show-validity-label-when-published"), // NetCDF SETTINGS SCOPE_NETCDF(PREFIX, "netcdf"), diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index 627cef08d8b..a1975b0b975 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -45,6 +45,7 @@ public class SettingsServiceBean { * over your shoulder when typing strings in various places of a large app. * So there. */ + @SuppressWarnings("java:S115") public enum Key { AllowApiTokenLookupViaApi, /** @@ -54,6 +55,10 @@ public enum Key { CustomDatasetSummaryFields, /** * Defines a public installation -- all datafiles are unrestricted + * + * This was added along with CloudEnvironmentName and ComputeBaseUrl. + * See https://github.com/IQSS/dataverse/issues/3776 and + * https://github.com/IQSS/dataverse/pull/3967 */ PublicInstall, /** @@ -74,9 +79,12 @@ public enum Key { /** * For example, https://datacapture.example.org */ + @Deprecated(forRemoval = true, since = "2024-07-07") DataCaptureModuleUrl, + @Deprecated(forRemoval = true, since = "2024-07-07") RepositoryStorageAbstractionLayerUrl, UploadMethods, + @Deprecated(forRemoval = true, since = "2024-07-07") DownloadMethods, /** * If the data replicated around the world using RSAL (Repository @@ -86,7 +94,17 @@ public enum Key { * TODO: Think about if it makes sense to make this a column in the * StorageSite database table. */ + @Deprecated(forRemoval = true, since = "2024-07-07") LocalDataAccessPath, + /** + * The algorithm used to generate PIDs, randomString (default) or + * storedProcedure + * + * @deprecated New installations should not use this database setting, but use + * the settings within {@link JvmSettings#SCOPE_PID}. + * + */ + @Deprecated(forRemoval = true, since = "2024-02-13") IdentifierGenerationStyle, OAuth2CallbackUrl, DefaultAuthProvider, @@ -189,24 +207,51 @@ public enum Key { SignUpUrl, /** Key for whether we allow users to sign up */ AllowSignUp, - /** protocol for global id */ + /** + * protocol for global id + * + * @deprecated New installations should not use this database setting, but use + * the settings within {@link JvmSettings#SCOPE_PID}. + * + */ + @Deprecated(forRemoval = true, since = "2024-02-13") Protocol, - /** authority for global id */ + /** + * authority for global id + * + * @deprecated New installations should not use this database setting, but use + * the settings within {@link JvmSettings#SCOPE_PID}. + * + */ + @Deprecated(forRemoval = true, since = "2024-02-13") Authority, - /** DoiProvider for global id */ + /** + * DoiProvider for global id + * + * @deprecated New installations should not use this database setting, but use + * the settings within {@link JvmSettings#SCOPE_PID}. + * + */ + @Deprecated(forRemoval = true, since = "2024-02-13") DoiProvider, - /** Shoulder for global id - used to create a common prefix on identifiers */ + /** + * Shoulder for global id - used to create a common prefix on identifiers + * + * @deprecated New installations should not use this database setting, but use + * the settings within {@link JvmSettings#SCOPE_PID}. + * + */ + @Deprecated(forRemoval = true, since = "2024-02-13") Shoulder, - /* Removed for now - tried to add here but DOI Service Bean didn't like it at start-up - DoiUsername, - DoiPassword, - DoiBaseurlstring, - */ /** Optionally override http://guides.dataverse.org . */ GuidesBaseUrl, CVocConf, + // Default calls per hour for each tier. csv format (30,60,...) + RateLimitingDefaultCapacityTiers, + // json defined list of capacities by tier and action list. See RateLimitSetting.java + RateLimitingCapacityByTierAndAction, /** * A link to an installation of https://github.com/IQSS/miniverse or * some other metrics app. @@ -227,7 +272,12 @@ public enum Key { /* the number of files the GUI user is allowed to upload in one batch, via drag-and-drop, or through the file select dialog */ MultipleUploadFilesLimit, - /* return email address for system emails such as notifications */ + /** + * Return email address for system emails such as notifications + * @deprecated Please replace usages with {@link edu.harvard.iq.dataverse.MailServiceBean#getSystemAddress}, + * which is backward compatible with this setting. + */ + @Deprecated(since = "6.2", forRemoval = true) SystemEmail, /* size limit for Tabular data file ingests */ /* (can be set separately for specific ingestable formats; in which @@ -347,10 +397,16 @@ Whether Harvesting (OAI) service is enabled */ PVCustomPasswordResetAlertMessage, /* - String to describe DOI format for data files. Default is DEPENDENT. - 'DEPENEDENT' means the DOI will be the Dataset DOI plus a file DOI with a slash in between. - 'INDEPENDENT' means a new global id, completely independent from the dataset-level global id. - */ + * String to describe DOI format for data files. Default is DEPENDENT. + * 'DEPENDENT' means the DOI will be the Dataset DOI plus a file DOI with a + * slash in between. 'INDEPENDENT' means a new global id, completely independent + * from the dataset-level global id. + * + * @deprecated New installations should not use this database setting, but use + * the settings within {@link JvmSettings#SCOPE_PID}. + * + */ + @Deprecated(forRemoval = true, since = "2024-02-13") DataFilePIDFormat, /* Json array of supported languages */ @@ -358,7 +414,7 @@ Whether Harvesting (OAI) service is enabled /* Number for the minimum number of files to send PID registration to asynchronous workflow */ - PIDAsynchRegFileCount, + //PIDAsynchRegFileCount, /** * */ @@ -366,12 +422,22 @@ Whether Harvesting (OAI) service is enabled /** * Indicates if the Handle service is setup to work 'independently' (No communication with the Global Handle Registry) + * + * @deprecated New installations should not use this database setting, but use + * the settings within {@link JvmSettings#SCOPE_PID}. + * */ + @Deprecated(forRemoval = true, since = "2024-02-13") IndependentHandleService, /** Handle to use for authentication if the default is not being used - */ + * + * @deprecated New installations should not use this database setting, but use + * the settings within {@link JvmSettings#SCOPE_PID}. + * + */ + @Deprecated(forRemoval = true, since = "2024-02-13") HandleAuthHandle, /** @@ -528,6 +594,12 @@ Whether Harvesting (OAI) service is enabled * n: embargo enabled with n months the maximum allowed duration */ MaxEmbargoDurationInMonths, + /** This setting enables Retention capabilities in Dataverse and sets the minimum Retention duration allowed. + * 0 or not set: new retentions disabled + * -1: retention enabled, no time limit + * n: retention enabled with n months the minimum allowed duration + */ + MinRetentionDurationInMonths, /* * Include "Custom Terms" as an item in the license drop-down or not. */ @@ -598,7 +670,12 @@ Whether Harvesting (OAI) service is enabled * Allows an instance admin to disable Solr search facets on the collection * and dataset pages instantly */ - DisableSolrFacets + DisableSolrFacets, + /** + * When ingesting tabular data files, store the generated tab-delimited + * files *with* the variable names line up top. + */ + StoreIngestedTabularFilesWithVarHeaders ; @Override diff --git a/src/main/java/edu/harvard/iq/dataverse/sitemap/SiteMapUtil.java b/src/main/java/edu/harvard/iq/dataverse/sitemap/SiteMapUtil.java index 86ae697f771..8408e7d91f2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/sitemap/SiteMapUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/sitemap/SiteMapUtil.java @@ -1,194 +1,140 @@ package edu.harvard.iq.dataverse.sitemap; -import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.Dataverse; -import edu.harvard.iq.dataverse.DvObjectContainer; -import edu.harvard.iq.dataverse.settings.ConfigCheckService; -import edu.harvard.iq.dataverse.settings.JvmSettings; -import edu.harvard.iq.dataverse.util.SystemConfig; -import edu.harvard.iq.dataverse.util.xml.XmlValidator; import java.io.File; import java.io.IOException; import java.net.MalformedURLException; -import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; -import java.text.SimpleDateFormat; +import java.text.ParseException; +import java.time.format.DateTimeFormatter; import java.util.List; import java.util.logging.Logger; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; -import javax.xml.transform.OutputKeys; -import javax.xml.transform.Transformer; -import javax.xml.transform.TransformerConfigurationException; -import javax.xml.transform.TransformerException; -import javax.xml.transform.TransformerFactory; -import javax.xml.transform.dom.DOMSource; -import javax.xml.transform.stream.StreamResult; -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.xml.sax.SAXException; + +import com.redfin.sitemapgenerator.W3CDateFormat; +import com.redfin.sitemapgenerator.W3CDateFormat.Pattern; +import com.redfin.sitemapgenerator.WebSitemapGenerator; +import com.redfin.sitemapgenerator.WebSitemapUrl; + +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.DvObjectContainer; +import edu.harvard.iq.dataverse.settings.ConfigCheckService; +import edu.harvard.iq.dataverse.settings.JvmSettings; +import edu.harvard.iq.dataverse.util.SystemConfig; public class SiteMapUtil { + static final String DATE_PATTERN = "yyyy-MM-dd"; + static final String SITEMAP_FILENAME_STAGED = "sitemap.xml.staged"; + /** @see https://www.sitemaps.org/protocol.html#index */ + static final int SITEMAP_LIMIT = 50000; + private static final Logger logger = Logger.getLogger(SiteMapUtil.class.getCanonicalName()); + private static DateTimeFormatter formatter = DateTimeFormatter.ofPattern(DATE_PATTERN); - static final String SITEMAP_FILENAME_FINAL = "sitemap.xml"; - static final String SITEMAP_FILENAME_STAGED = "sitemap.xml.staged"; - /** - * TODO: Handle more than 50,000 entries in the sitemap. - * - * (As of this writing Harvard Dataverse only has ~3000 dataverses and - * ~30,000 datasets.) - * - * "each Sitemap file that you provide must have no more than 50,000 URLs" - * https://www.sitemaps.org/protocol.html - * - * Consider using a third party library: "One sitemap can contain a maximum - * of 50,000 URLs. (Some sitemaps, like Google News sitemaps, can contain - * only 1,000 URLs.) If you need to put more URLs than that in a sitemap, - * you'll have to use a sitemap index file. Fortunately, WebSitemapGenerator - * can manage the whole thing for you." - * https://github.com/dfabulich/sitemapgen4j - */ public static void updateSiteMap(List dataverses, List datasets) { logger.info("BEGIN updateSiteMap"); - String sitemapPathString = getSitemapPathString(); - String stagedSitemapPathAndFileString = sitemapPathString + File.separator + SITEMAP_FILENAME_STAGED; - String finalSitemapPathAndFileString = sitemapPathString + File.separator + SITEMAP_FILENAME_FINAL; - - Path stagedPath = Paths.get(stagedSitemapPathAndFileString); - if (Files.exists(stagedPath)) { - logger.warning("Unable to update sitemap! The staged file from a previous run already existed. Delete " + stagedSitemapPathAndFileString + " and try again."); + final String dataverseSiteUrl = SystemConfig.getDataverseSiteUrlStatic(); + final String msgErrorFormat = "Problem with %s : %s. The exception is %s"; + final String msgErrorW3CFormat = "%s isn't a valid W3C date time for %s. The exception is %s"; + final String sitemapPathString = getSitemapPathString(); + final String stagedSitemapPathAndFileString = sitemapPathString + File.separator + SITEMAP_FILENAME_STAGED; + final Path stagedSitemapPath = Paths.get(stagedSitemapPathAndFileString); + + if (Files.exists(stagedSitemapPath)) { + logger.warning(String.format( + "Unable to update sitemap! The staged file from a previous run already existed. Delete %s and try again.", + stagedSitemapPathAndFileString)); return; } - DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance(); - DocumentBuilder documentBuilder = null; + final File directory = new File(sitemapPathString); + if (!directory.exists()) { + directory.mkdir(); + } + + // Use DAY pattern (YYYY-MM-DD), local machine timezone + final W3CDateFormat dateFormat = new W3CDateFormat(Pattern.DAY); + WebSitemapGenerator wsg = null; try { - documentBuilder = documentBuilderFactory.newDocumentBuilder(); - } catch (ParserConfigurationException ex) { - logger.warning("Unable to update sitemap! ParserConfigurationException: " + ex.getLocalizedMessage()); + // All sitemap files are in "sitemap" folder, see "getSitemapPathString" method. + // But with pretty-faces configuration, "sitemap.xml" and "sitemap_index.xml" are accessible directly, + // like "https://demo.dataverse.org/sitemap.xml". So "/sitemap/" need to be added on "WebSitemapGenerator" + // in order to have valid URL for sitemap location. + wsg = WebSitemapGenerator.builder(dataverseSiteUrl + "/sitemap/", directory).autoValidate(true).dateFormat(dateFormat) + .build(); + } catch (MalformedURLException e) { + logger.warning(String.format(msgErrorFormat, "Dataverse site URL", dataverseSiteUrl, e.getLocalizedMessage())); return; } - Document document = documentBuilder.newDocument(); - - Element urlSet = document.createElement("urlset"); - urlSet.setAttribute("xmlns", "http://www.sitemaps.org/schemas/sitemap/0.9"); - urlSet.setAttribute("xmlns:xhtml", "http://www.w3.org/1999/xhtml"); - document.appendChild(urlSet); for (Dataverse dataverse : dataverses) { if (!dataverse.isReleased()) { continue; } - Element url = document.createElement("url"); - urlSet.appendChild(url); - - Element loc = document.createElement("loc"); - String dataverseAlias = dataverse.getAlias(); - loc.appendChild(document.createTextNode(SystemConfig.getDataverseSiteUrlStatic() + "/dataverse/" + dataverseAlias)); - url.appendChild(loc); - - Element lastmod = document.createElement("lastmod"); - lastmod.appendChild(document.createTextNode(getLastModDate(dataverse))); - url.appendChild(lastmod); + final String dvAlias = dataverse.getAlias(); + final String dataverseUrl = dataverseSiteUrl + "/dataverse/" + dvAlias; + final String lastModDate = getLastModDate(dataverse); + try { + final WebSitemapUrl url = new WebSitemapUrl.Options(dataverseUrl).lastMod(lastModDate).build(); + wsg.addUrl(url); + } catch (MalformedURLException e) { + logger.fine(String.format(msgErrorFormat, "dataverse URL", dataverseUrl, e.getLocalizedMessage())); + } catch (ParseException e) { + logger.fine(String.format(msgErrorW3CFormat, lastModDate, "dataverse alias " + dvAlias, e.getLocalizedMessage())); + } } for (Dataset dataset : datasets) { - if (!dataset.isReleased()) { - continue; - } - if (dataset.isHarvested()) { - continue; - } // The deaccessioned check is last because it has to iterate through dataset versions. - if (dataset.isDeaccessioned()) { + if (!dataset.isReleased() || dataset.isHarvested() || dataset.isDeaccessioned()) { continue; } - Element url = document.createElement("url"); - urlSet.appendChild(url); - - Element loc = document.createElement("loc"); - String datasetPid = dataset.getGlobalId().asString(); - loc.appendChild(document.createTextNode(SystemConfig.getDataverseSiteUrlStatic() + "/dataset.xhtml?persistentId=" + datasetPid)); - url.appendChild(loc); - - Element lastmod = document.createElement("lastmod"); - lastmod.appendChild(document.createTextNode(getLastModDate(dataset))); - url.appendChild(lastmod); - } - - TransformerFactory transformerFactory = TransformerFactory.newInstance(); - Transformer transformer = null; - try { - transformer = transformerFactory.newTransformer(); - } catch (TransformerConfigurationException ex) { - logger.warning("Unable to update sitemap! TransformerConfigurationException: " + ex.getLocalizedMessage()); - return; - } - transformer.setOutputProperty(OutputKeys.INDENT, "yes"); - transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2"); - DOMSource source = new DOMSource(document); - File directory = new File(sitemapPathString); - if (!directory.exists()) { - directory.mkdir(); - } - - boolean debug = false; - if (debug) { - logger.info("Writing sitemap to console/logs"); - StreamResult consoleResult = new StreamResult(System.out); + final String datasetPid = dataset.getGlobalId().asString(); + final String datasetUrl = dataverseSiteUrl + "/dataset.xhtml?persistentId=" + datasetPid; + final String lastModDate = getLastModDate(dataset); try { - transformer.transform(source, consoleResult); - } catch (TransformerException ex) { - logger.warning("Unable to print sitemap to the console: " + ex.getLocalizedMessage()); + final WebSitemapUrl url = new WebSitemapUrl.Options(datasetUrl).lastMod(lastModDate).build(); + wsg.addUrl(url); + } catch (MalformedURLException e) { + logger.fine(String.format(msgErrorFormat, "dataset URL", datasetUrl, e.getLocalizedMessage())); + } catch (ParseException e) { + logger.fine(String.format(msgErrorW3CFormat, lastModDate, "dataset " + datasetPid, e.getLocalizedMessage())); } } - logger.info("Writing staged sitemap to " + stagedSitemapPathAndFileString); - StreamResult result = new StreamResult(new File(stagedSitemapPathAndFileString)); - try { - transformer.transform(source, result); - } catch (TransformerException ex) { - logger.warning("Unable to update sitemap! Unable to write staged sitemap to " + stagedSitemapPathAndFileString + ". TransformerException: " + ex.getLocalizedMessage()); - return; - } - - logger.info("Checking staged sitemap for well-formedness. The staged file is " + stagedSitemapPathAndFileString); + logger.info(String.format("Writing and checking sitemap file into %s", sitemapPathString)); try { - XmlValidator.validateXmlWellFormed(stagedSitemapPathAndFileString); + wsg.write(); + if (dataverses.size() + datasets.size() > SITEMAP_LIMIT) { + wsg.writeSitemapsWithIndex(); + } } catch (Exception ex) { - logger.warning("Unable to update sitemap! Staged sitemap file is not well-formed XML! The exception for " + stagedSitemapPathAndFileString + " is " + ex.getLocalizedMessage()); - return; - } - - logger.info("Checking staged sitemap against XML schema. The staged file is " + stagedSitemapPathAndFileString); - URL schemaUrl = null; - try { - schemaUrl = new URL("https://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd"); - } catch (MalformedURLException ex) { - // This URL is hard coded and it's fine. We should never get MalformedURLException so we just swallow the exception and carry on. - } - try { - XmlValidator.validateXmlSchema(stagedSitemapPathAndFileString, schemaUrl); - } catch (SAXException | IOException ex) { - logger.warning("Unable to update sitemap! Exception caught while checking XML staged file (" + stagedSitemapPathAndFileString + " ) against XML schema: " + ex.getLocalizedMessage()); + final StringBuffer errorMsg = new StringBuffer("Unable to write or validate sitemap ! The exception is "); + errorMsg.append(ex.getLocalizedMessage()); + // Add causes messages exception + Throwable cause = ex.getCause(); + // Fix limit to 5 causes + final int causeLimit = 5; + int cpt = 0; + while (cause != null && cpt < causeLimit) { + errorMsg.append(" with cause ").append(cause.getLocalizedMessage()); + cause = ex.getCause(); + cpt = cpt + 1; + } + logger.warning(errorMsg.toString()); return; } - Path finalPath = Paths.get(finalSitemapPathAndFileString); - logger.info("Copying staged sitemap from " + stagedSitemapPathAndFileString + " to " + finalSitemapPathAndFileString); + logger.info(String.format("Remove staged sitemap %s", stagedSitemapPathAndFileString)); try { - Files.move(stagedPath, finalPath, StandardCopyOption.REPLACE_EXISTING); + Files.deleteIfExists(stagedSitemapPath); } catch (IOException ex) { - logger.warning("Unable to update sitemap! Unable to copy staged sitemap from " + stagedSitemapPathAndFileString + " to " + finalSitemapPathAndFileString + ". IOException: " + ex.getLocalizedMessage()); + logger.warning("Unable to delete sitemap staged file! IOException: " + ex.getLocalizedMessage()); return; } @@ -199,12 +145,11 @@ private static String getLastModDate(DvObjectContainer dvObjectContainer) { // TODO: Decide if YYYY-MM-DD is enough. https://www.sitemaps.org/protocol.html // says "The date of last modification of the file. This date should be in W3C Datetime format. // This format allows you to omit the time portion, if desired, and use YYYY-MM-DD." - return new SimpleDateFormat("yyyy-MM-dd").format(dvObjectContainer.getModificationTime()); + return dvObjectContainer.getModificationTime().toLocalDateTime().format(formatter); } public static boolean stageFileExists() { - String sitemapPathString = getSitemapPathString(); - String stagedSitemapPathAndFileString = sitemapPathString + File.separator + SITEMAP_FILENAME_STAGED; + String stagedSitemapPathAndFileString = getSitemapPathString() + File.separator + SITEMAP_FILENAME_STAGED; Path stagedPath = Paths.get(stagedSitemapPathAndFileString); if (Files.exists(stagedPath)) { logger.warning("Unable to update sitemap! The staged file from a previous run already existed. Delete " + stagedSitemapPathAndFileString + " and try again."); @@ -212,7 +157,7 @@ public static boolean stageFileExists() { } return false; } - + /** * Lookup the location where to generate the sitemap. * @@ -223,6 +168,6 @@ public static boolean stageFileExists() { */ private static String getSitemapPathString() { return JvmSettings.DOCROOT_DIRECTORY.lookup() + File.separator + "sitemap"; - } + } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java index 8decf74fe13..6c427672e6d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java @@ -21,14 +21,8 @@ package edu.harvard.iq.dataverse.util; -import edu.harvard.iq.dataverse.DataFile; +import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.DataFile.ChecksumType; -import edu.harvard.iq.dataverse.DataFileServiceBean; -import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.DatasetVersion; -import edu.harvard.iq.dataverse.Embargo; -import edu.harvard.iq.dataverse.FileMetadata; -import edu.harvard.iq.dataverse.TermsOfUseAndAccess; import edu.harvard.iq.dataverse.dataaccess.DataAccess; import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter; import edu.harvard.iq.dataverse.dataaccess.S3AccessIO; @@ -86,6 +80,7 @@ import java.util.HashMap; import java.util.List; import java.util.Optional; +import java.util.ResourceBundle; import java.util.UUID; import java.util.logging.Level; import java.util.logging.Logger; @@ -182,6 +177,7 @@ public class FileUtil implements java.io.Serializable { public static final String MIME_TYPE_NETCDF = "application/netcdf"; public static final String MIME_TYPE_XNETCDF = "application/x-netcdf"; public static final String MIME_TYPE_HDF5 = "application/x-hdf5"; + public static final String MIME_TYPE_RO_CRATE = "application/ld+json; profile=\"http://www.w3.org/ns/json-ld#flattened http://www.w3.org/ns/json-ld#compacted https://w3id.org/ro/crate\""; // File type "thumbnail classes" tags: @@ -278,6 +274,11 @@ public static String getUserFriendlyFileType(DataFile dataFile) { if (fileType.equalsIgnoreCase(ShapefileHandler.SHAPEFILE_FILE_TYPE)){ return ShapefileHandler.SHAPEFILE_FILE_TYPE_FRIENDLY_NAME; } + try { + return BundleUtil.getStringFromPropertyFile(fileType,"MimeTypeDisplay" ); + } catch (MissingResourceException e) { + //NOOP: we will try again after trimming ";" + } if (fileType.contains(";")) { fileType = fileType.substring(0, fileType.indexOf(";")); } @@ -292,6 +293,11 @@ public static String getUserFriendlyFileType(DataFile dataFile) { } public static String getIndexableFacetFileType(DataFile dataFile) { + try { + return BundleUtil.getStringFromDefaultPropertyFile(dataFile.getContentType(),"MimeTypeFacets" ); + } catch (MissingResourceException e) { + //NOOP: we will try again after trimming ";" + } String fileType = getFileType(dataFile); try { return BundleUtil.getStringFromDefaultPropertyFile(fileType,"MimeTypeFacets" ); @@ -421,7 +427,10 @@ public static String retestIngestableFileType(File file, String fileType) { } public static String determineFileType(File f, String fileName) throws IOException{ - String fileType = null; + String fileType = lookupFileTypeByFileName(fileName); + if (fileType != null) { + return fileType; + } String fileExtension = getFileExtension(fileName); @@ -480,17 +489,17 @@ public static String determineFileType(File f, String fileName) throws IOExcepti if (fileType != null && fileType.startsWith("text/plain") && STATISTICAL_FILE_EXTENSION.containsKey(fileExtension)) { fileType = STATISTICAL_FILE_EXTENSION.get(fileExtension); } else { - fileType = determineFileTypeByNameAndExtension(fileName); + fileType = lookupFileTypeByExtension(fileName); } logger.fine("mime type recognized by extension: "+fileType); } } else { logger.fine("fileExtension is null"); - String fileTypeByName = lookupFileTypeFromPropertiesFile(fileName); - if(!StringUtil.isEmpty(fileTypeByName)) { - logger.fine(String.format("mime type: %s recognized by filename: %s", fileTypeByName, fileName)); - fileType = fileTypeByName; + final String fileTypeByExtension = lookupFileTypeByExtensionFromPropertiesFile(fileName); + if(!StringUtil.isEmpty(fileTypeByExtension)) { + logger.fine(String.format("mime type: %s recognized by extension: %s", fileTypeByExtension, fileName)); + fileType = fileTypeByExtension; } } @@ -501,24 +510,15 @@ public static String determineFileType(File f, String fileName) throws IOExcepti if ("application/x-gzip".equals(fileType)) { logger.fine("we'll run additional checks on this gzipped file."); - // We want to be able to support gzipped FITS files, same way as - // if they were just regular FITS files: - FileInputStream gzippedIn = new FileInputStream(f); - // (new FileInputStream() can throw a "filen not found" exception; - // however, if we've made it this far, it really means that the - // file does exist and can be opened) - InputStream uncompressedIn = null; - try { - uncompressedIn = new GZIPInputStream(gzippedIn); + try (FileInputStream gzippedIn = new FileInputStream(f); + InputStream uncompressedIn = new GZIPInputStream(gzippedIn)) { if (isFITSFile(uncompressedIn)) { fileType = "application/fits-gzipped"; } } catch (IOException ioex) { - if (uncompressedIn != null) { - try {uncompressedIn.close();} catch (IOException e) {} - } + logger.warning("IOException while processing gzipped FITS file: " + ioex.getMessage()); } - } + } if ("application/zip".equals(fileType)) { // Is this a zipped Shapefile? @@ -544,33 +544,41 @@ public static String determineFileType(File f, String fileName) throws IOExcepti return fileType; } - public static String determineFileTypeByNameAndExtension(String fileName) { - String mimetypesFileTypeMapResult = MIME_TYPE_MAP.getContentType(fileName); + public static String determineFileTypeByNameAndExtension(final String fileName) { + final String fileType = lookupFileTypeByFileName(fileName); + if (fileType != null) { + return fileType; + } + return lookupFileTypeByExtension(fileName); + } + + private static String lookupFileTypeByExtension(final String fileName) { + final String mimetypesFileTypeMapResult = MIME_TYPE_MAP.getContentType(fileName); logger.fine("MimetypesFileTypeMap type by extension, for " + fileName + ": " + mimetypesFileTypeMapResult); - if (mimetypesFileTypeMapResult != null) { - if ("application/octet-stream".equals(mimetypesFileTypeMapResult)) { - return lookupFileTypeFromPropertiesFile(fileName); - } else { - return mimetypesFileTypeMapResult; - } - } else { + if (mimetypesFileTypeMapResult == null) { return null; } + if ("application/octet-stream".equals(mimetypesFileTypeMapResult)) { + return lookupFileTypeByExtensionFromPropertiesFile(fileName); + } + return mimetypesFileTypeMapResult; } - public static String lookupFileTypeFromPropertiesFile(String fileName) { - String fileKey = FilenameUtils.getExtension(fileName); - String propertyFileName = "MimeTypeDetectionByFileExtension"; - if(fileKey == null || fileKey.isEmpty()) { - fileKey = fileName; - propertyFileName = "MimeTypeDetectionByFileName"; + private static String lookupFileTypeByFileName(final String fileName) { + return lookupFileTypeFromPropertiesFile("MimeTypeDetectionByFileName", fileName); + } - } - String propertyFileNameOnDisk = propertyFileName + ".properties"; + private static String lookupFileTypeByExtensionFromPropertiesFile(final String fileName) { + final String fileKey = FilenameUtils.getExtension(fileName); + return lookupFileTypeFromPropertiesFile("MimeTypeDetectionByFileExtension", fileKey); + } + + private static String lookupFileTypeFromPropertiesFile(final String propertyFileName, final String fileKey) { + final String propertyFileNameOnDisk = propertyFileName + ".properties"; try { logger.fine("checking " + propertyFileNameOnDisk + " for file key " + fileKey); return BundleUtil.getStringFromPropertyFile(fileKey, propertyFileName); - } catch (MissingResourceException ex) { + } catch (final MissingResourceException ex) { logger.info(fileKey + " is a filename/extension Dataverse doesn't know about. Consider adding it to the " + propertyFileNameOnDisk + " file."); return null; } @@ -825,7 +833,8 @@ public static boolean useRecognizedType(String suppliedContentType, String recog || canIngestAsTabular(recognizedType) || recognizedType.equals("application/fits-gzipped") || recognizedType.equalsIgnoreCase(ShapefileHandler.SHAPEFILE_FILE_TYPE) || recognizedType.equalsIgnoreCase(BagItFileHandler.FILE_TYPE) - || recognizedType.equals(MIME_TYPE_ZIP)) { + || recognizedType.equals(MIME_TYPE_ZIP) + || recognizedType.equals(MIME_TYPE_RO_CRATE)) { return true; } return false; @@ -1223,6 +1232,9 @@ public static boolean isPubliclyDownloadable(FileMetadata fileMetadata) { if (isActivelyEmbargoed(fileMetadata)) { return false; } + if (isRetentionExpired(fileMetadata)) { + return false; + } boolean popupReasons = isDownloadPopupRequired(fileMetadata.getDatasetVersion()); if (popupReasons == true) { /** @@ -1776,6 +1788,29 @@ public static boolean isActivelyEmbargoed(List fmdList) { return false; } + public static boolean isRetentionExpired(DataFile df) { + Retention e = df.getRetention(); + if (e != null) { + LocalDate endDate = e.getDateUnavailable(); + if (endDate != null && endDate.isBefore(LocalDate.now())) { + return true; + } + } + return false; + } + + public static boolean isRetentionExpired(FileMetadata fileMetadata) { + return isRetentionExpired(fileMetadata.getDataFile()); + } + + public static boolean isRetentionExpired(List fmdList) { + for (FileMetadata fmd : fmdList) { + if (isRetentionExpired(fmd)) { + return true; + } + } + return false; + } public static String getStorageDriver(DataFile dataFile) { String storageIdentifier = dataFile.getStorageIdentifier(); diff --git a/src/main/java/edu/harvard/iq/dataverse/util/MailSessionProducer.java b/src/main/java/edu/harvard/iq/dataverse/util/MailSessionProducer.java new file mode 100644 index 00000000000..149f92761d2 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/util/MailSessionProducer.java @@ -0,0 +1,150 @@ +package edu.harvard.iq.dataverse.util; + +import edu.harvard.iq.dataverse.settings.JvmSettings; +import jakarta.annotation.Resource; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.inject.Produces; +import jakarta.inject.Named; +import jakarta.mail.Authenticator; +import jakarta.mail.PasswordAuthentication; +import jakarta.mail.Session; + +import javax.naming.Context; +import javax.naming.InitialContext; +import javax.naming.NamingException; +import java.util.List; +import java.util.Properties; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +@ApplicationScoped +public class MailSessionProducer { + + // NOTE: We do not allow "from" here, as we want the transport to get it from the message being sent, enabling + // matching addresses. If "from" in transport and "from" in the message differ, some MTAs may reject or + // classify as spam. + // NOTE: Complete list including descriptions at https://eclipse-ee4j.github.io/angus-mail/docs/api/org.eclipse.angus.mail/org/eclipse/angus/mail/smtp/package-summary.html + static final List smtpStringProps = List.of( + "host", "localhost", "localaddress", "auth.mechanisms", "auth.ntlm.domain", "submitter", "dsn.notify", "dsn.ret", + "sasl.mechanisms", "sasl.authorizationid", "sasl.realm", "ssl.trust", "ssl.protocols", "ssl.ciphersuites", + "proxy.host", "proxy.port", "proxy.user", "proxy.password", "socks.host", "socks.port", "mailextension" + ); + static final List smtpIntProps = List.of( + "port", "connectiontimeout", "timeout", "writetimeout", "localport", "auth.ntlm.flag" + ); + static final List smtpBoolProps = List.of( + "auth", "ehlo", "auth.login.disable", "auth.plain.disable", "auth.digest-md5.disable", "auth.ntlm.disable", + "auth.xoauth2.disable", "allow8bitmime", "sendpartial", "sasl.enable", "sasl.usecanonicalhostname", + "quitwait", "quitonsessionreject", "ssl.enable", "ssl.checkserveridentity", "starttls.enable", + "starttls.required", "userset", "noop.strict" + ); + + private static final String PREFIX = "mail.smtp."; + private static final Logger logger = Logger.getLogger(MailSessionProducer.class.getCanonicalName()); + + static { + if (Boolean.TRUE.equals(JvmSettings.MAIL_DEBUG.lookup(Boolean.class))) { + logger.setLevel(Level.FINE); + } + } + + Session systemMailSession; + + /** + * Cache the application server provided (user defined) javamail resource to enable backwards compatibility. + * No direct JNDI lookup on the field to avoid deployment failures when not present. + * @deprecated This should be removed with the next major release of Dataverse, as it would be a breaking change. + */ + @Deprecated(forRemoval = true, since = "6.2") + Session appserverProvidedSession; + + public MailSessionProducer() { + try { + // Do JNDI lookup of legacy mail session programmatically to avoid deployment errors when not found. + Context initialContext = new InitialContext(); + this.appserverProvidedSession = (Session)initialContext.lookup("mail/notifyMailSession"); + } catch (NamingException e) { + // This exception simply means the appserver did not provide the legacy mail session. + // Debug level output is just fine. + logger.log(Level.FINER, "Error during legacy appserver-level mail resource lookup", e); + } + } + + @Produces + @Named("mail/systemSession") + public Session getSession() { + // For backward compatibility, prefer to return the mail resource configured on the appserver. + if (appserverProvidedSession != null) { + logger.warning("The configuration of mail transfer agents using asadmin create-javamail-resource is" + + " deprecated. Please migrate to using JVM options, see Dataverse guides for details"); + return appserverProvidedSession; + } + + if (systemMailSession == null) { + logger.fine("Setting up new mail session"); + + // Initialize with null (= no authenticator) is a valid argument for the session factory method. + Authenticator authenticator = null; + + // In case we want auth, create an authenticator (default = false from microprofile-config.properties) + if (Boolean.TRUE.equals(JvmSettings.MAIL_MTA_AUTH.lookup(Boolean.class))) { + logger.fine("Mail Authentication is enabled, building authenticator"); + authenticator = new Authenticator() { + @Override + protected PasswordAuthentication getPasswordAuthentication() { + logger.fine(() -> + String.format("Returning PasswordAuthenticator with username='%s', password='%s'", + JvmSettings.MAIL_MTA_USER.lookup(), + "*".repeat(JvmSettings.MAIL_MTA_PASSWORD.lookup().length()))); + return new PasswordAuthentication(JvmSettings.MAIL_MTA_USER.lookup(), JvmSettings.MAIL_MTA_PASSWORD.lookup()); + } + }; + } + + this.systemMailSession = Session.getInstance(getMailProperties(), authenticator); + } + return systemMailSession; + } + + Properties getMailProperties() { + Properties configuration = new Properties(); + + // See https://jakarta.ee/specifications/mail/2.1/apidocs/jakarta.mail/jakarta/mail/package-summary + configuration.put("mail.transport.protocol", "smtp"); + configuration.put("mail.debug", JvmSettings.MAIL_DEBUG.lookupOptional(Boolean.class).orElse(false).toString()); + // Only enable if your MTA properly supports UTF-8 mail addresses following RFC 6530/6531/6532. + // Before, we used a hack to put the raw UTF-8 mail address into the system. + // Now, make it proper, but make it possible to disable it - see also EMailValidator. + // Default = true from microprofile-config.properties as most MTAs these days support SMTPUTF8 extension + configuration.put("mail.mime.allowutf8", JvmSettings.MAIL_MTA_SUPPORT_UTF8.lookup(Boolean.class).toString()); + + // Map properties 1:1 to mail.smtp properties for the mail session. + smtpStringProps.forEach( + prop -> JvmSettings.MAIL_MTA_SETTING.lookupOptional(prop).ifPresent( + string -> configuration.put(PREFIX + prop, string))); + smtpBoolProps.forEach( + prop -> JvmSettings.MAIL_MTA_SETTING.lookupOptional(Boolean.class, prop).ifPresent( + bool -> configuration.put(PREFIX + prop, bool.toString()))); + smtpIntProps.forEach( + prop -> JvmSettings.MAIL_MTA_SETTING.lookupOptional(Integer.class, prop).ifPresent( + number -> configuration.put(PREFIX + prop, number.toString()))); + + logger.fine(() -> "Compiled properties:" + configuration.entrySet().stream() + .map(entry -> "\"" + entry.getKey() + "\": \"" + entry.getValue() + "\"") + .collect(Collectors.joining(",\n"))); + + return configuration; + } + + /** + * Determine if the session returned by {@link #getSession()} has been provided by the application server + * @return True if injected as resource from app server, false otherwise + * @deprecated This is supposed to be removed when {@link #appserverProvidedSession} is removed. + */ + @Deprecated(forRemoval = true, since = "6.2") + public boolean hasSessionFromAppServer() { + return this.appserverProvidedSession != null; + } + +} diff --git a/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java index 0724e53700b..ccec3e5f09b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java @@ -5,32 +5,14 @@ import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.UserNotification; import edu.harvard.iq.dataverse.branding.BrandingUtil; -import static edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key.SystemEmail; import java.util.Arrays; import java.util.List; import java.util.logging.Logger; -import jakarta.mail.internet.AddressException; -import jakarta.mail.internet.InternetAddress; public class MailUtil { private static final Logger logger = Logger.getLogger(MailUtil.class.getCanonicalName()); - public static InternetAddress parseSystemAddress(String systemEmail) { - if (systemEmail != null) { - try { - InternetAddress parsedSystemEmail = new InternetAddress(systemEmail); - logger.fine("parsed system email: " + parsedSystemEmail); - return parsedSystemEmail; - } catch (AddressException ex) { - logger.info("Email will not be sent due to invalid value in " + SystemEmail + " setting: " + ex); - return null; - } - } - logger.fine("Email will not be sent because the " + SystemEmail + " setting is null."); - return null; - } - public static String getSubjectTextBasedOnNotification(UserNotification userNotification, Object objectOfNotification) { List rootDvNameAsList = Arrays.asList(BrandingUtil.getInstallationBrandName()); String datasetDisplayName = ""; diff --git a/src/main/java/edu/harvard/iq/dataverse/util/ShapefileHandler.java b/src/main/java/edu/harvard/iq/dataverse/util/ShapefileHandler.java index 3af562882f3..9786fda4217 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/ShapefileHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/ShapefileHandler.java @@ -72,7 +72,7 @@ public class ShapefileHandler{ public final static List SHAPEFILE_MANDATORY_EXTENSIONS = Arrays.asList("shp", "shx", "dbf", "prj"); public final static String SHP_XML_EXTENSION = "shp.xml"; public final static String BLANK_EXTENSION = "__PLACEHOLDER-FOR-BLANK-EXTENSION__"; - public final static List SHAPEFILE_ALL_EXTENSIONS = Arrays.asList("shp", "shx", "dbf", "prj", "sbn", "sbx", "fbn", "fbx", "ain", "aih", "ixs", "mxs", "atx", "cpg", SHP_XML_EXTENSION); + public final static List SHAPEFILE_ALL_EXTENSIONS = Arrays.asList("shp", "shx", "dbf", "prj", "sbn", "sbx", "fbn", "fbx", "ain", "aih", "ixs", "mxs", "atx", "cpg", "qpj", "qmd", SHP_XML_EXTENSION); public boolean DEBUG = false; diff --git a/src/main/java/edu/harvard/iq/dataverse/util/StringUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/StringUtil.java index 33c87563104..137ae21d793 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/StringUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/StringUtil.java @@ -2,9 +2,13 @@ import edu.harvard.iq.dataverse.authorization.providers.oauth2.OAuth2LoginBackingBean; import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; +import java.security.InvalidAlgorithmParameterException; import java.security.InvalidKeyException; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.security.spec.AlgorithmParameterSpec; import java.util.ArrayList; import java.util.Arrays; import java.util.Base64; @@ -19,6 +23,7 @@ import javax.crypto.Cipher; import javax.crypto.IllegalBlockSizeException; import javax.crypto.NoSuchPaddingException; +import javax.crypto.spec.GCMParameterSpec; import javax.crypto.spec.SecretKeySpec; import org.apache.commons.lang3.StringUtils; @@ -117,6 +122,10 @@ public static List htmlArray2textArray(List htmlArray) { return cleanTextArray; } + private final static SecureRandom secureRandom = new SecureRandom(); + // 12 bytes is recommended by GCM spec + private final static int GCM_IV_LENGTH = 12; + /** * Generates an AES-encrypted version of the string. Resultant string is URL safe. * @param value The value to encrypt. @@ -124,19 +133,26 @@ public static List htmlArray2textArray(List htmlArray) { * @return encrypted string, URL-safe. */ public static String encrypt(String value, String password ) { + byte[] baseBytes = value.getBytes(); try { - Cipher aes = Cipher.getInstance("AES"); + byte[] iv = new byte[GCM_IV_LENGTH]; //NEVER REUSE THIS IV WITH SAME KEY + secureRandom.nextBytes(iv); + Cipher aes = Cipher.getInstance("AES/GCM/NoPadding"); final SecretKeySpec secretKeySpec = generateKeyFromString(password); - aes.init(Cipher.ENCRYPT_MODE, secretKeySpec); + GCMParameterSpec parameterSpec = new GCMParameterSpec(128, iv); + aes.init(Cipher.ENCRYPT_MODE, secretKeySpec, parameterSpec); byte[] encrypted = aes.doFinal(baseBytes); - String base64ed = new String(Base64.getEncoder().encode(encrypted)); + ByteBuffer byteBuffer = ByteBuffer.allocate(iv.length + encrypted.length); + byteBuffer.put(iv); + byteBuffer.put(encrypted); + String base64ed = new String(Base64.getEncoder().encode(byteBuffer.array())); return base64ed.replaceAll("\\+", ".") .replaceAll("=", "-") .replaceAll("/", "_"); } catch ( InvalidKeyException | NoSuchAlgorithmException | BadPaddingException - | IllegalBlockSizeException | NoSuchPaddingException | UnsupportedEncodingException ex) { + | IllegalBlockSizeException | NoSuchPaddingException | UnsupportedEncodingException | InvalidAlgorithmParameterException ex) { Logger.getLogger(OAuth2LoginBackingBean.class.getName()).log(Level.SEVERE, null, ex); throw new RuntimeException(ex); } @@ -149,13 +165,15 @@ public static String decrypt(String value, String password ) { byte[] baseBytes = Base64.getDecoder().decode(base64); try { - Cipher aes = Cipher.getInstance("AES"); - aes.init( Cipher.DECRYPT_MODE, generateKeyFromString(password)); - byte[] decrypted = aes.doFinal(baseBytes); + Cipher aes = Cipher.getInstance("AES/GCM/NoPadding"); + //use first 12 bytes for iv + AlgorithmParameterSpec gcmIv = new GCMParameterSpec(128, baseBytes, 0, GCM_IV_LENGTH); + aes.init( Cipher.DECRYPT_MODE, generateKeyFromString(password),gcmIv); + byte[] decrypted = aes.doFinal(baseBytes,GCM_IV_LENGTH, baseBytes.length - GCM_IV_LENGTH); return new String(decrypted); } catch ( InvalidKeyException | NoSuchAlgorithmException | BadPaddingException - | IllegalBlockSizeException | NoSuchPaddingException | UnsupportedEncodingException ex) { + | IllegalBlockSizeException | NoSuchPaddingException | UnsupportedEncodingException | InvalidAlgorithmParameterException ex) { Logger.getLogger(OAuth2LoginBackingBean.class.getName()).log(Level.SEVERE, null, ex); throw new RuntimeException(ex); } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java index 3c6992f8ec3..f9801419e47 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java @@ -752,6 +752,7 @@ public enum FileUploadMethods { * DCM stands for Data Capture Module. Right now it supports upload over * rsync+ssh but DCM may support additional methods in the future. */ + @Deprecated(forRemoval = true, since = "2024-07-07") RSYNC("dcm/rsync+ssh"), /** * Traditional Dataverse file handling, which tends to involve users @@ -809,6 +810,7 @@ public enum FileDownloadMethods { * RSAL stands for Repository Storage Abstraction Layer. Downloads don't * go through Glassfish. */ + @Deprecated(forRemoval = true, since = "2024-07-07") RSYNC("rsal/rsync"), NATIVE("native/http"), GLOBUS("globus") @@ -862,6 +864,7 @@ public String toString() { */ public enum TransferProtocols { + @Deprecated(forRemoval = true, since = "2024-07-07") RSYNC("rsync"), /** * POSIX includes NFS. This is related to Key.LocalDataAccessPath in @@ -898,7 +901,8 @@ public boolean isPublicInstall(){ boolean saneDefault = false; return settingsService.isTrueForKey(SettingsServiceBean.Key.PublicInstall, saneDefault); } - + + @Deprecated(forRemoval = true, since = "2024-07-07") public boolean isRsyncUpload(){ return getMethodAvailable(SystemConfig.FileUploadMethods.RSYNC.toString(), true); } @@ -915,7 +919,8 @@ public boolean isWebloaderUpload(){ public boolean isHTTPUpload(){ return getMethodAvailable(SystemConfig.FileUploadMethods.NATIVE.toString(), true); } - + + @Deprecated(forRemoval = true, since = "2024-07-07") public boolean isRsyncOnly(){ String downloadMethods = settingsService.getValueForKey(SettingsServiceBean.Key.DownloadMethods); if(downloadMethods == null){ @@ -931,11 +936,12 @@ public boolean isRsyncOnly(){ return Arrays.asList(uploadMethods.toLowerCase().split("\\s*,\\s*")).size() == 1 && uploadMethods.toLowerCase().equals(SystemConfig.FileUploadMethods.RSYNC.toString()); } } - + + @Deprecated(forRemoval = true, since = "2024-07-07") public boolean isRsyncDownload() { return getMethodAvailable(SystemConfig.FileUploadMethods.RSYNC.toString(), false); } - + public boolean isHTTPDownload() { return getMethodAvailable(SystemConfig.FileUploadMethods.NATIVE.toString(), false); } @@ -966,25 +972,6 @@ public Integer getUploadMethodCount(){ return Arrays.asList(uploadMethods.toLowerCase().split("\\s*,\\s*")).size(); } } - public boolean isDataFilePIDSequentialDependent(){ - String doiIdentifierType = settingsService.getValueForKey(SettingsServiceBean.Key.IdentifierGenerationStyle, "randomString"); - String doiDataFileFormat = settingsService.getValueForKey(SettingsServiceBean.Key.DataFilePIDFormat, "DEPENDENT"); - if (doiIdentifierType.equals("storedProcGenerated") && doiDataFileFormat.equals("DEPENDENT")){ - return true; - } - return false; - } - - public int getPIDAsynchRegFileCount() { - String fileCount = settingsService.getValueForKey(SettingsServiceBean.Key.PIDAsynchRegFileCount, "10"); - int retVal = 10; - try { - retVal = Integer.parseInt(fileCount); - } catch (NumberFormatException e) { - //if no number in the setting we'll return 10 - } - return retVal; - } public boolean isAllowCustomTerms() { boolean safeDefaultIfKeyNotFound = true; @@ -1016,16 +1003,7 @@ public boolean isFilePIDsEnabledForCollection(Dataverse collection) { return thisCollection.getFilePIDsEnabled(); } - public boolean isIndependentHandleService() { - boolean safeDefaultIfKeyNotFound = false; - return settingsService.isTrueForKey(SettingsServiceBean.Key.IndependentHandleService, safeDefaultIfKeyNotFound); - - } - - public String getHandleAuthHandle() { - String handleAuthHandle = settingsService.getValueForKey(SettingsServiceBean.Key.HandleAuthHandle, null); - return handleAuthHandle; - } + public String getMDCLogPath() { String mDCLogPath = settingsService.getValueForKey(SettingsServiceBean.Key.MDCLogPath, null); @@ -1173,4 +1151,22 @@ public boolean isStorageQuotasEnforced() { public Long getTestStorageQuotaLimit() { return settingsService.getValueForKeyAsLong(SettingsServiceBean.Key.StorageQuotaSizeInBytes); } + /** + * Should we store tab-delimited files produced during ingest *with* the + * variable name header line included? + * @return boolean - defaults to false. + */ + public boolean isStoringIngestedFilesWithHeaders() { + return settingsService.isTrueForKey(SettingsServiceBean.Key.StoreIngestedTabularFilesWithVarHeaders, false); + } + + /** + * RateLimitUtil will parse the json to create a List + */ + public String getRateLimitsJson() { + return settingsService.getValueForKey(SettingsServiceBean.Key.RateLimitingCapacityByTierAndAction, ""); + } + public String getRateLimitingDefaultCapacityTiers() { + return settingsService.getValueForKey(SettingsServiceBean.Key.RateLimitingDefaultCapacityTiers, ""); + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/URLTokenUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/URLTokenUtil.java index a3293e0cd28..90557a530c9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/URLTokenUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/URLTokenUtil.java @@ -1,6 +1,7 @@ package edu.harvard.iq.dataverse.util; import java.util.Arrays; +import java.util.Random; import java.util.logging.Logger; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -261,7 +262,9 @@ public JsonObject getParams(JsonObject toolParameters) { public static String getScriptForUrl(String url) { String msg = BundleUtil.getStringFromBundle("externaltools.enable.browser.popups"); - String script = "const newWin = window.open('" + url + "', target='_blank'); if (!newWin || newWin.closed || typeof newWin.closed == \"undefined\") {alert(\"" + msg + "\");}"; + String newWin = "newWin" + (new Random()).nextInt(1000000000); + //Always use a unique identifier so that more than one script can run (or one can be rerun) without conflicts + String script = String.format("const %1$s = window.open('" + url + "', target='_blank'); if (!%1$s || %1$s.closed || typeof %1$s.closed == \"undefined\") {alert(\"" + msg + "\");}", newWin); return script; } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java index aa653a6e360..84bc7834ab9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java @@ -1,19 +1,7 @@ package edu.harvard.iq.dataverse.util.bagit; -import edu.harvard.iq.dataverse.DataFile; -import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.DatasetField; -import edu.harvard.iq.dataverse.DatasetFieldCompoundValue; -import edu.harvard.iq.dataverse.DatasetFieldConstant; -import edu.harvard.iq.dataverse.DatasetFieldServiceBean; -import edu.harvard.iq.dataverse.DatasetFieldType; -import edu.harvard.iq.dataverse.DatasetVersion; +import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.DatasetVersion.VersionState; -import edu.harvard.iq.dataverse.Dataverse; -import edu.harvard.iq.dataverse.DvObjectContainer; -import edu.harvard.iq.dataverse.Embargo; -import edu.harvard.iq.dataverse.FileMetadata; -import edu.harvard.iq.dataverse.TermsOfUseAndAccess; import edu.harvard.iq.dataverse.branding.BrandingUtil; import edu.harvard.iq.dataverse.export.OAI_OREExporter; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; @@ -236,6 +224,17 @@ public JsonObjectBuilder getOREMapBuilder(boolean aggregationOnly) { } aggRes.add(JsonLDTerm.DVCore("embargoed").getLabel(), embargoObject); } + Retention retention = df.getRetention(); + if(retention!=null) { + String date = retention.getFormattedDateUnavailable(); + String reason= retention.getReason(); + JsonObjectBuilder retentionObject = Json.createObjectBuilder(); + retentionObject.add(JsonLDTerm.DVCore("dateUnavailable").getLabel(), date); + if(reason!=null) { + retentionObject.add(JsonLDTerm.DVCore("reason").getLabel(), reason); + } + aggRes.add(JsonLDTerm.DVCore("retained").getLabel(), retentionObject); + } addIfNotNull(aggRes, JsonLDTerm.directoryLabel, fmd.getDirectoryLabel()); addIfNotNull(aggRes, JsonLDTerm.schemaOrg("version"), fmd.getVersion()); addIfNotNull(aggRes, JsonLDTerm.datasetVersionId, fmd.getDatasetVersion().getId()); diff --git a/src/main/java/edu/harvard/iq/dataverse/util/cache/CacheFactoryBean.java b/src/main/java/edu/harvard/iq/dataverse/util/cache/CacheFactoryBean.java new file mode 100644 index 00000000000..36b2b35b48f --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/util/cache/CacheFactoryBean.java @@ -0,0 +1,60 @@ +package edu.harvard.iq.dataverse.util.cache; + +import edu.harvard.iq.dataverse.authorization.users.User; +import edu.harvard.iq.dataverse.engine.command.Command; +import edu.harvard.iq.dataverse.util.SystemConfig; +import jakarta.annotation.PostConstruct; +import jakarta.ejb.EJB; +import jakarta.ejb.Singleton; +import jakarta.ejb.Startup; +import jakarta.inject.Inject; + +import javax.cache.Cache; +import javax.cache.CacheManager; +import javax.cache.configuration.CompleteConfiguration; +import javax.cache.configuration.MutableConfiguration; +import javax.cache.spi.CachingProvider; +import java.util.logging.Logger; + +@Singleton +@Startup +public class CacheFactoryBean implements java.io.Serializable { + private static final Logger logger = Logger.getLogger(CacheFactoryBean.class.getCanonicalName()); + // Retrieved from Hazelcast, implements ConcurrentMap and is threadsafe + Cache rateLimitCache; + @EJB + SystemConfig systemConfig; + @Inject + CacheManager manager; + @Inject + CachingProvider provider; + public final static String RATE_LIMIT_CACHE = "rateLimitCache"; + + @PostConstruct + public void init() { + rateLimitCache = manager.getCache(RATE_LIMIT_CACHE); + if (rateLimitCache == null) { + CompleteConfiguration config = + new MutableConfiguration() + .setTypes( String.class, String.class ); + rateLimitCache = manager.createCache(RATE_LIMIT_CACHE, config); + } + } + + /** + * Check if user can make this call or if they are rate limited + * @param user + * @param command + * @return true if user is superuser or rate not limited + */ + public boolean checkRate(User user, Command command) { + final String action = command.getClass().getSimpleName(); + int capacity = RateLimitUtil.getCapacity(systemConfig, user, action); + if (capacity == RateLimitUtil.NO_LIMIT) { + return true; + } else { + String cacheKey = RateLimitUtil.generateCacheKey(user, action); + return (!RateLimitUtil.rateLimited(rateLimitCache, cacheKey, capacity)); + } + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/util/cache/RateLimitSetting.java b/src/main/java/edu/harvard/iq/dataverse/util/cache/RateLimitSetting.java new file mode 100644 index 00000000000..54da5a46670 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/util/cache/RateLimitSetting.java @@ -0,0 +1,40 @@ +package edu.harvard.iq.dataverse.util.cache; + +import java.util.ArrayList; +import java.util.List; + +public class RateLimitSetting { + + private int tier; + private int limitPerHour = RateLimitUtil.NO_LIMIT; + private List actions = new ArrayList<>(); + + private int defaultLimitPerHour; + + public RateLimitSetting() {} + + public void setTier(int tier) { + this.tier = tier; + } + public int getTier() { + return this.tier; + } + public void setLimitPerHour(int limitPerHour) { + this.limitPerHour = limitPerHour; + } + public int getLimitPerHour() { + return this.limitPerHour; + } + public void setActions(List actions) { + this.actions = actions; + } + public List getActions() { + return this.actions; + } + public void setDefaultLimit(int defaultLimitPerHour) { + this.defaultLimitPerHour = defaultLimitPerHour; + } + public int getDefaultLimitPerHour() { + return this.defaultLimitPerHour; + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/util/cache/RateLimitUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/cache/RateLimitUtil.java new file mode 100644 index 00000000000..b566cd42fe1 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/util/cache/RateLimitUtil.java @@ -0,0 +1,134 @@ +package edu.harvard.iq.dataverse.util.cache; + +import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import edu.harvard.iq.dataverse.authorization.users.GuestUser; +import edu.harvard.iq.dataverse.authorization.users.User; +import edu.harvard.iq.dataverse.util.SystemConfig; +import jakarta.json.bind.Jsonb; +import jakarta.json.bind.JsonbBuilder; +import jakarta.json.bind.JsonbException; + +import javax.cache.Cache; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.logging.Logger; + +import static java.lang.Math.max; +import static java.lang.Math.min; + +public class RateLimitUtil { + private static final Logger logger = Logger.getLogger(RateLimitUtil.class.getCanonicalName()); + static final List rateLimits = new CopyOnWriteArrayList<>(); + static final Map rateLimitMap = new ConcurrentHashMap<>(); + public static final int NO_LIMIT = -1; + + static String generateCacheKey(final User user, final String action) { + return (user != null ? user.getIdentifier() : GuestUser.get().getIdentifier()) + + (action != null ? ":" + action : ""); + } + static int getCapacity(SystemConfig systemConfig, User user, String action) { + if (user != null && user.isSuperuser()) { + return NO_LIMIT; + } + // get the capacity, i.e. calls per hour, from config + return (user instanceof AuthenticatedUser authUser) ? + getCapacityByTierAndAction(systemConfig, authUser.getRateLimitTier(), action) : + getCapacityByTierAndAction(systemConfig, 0, action); + } + static boolean rateLimited(final Cache rateLimitCache, final String key, int capacityPerHour) { + if (capacityPerHour == NO_LIMIT) { + return false; + } + long currentTime = System.currentTimeMillis() / 60000L; // convert to minutes + double tokensPerMinute = (capacityPerHour / 60.0); + // Get the last time this bucket was added to + final String keyLastUpdate = String.format("%s:last_update",key); + long lastUpdate = longFromKey(rateLimitCache, keyLastUpdate); + long deltaTime = currentTime - lastUpdate; + // Get the current number of tokens in the bucket + long tokens = longFromKey(rateLimitCache, key); + long tokensToAdd = (long) (deltaTime * tokensPerMinute); + if (tokensToAdd > 0) { // Don't update timestamp if we aren't adding any tokens to the bucket + tokens = min(capacityPerHour, tokens + tokensToAdd); + rateLimitCache.put(keyLastUpdate, String.valueOf(currentTime)); + } + // Update with any added tokens and decrement 1 token for this call if not rate limited (0 tokens) + rateLimitCache.put(key, String.valueOf(max(0, tokens-1))); + return tokens < 1; + } + + static int getCapacityByTierAndAction(SystemConfig systemConfig, Integer tier, String action) { + if (rateLimits.isEmpty()) { + init(systemConfig); + } + + if (rateLimitMap.containsKey(getMapKey(tier, action))) { + return rateLimitMap.get(getMapKey(tier,action)); + } else if (rateLimitMap.containsKey(getMapKey(tier))) { + return rateLimitMap.get(getMapKey(tier)); + } else { + return getCapacityByTier(systemConfig, tier); + } + } + static int getCapacityByTier(SystemConfig systemConfig, int tier) { + int value = NO_LIMIT; + String csvString = systemConfig.getRateLimitingDefaultCapacityTiers(); + try { + if (!csvString.isEmpty()) { + int[] values = Arrays.stream(csvString.split(",")).mapToInt(Integer::parseInt).toArray(); + if (tier < values.length) { + value = values[tier]; + } + } + } catch (NumberFormatException nfe) { + logger.warning(nfe.getMessage()); + } + return value; + } + static void init(SystemConfig systemConfig) { + getRateLimitsFromJson(systemConfig); + /* Convert the List of Rate Limit Settings containing a list of Actions to a fast lookup Map where the key is: + for default if no action defined: "{tier}:" and the value is the default limit for the tier + for each action: "{tier}:{action}" and the value is the limit defined in the setting + */ + rateLimitMap.clear(); + rateLimits.forEach(r -> { + r.setDefaultLimit(getCapacityByTier(systemConfig, r.getTier())); + rateLimitMap.put(getMapKey(r.getTier()), r.getDefaultLimitPerHour()); + r.getActions().forEach(a -> rateLimitMap.put(getMapKey(r.getTier(), a), r.getLimitPerHour())); + }); + } + + @SuppressWarnings("java:S2133") // <- To enable casting to generic in JSON-B we need a class instance, false positive + static void getRateLimitsFromJson(SystemConfig systemConfig) { + String setting = systemConfig.getRateLimitsJson(); + rateLimits.clear(); + if (!setting.isEmpty()) { + try (Jsonb jsonb = JsonbBuilder.create()) { + rateLimits.addAll(jsonb.fromJson(setting, + new ArrayList() {}.getClass().getGenericSuperclass())); + } catch (JsonbException e) { + logger.warning("Unable to parse Rate Limit Json: " + e.getLocalizedMessage() + " Json:(" + setting + ")"); + rateLimits.add(new RateLimitSetting()); // add a default entry to prevent re-initialization + // Note: Usually using Exception in a catch block is an antipattern and should be avoided. + // As the JSON-B interface does not specify a non-generic type, we have to use this. + } catch (Exception e) { + logger.warning("Could not close JSON-B reader"); + } + } + } + static String getMapKey(int tier) { + return getMapKey(tier, null); + } + static String getMapKey(int tier, String action) { + return tier + ":" + (action != null ? action : ""); + } + static long longFromKey(Cache cache, String key) { + Object l = cache.get(key); + return l != null ? Long.parseLong(String.valueOf(l)) : 0L; + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/BriefJsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/BriefJsonPrinter.java index 3fcaf6b11ff..c16a46a1765 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/BriefJsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/BriefJsonPrinter.java @@ -28,6 +28,7 @@ public JsonObjectBuilder json( MetadataBlock blk ) { ? null : jsonObjectBuilder().add("id", blk.getId()) .add("displayName", blk.getDisplayName()) + .add("displayOnCreate", blk.isDisplayOnCreate()) .add("name", blk.getName()) ; } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 4fb3ffe6c14..52491a5a7e1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -39,7 +39,6 @@ import edu.harvard.iq.dataverse.DatasetFieldValue; import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.GlobalId; -import edu.harvard.iq.dataverse.GlobalIdServiceBean; import edu.harvard.iq.dataverse.MetadataBlock; import edu.harvard.iq.dataverse.MetadataBlockServiceBean; import edu.harvard.iq.dataverse.TermsOfUseAndAccess; @@ -52,6 +51,7 @@ import edu.harvard.iq.dataverse.DatasetVersion.VersionState; import edu.harvard.iq.dataverse.license.License; import edu.harvard.iq.dataverse.license.LicenseServiceBean; +import edu.harvard.iq.dataverse.pidproviders.PidProvider; import jakarta.json.JsonReader; public class JSONLDUtil { @@ -83,7 +83,7 @@ public static Dataset updateDatasetMDFromJsonLD(Dataset ds, String jsonLDBody, JsonObject jsonld = decontextualizeJsonLD(jsonLDBody); if (migrating) { - Optional maybePid = GlobalIdServiceBean.parse(jsonld.getString("@id")); + Optional maybePid = PidProvider.parse(jsonld.getString("@id")); if (maybePid.isPresent()) { ds.setGlobalId(maybePid.get()); } else { @@ -466,7 +466,6 @@ private static void addField(DatasetField dsf, JsonArray valArray, DatasetFieldT if(!datasetFieldSvc.isValidCVocValue(dsft, strValue)) { throw new BadRequestException("Invalid values submitted for " + dsft.getName() + " which is limited to specific vocabularies."); } - datasetFieldSvc.registerExternalTerm(cvocMap.get(dsft.getId()), strValue); } DatasetFieldValue datasetFieldValue = new DatasetFieldValue(); diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java index 984c607aac7..addccc93fe0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java @@ -38,7 +38,6 @@ import java.text.ParseException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.HashSet; @@ -69,7 +68,8 @@ public class JsonParser { MetadataBlockServiceBean blockService; SettingsServiceBean settingsService; LicenseServiceBean licenseService; - HarvestingClient harvestingClient = null; + HarvestingClient harvestingClient = null; + boolean allowHarvestingMissingCVV = false; /** * if lenient, we will accept alternate spellings for controlled vocabulary values @@ -93,6 +93,7 @@ public JsonParser(DatasetFieldServiceBean datasetFieldSvc, MetadataBlockServiceB this.settingsService = settingsService; this.licenseService = licenseService; this.harvestingClient = harvestingClient; + this.allowHarvestingMissingCVV = harvestingClient != null && harvestingClient.getAllowHarvestingMissingCVV(); } public JsonParser() { @@ -318,8 +319,8 @@ public DatasetVersion parseDatasetVersion(JsonObject obj) throws JsonParseExcept public Dataset parseDataset(JsonObject obj) throws JsonParseException { Dataset dataset = new Dataset(); - dataset.setAuthority(obj.getString("authority", null) == null ? settingsService.getValueForKey(SettingsServiceBean.Key.Authority) : obj.getString("authority")); - dataset.setProtocol(obj.getString("protocol", null) == null ? settingsService.getValueForKey(SettingsServiceBean.Key.Protocol) : obj.getString("protocol")); + dataset.setAuthority(obj.getString("authority", null)); + dataset.setProtocol(obj.getString("protocol", null)); dataset.setIdentifier(obj.getString("identifier",null)); String mdl = obj.getString("metadataLanguage",null); if(mdl==null || settingsService.getBaseMetadataLanguageMap(new HashMap(), true).containsKey(mdl)) { @@ -737,39 +738,23 @@ public DatasetField parseField(JsonObject json, Boolean testType) throws JsonPar ret.setDatasetFieldType(type); - - if (type.isCompound()) { - List vals = parseCompoundValue(type, json, testType); - for (DatasetFieldCompoundValue dsfcv : vals) { - dsfcv.setParentDatasetField(ret); - } - ret.setDatasetFieldCompoundValues(vals); + if (type.isCompound()) { + parseCompoundValue(ret, type, json, testType); } else if (type.isControlledVocabulary()) { - List vals = parseControlledVocabularyValue(type, json); - for (ControlledVocabularyValue cvv : vals) { - cvv.setDatasetFieldType(type); - } - ret.setControlledVocabularyValues(vals); - + parseControlledVocabularyValue(ret, type, json); } else { - // primitive - - List values = parsePrimitiveValue(type, json); - for (DatasetFieldValue val : values) { - val.setDatasetField(ret); - } - ret.setDatasetFieldValues(values); - } + parsePrimitiveValue(ret, type, json); + } return ret; } - public List parseCompoundValue(DatasetFieldType compoundType, JsonObject json) throws JsonParseException { - return parseCompoundValue(compoundType, json, true); + public void parseCompoundValue(DatasetField dsf, DatasetFieldType compoundType, JsonObject json) throws JsonParseException { + parseCompoundValue(dsf, compoundType, json, true); } - public List parseCompoundValue(DatasetFieldType compoundType, JsonObject json, Boolean testType) throws JsonParseException { + public void parseCompoundValue(DatasetField dsf, DatasetFieldType compoundType, JsonObject json, Boolean testType) throws JsonParseException { List vocabExceptions = new ArrayList<>(); List vals = new LinkedList<>(); if (compoundType.isAllowMultiples()) { @@ -836,18 +821,17 @@ public List parseCompoundValue(DatasetFieldType compo if (!vocabExceptions.isEmpty()) { throw new CompoundVocabularyException( "Invalid controlled vocabulary in compound field ", vocabExceptions, vals); } - return vals; + + for (DatasetFieldCompoundValue dsfcv : vals) { + dsfcv.setParentDatasetField(dsf); + } + dsf.setDatasetFieldCompoundValues(vals); } - public List parsePrimitiveValue(DatasetFieldType dft , JsonObject json) throws JsonParseException { + public void parsePrimitiveValue(DatasetField dsf, DatasetFieldType dft , JsonObject json) throws JsonParseException { Map cvocMap = datasetFieldSvc.getCVocConf(true); - boolean extVocab=false; - if(cvocMap.containsKey(dft.getId())) { - extVocab=true; - } - - + boolean extVocab = cvocMap.containsKey(dft.getId()); List vals = new LinkedList<>(); if (dft.isAllowMultiples()) { try { @@ -856,14 +840,13 @@ public List parsePrimitiveValue(DatasetFieldType dft , JsonOb throw new JsonParseException("Invalid values submitted for " + dft.getName() + ". It should be an array of values."); } for (JsonString val : json.getJsonArray("value").getValuesAs(JsonString.class)) { - DatasetFieldValue datasetFieldValue = new DatasetFieldValue(); + DatasetFieldValue datasetFieldValue = new DatasetFieldValue(dsf); datasetFieldValue.setDisplayOrder(vals.size() - 1); datasetFieldValue.setValue(val.getString().trim()); if(extVocab) { if(!datasetFieldSvc.isValidCVocValue(dft, datasetFieldValue.getValue())) { throw new JsonParseException("Invalid values submitted for " + dft.getName() + " which is limited to specific vocabularies."); } - datasetFieldSvc.registerExternalTerm(cvocMap.get(dft.getId()), datasetFieldValue.getValue()); } vals.add(datasetFieldValue); } @@ -875,16 +858,16 @@ public List parsePrimitiveValue(DatasetFieldType dft , JsonOb } DatasetFieldValue datasetFieldValue = new DatasetFieldValue(); datasetFieldValue.setValue(json.getString("value", "").trim()); + datasetFieldValue.setDatasetField(dsf); if(extVocab) { if(!datasetFieldSvc.isValidCVocValue(dft, datasetFieldValue.getValue())) { throw new JsonParseException("Invalid values submitted for " + dft.getName() + " which is limited to specific vocabularies."); } - datasetFieldSvc.registerExternalTerm(cvocMap.get(dft.getId()), datasetFieldValue.getValue()); } vals.add(datasetFieldValue); } - return vals; + dsf.setDatasetFieldValues(vals); } public Workflow parseWorkflow(JsonObject json) throws JsonParseException { @@ -929,31 +912,35 @@ private String jsonValueToString(JsonValue jv) { default: return jv.toString(); } } - - public List parseControlledVocabularyValue(DatasetFieldType cvvType, JsonObject json) throws JsonParseException { + + public void parseControlledVocabularyValue(DatasetField dsf, DatasetFieldType cvvType, JsonObject json) throws JsonParseException { + List vals = new LinkedList<>(); try { if (cvvType.isAllowMultiples()) { try { json.getJsonArray("value").getValuesAs(JsonObject.class); } catch (ClassCastException cce) { throw new JsonParseException("Invalid values submitted for " + cvvType.getName() + ". It should be an array of values."); - } - List vals = new LinkedList<>(); + } for (JsonString strVal : json.getJsonArray("value").getValuesAs(JsonString.class)) { String strValue = strVal.getString(); ControlledVocabularyValue cvv = datasetFieldSvc.findControlledVocabularyValueByDatasetFieldTypeAndStrValue(cvvType, strValue, lenient); if (cvv == null) { - throw new ControlledVocabularyException("Value '" + strValue + "' does not exist in type '" + cvvType.getName() + "'", cvvType, strValue); - } - // Only add value to the list if it is not a duplicate - if (strValue.equals("Other")) { - System.out.println("vals = " + vals + ", contains: " + vals.contains(cvv)); + if (allowHarvestingMissingCVV) { + // we need to process these as primitive values + logger.warning("Value '" + strValue + "' does not exist in type '" + cvvType.getName() + "'. Processing as primitive per setting override."); + parsePrimitiveValue(dsf, cvvType, json); + return; + } else { + throw new ControlledVocabularyException("Value '" + strValue + "' does not exist in type '" + cvvType.getName() + "'", cvvType, strValue); + } } + cvv.setDatasetFieldType(cvvType); + // Only add value to the list if it is not a duplicate if (!vals.contains(cvv)) { vals.add(cvv); } } - return vals; } else { try { @@ -964,13 +951,23 @@ public List parseControlledVocabularyValue(DatasetFie String strValue = json.getString("value", ""); ControlledVocabularyValue cvv = datasetFieldSvc.findControlledVocabularyValueByDatasetFieldTypeAndStrValue(cvvType, strValue, lenient); if (cvv == null) { - throw new ControlledVocabularyException("Value '" + strValue + "' does not exist in type '" + cvvType.getName() + "'", cvvType, strValue); + if (allowHarvestingMissingCVV) { + // we need to process this as a primitive value + logger.warning(">>>> Value '" + strValue + "' does not exist in type '" + cvvType.getName() + "'. Processing as primitive per setting override."); + parsePrimitiveValue(dsf, cvvType , json); + return; + } else { + throw new ControlledVocabularyException("Value '" + strValue + "' does not exist in type '" + cvvType.getName() + "'", cvvType, strValue); + } } - return Collections.singletonList(cvv); + cvv.setDatasetFieldType(cvvType); + vals.add(cvv); } } catch (ClassCastException cce) { throw new JsonParseException("Invalid values submitted for " + cvvType.getName()); } + + dsf.setControlledVocabularyValues(vals); } Date parseDate(String str) throws ParseException { @@ -1001,6 +998,7 @@ public String parseHarvestingClient(JsonObject obj, HarvestingClient harvestingC harvestingClient.setMetadataPrefix(obj.getString("metadataFormat",null)); harvestingClient.setHarvestingSet(obj.getString("set",null)); harvestingClient.setCustomHttpHeaders(obj.getString("customHeaders", null)); + harvestingClient.setAllowHarvestingMissingCVV(obj.getBoolean("allowHarvestingMissingCVV", false)); return dataverseAlias; } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index 2eaf6b64579..95f14b79ece 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -55,6 +55,7 @@ import jakarta.ejb.Singleton; import jakarta.json.JsonArray; import jakarta.json.JsonObject; +import java.math.BigDecimal; /** * Convert objects to Json. @@ -257,11 +258,11 @@ public static JsonObjectBuilder json(Workflow wf){ } public static JsonObjectBuilder json(Dataverse dv) { - return json(dv, false); + return json(dv, false, false); } //TODO: Once we upgrade to Java EE 8 we can remove objects from the builder, and this email removal can be done in a better place. - public static JsonObjectBuilder json(Dataverse dv, Boolean hideEmail) { + public static JsonObjectBuilder json(Dataverse dv, Boolean hideEmail, Boolean returnOwners) { JsonObjectBuilder bld = jsonObjectBuilder() .add("id", dv.getId()) .add("alias", dv.getAlias()) @@ -270,7 +271,9 @@ public static JsonObjectBuilder json(Dataverse dv, Boolean hideEmail) { if(!hideEmail) { bld.add("dataverseContacts", JsonPrinter.json(dv.getDataverseContacts())); } - + if (returnOwners){ + bld.add("isPartOf", getOwnersFromDvObject(dv)); + } bld.add("permissionRoot", dv.isPermissionRoot()) .add("description", dv.getDescription()) .add("dataverseType", dv.getDataverseType().name()); @@ -289,6 +292,12 @@ public static JsonObjectBuilder json(Dataverse dv, Boolean hideEmail) { if (dv.getFilePIDsEnabled() != null) { bld.add("filePIDsEnabled", dv.getFilePIDsEnabled()); } + bld.add("isReleased", dv.isReleased()); + + List inputLevels = dv.getDataverseFieldTypeInputLevels(); + if(!inputLevels.isEmpty()) { + bld.add("inputLevels", JsonPrinter.jsonDataverseFieldTypeInputLevels(inputLevels)); + } return bld; } @@ -303,7 +312,57 @@ public static JsonArrayBuilder json(List dataverseContacts) { } return jsonArrayOfContacts; } + + public static JsonObjectBuilder getOwnersFromDvObject(DvObject dvObject){ + return getOwnersFromDvObject(dvObject, null); + } + + public static JsonObjectBuilder getOwnersFromDvObject(DvObject dvObject, DatasetVersion dsv) { + List ownerList = new ArrayList(); + dvObject = dvObject.getOwner(); // We're going to ignore the object itself + //Get "root" to top of list + while (dvObject != null) { + ownerList.add(0, dvObject); + dvObject = dvObject.getOwner(); + } + //then work "inside out" + JsonObjectBuilder saved = null; + for (DvObject dvo : ownerList) { + saved = addEmbeddedOwnerObject(dvo, saved, dsv); + } + return saved; + } + + private static JsonObjectBuilder addEmbeddedOwnerObject(DvObject dvo, JsonObjectBuilder isPartOf, DatasetVersion dsv ) { + JsonObjectBuilder ownerObject = jsonObjectBuilder(); + + if (dvo.isInstanceofDataverse()) { + ownerObject.add("type", "DATAVERSE"); + Dataverse in = (Dataverse) dvo; + ownerObject.add("identifier", in.getAlias()); + } + if (dvo.isInstanceofDataset()) { + ownerObject.add("type", "DATASET"); + if (dvo.getGlobalId() != null) { + ownerObject.add("persistentIdentifier", dvo.getGlobalId().asString()); + } + ownerObject.add("identifier", dvo.getId()); + String versionString = dsv == null ? "" : dsv.getFriendlyVersionNumber(); + if (!versionString.isEmpty()){ + ownerObject.add("version", versionString); + } + } + + ownerObject.add("displayName", dvo.getDisplayName()); + + if (isPartOf != null) { + ownerObject.add("isPartOf", isPartOf); + } + + return ownerObject; + } + public static JsonObjectBuilder json( DataverseTheme theme ) { final NullSafeJsonBuilder baseObject = jsonObjectBuilder() .add("id", theme.getId() ) @@ -326,8 +385,12 @@ public static JsonObjectBuilder json(BuiltinUser user) { .add("id", user.getId()) .add("userName", user.getUserName()); } + + public static JsonObjectBuilder json(Dataset ds){ + return json(ds, false); + } - public static JsonObjectBuilder json(Dataset ds) { + public static JsonObjectBuilder json(Dataset ds, Boolean returnOwners) { JsonObjectBuilder bld = jsonObjectBuilder() .add("id", ds.getId()) .add("identifier", ds.getIdentifier()) @@ -340,6 +403,9 @@ public static JsonObjectBuilder json(Dataset ds) { if (DvObjectContainer.isMetadataLanguageSet(ds.getMetadataLanguage())) { bld.add("metadataLanguage", ds.getMetadataLanguage()); } + if (returnOwners){ + bld.add("isPartOf", getOwnersFromDvObject(ds)); + } return bld; } @@ -352,30 +418,33 @@ public static JsonObjectBuilder json(FileDetailsHolder ds) { } public static JsonObjectBuilder json(DatasetVersion dsv, boolean includeFiles) { - return json(dsv, null, includeFiles); + return json(dsv, null, includeFiles, false); } - public static JsonObjectBuilder json(DatasetVersion dsv, List anonymizedFieldTypeNamesList, boolean includeFiles) { - /* return json(dsv, null, includeFiles, null); - } - public static JsonObjectBuilder json(DatasetVersion dsv, List anonymizedFieldTypeNamesList, boolean includeFiles, Long numberOfFiles) {*/ + public static JsonObjectBuilder json(DatasetVersion dsv, List anonymizedFieldTypeNamesList, + boolean includeFiles, boolean returnOwners) { Dataset dataset = dsv.getDataset(); JsonObjectBuilder bld = jsonObjectBuilder() .add("id", dsv.getId()).add("datasetId", dataset.getId()) .add("datasetPersistentId", dataset.getGlobalId().asString()) .add("storageIdentifier", dataset.getStorageIdentifier()) - .add("versionNumber", dsv.getVersionNumber()).add("versionMinorNumber", dsv.getMinorVersionNumber()) - .add("versionState", dsv.getVersionState().name()).add("versionNote", dsv.getVersionNote()) - .add("archiveNote", dsv.getArchiveNote()).add("deaccessionLink", dsv.getDeaccessionLink()) - .add("distributionDate", dsv.getDistributionDate()).add("productionDate", dsv.getProductionDate()) + .add("versionNumber", dsv.getVersionNumber()) + .add("versionMinorNumber", dsv.getMinorVersionNumber()) + .add("versionState", dsv.getVersionState().name()) + .add("latestVersionPublishingState", dataset.getLatestVersion().getVersionState().name()) + .add("versionNote", dsv.getVersionNote()) + .add("archiveNote", dsv.getArchiveNote()) + .add("deaccessionLink", dsv.getDeaccessionLink()) + .add("distributionDate", dsv.getDistributionDate()) + .add("productionDate", dsv.getProductionDate()) .add("UNF", dsv.getUNF()).add("archiveTime", format(dsv.getArchiveTime())) - .add("lastUpdateTime", format(dsv.getLastUpdateTime())).add("releaseTime", format(dsv.getReleaseTime())) + .add("lastUpdateTime", format(dsv.getLastUpdateTime())) + .add("releaseTime", format(dsv.getReleaseTime())) .add("createTime", format(dsv.getCreateTime())) .add("alternativePersistentId", dataset.getAlternativePersistentIdentifier()) .add("publicationDate", dataset.getPublicationDateFormattedYYYYMMDD()) .add("citationDate", dataset.getCitationDateFormattedYYYYMMDD()); - //.add("numberOfFiles", numberOfFiles); - + License license = DatasetUtil.getLicense(dsv); if (license != null) { bld.add("license", jsonLicense(dsv)); @@ -402,7 +471,10 @@ public static JsonObjectBuilder json(DatasetVersion dsv, List anonymized bld.add("metadataBlocks", (anonymizedFieldTypeNamesList != null) ? jsonByBlocks(dsv.getDatasetFields(), anonymizedFieldTypeNamesList) : jsonByBlocks(dsv.getDatasetFields()) - ); + ); + if(returnOwners){ + bld.add("isPartOf", getOwnersFromDvObject(dataset)); + } if (includeFiles) { bld.add("files", jsonFileMetadatas(dsv.getFileMetadatas())); } @@ -521,6 +593,18 @@ public static JsonObjectBuilder json(MetadataBlock block, List fie return blockBld; } + public static JsonArrayBuilder json(List metadataBlocks, boolean returnDatasetFieldTypes, boolean printOnlyDisplayedOnCreateDatasetFieldTypes) { + return json(metadataBlocks, returnDatasetFieldTypes, printOnlyDisplayedOnCreateDatasetFieldTypes, null); + } + + public static JsonArrayBuilder json(List metadataBlocks, boolean returnDatasetFieldTypes, boolean printOnlyDisplayedOnCreateDatasetFieldTypes, Dataverse ownerDataverse) { + JsonArrayBuilder arrayBuilder = Json.createArrayBuilder(); + for (MetadataBlock metadataBlock : metadataBlocks) { + arrayBuilder.add(returnDatasetFieldTypes ? json(metadataBlock, printOnlyDisplayedOnCreateDatasetFieldTypes, ownerDataverse) : brief.json(metadataBlock)); + } + return arrayBuilder; + } + public static String typeClassString(DatasetFieldType typ) { if (typ.isControlledVocabulary()) { return "controlledVocabulary"; @@ -543,26 +627,43 @@ public static JsonObject json(DatasetField dfv) { } } - public static JsonObjectBuilder json(MetadataBlock blk) { - JsonObjectBuilder bld = jsonObjectBuilder(); - bld.add("id", blk.getId()); - bld.add("name", blk.getName()); - bld.add("displayName", blk.getDisplayName()); + public static JsonObjectBuilder json(MetadataBlock metadataBlock) { + return json(metadataBlock, false, null); + } - JsonObjectBuilder fieldsBld = jsonObjectBuilder(); - for (DatasetFieldType df : new TreeSet<>(blk.getDatasetFieldTypes())) { - fieldsBld.add(df.getName(), JsonPrinter.json(df)); + public static JsonObjectBuilder json(MetadataBlock metadataBlock, boolean printOnlyDisplayedOnCreateDatasetFieldTypes, Dataverse ownerDataverse) { + JsonObjectBuilder jsonObjectBuilder = jsonObjectBuilder(); + jsonObjectBuilder.add("id", metadataBlock.getId()); + jsonObjectBuilder.add("name", metadataBlock.getName()); + jsonObjectBuilder.add("displayName", metadataBlock.getDisplayName()); + jsonObjectBuilder.add("displayOnCreate", metadataBlock.isDisplayOnCreate()); + + JsonObjectBuilder fieldsBuilder = Json.createObjectBuilder(); + Set datasetFieldTypes = new TreeSet<>(metadataBlock.getDatasetFieldTypes()); + for (DatasetFieldType datasetFieldType : datasetFieldTypes) { + boolean requiredInOwnerDataverse = ownerDataverse != null && ownerDataverse.isDatasetFieldTypeRequiredAsInputLevel(datasetFieldType.getId()); + boolean displayCondition = !printOnlyDisplayedOnCreateDatasetFieldTypes || + datasetFieldType.isDisplayOnCreate() || + requiredInOwnerDataverse; + if (displayCondition) { + fieldsBuilder.add(datasetFieldType.getName(), json(datasetFieldType, ownerDataverse)); + } } - bld.add("fields", fieldsBld); + jsonObjectBuilder.add("fields", fieldsBuilder); - return bld; + return jsonObjectBuilder; } public static JsonObjectBuilder json(DatasetFieldType fld) { + return json(fld, null); + } + + public static JsonObjectBuilder json(DatasetFieldType fld, Dataverse ownerDataverse) { JsonObjectBuilder fieldsBld = jsonObjectBuilder(); fieldsBld.add("name", fld.getName()); fieldsBld.add("displayName", fld.getDisplayName()); + fieldsBld.add("displayOnCreate", fld.isDisplayOnCreate()); fieldsBld.add("title", fld.getTitle()); fieldsBld.add("type", fld.getFieldType().toString()); fieldsBld.add("typeClass", typeClassString(fld)); @@ -571,8 +672,11 @@ public static JsonObjectBuilder json(DatasetFieldType fld) { fieldsBld.add("multiple", fld.isAllowMultiples()); fieldsBld.add("isControlledVocabulary", fld.isControlledVocabulary()); fieldsBld.add("displayFormat", fld.getDisplayFormat()); - fieldsBld.add("isRequired", fld.isRequired()); fieldsBld.add("displayOrder", fld.getDisplayOrder()); + + boolean requiredInOwnerDataverse = ownerDataverse != null && ownerDataverse.isDatasetFieldTypeRequiredAsInputLevel(fld.getId()); + fieldsBld.add("isRequired", requiredInOwnerDataverse || fld.isRequired()); + if (fld.isControlledVocabulary()) { // If the field has a controlled vocabulary, // add all values to the resulting JSON @@ -582,40 +686,52 @@ public static JsonObjectBuilder json(DatasetFieldType fld) { } fieldsBld.add("controlledVocabularyValues", jab); } + if (!fld.getChildDatasetFieldTypes().isEmpty()) { JsonObjectBuilder subFieldsBld = jsonObjectBuilder(); for (DatasetFieldType subFld : fld.getChildDatasetFieldTypes()) { - subFieldsBld.add(subFld.getName(), JsonPrinter.json(subFld)); + subFieldsBld.add(subFld.getName(), JsonPrinter.json(subFld, ownerDataverse)); } fieldsBld.add("childFields", subFieldsBld); } return fieldsBld; } + + public static JsonObjectBuilder json(FileMetadata fmd){ + return json(fmd, false, false); + } + + public static JsonObjectBuilder json(FileMetadata fmd, boolean returnOwners, boolean printDatasetVersion) { + NullSafeJsonBuilder builder = jsonObjectBuilder(); - public static JsonObjectBuilder json(FileMetadata fmd) { - return jsonObjectBuilder() // deprecated: .add("category", fmd.getCategory()) - // TODO: uh, figure out what to do here... it's deprecated - // in a sense that there's no longer the category field in the - // fileMetadata object; but there are now multiple, oneToMany file + // TODO: uh, figure out what to do here... it's deprecated + // in a sense that there's no longer the category field in the + // fileMetadata object; but there are now multiple, oneToMany file // categories - and we probably need to export them too!) -- L.A. 4.5 - // DONE: catgegories by name - .add("description", fmd.getDescription()) + // DONE: catgegories by name + builder.add("description", fmd.getDescription()) .add("label", fmd.getLabel()) // "label" is the filename - .add("restricted", fmd.isRestricted()) + .add("restricted", fmd.isRestricted()) .add("directoryLabel", fmd.getDirectoryLabel()) .add("version", fmd.getVersion()) .add("datasetVersionId", fmd.getDatasetVersion().getId()) .add("categories", getFileCategories(fmd)) - .add("dataFile", JsonPrinter.json(fmd.getDataFile(), fmd, false)); + .add("dataFile", JsonPrinter.json(fmd.getDataFile(), fmd, false, returnOwners)); + + if (printDatasetVersion) { + builder.add("datasetVersion", json(fmd.getDatasetVersion(), false)); + } + + return builder; } - public static JsonObjectBuilder json(AuxiliaryFile auxFile) { + public static JsonObjectBuilder json(AuxiliaryFile auxFile) { return jsonObjectBuilder() .add("formatTag", auxFile.getFormatTag()) .add("formatVersion", auxFile.getFormatVersion()) // "label" is the filename - .add("origin", auxFile.getOrigin()) + .add("origin", auxFile.getOrigin()) .add("isPublic", auxFile.getIsPublic()) .add("type", auxFile.getType()) .add("contentType", auxFile.getContentType()) @@ -623,11 +739,16 @@ public static JsonObjectBuilder json(AuxiliaryFile auxFile) { .add("checksum", auxFile.getChecksum()) .add("dataFile", JsonPrinter.json(auxFile.getDataFile())); } + public static JsonObjectBuilder json(DataFile df) { return JsonPrinter.json(df, null, false); } - public static JsonObjectBuilder json(DataFile df, FileMetadata fileMetadata, boolean forExportDataProvider) { + public static JsonObjectBuilder json(DataFile df, FileMetadata fileMetadata, boolean forExportDataProvider){ + return json(df, fileMetadata, forExportDataProvider, false); + } + + public static JsonObjectBuilder json(DataFile df, FileMetadata fileMetadata, boolean forExportDataProvider, boolean returnOwners) { // File names are no longer stored in the DataFile entity; // (they are instead in the FileMetadata (as "labels") - this way // the filename can change between versions... @@ -651,6 +772,7 @@ public static JsonObjectBuilder json(DataFile df, FileMetadata fileMetadata, boo String pidString = (filePid!=null)? filePid.asString(): ""; JsonObjectBuilder embargo = df.getEmbargo() != null ? JsonPrinter.json(df.getEmbargo()) : null; + JsonObjectBuilder retention = df.getRetention() != null ? JsonPrinter.json(df.getRetention()) : null; NullSafeJsonBuilder builder = jsonObjectBuilder() .add("id", df.getId()) @@ -663,6 +785,7 @@ public static JsonObjectBuilder json(DataFile df, FileMetadata fileMetadata, boo .add("description", fileMetadata.getDescription()) .add("categories", getFileCategories(fileMetadata)) .add("embargo", embargo) + .add("retention", retention) //.add("released", df.isReleased()) .add("storageIdentifier", df.getStorageIdentifier()) .add("originalFileFormat", df.getOriginalFileFormat()) @@ -703,6 +826,9 @@ public static JsonObjectBuilder json(DataFile df, FileMetadata fileMetadata, boo ? JsonPrinter.jsonVarGroup(fileMetadata.getVarGroups()) : null); } + if (returnOwners){ + builder.add("isPartOf", getOwnersFromDvObject(df, fileMetadata.getDatasetVersion())); + } return builder; } @@ -871,6 +997,7 @@ public static JsonObjectBuilder json(HarvestingClient harvestingClient) { add("schedule", harvestingClient.isScheduled() ? harvestingClient.getScheduleDescription() : "none"). add("status", harvestingClient.isHarvestingNow() ? "inProgress" : "inActive"). add("customHeaders", harvestingClient.getCustomHttpHeaders()). + add("allowHarvestingMissingCVV", harvestingClient.getAllowHarvestingMissingCVV()). add("lastHarvest", harvestingClient.getLastHarvestTime() == null ? null : harvestingClient.getLastHarvestTime().toString()). add("lastResult", harvestingClient.getLastResult()). add("lastSuccessful", harvestingClient.getLastSuccessfulHarvestTime() == null ? null : harvestingClient.getLastSuccessfulHarvestTime().toString()). @@ -1064,6 +1191,11 @@ public static JsonObjectBuilder json(Embargo embargo) { embargo.getReason()); } + public static JsonObjectBuilder json(Retention retention) { + return jsonObjectBuilder().add("dateUnavailable", retention.getDateUnavailable().toString()).add("reason", + retention.getReason()); + } + public static JsonObjectBuilder json(License license) { return jsonObjectBuilder() .add("id", license.getId()) @@ -1232,4 +1364,16 @@ private static JsonObjectBuilder jsonLicense(DatasetVersion dsv) { } return licenseJsonObjectBuilder; } + + public static JsonArrayBuilder jsonDataverseFieldTypeInputLevels(List inputLevels) { + JsonArrayBuilder jsonArrayOfInputLevels = Json.createArrayBuilder(); + for (DataverseFieldTypeInputLevel inputLevel : inputLevels) { + NullSafeJsonBuilder inputLevelJsonObject = NullSafeJsonBuilder.jsonObjectBuilder(); + inputLevelJsonObject.add("datasetFieldTypeName", inputLevel.getDatasetFieldType().getName()); + inputLevelJsonObject.add("required", inputLevel.isRequired()); + inputLevelJsonObject.add("include", inputLevel.isInclude()); + jsonArrayOfInputLevels.add(inputLevelJsonObject); + } + return jsonArrayOfInputLevels; + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/validation/EMailValidator.java b/src/main/java/edu/harvard/iq/dataverse/validation/EMailValidator.java index 624e49623f2..446f55a193f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/validation/EMailValidator.java +++ b/src/main/java/edu/harvard/iq/dataverse/validation/EMailValidator.java @@ -1,5 +1,6 @@ package edu.harvard.iq.dataverse.validation; +import edu.harvard.iq.dataverse.settings.JvmSettings; import jakarta.validation.ConstraintValidator; import jakarta.validation.ConstraintValidatorContext; @@ -10,7 +11,7 @@ * @author skraffmi */ public class EMailValidator implements ConstraintValidator { - + @Override public boolean isValid(String value, ConstraintValidatorContext context) { return isEmailValid(value); @@ -23,6 +24,12 @@ public boolean isValid(String value, ConstraintValidatorContext context) { * @return true when valid, false when invalid (null = valid!) */ public static boolean isEmailValid(String value) { - return value == null || EmailValidator.getInstance().isValid(value); + // Must be looked up here - otherwise changes are not picked up (tests, live config, ...) + final boolean mtaSupportsUTF8 = JvmSettings.MAIL_MTA_SUPPORT_UTF8.lookup(Boolean.class); + return value == null || (EmailValidator.getInstance().isValid(value) && + // If the MTA isn't able to handle UTF-8 mail addresses following RFC 6530/6531/6532, we can only declare + // mail addresses using 7bit ASCII (RFC 821) as valid. + // Beyond scope for Apache Commons Validator, see also https://issues.apache.org/jira/browse/VALIDATOR-487 + (value.codePoints().noneMatch(c -> c > 127) || mtaSupportsUTF8) ); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/validation/URLValidator.java b/src/main/java/edu/harvard/iq/dataverse/validation/URLValidator.java index 285f34d3f8c..8fde76d84e1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/validation/URLValidator.java +++ b/src/main/java/edu/harvard/iq/dataverse/validation/URLValidator.java @@ -41,7 +41,7 @@ public static boolean isURLValid(String value) { * @return true when valid (null is also valid) or false */ public static boolean isURLValid(String value, String[] schemes) { - UrlValidator urlValidator = new UrlValidator(schemes); + UrlValidator urlValidator = new UrlValidator(schemes, UrlValidator.ALLOW_2_SLASHES); return value == null || urlValidator.isValid(value); } diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java index 47f24c9b8bd..757d447b60a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java @@ -1,8 +1,10 @@ package edu.harvard.iq.dataverse.workflow; +import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetLock; import edu.harvard.iq.dataverse.DatasetServiceBean; import edu.harvard.iq.dataverse.DataverseRequestServiceBean; +import edu.harvard.iq.dataverse.DvObjectServiceBean; import edu.harvard.iq.dataverse.EjbDataverseEngine; import edu.harvard.iq.dataverse.RoleAssigneeServiceBean; import edu.harvard.iq.dataverse.UserNotification; @@ -58,6 +60,9 @@ public class WorkflowServiceBean { @EJB DatasetServiceBean datasets; + + @EJB + DvObjectServiceBean dvObjects; @EJB SettingsServiceBean settings; @@ -387,16 +392,11 @@ private void workflowCompleted(Workflow wf, WorkflowContext ctxt) { //Now lock for FinalizePublication - this block mirrors that in PublishDatasetCommand AuthenticatedUser user = ctxt.getRequest().getAuthenticatedUser(); DatasetLock lock = new DatasetLock(DatasetLock.Reason.finalizePublication, user); - lock.setDataset(ctxt.getDataset()); - String currentGlobalIdProtocol = settings.getValueForKey(SettingsServiceBean.Key.Protocol, ""); - String currentGlobalAuthority= settings.getValueForKey(SettingsServiceBean.Key.Authority, ""); - String dataFilePIDFormat = settings.getValueForKey(SettingsServiceBean.Key.DataFilePIDFormat, "DEPENDENT"); + Dataset dataset = ctxt.getDataset(); + lock.setDataset(dataset); boolean registerGlobalIdsForFiles = - (currentGlobalIdProtocol.equals(ctxt.getDataset().getProtocol()) || dataFilePIDFormat.equals("INDEPENDENT")) - && systemConfig.isFilePIDsEnabledForCollection(ctxt.getDataset().getOwner()); - if ( registerGlobalIdsForFiles ){ - registerGlobalIdsForFiles = currentGlobalAuthority.equals( ctxt.getDataset().getAuthority() ); - } + systemConfig.isFilePIDsEnabledForCollection(ctxt.getDataset().getOwner()) && + dvObjects.getEffectivePidGenerator(dataset).canCreatePidsLike(dataset.getGlobalId()); boolean validatePhysicalFiles = systemConfig.isDatafileValidationOnPublishEnabled(); String info = "Publishing the dataset; "; diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 157f2ecaf54..2996ccb509b 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -15,6 +15,7 @@ embargoed=Embargoed embargoedaccess=Embargoed with Access embargoedandrestricted=Embargoed and then Restricted embargoedandrestrictedaccess=Embargoed and then Restricted with Access +retentionExpired=Retention Period Expired incomplete=Incomplete metadata valid=Valid find=Find @@ -30,6 +31,12 @@ embargoed.wasthrough=Was embargoed until embargoed.willbeuntil=Draft: will be embargoed until embargo.date.invalid=Date is outside the allowed range: ({0} to {1}) embargo.date.required=An embargo date is required +retention.after=Was retained until +retention.isfrom=Is retained until +retention.willbeafter=Draft: will be retained until +retention.enddateinfo=after which it will no longer be accessible +retention.date.invalid=Date is outside the allowed range: ({0} to {1}) +retention.date.required=A retention period end date is required cancel=Cancel ok=OK saveChanges=Save Changes @@ -56,6 +63,7 @@ storage=Storage curationLabels=Curation Labels metadataLanguage=Dataset Metadata Language guestbookEntryOption=Guestbook Entry Option +pidProviderOption=PID Provider Option createDataverse=Create Dataverse remove=Remove done=Done @@ -783,10 +791,12 @@ notification.email.rejectFileAccess=Your request for access was rejected for the notification.email.createDataverse=Your new dataverse named {0} (view at {1} ) was created in {2} (view at {3} ). To learn more about what you can do with your dataverse, check out the Dataverse Management - User Guide at {4}/{5}/user/dataverse-management.html . # Bundle file editors, please note that "notification.email.createDataset" is used in a unit test notification.email.createDataset=Your new dataset named {0} (view at {1} ) was created in {2} (view at {3} ). To learn more about what you can do with a dataset, check out the Dataset Management - User Guide at {4}/{5}/user/dataset-management.html . -notification.email.wasSubmittedForReview={0} (view at {1} ) was submitted for review to be published in {2} (view at {3} ). Don''t forget to publish it or send it back to the contributor, {4} ({5})\! -notification.email.wasReturnedByReviewer={0} (view at {1} ) was returned by the curator of {2} (view at {3} ). -notification.email.wasPublished={0} (view at {1} ) was published in {2} (view at {3} ). -notification.email.publishFailedPidReg={0} (view at {1} ) in {2} (view at {3} ) could not be published due to a failure to register, or update the Global Identifier for the dataset or one of the files in it. Contact support if this continues to happen. +notification.email.wasSubmittedForReview=Your dataset named {0} (view at {1} ) was submitted for review to be published in {2} (view at {3} ). Don''t forget to publish it or send it back to the contributor, {4} ({5})\! +notification.email.wasReturnedByReviewer=Your dataset named {0} (view at {1} ) was returned by the curator of {2} (view at {3} ). +notification.email.wasReturnedByReviewerReason=Here is the curator comment: {0} +notification.email.wasReturnedByReviewer.collectionContacts=You may contact the collection administrator for more information: {0} +notification.email.wasPublished=Your dataset named {0} (view at {1} ) was published in {2} (view at {3} ). +notification.email.publishFailedPidReg=Your dataset named {0} (view at {1} ) in {2} (view at {3} ) could not be published due to a failure to register, or update the Global Identifier for the dataset or one of the files in it. Contact support if this continues to happen. notification.email.closing=\n\nYou may contact us for support at {0}.\n\nThank you,\n{1} notification.email.closing.html=

You may contact us for support at {0}.

Thank you,
{1} notification.email.assignRole=You are now {0} for the {1} "{2}" (view at {3} ). @@ -831,6 +841,7 @@ dataverse.curationLabels.disabled=Disabled dataverse.category=Category dataverse.category.title=The type that most closely reflects this dataverse. dataverse.guestbookentryatrequest.title=Whether Guestbooks are displayed to users when they request file access or when they download files. +dataverse.pidProvider.title=The source of PIDs (DOIs, Handles, etc.) when a new PID is created. dataverse.type.selectTab.top=Select one... dataverse.type.selectTab.researchers=Researcher dataverse.type.selectTab.researchProjects=Research Project @@ -875,7 +886,7 @@ dataverse.option.deleteDataverse=Delete Dataverse dataverse.publish.btn=Publish dataverse.publish.header=Publish Dataverse dataverse.nopublished=No Published Dataverses -dataverse.nopublished.tip=In order to use this feature you must have at least one published dataverse. +dataverse.nopublished.tip=In order to use this feature you must have at least one published or linked dataverse. dataverse.contact=Email Dataverse Contact dataverse.link=Link Dataverse dataverse.link.btn.tip=Link to Your Dataverse @@ -936,6 +947,7 @@ dataverse.default=(Default) dataverse.metadatalanguage.setatdatasetcreation=Chosen at Dataset Creation dataverse.guestbookentry.atdownload=Guestbook Entry At Download dataverse.guestbookentry.atrequest=Guestbook Entry At Access Request +dataverse.updateinputlevels.error.invalidfieldtypename=Invalid dataset field type name: {0} # rolesAndPermissionsFragment.xhtml # advanced.xhtml @@ -1023,7 +1035,7 @@ dataverse.theme.inheritCustomization.title=For this dataverse, use the same them dataverse.theme.inheritCustomization.label=Inherit Theme dataverse.theme.inheritCustomization.checkbox=Inherit theme from {0} dataverse.theme.logo=Logo -dataverse.theme.logo.tip=Supported image types are JPG, TIF, or PNG and should be no larger than 500 KB. The maximum display size for an image file in a dataverse's theme is 940 pixels wide by 120 pixels high. +dataverse.theme.logo.tip=Supported image types are JPG and PNG, must be no larger than 500 KB. The maximum display size for an image file in a dataverse's theme is 940 pixels wide by 120 pixels high. dataverse.theme.logo.format=Logo Format dataverse.theme.logo.format.selectTab.square=Square dataverse.theme.logo.format.selectTab.rectangle=Rectangle @@ -1474,7 +1486,7 @@ dataset.submit.failure.inReview=You cannot submit this dataset for review becaus dataset.status.failure.notallowed=Status update failed - label not allowed dataset.status.failure.disabled=Status labeling disabled for this dataset dataset.status.failure.isReleased=Latest version of dataset is already released. Status can only be set on draft versions -dataset.rejectMessage=Return this dataset to contributor for modification. +dataset.rejectMessage=Return this dataset to contributor for modification. The reason for return entered below will be sent by email to the author. dataset.rejectMessage.label=Return to Author Reason dataset.rejectWatermark=Please enter a reason for returning this dataset to its author(s). dataset.reject.enterReason.error=Reason for return to author is required. @@ -1482,6 +1494,7 @@ dataset.reject.success=This dataset has been sent back to the contributor. dataset.reject.failure=Dataset Submission Return Failed - {0} dataset.reject.datasetNull=Cannot return the dataset to the author(s) because it is null. dataset.reject.datasetNotInReview=This dataset cannot be return to the author(s) because the latest version is not In Review. The author(s) needs to click Submit for Review first. +dataset.reject.commentNull=You must enter a reason for returning a dataset to the author(s). dataset.publish.tip=Are you sure you want to publish this dataset? Once you do so it must remain published. dataset.publish.terms.tip=This version of the dataset will be published with the following terms: dataset.publish.terms.help.tip=To change the terms for this version, click the Cancel button and go to the Terms tab for this dataset. @@ -1660,17 +1673,19 @@ dataset.noSelectedFiles=Please select one or more files. dataset.noSelectedFilesForDownload=Please select a file or files to be downloaded. dataset.noSelectedFilesForRequestAccess=Please select a file or files for access request. dataset.embargoedSelectedFilesForRequestAccess=Embargoed files cannot be accessed. Please select an unembargoed file or files for your access request. -dataset.inValidSelectedFilesForDownload=Restricted Files Selected -dataset.inValidSelectedFilesForDownloadWithEmbargo=Embargoed and/or Restricted Files Selected -dataset.noValidSelectedFilesForDownload=The selected file(s) may not be downloaded because you have not been granted access. -dataset.mixedSelectedFilesForDownload=The restricted file(s) selected may not be downloaded because you have not been granted access. -dataset.mixedSelectedFilesForDownloadWithEmbargo=The embargoed and/or restricted file(s) selected may not be downloaded because you have not been granted access. -dataset.mixedSelectedFilesForTransfer=Some file(s) cannot be transferred. (They are restricted, embargoed, or not Globus accessible.) +dataset.inValidSelectedFilesForDownload=Inaccessible Files Selected +dataset.inValidSelectedFilesForDownloadWithEmbargo=Inaccessible Files Selected +dataset.inValidSelectedFilesForTransferWithEmbargo=Inaccessible Files Selected +dataset.noValidSelectedFilesForDownload=The selected file(s) may not be downloaded because you have not been granted access or the file(s) have a retention period that has expired or the files can only be transferred via Globus. +dataset.noValidSelectedFilesForTransfer=The selected file(s) may not be transferred because you have not been granted access or the file(s) have a retention period that has expired or the files are not Globus accessible. +dataset.mixedSelectedFilesForDownload=The selected file(s) may not be downloaded because you have not been granted access or the file(s) have a retention period that has expired. +dataset.mixedSelectedFilesForDownloadWithEmbargo=Any embargoed and/or restricted file(s) selected may not be downloaded because you have not been granted access. Some files may have a retention period that has expired. Some files may only be accessible via Globus. +dataset.mixedSelectedFilesForTransfer=Some file(s) cannot be transferred. (They are restricted, embargoed, with an expired retention period, or not Globus accessible.) dataset.inValidSelectedFilesForTransfer=Ineligible Files Selected dataset.downloadUnrestricted=Click Continue to download the files you have access to download. dataset.transferUnrestricted=Click Continue to transfer the elligible files. -dataset.requestAccessToRestrictedFiles=You may request access to the restricted file(s) by clicking the Request Access button. +dataset.requestAccessToRestrictedFiles=You may request access to any restricted file(s) by clicking the Request Access button. dataset.requestAccessToRestrictedFilesWithEmbargo=Embargoed files cannot be accessed during the embargo period. If your selection contains restricted files, you may request access to them by clicking the Request Access button. dataset.privateurl.infoMessageAuthor=Privately share this dataset before it is published: {0} dataset.privateurl.infoMessageReviewer=This unpublished dataset is being privately shared. @@ -1844,6 +1859,18 @@ file.editEmbargoDialog.newReason=Add a reason... file.editEmbargoDialog.newDate=Select the embargo end-date file.editEmbargoDialog.remove=Remove existing embargo(es) on selected files +file.retention=Retention Period +file.editRetention=Edit Retention Period +file.editRetention.add=Add or Change +file.editRetention.delete=Remove +file.editRetentionDialog.tip=Edit the planned retention period for the selected file or files. Once this dataset version is published, you will need to contact an administrator to change the retention period end date or reason of the file or files. \n After the retention period expires the files become unavailable for download. +file.editRetentionDialog.some.tip=One or more of the selected files have already been published. Contact an administrator to change the retention period date or reason of the file or files. +file.editRetentionDialog.none.tip=The selected file or files have already been published. Contact an administrator to change the retention period date or reason of the file or files. +file.editRetentionDialog.partial.tip=Any changes you make here will not be made to these files. +file.editRetentionDialog.reason.tip=Enter a short reason why this retention period exists +file.editRetentionDialog.newReason=Add a reason... +file.editRetentionDialog.newDate=Select the retention period end date +file.editRetentionDialog.remove=Remove existing retention period(s) on selected files file.setThumbnail=Set Thumbnail file.setThumbnail.header=Set Dataset Thumbnail @@ -1856,6 +1883,7 @@ file.advancedIngestOptions=Advanced Ingest Options file.assignedDataverseImage.success={0} has been saved as the thumbnail for this dataset. file.assignedTabFileTags.success=The tags were successfully added for {0}. file.assignedEmbargo.success=An Embargo was successfully added for {0}. +file.assignedRetention.success=A Retention Period was successfully added for {0}. file.tabularDataTags=Tabular Data Tags file.tabularDataTags.tip=Select a tag to describe the type(s) of data this is (survey, time series, geospatial, etc). file.spss-savEncoding=Language Encoding @@ -2002,7 +2030,8 @@ file.deleteFileDialog.immediate=The file will be deleted after you click on the file.deleteFileDialog.multiple.immediate=The file(s) will be deleted after you click on the Delete button. file.deleteFileDialog.header=Delete Files file.deleteFileDialog.failed.tip=Files will not be removed from previously published versions of the dataset. -file.deaccessionDialog.tip=Once you deaccession this dataset it will no longer be viewable by the public. +file.deaccessionDialog.tip.permanent=Deaccession is permanent. +file.deaccessionDialog.tip=This dataset will no longer be public and a tumbstone will display the reason for deaccessioning.
Please read the documentation if you have any questions. file.deaccessionDialog.version=Version file.deaccessionDialog.reason.question1=Which version(s) do you want to deaccession? file.deaccessionDialog.reason.question2=What is the reason for deaccession? @@ -2016,8 +2045,8 @@ file.deaccessionDialog.reason.selectItem.other=Other (Please type reason in spac file.deaccessionDialog.enterInfo=Please enter additional information about the reason for deaccession. file.deaccessionDialog.leaveURL=If applicable, please leave a URL where this dataset can be accessed after deaccessioning. file.deaccessionDialog.leaveURL.watermark=Optional dataset site, http://... -file.deaccessionDialog.deaccession.tip=Are you sure you want to deaccession? The selected version(s) will no longer be viewable by the public. -file.deaccessionDialog.deaccessionDataset.tip=Are you sure you want to deaccession this dataset? It will no longer be viewable by the public. +file.deaccessionDialog.deaccession.tip=Are you sure you want to deaccession? This is permanent and the selected version(s) will no longer be viewable by the public. +file.deaccessionDialog.deaccessionDataset.tip=Are you sure you want to deaccession this dataset? This is permanent an it will no longer be viewable by the public. file.deaccessionDialog.dialog.selectVersion.error=Please select version(s) for deaccessioning. file.deaccessionDialog.dialog.reason.error=Please select reason for deaccessioning. file.deaccessionDialog.dialog.url.error=Please enter valid forwarding URL. @@ -2113,7 +2142,7 @@ dataset.thumbnailsAndWidget.thumbnails.title=Thumbnail dataset.thumbnailsAndWidget.widgets.title=Widgets dataset.thumbnailsAndWidget.thumbnailImage=Thumbnail Image dataset.thumbnailsAndWidget.thumbnailImage.title=The logo or image file you wish to display as the thumbnail of this dataset. -dataset.thumbnailsAndWidget.thumbnailImage.tip=Supported image types are JPG, TIF, or PNG and should be no larger than {0} KB. The maximum display size for an image file as a dataset thumbnail is 48 pixels wide by 48 pixels high. +dataset.thumbnailsAndWidget.thumbnailImage.tip=Supported image types are JPG and PNG, must be no larger than {0} KB. The maximum display size for an image file as a dataset thumbnail is 140 pixels wide by 140 pixels high. dataset.thumbnailsAndWidget.thumbnailImage.default=Default Icon dataset.thumbnailsAndWidget.thumbnailImage.selectAvailable=Select Available File dataset.thumbnailsAndWidget.thumbnailImage.selectThumbnail=Select Thumbnail @@ -2174,6 +2203,8 @@ file.metadataTab.fileMetadata.type.label=Type file.metadataTab.fileMetadata.description.label=Description file.metadataTab.fileMetadata.publicationDate.label=Publication Date file.metadataTab.fileMetadata.embargoReason.label=Embargo Reason +file.metadataTab.fileMetadata.retentionDate.label=Retention End Date +file.metadataTab.fileMetadata.retentionReason.label=Retention Reason file.metadataTab.fileMetadata.metadataReleaseDate.label=Metadata Release Date file.metadataTab.fileMetadata.depositDate.label=Deposit Date file.metadataTab.fileMetadata.hierarchy.label=File Path @@ -2203,6 +2234,15 @@ ingest.csv.lineMismatch=Mismatch between line counts in first and final passes!, ingest.csv.recordMismatch=Reading mismatch, line {0} of the Data file: {1} delimited values expected, {2} found. ingest.csv.nullStream=Stream can't be null. +file.ingest=Ingest +file.uningest=Uningest +file.ingest.alreadyIngestedWarning=This file has already been ingested +file.ingest.ingestInProgressWarning=Ingestion of this file is already in progress +file.ingest.cantIngestFileWarning=Ingest not supported for this file type +file.ingest.ingestQueued=Ingestion has been requested +file.ingest.cantUningestFileWarning=This file cannot be uningested +file.uningest.complete=Uningestion of this file has been completed + # editdatafile.xhtml # editFilesFragment.xhtml @@ -2614,6 +2654,7 @@ pid.allowedCharacters=^[A-Za-z0-9._/:\\-]* command.exception.only.superusers={1} can only be called by superusers. command.exception.user.deactivated={0} failed: User account has been deactivated. command.exception.user.deleted={0} failed: User account has been deleted. +command.exception.user.ratelimited={0} failed: Rate limited due to too many requests. #Admin-API admin.api.auth.mustBeSuperUser=Forbidden. You must be a superuser. @@ -2633,7 +2674,10 @@ admin.api.deleteUser.success=Authenticated User {0} deleted. #Files.java files.api.metadata.update.duplicateFile=Filename already exists at {0} files.api.no.draft=No draft available for this file +files.api.no.draftOrUnauth=Dataset version cannot be found or unauthorized. +files.api.notFoundInVersion="File metadata for file with id {0} in dataset version {1} not found" files.api.only.tabular.supported=This operation is only available for tabular files. +files.api.fileNotFound=File could not be found. #Datasets.java datasets.api.updatePIDMetadata.failure.dataset.must.be.released=Modify Registration Metadata must be run on a published dataset. @@ -2650,6 +2694,7 @@ datasets.api.datasize.ioerror=Fatal IO error while trying to determine the total datasets.api.grant.role.not.found.error=Cannot find role named ''{0}'' in dataverse {1} datasets.api.grant.role.cant.create.assignment.error=Cannot create assignment: {0} datasets.api.grant.role.assignee.not.found.error=Assignee not found +datasets.api.grant.role.assignee.has.role.error=User already has this role for this dataset datasets.api.revoke.role.not.found.error="Role assignment {0} not found" datasets.api.revoke.role.success=Role {0} revoked for assignee {1} in {2} datasets.api.privateurl.error.datasetnotfound=Could not find dataset. @@ -2666,6 +2711,18 @@ datasets.api.deaccessionDataset.invalid.forward.url=Invalid deaccession forward datasets.api.globusdownloaddisabled=File transfer from Dataverse via Globus is not available for this dataset. datasets.api.globusdownloadnotfound=List of files to transfer not found. datasets.api.globusuploaddisabled=File transfer to Dataverse via Globus is not available for this dataset. +datasets.api.pidgenerator.notfound=No PID Generator configured for the give id. +datasets.api.thumbnail.fileToLarge=File is larger than maximum size: {0} +datasets.api.thumbnail.nonDatasetFailed=In setNonDatasetFileAsThumbnail could not generate thumbnail from uploaded file. +datasets.api.thumbnail.notDeleted=User wanted to remove the thumbnail it still has one! +datasets.api.thumbnail.actionNotSupported=Whatever you are trying to do to the dataset thumbnail is not supported. +datasets.api.thumbnail.nonDatasetsFileIsNull=In setNonDatasetFileAsThumbnail uploadedFile was null. +datasets.api.thumbnail.inputStreamToFile.exception=In setNonDatasetFileAsThumbnail caught exception calling inputStreamToFile: {0} +datasets.api.thumbnail.missing=Dataset thumbnail is unexpectedly absent. +datasets.api.thumbnail.basedOnWrongFileId=Dataset thumbnail should be based on file id {0} but instead it is {1} +datasets.api.thumbnail.fileNotFound=Could not find file based on id supplied: {0} +datasets.api.thumbnail.fileNotSupplied=A file was not selected to be the new dataset thumbnail. +datasets.api.thumbnail.noChange=No changes to save. #Dataverses.java dataverses.api.update.default.contributor.role.failure.role.not.found=Role {0} not found. @@ -2678,6 +2735,7 @@ dataverses.api.move.dataverse.failure.not.published=Published dataverse may not dataverses.api.move.dataverse.error.guestbook=Dataset guestbook is not in target dataverse. dataverses.api.move.dataverse.error.template=Dataverse template is not in target dataverse. dataverses.api.move.dataverse.error.featured=Dataverse is featured in current dataverse. +dataverses.api.delete.featured.collections.successful=Featured dataverses have been removed dataverses.api.move.dataverse.error.metadataBlock=Dataverse metadata block is not in target dataverse. dataverses.api.move.dataverse.error.dataverseLink=Dataverse is linked to target dataverse or one of its parents. dataverses.api.move.dataverse.error.datasetLink=Dataset is linked to target dataverse or one of its parents. @@ -2700,6 +2758,8 @@ access.api.fileAccess.failure.noUser=Could not find user to execute command: {0} access.api.requestAccess.failure.commandError=Problem trying request access on {0} : {1} access.api.requestAccess.failure.requestExists=An access request for this file on your behalf already exists. access.api.requestAccess.failure.invalidRequest=You may not request access to this file. It may already be available to you. +access.api.requestAccess.failure.retentionExpired=You may not request access to this file. It is not available because its retention period has ended. + access.api.requestAccess.noKey=You must provide a key to request access to a file. access.api.requestAccess.fileNotFound=Could not find datafile with id {0}. access.api.requestAccess.invalidRequest=This file is already available to you for download or you have a pending request @@ -2839,7 +2899,22 @@ passwdVal.passwdReq.lowercase=lowercase passwdVal.passwdReq.letter=letter passwdVal.passwdReq.numeral=numeral passwdVal.passwdReq.special=special +#mydata API (DataRetriverAPI.java and MyDataFinder.java) dataretrieverAPI.noMsgResultsFound=Sorry, no results were found. +dataretrieverAPI.authentication.required=Requires authentication. Please login. +dataretrieverAPI.authentication.required.opt=retrieveMyDataAsJsonString. User not found! Shouldn't be using this anyway. +dataretrieverAPI.user.not.found=No user found for: "{0}" +dataretrieverAPI.solr.error=Sorry! There was an error with the search service. +dataretrieverAPI.solr.error.opt=Sorry! There was a Solr Error. +myDataFilterParams.error.no.user=Sorry! No user was found! +myDataFilterParams.error.result.no.role=No results. Please select at least one Role. +myDataFilterParams.error.result.no.dvobject=No results. Please select one of Dataverses, Datasets, Files. +myDataFilterParams.error.result.no.publicationStatus=No results. Please select one of {0}. +myDataFinder.error.result.null=Sorry, the authenticated user ID could not be retrieved. +myDataFinder.error.result.no.role=Sorry, you have no assigned roles. +myDataFinder.error.result.role.empty=Sorry, nothing was found for this role: {0} +myDataFinder.error.result.roles.empty=Sorry, nothing was found for these roles: {0} +myDataFinder.error.result.no.dvobject=Sorry, you have no assigned Dataverses, Datasets, or Files. #xlsxfilereader.java xlsxfilereader.ioexception.parse=Could not parse Excel/XLSX spreadsheet. {0} @@ -2896,6 +2971,7 @@ Public=Public Restricted=Restricted EmbargoedThenPublic=Embargoed then Public EmbargoedThenRestricted=Embargoed then Restricted +RetentionPeriodExpired=Retention Period Expired #metadata source - Facet Label Harvested=Harvested @@ -2948,3 +3024,8 @@ publishDatasetCommand.pidNotReserved=Cannot publish dataset because its persiste api.errors.invalidApiToken=Invalid API token. api.ldninbox.citation.alert={0},

The {1} has just been notified that the {2}, {3}, cites "{6}" in this repository. api.ldninbox.citation.subject={0}: A Dataset Citation has been reported! + +#Info.java +openapi.exception.invalid.format=Invalid format {0}, currently supported formats are YAML and JSON. +openapi.exception=Supported format definition not found. +openapi.exception.unaligned=Unaligned parameters on Headers [{0}] and Request [{1}] diff --git a/src/main/java/propertyFiles/License.properties b/src/main/java/propertyFiles/License.properties index 6ded8c41d5b..6602f408a12 100644 --- a/src/main/java/propertyFiles/License.properties +++ b/src/main/java/propertyFiles/License.properties @@ -2,3 +2,5 @@ license.cc0_1.0.description=Creative Commons CC0 1.0 Universal Public Domain Ded license.cc_by_4.0.description=Creative Commons Attribution 4.0 International License. license.cc0_1.0.name=CC0 1.0 license.cc_by_4.0.name=CC BY 4.0 +license.custom_terms.name=Custom Terms + diff --git a/src/main/java/propertyFiles/MimeTypeDetectionByFileName.properties b/src/main/java/propertyFiles/MimeTypeDetectionByFileName.properties index 70b0c4e371e..5c1a22bfd5f 100644 --- a/src/main/java/propertyFiles/MimeTypeDetectionByFileName.properties +++ b/src/main/java/propertyFiles/MimeTypeDetectionByFileName.properties @@ -2,3 +2,5 @@ Makefile=text/x-makefile Snakemake=text/x-snakemake Dockerfile=application/x-docker-file Vagrantfile=application/x-vagrant-file +ro-crate-metadata.json=application/ld+json; profile="http://www.w3.org/ns/json-ld#flattened http://www.w3.org/ns/json-ld#compacted https://w3id.org/ro/crate" +ro-crate-metadata.jsonld=application/ld+json; profile="http://www.w3.org/ns/json-ld#flattened http://www.w3.org/ns/json-ld#compacted https://w3id.org/ro/crate" diff --git a/src/main/java/propertyFiles/MimeTypeDisplay.properties b/src/main/java/propertyFiles/MimeTypeDisplay.properties index 295ac226fa1..8e5a251abbf 100644 --- a/src/main/java/propertyFiles/MimeTypeDisplay.properties +++ b/src/main/java/propertyFiles/MimeTypeDisplay.properties @@ -207,6 +207,7 @@ audio/ogg=OGG Audio audio/wav=Waveform Audio audio/x-wav=Waveform Audio audio/x-wave=Waveform Audio +audio/vnd.wave=Waveform Audio # Video video/avi=AVI Video video/x-msvideo=AVI Video @@ -222,5 +223,6 @@ text/xml-graphml=GraphML Network Data application/octet-stream=Unknown application/x-docker-file=Docker Image File application/x-vagrant-file=Vagrant Image File +application/ld+json;\u0020profile\u003d\u0022http\u003a//www.w3.org/ns/json-ld#flattened\u0020http\u003a//www.w3.org/ns/json-ld#compacted\u0020https\u003a//w3id.org/ro/crate\u0022=RO-Crate metadata # Dataverse-specific application/vnd.dataverse.file-package=Dataverse Package diff --git a/src/main/java/propertyFiles/MimeTypeFacets.properties b/src/main/java/propertyFiles/MimeTypeFacets.properties index aaab66f20ae..0dad8daff4c 100644 --- a/src/main/java/propertyFiles/MimeTypeFacets.properties +++ b/src/main/java/propertyFiles/MimeTypeFacets.properties @@ -209,6 +209,7 @@ audio/ogg=Audio audio/wav=Audio audio/x-wav=Audio audio/x-wave=Audio +audio/vnd.wave=Audio # (anything else that looks like audio/* will also be indexed as facet type "Audio") # Video video/avi=Video @@ -224,5 +225,6 @@ video/webm=Video text/xml-graphml=Network Data # Other application/octet-stream=Unknown +application/ld+json;\u0020profile\u003d\u0022http\u003a//www.w3.org/ns/json-ld#flattened\u0020http\u003a//www.w3.org/ns/json-ld#compacted\u0020https\u003a//w3id.org/ro/crate\u0022=Metadata # Dataverse-specific application/vnd.dataverse.file-package=Data diff --git a/src/main/java/propertyFiles/citation.properties b/src/main/java/propertyFiles/citation.properties index f35ede79b50..cb864eb78e9 100644 --- a/src/main/java/propertyFiles/citation.properties +++ b/src/main/java/propertyFiles/citation.properties @@ -22,6 +22,7 @@ datasetfieldtype.dsDescriptionValue.title=Text datasetfieldtype.dsDescriptionDate.title=Date datasetfieldtype.subject.title=Subject datasetfieldtype.keyword.title=Keyword +datasetfieldtype.keywordTermURI.title=Term URI datasetfieldtype.keywordValue.title=Term datasetfieldtype.keywordVocabulary.title=Controlled Vocabulary Name datasetfieldtype.keywordVocabularyURI.title=Controlled Vocabulary URL @@ -100,6 +101,7 @@ datasetfieldtype.dsDescriptionValue.description=A summary describing the purpose datasetfieldtype.dsDescriptionDate.description=The date when the description was added to the Dataset. If the Dataset contains more than one description, e.g. the data producer supplied one description and the data repository supplied another, this date is used to distinguish between the descriptions datasetfieldtype.subject.description=The area of study relevant to the Dataset datasetfieldtype.keyword.description=A key term that describes an important aspect of the Dataset and information about any controlled vocabulary used +datasetfieldtype.keywordTermURI.description=A URI that points to the web presence of the Keyword Term datasetfieldtype.keywordValue.description=A key term that describes important aspects of the Dataset datasetfieldtype.keywordVocabulary.description=The controlled vocabulary used for the keyword term (e.g. LCSH, MeSH) datasetfieldtype.keywordVocabularyURI.description=The URL where one can access information about the term's controlled vocabulary @@ -178,6 +180,7 @@ datasetfieldtype.dsDescriptionValue.watermark= datasetfieldtype.dsDescriptionDate.watermark=YYYY-MM-DD datasetfieldtype.subject.watermark= datasetfieldtype.keyword.watermark= +datasetfieldtype.keywordTermURI.watermark=https:// datasetfieldtype.keywordValue.watermark= datasetfieldtype.keywordVocabulary.watermark= datasetfieldtype.keywordVocabularyURI.watermark=https:// diff --git a/src/main/java/propertyFiles/geospatial.properties b/src/main/java/propertyFiles/geospatial.properties index 86f297c29b9..2659c2a3cc9 100644 --- a/src/main/java/propertyFiles/geospatial.properties +++ b/src/main/java/propertyFiles/geospatial.properties @@ -10,8 +10,8 @@ datasetfieldtype.geographicUnit.title=Geographic Unit datasetfieldtype.geographicBoundingBox.title=Geographic Bounding Box datasetfieldtype.westLongitude.title=Westernmost (Left) Longitude datasetfieldtype.eastLongitude.title=Easternmost (Right) Longitude -datasetfieldtype.northLongitude.title=Northernmost (Top) Latitude -datasetfieldtype.southLongitude.title=Southernmost (Bottom) Latitude +datasetfieldtype.northLatitude.title=Northernmost (Top) Latitude +datasetfieldtype.southLatitude.title=Southernmost (Bottom) Latitude datasetfieldtype.geographicCoverage.description=Information on the geographic coverage of the data. Includes the total geographic scope of the data. datasetfieldtype.country.description=The country or nation that the Dataset is about. datasetfieldtype.state.description=The state or province that the Dataset is about. Use GeoNames for correct spelling and avoid abbreviations. @@ -19,10 +19,10 @@ datasetfieldtype.city.description=The name of the city that the Dataset is about datasetfieldtype.otherGeographicCoverage.description=Other information on the geographic coverage of the data. datasetfieldtype.geographicUnit.description=Lowest level of geographic aggregation covered by the Dataset, e.g., village, county, region. datasetfieldtype.geographicBoundingBox.description=The fundamental geometric description for any Dataset that models geography is the geographic bounding box. It describes the minimum box, defined by west and east longitudes and north and south latitudes, which includes the largest geographic extent of the Dataset's geographic coverage. This element is used in the first pass of a coordinate-based search. Inclusion of this element in the codebook is recommended, but is required if the bound polygon box is included. -datasetfieldtype.westLongitude.description=Westernmost coordinate delimiting the geographic extent of the Dataset. A valid range of values, expressed in decimal degrees, is -180,0 <= West Bounding Longitude Value <= 180,0. -datasetfieldtype.eastLongitude.description=Easternmost coordinate delimiting the geographic extent of the Dataset. A valid range of values, expressed in decimal degrees, is -180,0 <= East Bounding Longitude Value <= 180,0. -datasetfieldtype.northLongitude.description=Northernmost coordinate delimiting the geographic extent of the Dataset. A valid range of values, expressed in decimal degrees, is -90,0 <= North Bounding Latitude Value <= 90,0. -datasetfieldtype.southLongitude.description=Southernmost coordinate delimiting the geographic extent of the Dataset. A valid range of values, expressed in decimal degrees, is -90,0 <= South Bounding Latitude Value <= 90,0. +datasetfieldtype.westLongitude.description=Westernmost coordinate delimiting the geographic extent of the Dataset. A valid range of values, expressed in decimal degrees, is -180.0 <= West Bounding Longitude Value <= 180.0. +datasetfieldtype.eastLongitude.description=Easternmost coordinate delimiting the geographic extent of the Dataset. A valid range of values, expressed in decimal degrees, is -180.0 <= East Bounding Longitude Value <= 180.0. +datasetfieldtype.northLatitude.description=Northernmost coordinate delimiting the geographic extent of the Dataset. A valid range of values, expressed in decimal degrees, is -90.0 <= North Bounding Latitude Value <= 90.0. +datasetfieldtype.southLatitude.description=Southernmost coordinate delimiting the geographic extent of the Dataset. A valid range of values, expressed in decimal degrees, is -90.0 <= South Bounding Latitude Value <= 90.0. datasetfieldtype.geographicCoverage.watermark= datasetfieldtype.country.watermark= datasetfieldtype.state.watermark= @@ -32,8 +32,8 @@ datasetfieldtype.geographicUnit.watermark= datasetfieldtype.geographicBoundingBox.watermark= datasetfieldtype.westLongitude.watermark= datasetfieldtype.eastLongitude.watermark= -datasetfieldtype.northLongitude.watermark= -datasetfieldtype.southLongitude.watermark= +datasetfieldtype.northLatitude.watermark= +datasetfieldtype.southLatitude.watermark= controlledvocabulary.country.afghanistan=Afghanistan controlledvocabulary.country.albania=Albania controlledvocabulary.country.algeria=Algeria diff --git a/src/main/java/propertyFiles/staticSearchFields.properties b/src/main/java/propertyFiles/staticSearchFields.properties index ab03de64f23..53d0080b87c 100644 --- a/src/main/java/propertyFiles/staticSearchFields.properties +++ b/src/main/java/propertyFiles/staticSearchFields.properties @@ -3,6 +3,7 @@ staticSearchFields.metadata_type_ss=Dataset Feature staticSearchFields.dvCategory=Dataverse Category staticSearchFields.metadataSource=Metadata Source staticSearchFields.publicationDate=Publication Year +staticSearchFields.license=License staticSearchFields.fileTypeGroupFacet=File Type staticSearchFields.dvObjectType=Type staticSearchFields.fileTag=File Tag diff --git a/src/main/resources/META-INF/javamail.default.address.map b/src/main/resources/META-INF/javamail.default.address.map new file mode 100644 index 00000000000..b1115c9dc8c --- /dev/null +++ b/src/main/resources/META-INF/javamail.default.address.map @@ -0,0 +1,2 @@ +# See https://jakartaee.github.io/mail-api/docs/api/jakarta.mail/jakarta/mail/Session.html +rfc822=smtp diff --git a/src/main/resources/META-INF/microprofile-config.properties b/src/main/resources/META-INF/microprofile-config.properties index ec8427795ee..b0bc92cf975 100644 --- a/src/main/resources/META-INF/microprofile-config.properties +++ b/src/main/resources/META-INF/microprofile-config.properties @@ -42,6 +42,11 @@ dataverse.rserve.user=rserve dataverse.rserve.password=rserve dataverse.rserve.tempdir=/tmp/Rserv +# MAIL +dataverse.mail.debug=false +dataverse.mail.mta.auth=false +dataverse.mail.mta.allow-utf8-addresses=true + # OAI SERVER dataverse.oai.server.maxidentifiers=100 dataverse.oai.server.maxrecords=10 @@ -50,17 +55,6 @@ dataverse.oai.server.maxsets=100 # can be customized via the setting below: #dataverse.oai.server.repositoryname= -# PERSISTENT IDENTIFIER PROVIDERS -# EZID -dataverse.pid.ezid.api-url=https://ezid.cdlib.org - -# DataCite -dataverse.pid.datacite.mds-api-url=https://mds.test.datacite.org -dataverse.pid.datacite.rest-api-url=https://api.test.datacite.org - -# Handle.Net -dataverse.pid.handlenet.index=300 - # AUTHENTICATION dataverse.auth.oidc.pkce.max-cache-size=10000 dataverse.auth.oidc.pkce.max-cache-age=300 diff --git a/src/main/resources/db/migration/V6.1.0.2__8524-store-tabular-files-with-varheaders.sql b/src/main/resources/db/migration/V6.1.0.2__8524-store-tabular-files-with-varheaders.sql new file mode 100644 index 00000000000..7c52a00107a --- /dev/null +++ b/src/main/resources/db/migration/V6.1.0.2__8524-store-tabular-files-with-varheaders.sql @@ -0,0 +1 @@ +ALTER TABLE datatable ADD COLUMN IF NOT EXISTS storedWithVariableHeader BOOLEAN DEFAULT FALSE; diff --git a/src/main/resources/db/migration/V6.1.0.3__9983-missing-unique-constraints.sql b/src/main/resources/db/migration/V6.1.0.3__9983-missing-unique-constraints.sql new file mode 100644 index 00000000000..6cb3a455e4e --- /dev/null +++ b/src/main/resources/db/migration/V6.1.0.3__9983-missing-unique-constraints.sql @@ -0,0 +1,16 @@ +DO $$ +BEGIN + + BEGIN + ALTER TABLE externalvocabularyvalue ADD CONSTRAINT externalvocabularvalue_uri_key UNIQUE(uri); + EXCEPTION + WHEN duplicate_table THEN RAISE NOTICE 'Table unique constraint externalvocabularvalue_uri_key already exists'; + END; + + BEGIN + ALTER TABLE oaiset ADD CONSTRAINT oaiset_spec_key UNIQUE(spec); + EXCEPTION + WHEN duplicate_table THEN RAISE NOTICE 'Table unique constraint oaiset_spec_key already exists'; + END; + +END $$; \ No newline at end of file diff --git a/src/main/resources/db/migration/V6.1.0.4__5645-geospatial-fieldname-fix.sql b/src/main/resources/db/migration/V6.1.0.4__5645-geospatial-fieldname-fix.sql new file mode 100644 index 00000000000..2ab8cbc802e --- /dev/null +++ b/src/main/resources/db/migration/V6.1.0.4__5645-geospatial-fieldname-fix.sql @@ -0,0 +1,7 @@ +UPDATE datasetfieldtype +SET name = 'northLatitude' +WHERE name = 'northLongitude'; + +UPDATE datasetfieldtype +SET name = 'southLatitude' +WHERE name = 'southLongitude'; \ No newline at end of file diff --git a/src/main/resources/db/migration/V6.1.0.5__3623-multiple-pid-providers.sql b/src/main/resources/db/migration/V6.1.0.5__3623-multiple-pid-providers.sql new file mode 100644 index 00000000000..1d11e178abf --- /dev/null +++ b/src/main/resources/db/migration/V6.1.0.5__3623-multiple-pid-providers.sql @@ -0,0 +1,2 @@ +ALTER TABLE dataverse ADD COLUMN IF NOT EXISTS pidgeneratorspecs TEXT; +ALTER TABLE dataset ADD COLUMN IF NOT EXISTS pidgeneratorspecs TEXT; diff --git a/src/main/resources/db/migration/V6.1.0.6.sql b/src/main/resources/db/migration/V6.1.0.6.sql new file mode 100644 index 00000000000..c9942fb8480 --- /dev/null +++ b/src/main/resources/db/migration/V6.1.0.6.sql @@ -0,0 +1,2 @@ +-- Add flag to allow harvesting client to handle missing CVV values +ALTER TABLE harvestingclient ADD COLUMN IF NOT EXISTS allowharvestingmissingcvv BOOLEAN; diff --git a/src/main/resources/db/migration/V6.1.0.7.sql b/src/main/resources/db/migration/V6.1.0.7.sql new file mode 100644 index 00000000000..470483e2bf4 --- /dev/null +++ b/src/main/resources/db/migration/V6.1.0.7.sql @@ -0,0 +1 @@ +ALTER TABLE authenticateduser ADD COLUMN IF NOT EXISTS ratelimittier int DEFAULT 1; diff --git a/src/main/resources/db/migration/V6.2.0.1.sql b/src/main/resources/db/migration/V6.2.0.1.sql new file mode 100644 index 00000000000..cb23d589542 --- /dev/null +++ b/src/main/resources/db/migration/V6.2.0.1.sql @@ -0,0 +1 @@ +ALTER TABLE datafile ADD COLUMN IF NOT EXISTS retention_id BIGINT; \ No newline at end of file diff --git a/src/main/resources/edu/harvard/iq/dataverse/datacite_metadata_template.xml b/src/main/resources/edu/harvard/iq/dataverse/pidproviders/doi/datacite_metadata_template.xml similarity index 100% rename from src/main/resources/edu/harvard/iq/dataverse/datacite_metadata_template.xml rename to src/main/resources/edu/harvard/iq/dataverse/pidproviders/doi/datacite_metadata_template.xml diff --git a/src/main/webapp/WEB-INF/pretty-config.xml b/src/main/webapp/WEB-INF/pretty-config.xml index ab5f37a1051..5f8f4877af8 100644 --- a/src/main/webapp/WEB-INF/pretty-config.xml +++ b/src/main/webapp/WEB-INF/pretty-config.xml @@ -27,4 +27,9 @@ + + + + + \ No newline at end of file diff --git a/src/main/webapp/dataset-citation.xhtml b/src/main/webapp/dataset-citation.xhtml index b42dd5e563f..346e39dc463 100644 --- a/src/main/webapp/dataset-citation.xhtml +++ b/src/main/webapp/dataset-citation.xhtml @@ -33,13 +33,13 @@
  • - +
  • - +
  • - +
diff --git a/src/main/webapp/dataset-license-terms.xhtml b/src/main/webapp/dataset-license-terms.xhtml index c54d94442ea..6d5b0a5fe4f 100644 --- a/src/main/webapp/dataset-license-terms.xhtml +++ b/src/main/webapp/dataset-license-terms.xhtml @@ -482,7 +482,7 @@
- #{bundle['file.dataFilesTab.terms.list.guestbook']}   + #{bundle['file.dataFilesTab.terms.list.guestbook']}  
@@ -524,7 +524,7 @@ + update=":datasetForm:previewGuestbook" oncomplete="PF('viewGuestbook').show();"/>
@@ -563,7 +563,7 @@ + update=":datasetForm:previewGuestbook" oncomplete="PF('viewGuestbook').show();"/> diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index e50e68ec162..13faf9d7f20 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -32,7 +32,7 @@ and !permissionsWrapper.canIssuePublishDatasetCommand(DatasetPage.dataset)}"/> - + @@ -228,20 +236,21 @@ - -
  • - - #{bundle.transfer} - - - - - -
    • -     - + +
  • + + #{bundle.transfer} + + + + + +
    • +     + + @@ -585,7 +594,7 @@
    -
    +
    @@ -996,8 +1005,10 @@ - + + +

    #{bundle['dataset.share.datasetShare.tip']}

    @@ -1046,10 +1057,10 @@
    - +

    #{bundle['dataset.noValidSelectedFilesForDownload']}

    -

    #{DatasetPage.cantDownloadDueToEmbargo ? bundle['dataset.requestAccessToRestrictedFilesWithEmbargo'] : bundle['dataset.requestAccessToRestrictedFiles']}

    +

    #{DatasetPage.cantDownloadDueToEmbargoOrDVAccess ? bundle['dataset.requestAccessToRestrictedFilesWithEmbargo'] : bundle['dataset.requestAccessToRestrictedFiles']}
    + +

    #{bundle['dataset.noValidSelectedFilesForTransfer']}

    + +

    #{DatasetPage.cantDownloadDueToEmbargoOrDVAccess ? bundle['dataset.requestAccessToRestrictedFilesWithEmbargo'] : bundle['dataset.requestAccessToRestrictedFiles']}

    +     +
    + +
    +

    #{bundle['file.zip.download.exceeds.limit.info']}

    @@ -1078,8 +1100,8 @@
    - -

    #{DatasetPage.cantDownloadDueToEmbargo ? bundle['dataset.mixedSelectedFilesForDownloadWithEmbargo'] : bundle['dataset.mixedSelectedFilesForDownload']}

    + +

    #{DatasetPage.cantDownloadDueToEmbargoOrDVAccess ? bundle['dataset.mixedSelectedFilesForDownloadWithEmbargo'] : bundle['dataset.mixedSelectedFilesForDownload']}

    @@ -1221,7 +1243,12 @@ -

    #{bundle['file.deaccessionDialog.tip']}

    +
    +   +


    + +

    +
    #{bundle['dataset.rejectMessage']}

    - - - - -

    - -

    - -     + + +

    + +

    + + +
    - + + +
    - + @@ -1955,7 +1985,11 @@ PF('downloadTooLarge').show(); } if (outcome ==='FailRestricted'){ - PF('downloadInvalid').show(); + if(isTransfer) { + PF('transferInvalid').show(); + } else { + PF('downloadInvalid').show(); + } } if (outcome ==='GuestbookRequired'){ PF('guestbookAndTermsPopup').show(); diff --git a/src/main/webapp/datasetFieldForEditFragment.xhtml b/src/main/webapp/datasetFieldForEditFragment.xhtml index e72ee351ea0..d8c005366cb 100644 --- a/src/main/webapp/datasetFieldForEditFragment.xhtml +++ b/src/main/webapp/datasetFieldForEditFragment.xhtml @@ -42,6 +42,7 @@ + diff --git a/src/main/webapp/dataverse.xhtml b/src/main/webapp/dataverse.xhtml index 41e2807c4fd..7f70f28e194 100644 --- a/src/main/webapp/dataverse.xhtml +++ b/src/main/webapp/dataverse.xhtml @@ -283,6 +283,19 @@
    +
    + + #{bundle.pidProviderOption} + + +
    + + + + +
    +
    diff --git a/src/main/webapp/dataverseuser.xhtml b/src/main/webapp/dataverseuser.xhtml index 2426cf980d3..9ed8b5209b6 100644 --- a/src/main/webapp/dataverseuser.xhtml +++ b/src/main/webapp/dataverseuser.xhtml @@ -178,9 +178,6 @@ #{DataverseUserPage.getRequestorEmail(item)} - - #{DataverseUserPage.getReasonForReturn(item.theObject)} - diff --git a/src/main/webapp/editdatafiles.xhtml b/src/main/webapp/editdatafiles.xhtml index 02acb224827..be78359e02b 100644 --- a/src/main/webapp/editdatafiles.xhtml +++ b/src/main/webapp/editdatafiles.xhtml @@ -75,8 +75,10 @@ - + + + diff --git a/src/main/webapp/file-download-button-fragment.xhtml b/src/main/webapp/file-download-button-fragment.xhtml index 9c29fd777a1..cd6a6b06523 100644 --- a/src/main/webapp/file-download-button-fragment.xhtml +++ b/src/main/webapp/file-download-button-fragment.xhtml @@ -14,16 +14,18 @@
  • #{bundle['file.accessBtn.header.access']}
  • - + value=" #{dataFileServiceBean.isRetentionExpired(fileMetadata) ? bundle['retentionExpired'] : !fileDownloadHelper.isRestrictedOrEmbargoed(fileMetadata) ? bundle['public'] : (!fileDownloadHelper.canDownloadFile(fileMetadata) ? (!dataFileServiceBean.isActivelyEmbargoed(fileMetadata) ? bundle['restricted'] : bundle['embargoed']) : (!dataFileServiceBean.isActivelyEmbargoed(fileMetadata) ? bundle['restrictedaccess'] : bundle['embargoed']) )}" + styleClass="#{dataFileServiceBean.isRetentionExpired(fileMetadata) ? 'text-danger' : !fileDownloadHelper.isRestrictedOrEmbargoed(fileMetadata) ? 'text-success' : (!fileDownloadHelper.canDownloadFile(fileMetadata) ? 'text-danger' : 'text-success')}"/>
    • + and fileMetadata.dataFile.owner.fileAccessRequest + and !dataFileServiceBean.isActivelyEmbargoed(fileMetadata) + and !dataFileServiceBean.isRetentionExpired(fileMetadata)}"> + + + +
  • + + + +
    • +     + +
  • + + + +
    • +     @@ -102,6 +118,18 @@ + +
  • + + + + +
    • +     +
  • diff --git a/src/main/webapp/file-edit-popup-fragment.xhtml b/src/main/webapp/file-edit-popup-fragment.xhtml index ffc4a1fcef7..3b1141816c8 100644 --- a/src/main/webapp/file-edit-popup-fragment.xhtml +++ b/src/main/webapp/file-edit-popup-fragment.xhtml @@ -168,7 +168,83 @@ PF('blockDatasetForm').hide();" action="#{bean.clearEmbargoPopup()}" update="#{updateElements}" immediate="true"/> - + + + + +

    #{bundle['file.editRetentionDialog.tip']}

    +

    #{bundle['file.editRetentionDialog.some.tip']} #{bundle['file.editRetentionDialog.partial.tip']}

    +

    #{bundle['file.editRetentionDialog.none.tip']}

    + + +
    + +
    +
    + +
    +
    +
    + + + + +
    + +
    +
    +
    +

    #{bundle['file.editRetentionDialog.reason.tip']}

    + +
    +
    +
    +
    +
    +
    +
    + +
    +
    + + + +
    +
    +
    +     +
    + + +
    +     +

    #{bundle['file.deleteFileDialog.immediate']}

    diff --git a/src/main/webapp/file-info-fragment.xhtml b/src/main/webapp/file-info-fragment.xhtml index 72fe279fbf8..dca5c4a8cec 100644 --- a/src/main/webapp/file-info-fragment.xhtml +++ b/src/main/webapp/file-info-fragment.xhtml @@ -64,6 +64,7 @@
    +
    diff --git a/src/main/webapp/file.xhtml b/src/main/webapp/file.xhtml index f69b5c35afd..835764d9cf5 100644 --- a/src/main/webapp/file.xhtml +++ b/src/main/webapp/file.xhtml @@ -43,7 +43,7 @@
    #{FilePage.fileMetadata.label} - +
    @@ -64,22 +64,23 @@

    - + - + + - + - + + value="#{bundle['file.DatasetVersion']} #{FilePage.fileMetadata.datasetVersion.versionNumber}.#{FilePage.fileMetadata.datasetVersion.minorVersionNumber}"/>
    @@ -98,9 +99,9 @@ - -
    @@ -112,19 +113,19 @@
    • @@ -145,11 +146,11 @@ - - -
    @@ -161,19 +162,19 @@
    • @@ -192,7 +193,7 @@
    -
      - @@ -227,12 +228,12 @@
      #{bundle['file.editBtn']} - +
      #{bundle['metrics.file.title']} - -
      @@ -317,7 +318,7 @@
      -
      @@ -351,7 +352,7 @@ - @@ -376,7 +377,7 @@ - +
        @@ -393,8 +394,8 @@
        - -
        @@ -443,7 +444,7 @@ #{bundle['file.dataFilesTab.metadata.addBtn']}
      @@ -483,7 +484,7 @@
      -
      @@ -495,7 +496,7 @@ #{bundle['file.metadata.filetags']}
    + - + - @@ -566,6 +574,13 @@ + + + +
    -
    +
    @@ -504,7 +505,7 @@
    -
    @@ -552,11 +553,18 @@ #{FilePage.file.publicationDateFormattedYYYYMMDD}
    #{bundle['file.metadataTab.fileMetadata.publicationDate.label']} #{!(empty FilePage.file.embargo) ? FilePage.embargoPhrase: ''} #{!(empty FilePage.file.embargo) ? FilePage.file.embargo.dateAvailable : FilePage.file.publicationDateFormattedYYYYMMDD} + #{(empty FilePage.file.embargo) and !(empty FilePage.file.released) and !(empty FilePage.file.retention) ? ''.concat(FilePage.file.publicationDateFormattedYYYYMMDD).concat('; ') : ''} + #{!(empty FilePage.file.embargo) ? ''.concat(FilePage.embargoPhrase).concat(' ').concat(FilePage.file.embargo.dateAvailable) : ''} + #{!(empty FilePage.file.embargo) and !(empty FilePage.file.retention) ? '; ': ''} + #{!(empty FilePage.file.retention) ? ''.concat(FilePage.retentionPhrase) + .concat(' ').concat(FilePage.file.retention.dateUnavailable) + .concat(', ').concat(bundle['retention.enddateinfo']) + : ''} + #{(empty FilePage.file.embargo and empty FilePage.file.retention)? FilePage.file.publicationDateFormattedYYYYMMDD : ''}
    #{FilePage.file.embargo.reason}
    + #{bundle['file.metadataTab.fileMetadata.retentionReason.label']} + #{FilePage.file.retention.reason} +
    #{bundle['file.metadataTab.fileMetadata.size.label']} @@ -623,9 +638,11 @@ + - - + + +

    #{bundle['file.share.tip']}

    @@ -634,7 +651,7 @@ #{bundle.close} -     + @@ -645,7 +662,7 @@ - + @@ -657,7 +674,7 @@ - + @@ -678,7 +695,7 @@ #{bundle.close} - +

    #{bundle['file.compute.fileAccessDenied']}

    @@ -690,7 +707,7 @@