Merge branch 'develop' into 4371-schemaorg #4371

conf/solr/7.3.0/schema.xml

@@ -147,6 +147,7 @@
     <field name="isHarvested" type="boolean" stored="true" indexed="true" multiValued="false"/>
 
     <field name="dvName" type="text_en" stored="true" indexed="true" multiValued="false"/>
+    <field name="dvAlias" type="text_en" stored="true" indexed="true" multiValued="false"/>
     <field name="dvAffiliation" type="text_en" stored="true" indexed="true" multiValued="false"/>
     <field name="dvDescription" type="text_en" stored="true" indexed="true" multiValued="false"/>
 
@@ -456,6 +457,7 @@
     <copyField source="variableLabel" dest="_text_" maxChars="3000"/>
     <!-- Make dataverse subject and affiliation searchable from basic search: https://github.com/IQSS/dataverse/issues/1431 -->
     <copyField source="dvSubject" dest="_text_" maxChars="3000"/>
+    <copyField source="dvAlias" dest="_text_" maxChars="3000"/>
     <copyField source="dvAffiliation" dest="_text_" maxChars="3000"/>
     <copyField source="dsPersistentId" dest="_text_" maxChars="3000"/>
     <!-- copyField commands copy one field to another at the time a document

doc/release-notes/5011-search-by-dataverse-alias.md

@@ -0,0 +1 @@
+The Solr schema.xml file must be updated due to the addition of the "dvAlias" field.

doc/sphinx-guides/source/admin/dataverses-datasets.rst

@@ -29,7 +29,7 @@ Removes a link between a dataverse and another dataverse. Only accessible to sup
     curl -H "X-Dataverse-key: $API_TOKEN" -X DELETE http://$SERVER/api/dataverses/$linked-dataverse-alias/deleteLink/$linking-dataverse-alias
 
 Add Dataverse RoleAssignments to Child Dataverses
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Recursively assigns the users and groups having a role(s),that are in the set configured to be inheritable via the :InheritParentRoleAssignments setting, on a specified dataverse to have the same role assignments on all of the dataverses that have been created within it. The response indicates success or failure and lists the individuals/groups and dataverses involved in the update. Only accessible to superusers. ::
  

doc/sphinx-guides/source/api/apps.rst

@@ -53,6 +53,14 @@ metrics.dataverse.org_ aggregates metrics across multiple Dataverse installation
 Java
 ----
 
+DVUploader
+~~~~~~~~~~~~~~~~~~~~~
+
+The open-source DVUploader tool is a stand-alone command-line Java application that uses the Dataverse API to upload files to a specified Dataset. Files can be specified by name, or the DVUploader can upload all files in a directory or recursively from a directory tree. The DVUploader can also verify that uploaded files match their local sources by comparing the local and remote fixity checksums. Source code, release 1.0.0- jar file, and documentation are available on GitHub. DVUploader's creation was supported by the Texas Digital Library.
+
+https://github.com/IQSS/dataverse-uploader
+
+
 Dataverse for Android
 ~~~~~~~~~~~~~~~~~~~~~
 

doc/sphinx-guides/source/api/sword.rst

@@ -11,7 +11,7 @@ About
 
 Introduced in Dataverse Network (DVN) `3.6 <http://guides.dataverse.org/en/3.6.2/dataverse-api-main.html#data-deposit-api>`_, the SWORD API was formerly known as the "Data Deposit API" and ``data-deposit/v1`` appeared in the URLs. For backwards compatibility these URLs continue to work (with deprecation warnings). Due to architectural changes and security improvements (especially the introduction of API tokens) in Dataverse 4.0, a few backward incompatible changes were necessarily introduced and for this reason the version has been increased to ``v1.1``. For details, see :ref:`incompatible`.
 
-Dataverse implements most of SWORDv2_, which is specified at http://swordapp.github.io/SWORDv2-Profile/SWORDProfile.html . Please reference the `SWORDv2 specification`_ for expected HTTP status codes (i.e. 201, 204, 404, etc.), headers (i.e. "Location"), etc. For a quick introduction to SWORD, the two minute video at http://cottagelabs.com/news/intro-to-sword-2 is recommended.
+Dataverse implements most of SWORDv2_, which is specified at http://swordapp.github.io/SWORDv2-Profile/SWORDProfile.html . Please reference the `SWORDv2 specification`_ for expected HTTP status codes (i.e. 201, 204, 404, etc.), headers (i.e. "Location"), etc.
 
 As a profile of AtomPub, XML is used throughout SWORD. As of Dataverse 4.0 datasets can also be created via JSON using the "native" API. SWORD is limited to the dozen or so fields listed below in the crosswalk, but the native API allows you to populate all metadata fields available in Dataverse.
 

doc/sphinx-guides/source/developers/dev-environment.rst

@@ -166,6 +166,11 @@ After the script has finished, you should be able to log into Dataverse with the
 - username: dataverseAdmin
 - password: admin
 
+Configure Your Development Environment for Publishing
+-----------------------------------------------------
+
+In order to publish datasets, you must configure Dataverse with a username and password for a persistent ID provider. The installer configures your development environment to use DOIs (rather than Handles) for persistent IDs with DataCite's test server at https://mds.test.datacite.org as the provider. In order to publish datasets with this provider, you must email [email protected] and ask for a test account. Once you have your DataCite username and password, you must add them as JVM options (``doi.username`` and ``doi.password``) as described under "Persistent Identifiers and Publishing Datasets" in the :doc:`/installation/config` section of the Installation Guide.
+
 Next Steps
 ----------
 

doc/sphinx-guides/source/developers/intro.rst

@@ -55,6 +55,7 @@ As a developer, you also may be interested in these projects related to Datavers
 
 - External Tools - add additional features to Dataverse: See the :doc:`/installation/external-tools` section of the Installation Guide.
 - Dataverse API client libraries - use Dataverse APIs from various languages: :doc:`/api/client-libraries`
+- DVUploader - a stand-alone command-line Java application that uses the Dataverse API to support upload of files from local disk to a Dataset: https://github.com/IQSS/dataverse-uploader 
 - Miniverse - expose metrics from a Dataverse database: https://github.com/IQSS/miniverse
 - Configuration management scripts - Ansible, Puppet, etc.: See "Advanced Installation" in the :doc:`/installation/prep` section of the Installation Guide.
 - :doc:`/developers/unf/index` (Java) -  a Universal Numerical Fingerprint: https://github.com/IQSS/UNF

doc/sphinx-guides/source/installation/config.rst

@@ -1175,6 +1175,10 @@ Set custom text a user will view when publishing a dataset. Note that this text
 
 ``curl -X PUT -d "Deposit License Requirements" http://localhost:8080/api/admin/settings/:DatasetPublishPopupCustomText``
 
+If you have a long text string, you can upload it as a file as in the example below.
+
+``curl -X PUT --upload-file /tmp/long.txt http://localhost:8080/api/admin/settings/:DatasetPublishPopupCustomText``
+
 :DatasetPublishPopupCustomTextOnAllVersions
 +++++++++++++++++++++++++++++++++++++++++++
 

doc/sphinx-guides/source/user/dataset-management.rst

@@ -104,6 +104,23 @@ An rsync-enabled Dataverse installation has a file upload process that differs f
 
 **Note:** A dataset can only hold one data package. If you need to replace the data package in your dataset, contact Support.
 
+Command-line DVUploader
+-----------------------
+
+The open-source DVUploader tool is a stand-alone command-line Java application that uses the Dataverse API to upload files to a specified Dataset. Since it can be installed by users, and requires no server-side configuration, it can be used with any Dataverse installation. It is intended as an alternative to uploading files through the Dataverse web interface in situations where the web interface is inconvenient due to the number of files or file locations (spread across multiple directories, mixed with files that have already been uploaded or file types that should be excluded) or the need to automate uploads. Since it uses the Dataverse API, transfers are limited in the same ways as HTTP uploads through the Dataverse web interface in terms of size and performance. The DVUploader logs its activity and can be killed and restarted as desired. If stopped and resumed, it will continue processing from where it left off.
+
+Usage
+~~~~~
+
+The DVUploader is open source and is available as source, as a Java jar, and with documentation at https://github.com/IQSS/dataverse-uploader. The DVUploader requires Java 1.8+. Users will need to install Java if they don't already have it and then download the DVUploader-v1.0.0.jar file. Users will need to know the URL of the Dataverse server, the DOI of their existing Dataverse Dataset, and have generated a Dataverse API Key (an option in the user's profile menu). 
+
+Basic usage is to run the command: ::
+
+    java -jar DVUploader-v1.0.0.jar -server=<Dataverse server URL> -did=<Dataset DOI> -key=<User's API Key> <file or directory list>
+
+Additional command line arguments are available to make the DVUploader list what it would do without uploading, limit the number of files it uploads, recurse through sub-directories, verify fixity, exclude files with specific extensions or name patterns, and/or wait longer than 60 seconds for any Dataverse ingest lock to clear (e.g. while the previously uploaded file is processed, as discussed in the File Handling section below). 
+
+DVUploader is a community-developed tool, and its creation was primarily supported by the Texas Digital Library. Further information and support for DVUploader can be sought at `the project's GitHub repository <https://github.com/IQSS/dataverse-uploader>`_ . 
 
 File Handling
 =============

scripts/database/upgrades/upgrade_v4.9.4_to_v4.10.sql

@@ -7,3 +7,5 @@ ALTER TABLE pendingworkflowinvocation ADD COLUMN datasetexternallyreleased BOOLE
 INSERT INTO setting(
            name, content)
    VALUES (':UploadMethods', 'native/http');
+
+ALTER TABLE actionlogrecord ALTER COLUMN info TYPE text;

src/main/java/Bundle.properties

@@ -265,6 +265,9 @@ login.institution.support.afterLink=for assistance.
 login.builtin.credential.usernameOrEmail=Username/Email
 login.builtin.credential.password=Password
 login.builtin.invalidUsernameEmailOrPassword=The username, email address, or password you entered is invalid. Need assistance accessing your account?
+login.echo.credential.name=Name
+login.echo.credential.email=Email
+login.echo.credential.affiliation=Affiliation
 # how do we exercise login.error? Via a password upgrade failure? See https://github.com/IQSS/dataverse/pull/2922
 login.error=Error validating the username, email address, or password. Please try again. If the problem persists, contact an administrator.
 user.error.cannotChangePassword=Sorry, your password cannot be changed. Please contact your system administrator.

src/main/java/Bundle_fr.properties

@@ -264,6 +264,9 @@ login.institution.support.afterLink=pour obtenir de l'aide.
 login.builtin.credential.usernameOrEmail=Nom d'utilisateur/courriel
 login.builtin.credential.password=Mot de passe
 login.builtin.invalidUsernameEmailOrPassword=Le nom d'utilisateur, le courriel ou le mot de passe indiqué n'est pas valide. Avez-vous besoin d'aide pour accéder à votre compte?
+login.echo.credential.name=Nom
+login.echo.credential.email=Email
+login.echo.credential.affiliation=Affiliation
 # how do we exercise login.error? Via a password upgrade failure? See https://github.com/IQSS/dataverse/pull/2922
 login.error=Une erreur s'est produite au moment de la validation du nom d'utilisateur ou du mot de passe. Veuillez essayer à nouveau. Si le problème persiste, communiquez avec un administrateur.
 user.error.cannotChangePassword=Désolé, votre mot de passe ne peut pas être modifié. Veuillez contacter votre administrateur système.

src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java

@@ -42,11 +42,19 @@ public class DOIDataCiteRegisterService {
 
     @EJB
     DataverseServiceBean dataverseService;
+
     @EJB
     DOIDataCiteServiceBean doiDataCiteServiceBean;
 
-    private DataCiteRESTfullClient openClient() throws IOException {
-        return new DataCiteRESTfullClient(System.getProperty("doi.baseurlstring"), System.getProperty("doi.username"), System.getProperty("doi.password"));
+
+    //A singleton since it, and the httpClient in it can be reused.
+    private DataCiteRESTfullClient client=null;
+
+    private DataCiteRESTfullClient getClient() throws IOException {
+        if (client == null) {
+            client = new DataCiteRESTfullClient(System.getProperty("doi.baseurlstring"), System.getProperty("doi.username"), System.getProperty("doi.password"));
+        }
+        return client;
     }
 
     public String createIdentifierLocal(String identifier, Map<String, String> metadata, DvObject dvObject) {
@@ -88,14 +96,16 @@ public String registerIdentifier(String identifier, Map<String, String> metadata
             } else {
                 rc.setUrl(target);
             }
-            try (DataCiteRESTfullClient client = openClient()) {
+            try {
+                DataCiteRESTfullClient client = getClient();
                 retString = client.postMetadata(xmlMetadata);
                 client.postUrl(identifier.substring(identifier.indexOf(":") + 1), target);
             } catch (UnsupportedEncodingException ex) {
                 Logger.getLogger(DOIDataCiteRegisterService.class.getName()).log(Level.SEVERE, null, ex);
             }
         } else {
-            try (DataCiteRESTfullClient client = openClient()) {
+            try {
+                DataCiteRESTfullClient client = getClient();
                 retString = client.postMetadata(xmlMetadata);
                 client.postUrl(identifier.substring(identifier.indexOf(":") + 1), target);
             } catch (UnsupportedEncodingException ex) {
@@ -108,7 +118,8 @@ public String registerIdentifier(String identifier, Map<String, String> metadata
     public String deactivateIdentifier(String identifier, HashMap<String, String> metadata, DvObject dvObject) {
         String retString = "";
         DOIDataCiteRegisterCache rc = findByDOI(identifier);
-        try (DataCiteRESTfullClient client = openClient()) {
+        try {
+            DataCiteRESTfullClient client = getClient();
             if (rc != null) {
                 rc.setStatus("unavailable");
                 retString = client.inactiveDataset(identifier.substring(identifier.indexOf(":") + 1));
@@ -128,7 +139,7 @@ public static String getMetadataFromDvObject(String identifier, Map<String, Stri
         } else {
             dataset = (Dataset) dvObject.getOwner();
         }
-        
+
         DataCiteMetadataTemplate metadataTemplate = new DataCiteMetadataTemplate();
         metadataTemplate.setIdentifier(identifier.substring(identifier.indexOf(':') + 1));
         metadataTemplate.setCreators(Util.getListFromStr(metadata.get("datacite.creator")));
@@ -197,7 +208,8 @@ public String modifyIdentifier(String identifier, HashMap<String, String> metada
                 } else {
                     rc.setUrl(target);
                 }
-                try (DataCiteRESTfullClient client = openClient()) {
+                try {
+                    DataCiteRESTfullClient client = getClient();
                     retString = client.postMetadata(xmlMetadata);
                     client.postUrl(identifier.substring(identifier.indexOf(":") + 1), target);
 
@@ -212,7 +224,8 @@ public String modifyIdentifier(String identifier, HashMap<String, String> metada
             }
         } else if (status.equals("unavailable")) {
             DOIDataCiteRegisterCache rc = findByDOI(identifier);
-            try (DataCiteRESTfullClient client = openClient()) {
+            try {
+                DataCiteRESTfullClient client = getClient();
                 if (rc != null) {
                     rc.setStatus("unavailable");
                     retString = client.inactiveDataset(identifier.substring(identifier.indexOf(":") + 1));
@@ -226,7 +239,8 @@ public String modifyIdentifier(String identifier, HashMap<String, String> metada
 
     public boolean testDOIExists(String identifier) {
         boolean doiExists;
-        try (DataCiteRESTfullClient client = openClient()) {
+        try {
+            DataCiteRESTfullClient client = getClient();
             doiExists = client.testDOIExists(identifier.substring(identifier.indexOf(":") + 1));
         } catch (Exception e) {
             logger.log(Level.INFO, identifier, e);
@@ -237,7 +251,8 @@ public boolean testDOIExists(String identifier) {
 
     public HashMap<String, String> getMetadata(String identifier) throws IOException {
         HashMap<String, String> metadata = new HashMap<>();
-        try (DataCiteRESTfullClient client = openClient()) {
+        try {
+            DataCiteRESTfullClient client = getClient();
             String xmlMetadata = client.getMetadata(identifier.substring(identifier.indexOf(":") + 1));
             DOIDataCiteServiceBean.GlobalIdMetadataTemplate template = doiDataCiteServiceBean.new GlobalIdMetadataTemplate(xmlMetadata);
             metadata.put("datacite.creator", Util.getStrFromList(template.getCreators()));

src/main/java/edu/harvard/iq/dataverse/DataCitation.java

@@ -141,7 +141,7 @@ private void getCommonValuesFrom(DatasetVersion dsv) {
     }
 
     public String getAuthorsString() {
-        return String.join(";", authors);
+        return String.join("; ", authors);
     }
 
     public String getTitle() {
@@ -257,17 +257,16 @@ public void writeAsBibtexCitation(OutputStream os) throws IOException {
         out.write(publisher);
         out.write("},\r\n");
         if(getFileTitle() !=null && isDirect()) {
-        out.write("title = {");
-        out.write(fileTitle);
-        out.write("},\r\n");
-        out.write("booktitle = {");
-        out.write(title);
-        out.write("},\r\n");
+            out.write("title = {");
+            out.write(fileTitle);
+            out.write("},\r\n");
+            out.write("booktitle = {");
+            out.write(title);
+            out.write("},\r\n");
         } else {
             out.write("title = {");
             out.write(title);
             out.write("},\r\n");
-
         }
         out.write("year = {");
         out.write(year);
@@ -280,6 +279,7 @@ public void writeAsBibtexCitation(OutputStream os) throws IOException {
         out.write("url = {");
         out.write(persistentId.toURL().toString());
         out.write("}\r\n");
+        out.write("}\r\n");
         out.flush();
     }
 

src/main/java/edu/harvard/iq/dataverse/DataCiteRESTfullClient.java

@@ -64,9 +64,13 @@ public DataCiteRESTfullClient(String url, String username, String password) thro
         }
     }
 
-    public void close() throws IOException {
+    public void close() {
         if (this.httpClient != null) {
+            try {
            httpClient.close();
+            } catch (IOException io) {
+               logger.warning("IOException closing hhtpClient: " + io.getMessage());
+           }
         }
     }
 
@@ -154,8 +158,10 @@ public boolean testDOIExists(String doi) {
         try {
             HttpResponse response = httpClient.execute(httpGet,context);
             if (response.getStatusLine().getStatusCode() != 200) {
+                EntityUtils.consumeQuietly(response.getEntity());
                 return false;
             }
+            EntityUtils.consumeQuietly(response.getEntity());
             return true;
         } catch (IOException ioe) {
             logger.log(Level.SEVERE, "IOException when get metadata");

src/main/java/edu/harvard/iq/dataverse/LoginPage.java

@@ -162,9 +162,9 @@ public String login() {
         }
         for ( FilledCredential fc : filledCredentialsList ) {
             if(fc.getValue()==null || fc.getValue().isEmpty()){
-                JH.addMessage(FacesMessage.SEVERITY_ERROR, BundleUtil.getStringFromBundle("login."+fc.getCredential().getTitle()));
+                JH.addMessage(FacesMessage.SEVERITY_ERROR, BundleUtil.getStringFromBundle("login."+fc.getCredential().getKey()));
             }
-            authReq.putCredential(fc.getCredential().getTitle(), fc.getValue());
+            authReq.putCredential(fc.getCredential().getKey(), fc.getValue());
         }
         authReq.setIpAddress( dvRequestService.getDataverseRequest().getSourceAddress() );
         try {

src/main/java/edu/harvard/iq/dataverse/actionlogging/ActionLogRecord.java

@@ -67,7 +67,7 @@ public enum ActionType {
 
     private String actionSubType;
 
-    @Column(length = 1024)
+    @Column(columnDefinition="TEXT")
     private String info;
 
     public ActionLogRecord(){}