diff --git a/.github/actions/setup-maven/action.yml b/.github/actions/setup-maven/action.yml new file mode 100644 index 00000000000..4cf09f34231 --- /dev/null +++ b/.github/actions/setup-maven/action.yml @@ -0,0 +1,37 @@ +--- +name: "Setup Maven and Caches" +description: "Determine Java version and setup Maven, including necessary caches." +inputs: + git-reference: + description: 'The git reference (branch/tag) to check out' + required: false + default: '${{ github.ref }}' + pom-paths: + description: "List of paths to Maven POM(s) for cache dependency setup" + required: false + default: 'pom.xml' +runs: + using: composite + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: ${{ inputs.git-reference }} + - name: Determine Java version by reading the Maven property + shell: bash + run: | + echo "JAVA_VERSION=$(grep '' ${GITHUB_WORKSPACE}/modules/dataverse-parent/pom.xml | cut -f2 -d'>' | cut -f1 -d'<')" | tee -a ${GITHUB_ENV} + - name: Set up JDK ${{ env.JAVA_VERSION }} + id: setup-java + uses: actions/setup-java@v4 + with: + java-version: ${{ env.JAVA_VERSION }} + distribution: 'temurin' + cache: 'maven' + cache-dependency-path: ${{ inputs.pom-paths }} + - name: Download common cache on branch cache miss + if: ${{ steps.setup-java.outputs.cache-hit != 'true' }} + uses: actions/cache/restore@v4 + with: + key: dataverse-maven-cache + path: ~/.m2/repository diff --git a/.github/workflows/container_app_push.yml b/.github/workflows/container_app_push.yml index b3e247e376c..3b7ce066d73 100644 --- a/.github/workflows/container_app_push.yml +++ b/.github/workflows/container_app_push.yml @@ -5,6 +5,12 @@ on: # We are deliberately *not* running on push events here to avoid double runs. # Instead, push events will trigger from the base image and maven unit tests via workflow_call. workflow_call: + inputs: + base-image-ref: + type: string + description: "Reference of the base image to build on in full qualified form [/]/:" + required: false + default: "gdcc/base:unstable" pull_request: branches: - develop @@ -16,7 +22,6 @@ on: env: IMAGE_TAG: unstable - BASE_IMAGE_TAG: unstable REGISTRY: "" # Empty means default to Docker Hub PLATFORMS: "linux/amd64,linux/arm64" MASTER_BRANCH_TAG: alpha @@ -33,20 +38,24 @@ jobs: if: ${{ github.repository_owner == 'IQSS' }} steps: - - name: Checkout repository - uses: actions/checkout@v3 - - - name: Set up JDK - uses: actions/setup-java@v3 + - name: Checkout and Setup Maven + uses: IQSS/dataverse/.github/actions/setup-maven@develop with: - java-version: "17" - distribution: temurin - cache: maven + pom-paths: | + pom.xml + modules/container-configbaker/pom.xml + modules/dataverse-parent/pom.xml + + # TODO: Add a filter step here, that avoids building the image if this is a PR and there are other files touched than declared above. + # Use https://github.com/dorny/paths-filter to solve this. This will ensure we do not run this twice if this workflow + # will be triggered by the other workflows already (base image or java changes) + # To become a part of #10618. - name: Build app and configbaker container image with local architecture and submodules (profile will skip tests) run: > mvn -B -f modules/dataverse-parent -P ct -pl edu.harvard.iq:dataverse -am + $( [[ -n "${{ inputs.base-image-ref }}" ]] && echo "-Dbase.image=${{ inputs.base-image-ref }}" ) install # TODO: add smoke / integration testing here (add "-Pct -DskipIntegrationTests=false") @@ -106,11 +115,13 @@ jobs: if: needs.check-secrets.outputs.available == 'true' && ( github.event_name != 'push' || ( github.event_name == 'push' && contains(fromJSON('["develop", "master"]'), github.ref_name))) steps: - - uses: actions/checkout@v3 - - uses: actions/setup-java@v3 + - name: Checkout and Setup Maven + uses: IQSS/dataverse/.github/actions/setup-maven@develop with: - java-version: "17" - distribution: temurin + pom-paths: | + pom.xml + modules/container-configbaker/pom.xml + modules/dataverse-parent/pom.xml # Depending on context, we push to different targets. Login accordingly. - if: github.event_name != 'pull_request' @@ -146,11 +157,13 @@ jobs: run: > mvn -B -f modules/dataverse-parent -P ct -pl edu.harvard.iq:dataverse -am + $( [[ -n "${{ inputs.base-image-ref }}" ]] && echo "-Dbase.image=${{ inputs.base-image-ref }}" ) install - name: Deploy multi-arch application and configbaker container image run: > mvn - -Dapp.image.tag=${{ env.IMAGE_TAG }} -Dbase.image.tag=${{ env.BASE_IMAGE_TAG }} + -Dapp.image.tag=${{ env.IMAGE_TAG }} + $( [[ -n "${{ inputs.base-image-ref }}" ]] && echo "-Dbase.image=${{ inputs.base-image-ref }}" ) ${{ env.REGISTRY }} -Ddocker.platforms=${{ env.PLATFORMS }} -P ct deploy diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index b938851f816..c2340576c78 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -1,99 +1,130 @@ --- -name: Base Container Image +name: Container Images Releasing on: push: + tags: + - 'v[6-9].**' branches: - 'develop' - - 'master' + # "Path filters are not evaluated for pushes of tags" https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#onpushpull_requestpull_request_targetpathspaths-ignore paths: - 'modules/container-base/**' + - '!modules/container-base/src/backports/**' + - '!modules/container-base/README.md' - 'modules/dataverse-parent/pom.xml' - '.github/workflows/container_base_push.yml' - pull_request: - branches: - - 'develop' - - 'master' - paths: - - 'modules/container-base/**' - - 'modules/dataverse-parent/pom.xml' - - '.github/workflows/container_base_push.yml' - schedule: - - cron: '23 3 * * 0' # Run for 'develop' every Sunday at 03:23 UTC + + # These TODOs are left for #10618 + # TODO: we are missing a workflow_call option here, so we can trigger this flow from pr comments and maven tests (keep the secrets availability in mind!) + # TODO: we are missing a pull_request option here (filter for stuff that would trigger the maven runs!) so we can trigger preview builds for them when coming from the main repo (keep the secrets availability in mind!) env: - IMAGE_TAG: unstable PLATFORMS: linux/amd64,linux/arm64 + DEVELOPMENT_BRANCH: develop jobs: build: - name: Build image + name: Base Image runs-on: ubuntu-latest permissions: contents: read packages: read - strategy: - matrix: - jdk: [ '17' ] # Only run in upstream repo - avoid unnecessary runs in forks if: ${{ github.repository_owner == 'IQSS' }} + outputs: + base-image-ref: ${{ steps.finalize.outputs.base-image-ref }} steps: - - name: Checkout repository - uses: actions/checkout@v3 - - - name: Set up JDK ${{ matrix.jdk }} - uses: actions/setup-java@v3 + - name: Checkout and Setup Maven + uses: IQSS/dataverse/.github/actions/setup-maven@develop with: - java-version: ${{ matrix.jdk }} - distribution: 'adopt' - - name: Cache Maven packages - uses: actions/cache@v3 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2 - - - name: Build base container image with local architecture - run: mvn -f modules/container-base -Pct package + pom-paths: modules/container-base/pom.xml - # Run anything below only if this is not a pull request. - # Accessing, pushing tags etc. to DockerHub will only succeed in upstream because secrets. - - - if: ${{ github.event_name == 'push' && github.ref_name == 'develop' }} - name: Push description to DockerHub - uses: peter-evans/dockerhub-description@v3 + # Note: Accessing, pushing tags etc. to DockerHub will only succeed in upstream and + # on events in context of upstream because secrets. PRs run in context of forks by default! + - name: Log in to the Container registry + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - repository: gdcc/base - short-description: "Dataverse Base Container image providing Payara application server and optimized configuration" - readme-filepath: ./modules/container-base/README.md - - if: ${{ github.event_name != 'pull_request' }} - name: Log in to the Container registry - uses: docker/login-action@v2 + # In case this is a push to develop, we care about buildtime. + # Configure a remote ARM64 build host in addition to the local AMD64 in two steps. + - name: Setup SSH agent + if: ${{ github.event_name != 'schedule' }} + uses: webfactory/ssh-agent@v0.9.0 with: - registry: ${{ env.REGISTRY }} - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - if: ${{ github.event_name != 'pull_request' }} - name: Set up QEMU for multi-arch builds - uses: docker/setup-qemu-action@v2 - - name: Re-set image tag based on branch - if: ${{ github.ref_name == 'master' }} - run: echo "IMAGE_TAG=alpha" >> $GITHUB_ENV - - if: ${{ github.event_name != 'pull_request' }} - name: Deploy multi-arch base container image to Docker Hub - run: mvn -f modules/container-base -Pct deploy -Dbase.image.tag=${{ env.IMAGE_TAG }} -Ddocker.platforms=${{ env.PLATFORMS }} + ssh-private-key: ${{ secrets.BUILDER_ARM64_SSH_PRIVATE_KEY }} + - name: Provide the known hosts key and the builder config + if: ${{ github.event_name != 'schedule' }} + run: | + echo "${{ secrets.BUILDER_ARM64_SSH_HOST_KEY }}" > ~/.ssh/known_hosts + mkdir -p modules/container-base/target/buildx-state/buildx/instances + cat > modules/container-base/target/buildx-state/buildx/instances/maven << EOF + { "Name": "maven", + "Driver": "docker-container", + "Dynamic": false, + "Nodes": [{"Name": "maven0", + "Endpoint": "unix:///var/run/docker.sock", + "Platforms": [{"os": "linux", "architecture": "amd64"}], + "DriverOpts": null, + "Flags": ["--allow-insecure-entitlement=network.host"], + "Files": null}, + {"Name": "maven1", + "Endpoint": "ssh://${{ secrets.BUILDER_ARM64_SSH_CONNECTION }}", + "Platforms": [{"os": "linux", "architecture": "arm64"}], + "DriverOpts": null, + "Flags": ["--allow-insecure-entitlement=network.host"], + "Files": null}]} + EOF + + # Determine the base image name we are going to use from here on + - name: Determine base image name + run: | + if [[ "${{ github.ref_name }}" = "${{ env.DEVELOPMENT_BRANCH }}" ]]; then + echo "BASE_IMAGE=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout )" | tee -a "${GITHUB_ENV}" + echo "BASE_IMAGE_UPCOMING=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag.suffix="" -q -DforceStdout )" | tee -a "${GITHUB_ENV}" + else + echo "BASE_IMAGE=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag.suffix="" -q -DforceStdout )" | tee -a "${GITHUB_ENV}" + fi + - name: Calculate revision number for immutable tag (on release branches only) + if: ${{ github.ref_name != env.DEVELOPMENT_BRANCH }} + id: revision-tag + uses: ./.github/actions/get-image-revision + with: + image-ref: ${{ env.BASE_IMAGE }} + tag-options-prefix: "-Dbase.image.tag.suffix='' -Ddocker.tags.revision=" + - name: Configure update of "latest" tag for development branch + id: develop-tag + if: ${{ github.ref_name == env.DEVELOPMENT_BRANCH }} + run: | + echo "tag-options=-Ddocker.tags.develop=unstable -Ddocker.tags.upcoming=${BASE_IMAGE_UPCOMING#*:}" | tee -a "${GITHUB_OUTPUT}" + + - name: Deploy multi-arch base container image to Docker Hub + id: build + run: | + mvn -f modules/container-base -Pct deploy -Ddocker.noCache -Ddocker.platforms=${{ env.PLATFORMS }} \ + -Ddocker.imagePropertyConfiguration=override ${{ steps.develop-tag.outputs.tag-options }} ${{ steps.revision-tag.outputs.tag-options }} + + - name: Determine appropriate base image ref for app image + id: finalize + run: | + if [[ "${{ github.ref_name }}" = "${{ env.DEVELOPMENT_BRANCH }}" ]]; then + echo "base-image-ref=${BASE_IMAGE_UPCOMING}" | tee -a "$GITHUB_OUTPUT" + else + echo "base-image-ref=gdcc/base:${{ steps.revision-tag.outputs.revision-tag }}" | tee -a "$GITHUB_OUTPUT" + fi + push-app-img: name: "Rebase & Publish App Image" permissions: contents: read packages: write pull-requests: write - needs: build - # We do not release a new base image for pull requests, so do not trigger. - if: ${{ github.event_name != 'pull_request' }} - uses: ./.github/workflows/container_app_push.yml secrets: inherit + needs: + - build + uses: ./.github/workflows/container_app_push.yml + with: + base-image-ref: ${{ needs.build.outputs.base-image-ref }} diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml new file mode 100644 index 00000000000..986fe25cdf5 --- /dev/null +++ b/.github/workflows/container_maintenance.yml @@ -0,0 +1,119 @@ +--- +name: Container Images Scheduled Maintenance + +on: + # TODO: think about adding a (filtered) push event trigger here in case we change the patches + # --- + # Allow manual workflow triggers in case we need to repair images on Docker Hub (build and replace) + workflow_dispatch: + inputs: + force_build: + type: boolean + required: false + default: false + description: "Build and deploy even if no newer Java images or package updates are found." + schedule: + - cron: '23 3 * * 0' # Run for 'develop' every Sunday at 03:23 UTC + +env: + PLATFORMS: linux/amd64,linux/arm64 + NUM_PAST_RELEASES: 3 + +jobs: + build: + name: Base Image Matrix Build + runs-on: ubuntu-latest + permissions: + contents: read + packages: read + # Only run in upstream repo - avoid unnecessary runs in forks + if: ${{ github.repository_owner == 'IQSS' }} + outputs: + supported_tag_matrix: ${{ steps.execute.outputs.supported_tag_matrix }} + rebuilt_base_images: ${{ steps.execute.outputs.rebuilt_base_images }} + + steps: + - name: Checkout and Setup Maven + uses: IQSS/dataverse/.github/actions/setup-maven@develop + with: + pom-paths: modules/container-base/pom.xml + + # Note: Accessing, pushing tags etc. to DockerHub will only succeed in upstream and + # on events in context of upstream because secrets. PRs run in context of forks by default! + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Set up QEMU for multi-arch builds + uses: docker/setup-qemu-action@v3 + with: + platforms: ${{ env.PLATFORMS }} + + # Discover the releases we want to maintain + - name: Discover maintained releases + id: discover + run: | + echo "FORCE_BUILD=$( [[ "${{ inputs.force_build }}" = "true" ]] && echo 1 || echo 0 )" | tee -a "$GITHUB_ENV" + DEVELOPMENT_BRANCH=$( curl -f -sS https://api.github.com/repos/${{ github.repository }} | jq -r '.default_branch' ) + echo "DEVELOPMENT_BRANCH=$DEVELOPMENT_BRANCH" | tee -a "$GITHUB_ENV" + echo "branches=$( curl -f -sS https://api.github.com/repos/IQSS/dataverse/releases | jq -r " .[0:${{ env.NUM_PAST_RELEASES }}] | .[].tag_name, \"${DEVELOPMENT_BRANCH}\" " | tr "\n" " " )" | tee -a "${GITHUB_OUTPUT}" + + # Execute matrix build for the discovered branches + - name: Execute build matrix script + id: execute + run: | + .github/workflows/scripts/maintenance-job.sh ${{ steps.discover.outputs.branches }} + + # TODO: Use the needs.build.outputs.rebuilt_base_images with fromJSON() to create a matrix job. + # Must be a single rank matrix (vector), the branch and base image tag information ships as "branch=tag" string + # Will be part of working on #10618, app image versioned tags. + #push-app-img: + # name: "Rebase & Publish App Image" + # permissions: + # contents: read + # packages: write + # pull-requests: write + # secrets: inherit + # needs: + # - build + # strategy: + # fail-fast: false + # matrix: + # branch: ${{ fromJson(needs.discover.outputs.branches) }} + # uses: ./.github/workflows/container_app_push.yml + # with: + # branch: ${{ matrix.branch }} + + hub-description: + name: Push description to DockerHub + runs-on: ubuntu-latest + permissions: + contents: read + packages: read + needs: build + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Render README + id: render + run: | + TAGS_JSON='${{ needs.build.outputs.supported_tag_matrix }}' + echo "$TAGS_JSON" | jq -r 'keys | sort | reverse | .[]' | + while IFS= read -r branch; do + echo \ + "- \`$( echo "$TAGS_JSON" | jq --arg v "$branch" -r '.[$v] | join("`, `")' )\`" \ + "([Dockerfile](https://github.com/IQSS/dataverse/blob/${branch}/modules/container-base/src/main/docker/Dockerfile)," \ + "[Patches](https://github.com/IQSS/dataverse/blob/develop/modules/container-base/src/backports/${branch}))" \ + | tee -a "${GITHUB_WORKSPACE}/tags.md" + done + sed -i -e "/<\!-- TAG BLOCK HERE -->/r ${GITHUB_WORKSPACE}/tags.md" "./modules/container-base/README.md" + + - name: Push description to DockerHub + uses: peter-evans/dockerhub-description@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + repository: gdcc/base + short-description: "Dataverse Base Container image providing Payara application server and optimized configuration" + readme-filepath: ./modules/container-base/README.md \ No newline at end of file diff --git a/.github/workflows/maven_unit_test.yml b/.github/workflows/maven_unit_test.yml index 4ad4798bc64..a94b17a67ba 100644 --- a/.github/workflows/maven_unit_test.yml +++ b/.github/workflows/maven_unit_test.yml @@ -30,6 +30,7 @@ jobs: continue-on-error: ${{ matrix.experimental }} runs-on: ubuntu-latest steps: + # TODO: As part of #10618 change to setup-maven custom action # Basic setup chores - uses: actions/checkout@v3 - name: Set up JDK ${{ matrix.jdk }} @@ -95,6 +96,7 @@ jobs: # status: "Experimental" continue-on-error: ${{ matrix.experimental }} steps: + # TODO: As part of #10618 change to setup-maven custom action # Basic setup chores - uses: actions/checkout@v3 - name: Set up JDK ${{ matrix.jdk }} @@ -128,6 +130,7 @@ jobs: needs: integration-test name: Coverage Report Submission steps: + # TODO: As part of #10618 change to setup-maven custom action # Basic setup chores - uses: actions/checkout@v3 - uses: actions/setup-java@v3 @@ -156,6 +159,11 @@ jobs: # NOTE: this may be extended with adding a report to the build output, leave a comment, send to Sonarcloud, ... + # TODO: Add a filter step here, that avoids calling the app image release workflow if there are changes to the base image. + # Use https://github.com/dorny/paths-filter to solve this. Will require and additional job or adding to integration-test job. + # This way we ensure that we're not running the app image flow with a non-matching base image. + # To become a part of #10618. + push-app-img: name: Publish App Image permissions: diff --git a/.github/workflows/scripts/maintenance-job.sh b/.github/workflows/scripts/maintenance-job.sh new file mode 100755 index 00000000000..370988b9812 --- /dev/null +++ b/.github/workflows/scripts/maintenance-job.sh @@ -0,0 +1,180 @@ +#!/bin/bash + +# A matrix-like job to maintain a number of releases as well as the latest snap of Dataverse. + +# PREREQUISITES: +# - You have Java, Maven, QEMU and Docker all setup and ready to go +# - You obviously checked out the develop branch, otherwise you'd not be executing this script +# - You added all the branch names you want to run maintenance for as arguments +# Optional, but recommended: +# - You added a DEVELOPMENT_BRANCH env var to your runner/job env with the name of the development branch +# - You added a FORCE_BUILD=0|1 env var to indicate if the base image build should be forced +# - You added a PLATFORMS env var with all the target platforms you want to build for + +# NOTE: +# This script is a culmination of Github Action steps into a single script. +# The reason to put all of this in here is due to the complexity of the Github Action and the limitation of the +# matrix support in Github actions, where outputs cannot be aggregated or otherwise used further. + +set -euo pipefail + +# Get all the inputs +# If not within a runner, just print to stdout (duplicating the output in case of tee usage, but that's ok for testing) +GITHUB_OUTPUT=${GITHUB_OUTPUT:-"/proc/self/fd/1"} +GITHUB_ENV=${GITHUB_ENV:-"/proc/self/fd/1"} +GITHUB_WORKSPACE=${GITHUB_WORKSPACE:-"$(pwd)"} +GITHUB_SERVER_URL=${GITHUB_SERVER_URL:-"https://github.com"} +GITHUB_REPOSITORY=${GITHUB_REPOSITORY:-"IQSS/dataverse"} + +MAINTENANCE_WORKSPACE="${GITHUB_WORKSPACE}/maintenance-job" + +DEVELOPMENT_BRANCH="${DEVELOPMENT_BRANCH:-"develop"}" +FORCE_BUILD="${FORCE_BUILD:-"0"}" +PLATFORMS="${PLATFORMS:-"linux/amd64,linux/arm64"}" + +# Setup and validation +if [[ -z "$*" ]]; then + >&2 echo "You must give a list of branch names as arguments" + exit 1; +fi + +source "$( dirname "$0" )/utils.sh" + +# Delete old stuff if present +rm -rf "$MAINTENANCE_WORKSPACE" +mkdir -p "$MAINTENANCE_WORKSPACE" + +# Store the image tags we maintain in this array (same order as branches array!) +# This list will be used to build the support matrix within the Docker Hub image description +SUPPORTED_ROLLING_TAGS=() +# Store the tags of base images we are actually rebuilding to base new app images upon +# Takes the from "branch-name=base-image-ref" +REBUILT_BASE_IMAGES=() + +for BRANCH in "$@"; do + echo "::group::Running maintenance for $BRANCH" + + # 0. Determine if this is a development branch and the most current release + IS_DEV=0 + if [[ "$BRANCH" = "$DEVELOPMENT_BRANCH" ]]; then + IS_DEV=1 + fi + IS_CURRENT_RELEASE=0 + if [[ "$BRANCH" = $( curl -f -sS "https://api.github.com/repos/$GITHUB_REPOSITORY/releases" | jq -r '.[0].tag_name' ) ]]; then + IS_CURRENT_RELEASE=1 + fi + + # 1. Let's get the maintained sources + git clone -c advice.detachedHead=false --depth 1 --branch "$BRANCH" "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" "$MAINTENANCE_WORKSPACE/$BRANCH" + # Switch context + cd "$MAINTENANCE_WORKSPACE/$BRANCH" + + # 2. Now let's apply the patches (we have them checked out in $GITHUB_WORKSPACE, not necessarily in this local checkout) + echo "Checking for patches..." + if [[ -d ${GITHUB_WORKSPACE}/modules/container-base/src/backports/$BRANCH ]]; then + echo "Applying patches now." + find "${GITHUB_WORKSPACE}/modules/container-base/src/backports/$BRANCH" -type f -name '*.patch' -print0 | xargs -0 -n1 patch -p1 -s -i + fi + + # 3. Determine the base image ref (/:) + BASE_IMAGE_REF="" + # For the dev branch we want to full flexi stack tag, to detect stack upgrades requiring new build + if (( IS_DEV )); then + BASE_IMAGE_REF=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout ) + else + BASE_IMAGE_REF=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag.suffix="" -q -DforceStdout ) + fi + echo "Determined BASE_IMAGE_REF=$BASE_IMAGE_REF from Maven" + + # 4. Check for Temurin image updates + JAVA_IMAGE_REF=$( mvn help:evaluate -Pct -f modules/container-base -Dexpression=java.image -q -DforceStdout ) + echo "Determined JAVA_IMAGE_REF=$JAVA_IMAGE_REF from Maven" + NEWER_JAVA_IMAGE=0 + if check_newer_parent "$JAVA_IMAGE_REF" "$BASE_IMAGE_REF"; then + NEWER_JAVA_IMAGE=1 + fi + + # 5. Check for package updates in base image + PKGS="$( grep "ARG PKGS" modules/container-base/src/main/docker/Dockerfile | cut -f2 -d= | tr -d '"' )" + echo "Determined installed packages=\"$PKGS\" from Maven" + NEWER_PKGS=0 + # Don't bother with package checks if the java image is newer already + if ! (( NEWER_JAVA_IMAGE )); then + if check_newer_pkgs "$BASE_IMAGE_REF" "$PKGS"; then + NEWER_PKGS=1 + fi + fi + + # 6. Get current immutable revision tag if not on the dev branch + REV=$( current_revision "$BASE_IMAGE_REF" ) + CURRENT_REV_TAG="${BASE_IMAGE_REF#*:}-r$REV" + NEXT_REV_TAG="${BASE_IMAGE_REF#*:}-r$(( REV + 1 ))" + + # 7. Let's put together what tags we want added to this build run + TAG_OPTIONS="" + if ! (( IS_DEV )); then + TAG_OPTIONS="-Dbase.image=$BASE_IMAGE_REF -Ddocker.tags.revision=$NEXT_REV_TAG" + # In case of the current release, add the "latest" tag as well. + if (( IS_CURRENT_RELEASE )); then + TAG_OPTIONS="$TAG_OPTIONS -Ddocker.tags.latest=latest" + fi + else + UPCOMING_TAG=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image.tag -Dbase.image.tag.suffix="" -q -DforceStdout ) + TAG_OPTIONS="-Ddocker.tags.develop=unstable -Ddocker.tags.upcoming=$UPCOMING_TAG" + + # For the dev branch we only have rolling tags and can add them now already + SUPPORTED_ROLLING_TAGS+=("[\"unstable\", \"$UPCOMING_TAG\", \"${BASE_IMAGE_REF#*:}\"]") + fi + echo "Determined these additional Maven tag options: $TAG_OPTIONS" + + # 8. Let's build the base image if necessary + NEWER_IMAGE=0 + if (( NEWER_JAVA_IMAGE + NEWER_PKGS + FORCE_BUILD > 0 )); then + mvn -Pct -f modules/container-base deploy -Ddocker.noCache -Ddocker.platforms="${PLATFORMS}" \ + -Ddocker.imagePropertyConfiguration=override $TAG_OPTIONS + NEWER_IMAGE=1 + # Save the information about the immutable or rolling tag we just built + if ! (( IS_DEV )); then + REBUILT_BASE_IMAGES+=("$BRANCH=${BASE_IMAGE_REF%:*}:$NEXT_REV_TAG") + else + REBUILT_BASE_IMAGES+=("$BRANCH=$BASE_IMAGE_REF") + fi + else + echo "No rebuild necessary, we're done here." + fi + + # 9. Add list of rolling and immutable tags for release builds + if ! (( IS_DEV )); then + RELEASE_TAGS_LIST="[" + if (( IS_CURRENT_RELEASE )); then + RELEASE_TAGS_LIST+="\"latest\", " + fi + RELEASE_TAGS_LIST+="\"${BASE_IMAGE_REF#*:}\", " + if (( NEWER_IMAGE )); then + RELEASE_TAGS_LIST+="\"$NEXT_REV_TAG\"]" + else + RELEASE_TAGS_LIST+="\"$CURRENT_REV_TAG\"]" + fi + SUPPORTED_ROLLING_TAGS+=("${RELEASE_TAGS_LIST}") + fi + + echo "::endgroup::" +done + +# Built the output which base images have actually been rebuilt as JSON +REBUILT_IMAGES="[" +for IMAGE in "${REBUILT_BASE_IMAGES[@]}"; do + REBUILT_IMAGES+=" \"$IMAGE\" " +done +REBUILT_IMAGES+="]" +echo "rebuilt_base_images=${REBUILT_IMAGES// /, }" | tee -a "${GITHUB_OUTPUT}" + +# Built the supported rolling tags matrix as JSON +SUPPORTED_TAGS="{" +for (( i=0; i < ${#SUPPORTED_ROLLING_TAGS[@]} ; i++ )); do + j=$((i+1)) + SUPPORTED_TAGS+="\"${!j}\": ${SUPPORTED_ROLLING_TAGS[$i]}" + (( i < ${#SUPPORTED_ROLLING_TAGS[@]}-1 )) && SUPPORTED_TAGS+=", " +done +SUPPORTED_TAGS+="}" +echo "supported_tag_matrix=$SUPPORTED_TAGS" | tee -a "$GITHUB_OUTPUT" diff --git a/.github/workflows/scripts/utils.sh b/.github/workflows/scripts/utils.sh new file mode 100644 index 00000000000..987b58d8bb5 --- /dev/null +++ b/.github/workflows/scripts/utils.sh @@ -0,0 +1,108 @@ +#!/bin/bash + +set -euo pipefail + +function check_newer_parent() { + PARENT_IMAGE="$1" + # Get namespace, default to "library" if not found + PARENT_IMAGE_NS="${PARENT_IMAGE%/*}" + if [[ "$PARENT_IMAGE_NS" = "${PARENT_IMAGE}" ]]; then + PARENT_IMAGE_NS="library" + fi + PARENT_IMAGE_REPO="${PARENT_IMAGE%:*}" + PARENT_IMAGE_TAG="${PARENT_IMAGE#*:}" + + PARENT_IMAGE_LAST_UPDATE="$( curl -sS "https://hub.docker.com/v2/namespaces/${PARENT_IMAGE_NS}/repositories/${PARENT_IMAGE_REPO}/tags/${PARENT_IMAGE_TAG}" | jq -r .last_updated )" + if [[ "$PARENT_IMAGE_LAST_UPDATE" = "null" ]]; then + echo "::error title='Invalid PARENT Image'::Could not find ${PARENT_IMAGE} in the registry" + exit 1 + fi + + DERIVED_IMAGE="$2" + # Get namespace, default to "library" if not found + DERIVED_IMAGE_NS="${DERIVED_IMAGE%/*}" + if [[ "${DERIVED_IMAGE_NS}" = "${DERIVED_IMAGE}" ]]; then + DERIVED_IMAGE_NS="library" + fi + DERIVED_IMAGE_REPO="$( echo "${DERIVED_IMAGE%:*}" | cut -f2 -d/ )" + DERIVED_IMAGE_TAG="${DERIVED_IMAGE#*:}" + + DERIVED_IMAGE_LAST_UPDATE="$( curl -sS "https://hub.docker.com/v2/namespaces/${DERIVED_IMAGE_NS}/repositories/${DERIVED_IMAGE_REPO}/tags/${DERIVED_IMAGE_TAG}" | jq -r .last_updated )" + if [[ "$DERIVED_IMAGE_LAST_UPDATE" = "null" || "$DERIVED_IMAGE_LAST_UPDATE" < "$PARENT_IMAGE_LAST_UPDATE" ]]; then + echo "Parent image $PARENT_IMAGE has a newer release ($PARENT_IMAGE_LAST_UPDATE), which is more recent than $DERIVED_IMAGE ($DERIVED_IMAGE_LAST_UPDATE)" + return 0 + else + echo "Parent image $PARENT_IMAGE ($PARENT_IMAGE_LAST_UPDATE) is older than $DERIVED_IMAGE ($DERIVED_IMAGE_LAST_UPDATE)" + return 1 + fi +} + +function check_newer_pkgs() { + IMAGE="$1" + PKGS="$2" + + docker run --rm -u 0 "${IMAGE}" sh -c "apt update >/dev/null 2>&1 && apt install -s ${PKGS}" | tee /proc/self/fd/2 | grep -q "0 upgraded" + STATUS=$? + + if [[ $STATUS -eq 0 ]]; then + echo "Base image $IMAGE has no updates for our custom installed packages" + return 1 + else + echo "Base image $IMAGE needs updates for our custom installed packages" + return 0 + fi + + # TODO: In a future version of this script, we might want to include checking for other security updates, + # not just updates to the packages we installed. + # grep security /etc/apt/sources.list > /tmp/security.list + # apt-get update -oDir::Etc::Sourcelist=/tmp/security.list + # apt-get dist-upgrade -y -oDir::Etc::Sourcelist=/tmp/security.list -oDir::Etc::SourceParts=/bin/false -s + +} + +function current_revision() { + IMAGE="$1" + IMAGE_NS_REPO="${IMAGE%:*}" + IMAGE_TAG="${IMAGE#*:}" + + if [[ "$IMAGE_TAG" = "$IMAGE_NS_REPO" ]]; then + >&2 echo "You must provide an image reference in the format [/]:" + exit 1 + fi + + case "$IMAGE_NS_REPO" in + */*) :;; # namespace/repository syntax, leave as is + *) IMAGE_NS_REPO="library/$IMAGE_NS_REPO";; # bare repository name (docker official image); must convert to namespace/repository syntax + esac + + # Without such a token we may run into rate limits + # OB 2024-09-16: for some reason using this token stopped working. Let's go without and see if we really fall into rate limits. + # token=$( curl -s "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$IMAGE_NS_REPO:pull" ) + + ALL_TAGS="$( + i=0 + while [ $? == 0 ]; do + i=$((i+1)) + # OB 2024-09-16: for some reason using this token stopped working. Let's go without and see if we really fall into rate limits. + # RESULT=$( curl -s -H "Authorization: Bearer $token" "https://registry.hub.docker.com/v2/repositories/$IMAGE_NS_REPO/tags/?page=$i&page_size=100" ) + RESULT=$( curl -s "https://registry.hub.docker.com/v2/repositories/$IMAGE_NS_REPO/tags/?page=$i&page_size=100" ) + if [[ $( echo "$RESULT" | jq '.message' ) != "null" ]]; then + # If we run into an error on the first attempt, that means we have a problem. + if [[ "$i" == "1" ]]; then + >&2 echo "Error when retrieving tag data: $( echo "$RESULT" | jq '.message' )" + exit 2 + # Otherwise it will just mean we reached the last page already + else + break + fi + else + echo "$RESULT" | jq -r '."results"[]["name"]' + # DEBUG: + #echo "$RESULT" | >&2 jq -r '."results"[]["name"]' + fi + done + )" + + # Note: if a former tag could not be found, it just might not exist already. Start new series with rev 0 + echo "$ALL_TAGS" | grep "${IMAGE_TAG}-r" | sed -e "s#${IMAGE_TAG}-r##" | sort -h | tail -n1 || echo "-1" +} diff --git a/conf/solr/schema.xml b/conf/solr/schema.xml index b323d0c74af..1773837e39d 100644 --- a/conf/solr/schema.xml +++ b/conf/solr/schema.xml @@ -142,6 +142,7 @@ + diff --git a/doc/release-notes/10287-use-support-address-in-system-email-text.md b/doc/release-notes/10287-use-support-address-in-system-email-text.md new file mode 100644 index 00000000000..4c294404298 --- /dev/null +++ b/doc/release-notes/10287-use-support-address-in-system-email-text.md @@ -0,0 +1,4 @@ +### Notification Email Improvement + +The system email text has been improved to use the support email address (`dataverse.mail.support-email`) in the text where it states; "contact us for support at", instead of the default system email address. +Using the system email address here was particularly problematic when it was a 'noreply' address. diff --git a/doc/release-notes/10478-version-base-image.md b/doc/release-notes/10478-version-base-image.md new file mode 100644 index 00000000000..34f444a2122 --- /dev/null +++ b/doc/release-notes/10478-version-base-image.md @@ -0,0 +1,7 @@ +### Adding versioned tags to Container Base Images + +With this release we introduce a detailed maintenance workflow for our container images. +As output of the GDCC Containerization Working Group, the community takes another step towards production ready containers available directly from the core project. + +The maintenance workflow regularly updates the Container Base Image, which contains the operating system, Java, Payara Application Server, as well as tools and libraries required by the Dataverse application. +Shipping these rolling releases as well as immutable revisions is the foundation for secure and reliable Dataverse Application Container images. diff --git a/doc/release-notes/10810-search-api-payload-extensions.md b/doc/release-notes/10810-search-api-payload-extensions.md new file mode 100644 index 00000000000..5112d9f62ee --- /dev/null +++ b/doc/release-notes/10810-search-api-payload-extensions.md @@ -0,0 +1,52 @@ +Search API (/api/search) response will now include new fields for the different entities. + +For Dataverse: + +- "affiliation" +- "parentDataverseName" +- "parentDataverseIdentifier" +- "image_url" (optional) + +```javascript +"items": [ + { + "name": "Darwin's Finches", + ... + "affiliation": "Dataverse.org", + "parentDataverseName": "Root", + "parentDataverseIdentifier": "root", + "image_url":"data:image/png;base64,iVBORw0..." +(etc, etc) +``` + +For DataFile: + +- "releaseOrCreateDate" +- "image_url" (optional) + +```javascript +"items": [ + { + "name": "test.txt", + ... + "releaseOrCreateDate": "2016-05-10T12:53:39Z", + "image_url":"data:image/png;base64,iVBORw0..." +(etc, etc) +``` + +For Dataset: + +- "image_url" (optional) + +```javascript +"items": [ + { + ... + "image_url": "http://localhost:8080/api/datasets/2/logo" + ... +(etc, etc) +``` + +The image_url field was already part of the SolrSearchResult JSON (and incorrectly appeared in Search API documentation), but it wasn’t returned by the API because it was appended only after the Solr query was executed in the SearchIncludeFragment of JSF. Now, the field is set in SearchServiceBean, ensuring it is always returned by the API when an image is available. + +The schema.xml file for Solr has been updated to include a new field called dvParentAlias for supporting the new response field "parentDataverseIdentifier". So for the next Dataverse released version, a Solr reindex will be necessary to apply the new schema.xml version. diff --git a/doc/release-notes/8129-harvesting.md b/doc/release-notes/8129-harvesting.md new file mode 100644 index 00000000000..63ca8744941 --- /dev/null +++ b/doc/release-notes/8129-harvesting.md @@ -0,0 +1,18 @@ +### Remap oai_dc export and harvesting format fields: dc:type and dc:date + +The `oai_dc` export and harvesting format has had the following fields remapped: + +- dc:type was mapped to the field "Kind of Data". Now it is hard-coded to the word "Dataset". +- dc:date was mapped to the field "Production Date" when available and otherwise to "Publication Date". Now it is mapped the field "Publication Date" or the field used for the citation date, if set (see [Set Citation Date Field Type for a Dataset](https://guides.dataverse.org/en/6.3/api/native-api.html#set-citation-date-field-type-for-a-dataset)). + +In order for these changes to be reflected in existing datasets, a [reexport all](https://guides.dataverse.org/en/6.3/admin/metadataexport.html#batch-exports-through-the-api) should be run. + +For more information, please see #8129 and #10737. + +### Backward incompatible changes + +See the "Remap oai_dc export" section above. + +### Upgrade instructions + +In order for changes to the `oai_dc` metadata export format to be reflected in existing datasets, a [reexport all](https://guides.dataverse.org/en/6.3/admin/metadataexport.html#batch-exports-through-the-api) should be run. diff --git a/doc/release-notes/8581-add-crossref-pid-provider.md b/doc/release-notes/8581-add-crossref-pid-provider.md new file mode 100644 index 00000000000..3610aa6d7cc --- /dev/null +++ b/doc/release-notes/8581-add-crossref-pid-provider.md @@ -0,0 +1,3 @@ +Added CrossRef DOI Pid Provider + +See Installation Configuration document for JVM Settings to enable CrossRef as a Pid Provider diff --git a/doc/release-notes/permalink-base-urls.md b/doc/release-notes/permalink-base-urls.md new file mode 100644 index 00000000000..1dd74057351 --- /dev/null +++ b/doc/release-notes/permalink-base-urls.md @@ -0,0 +1,10 @@ +The configuration setting `dataverse.pid.*.permalink.base-url`, which is used for PermaLinks, has been updated to +support greater flexibility. Previously, the string "/citation?persistentId=" was automatically appended to the +configured base URL. With this update, the base URL will now be used exactly as configured, without any automatic +additions. + +**Upgrade instructions:** + +- If you currently use a PermaLink provider with a configured `base-url`: You must manually append + "/citation?persistentId=" to the existing base URL to maintain functionality. +- If you use a PermaLink provider without a configured `base-url`: No changes are required. \ No newline at end of file diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index ad1d217b9a1..117aceb141d 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1836,6 +1836,8 @@ The fully expanded example above (without environment variables) looks like this .. note:: You cannot deaccession a dataset more than once. If you call this endpoint twice for the same dataset version, you will get a not found error on the second call, since the dataset you are looking for will no longer be published since it is already deaccessioned. +.. _set-citation-date-field: + Set Citation Date Field Type for a Dataset ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/sphinx-guides/source/api/search.rst b/doc/sphinx-guides/source/api/search.rst index 297f1283ef7..359313ce1b5 100755 --- a/doc/sphinx-guides/source/api/search.rst +++ b/doc/sphinx-guides/source/api/search.rst @@ -61,25 +61,37 @@ https://demo.dataverse.org/api/search?q=trees "name":"Trees", "type":"dataverse", "url":"https://demo.dataverse.org/dataverse/trees", - "image_url":"https://demo.dataverse.org/api/access/dvCardImage/7", + "image_url":"data:image/png;base64,iVBORw0...", "identifier":"trees", "description":"A tree dataverse with some birds", - "published_at":"2016-05-10T12:53:38Z" + "published_at":"2016-05-10T12:53:38Z", + "publicationStatuses": [ + "Published" + ], + "affiliation": "Dataverse.org", + "parentDataverseName": "Root", + "parentDataverseIdentifier": "root" }, { "name":"Chestnut Trees", "type":"dataverse", "url":"https://demo.dataverse.org/dataverse/chestnuttrees", - "image_url":"https://demo.dataverse.org/api/access/dvCardImage/9", + "image_url":"data:image/png;base64,iVBORw0...", "identifier":"chestnuttrees", "description":"A dataverse with chestnut trees and an oriole", - "published_at":"2016-05-10T12:52:38Z" + "published_at":"2016-05-10T12:52:38Z", + "publicationStatuses": [ + "Published" + ], + "affiliation": "Dataverse.org", + "parentDataverseName": "Root", + "parentDataverseIdentifier": "root" }, { "name":"trees.png", "type":"file", "url":"https://demo.dataverse.org/api/access/datafile/12", - "image_url":"https://demo.dataverse.org/api/access/fileCardImage/12", + "image_url":"data:image/png;base64,iVBORw0...", "file_id":"12", "description":"", "published_at":"2016-05-10T12:53:39Z", @@ -91,16 +103,26 @@ https://demo.dataverse.org/api/search?q=trees "dataset_name": "Dataset One", "dataset_id": "32", "dataset_persistent_id": "doi:10.5072/FK2/XTT5BV", - "dataset_citation":"Spruce, Sabrina, 2016, \"Spruce Goose\", http://dx.doi.org/10.5072/FK2/XTT5BV, Root Dataverse, V1" + "dataset_citation":"Spruce, Sabrina, 2016, \"Spruce Goose\", http://dx.doi.org/10.5072/FK2/XTT5BV, Root Dataverse, V1", + "publicationStatuses": [ + "Published" + ], + "releaseOrCreateDate": "2016-05-10T12:53:39Z" }, { "name":"Birds", "type":"dataverse", "url":"https://demo.dataverse.org/dataverse/birds", - "image_url":"https://demo.dataverse.org/api/access/dvCardImage/2", + "image_url":"data:image/png;base64,iVBORw0...", "identifier":"birds", "description":"A bird Dataverse collection with some trees", - "published_at":"2016-05-10T12:57:27Z" + "published_at":"2016-05-10T12:57:27Z", + "publicationStatuses": [ + "Published" + ], + "affiliation": "Dataverse.org", + "parentDataverseName": "Root", + "parentDataverseIdentifier": "root" }, { "name":"Darwin's Finches", @@ -151,6 +173,8 @@ https://demo.dataverse.org/api/search?q=trees } } +Note that the image_url field, if exists, will be returned as a regular URL for Datasets, while for Files and Dataverses, it will be returned as a Base64 URL. We plan to standardize this behavior so that the field always returns a regular URL. (See: https://github.com/IQSS/dataverse/issues/10831) + .. _advancedsearch-example: Advanced Search Examples @@ -178,7 +202,7 @@ In this example, ``show_relevance=true`` matches per field are shown. Available "name":"Finches", "type":"dataverse", "url":"https://demo.dataverse.org/dataverse/finches", - "image_url":"https://demo.dataverse.org/api/access/dvCardImage/3", + "image_url":"data:image/png;base64,iVBORw0...", "identifier":"finches", "description":"A Dataverse collection with finches", "published_at":"2016-05-10T12:57:38Z", diff --git a/doc/sphinx-guides/source/container/base-image.rst b/doc/sphinx-guides/source/container/base-image.rst index 0005265fb1c..a0852a5465f 100644 --- a/doc/sphinx-guides/source/container/base-image.rst +++ b/doc/sphinx-guides/source/container/base-image.rst @@ -21,20 +21,70 @@ IQSS will not offer you support how to deploy or run it, please reach out to the You might be interested in taking a look at :doc:`../developers/containers`, linking you to some (community-based) efforts. +.. _base-supported-image-tags: + Supported Image Tags ++++++++++++++++++++ This image is sourced from the main upstream code `repository of the Dataverse software `_. Development and maintenance of the `image's code `_ -happens there (again, by the community). Community-supported image tags are based on the two most important -upstream branches: - -- The ``unstable`` tag corresponds to the ``develop`` branch, where pull requests are merged. - (`Dockerfile `__) -- The ``alpha`` tag corresponds to the ``master`` branch, where releases are cut from. - (`Dockerfile `__) - - +happens there (again, by the community). + +Our tagging is inspired by `Bitnami `_ and we offer two categories of tags: + +- rolling: images change over time +- immutable: images are fixed and never change + +In the tags below you'll see the term "flavor". This refers to flavor of Linux the container is built on. We use Ubuntu as the basis for our images and, for the time being, the only operating system flavors we use and support are ``noble`` (6.4+) and ``jammy`` (pre-6.4). + +You can find all the tags at https://hub.docker.com/r/gdcc/base/tags + +Tags for Production Use +^^^^^^^^^^^^^^^^^^^^^^^ + +The images of the three latest releases of the Dataverse project will receive updates such as security patches for the underlying operating system. +Content will be fairly stable as disruptive changes like Payara or Java upgrades will be handled in a new major or minor upgrade to Dataverse (a new ``.`` tag). +Expect disruptive changes in case of high risk security threats. + +- | **Latest** + | Definition: ``latest`` + | Summary: Rolling tag, always pointing to the latest revision of the most current Dataverse release. +- | **Rolling Production** + | Definition: ``.-`` + | Example: ``6.4-noble`` + | Summary: Rolling tag, pointing to the latest revision of an immutable production image for released versions of Dataverse. +- | **Immutable Production** + | Definition: ``.--r`` + | Example: ``6.4-noble-r1`` + | Summary: An **immutable tag** where the revision is incremented for rebuilds of the image. + | This image should be especially attractive if you want explict control over when your images are updated. + +Tags for Development Use +^^^^^^^^^^^^^^^^^^^^^^^^ + +All of the tags below are strongly recommended for development purposes only due to their fast changing nature. +In addition to updates due to PR merges, the most recent are undergoing scheduled maintenance to ensure timely security fixes. +When a development cycle of the Dataverse project finishes, maintenance ceases for any tags carrying version numbers. +For now, stale images will be kept on Docker Hub indefinitely. + +- | **Unstable** + | Definition: ``unstable`` + | Summary: Rolling tag, tracking the ``develop`` branch (see also :ref:`develop-branch`). (`Dockerfile `__) + | Please expect abrupt changes like new Payara or Java versions as well as OS updates or flavor switches when using this tag. +- | **Upcoming** + | Definition: ``.-`` + | Example: ``6.5-noble`` + | Summary: Rolling tag, equivalent to ``unstable`` for current development cycle. + Will roll over to the rolling production tag after a Dataverse release. +- | **Flexible Stack** + | Definition: ``.--p-j`` + | Example: ``6.5-noble-p6.2024.6-j17`` + | Summary: Rolling tag during a development cycle of the Dataverse software (`Dockerfile `__). + +**NOTE**: In these tags for development usage, the version number will always be 1 minor version ahead of existing Dataverse releases. +Example: Assume Dataverse ``6.x`` is released, ``6.(x+1)`` is underway. +The rolling tag in use during the cycle will be ``6.(x+1)-FFF`` and ``6.(x+1)-FFF-p6.202P.P-jJJ``. +See also: :doc:`/developers/making-releases`. Image Contents ++++++++++++++ @@ -387,4 +437,4 @@ from `run-java-sh recommendations`_. .. _MicroProfile Config Sources: https://docs.payara.fish/community/docs/Technical%20Documentation/MicroProfile/Config/Overview.html .. _run-java-sh recommendations: https://github.com/fabric8io-images/run-java-sh/blob/master/TUNING.md#recommandations .. _Domain Master Password: https://docs.payara.fish/community/docs/Technical%20Documentation/Payara%20Server%20Documentation/Security%20Guide/Administering%20System%20Security.html#to-change-the-master-password -.. _Payara Admin Console: https://docs.payara.fish/community/docs/Technical%20Documentation/Payara%20Server%20Documentation/General%20Administration/Overview.html#administration-console \ No newline at end of file +.. _Payara Admin Console: https://docs.payara.fish/community/docs/Technical%20Documentation/Payara%20Server%20Documentation/General%20Administration/Overview.html#administration-console diff --git a/doc/sphinx-guides/source/developers/making-releases.rst b/doc/sphinx-guides/source/developers/making-releases.rst index e436ba9e9d2..4936e942389 100755 --- a/doc/sphinx-guides/source/developers/making-releases.rst +++ b/doc/sphinx-guides/source/developers/making-releases.rst @@ -40,8 +40,8 @@ Use the GitHub issue number and the release tag for the name of the branch. (e.g Make the following changes in the release branch. -Bump Version Numbers --------------------- +Bump Version Numbers and Prepare Container Tags +----------------------------------------------- Increment the version number to the milestone (e.g. 5.10.1) in the following two files: @@ -52,6 +52,12 @@ Add the version being released to the lists in the following file: - doc/sphinx-guides/source/versions.rst (e.g. `versions.rst commit `_) +Return to the parent pom and make the following change, which is necessary for proper tagging of images: + +- modules/dataverse-parent/pom.xml -> ```` -> profile "ct" -> ```` -> Set ```` to ``${revision}`` + +(Before you make this change the value should be ``${parsedVersion.majorVersion}.${parsedVersion.nextMinorVersion}``. Later on, after cutting a release, we'll change it back to that value.) + Check in the Changes Above into a Release Branch and Merge It ------------------------------------------------------------- @@ -213,6 +219,17 @@ Now that we've published the release, close the milestone and create a new one. Note that for milestones we use just the number without the "v" (e.g. "5.10.1"). +Update the Container Base Image Version Property +------------------------------------------------ + +Create a new branch (any name is fine but ``prepare-next-iteration`` is suggested) and update the following files to prepare for the next development cycle: + +- modules/dataverse-parent/pom.xml -> ```` -> profile "ct" -> ```` -> Set ```` to ``${parsedVersion.majorVersion}.${parsedVersion.nextMinorVersion}`` + +Now create a pull request and merge it. + +For more background, see :ref:`base-supported-image-tags`. + Add the Release to the Dataverse Roadmap ---------------------------------------- diff --git a/doc/sphinx-guides/source/developers/version-control.rst b/doc/sphinx-guides/source/developers/version-control.rst index 127955a44ea..ecd2db6214d 100644 --- a/doc/sphinx-guides/source/developers/version-control.rst +++ b/doc/sphinx-guides/source/developers/version-control.rst @@ -97,10 +97,12 @@ In the issue you can simply leave a comment to say you're working on it. If you tell us your GitHub username we are happy to add you to the "read only" team at https://github.com/orgs/IQSS/teams/dataverse-readonly/members so that we can assign the issue to you while you're working on it. You can also tell us if you'd like to be added to the `Dataverse Community Contributors spreadsheet `_. +.. _create-branch-for-pr: + Create a New Branch Off the develop Branch ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Always create your feature branch from the latest code in develop, pulling the latest code if necessary. As mentioned above, your branch should have a name like "3728-doc-apipolicy-fix" that starts with the issue number you are addressing (e.g. `#3728 `_) and ends with a short, descriptive name. Dashes ("-") and underscores ("_") in your branch name are ok, but please try to avoid other special characters such as ampersands ("&") that have special meaning in Unix shells. +Always create your feature branch from the latest code in develop, pulling the latest code if necessary. As mentioned above, your branch should have a name like "3728-doc-apipolicy-fix" that starts with the issue number you are addressing (e.g. `#3728 `_) and ends with a short, descriptive name. Dashes ("-") and underscores ("_") in your branch name are ok, but please try to avoid other special characters such as ampersands ("&") that have special meaning in Unix shells. Please do not call your branch "develop" as it can cause maintainers :ref:`trouble `. Commit Your Change to Your New Branch ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -299,3 +301,40 @@ GitHub documents how to make changes to a fork at https://help.github.com/articl vim path/to/file.txt git commit git push OdumInstitute 4709-postgresql_96 + +.. _develop-into-develop: + +Handing a Pull Request from a "develop" Branch +---------------------------------------------- + +Note: this is something only maintainers of Dataverse need to worry about, typically. + +From time to time a pull request comes in from a fork of Dataverse that uses "develop" as the branch behind the PR. (We've started asking contributors not to do this. See :ref:`create-branch-for-pr`.) This is problematic because the "develop" branch is the main integration branch for the project. (See :ref:`develop-branch`.) + +If the PR is perfect and can be merged as-is, no problem. Just merge it. However, if you would like to push commits to the PR, you are likely to run into trouble with multiple "develop" branches locally. + +The following is a list of commands oriented toward the simple task of merging the latest from the "develop" branch into the PR but the same technique can be used to push other commits to the PR as well. In this example the PR is coming from username "coder123" on GitHub. At a high level, what we're doing is working in a safe place (/tmp) away from our normal copy of the repo. We clone the main repo from IQSS, check out coder123's version of "develop" (called "dev2" or "false develop"), merge the real "develop" into it, and push to the PR. + +If there's a better way to do this, please get in touch! + +.. code-block:: bash + + # do all this in /tmp away from your normal code + cd /tmp + git clone git@github.com:IQSS/dataverse.git + cd dataverse + git remote add coder123 git@github.com:coder123/dataverse.git + git fetch coder123 + # check out coder123's "develop" to a branch with a different name ("dev2") + git checkout coder123/develop -b dev2 + # merge IQSS "develop" into coder123's "develop" ("dev2") + git merge origin/develop + # delete the IQSS "develop" branch locally (!) + git branch -d develop + # checkout "dev2" (false "develop") as "develop" for now + git checkout -b develop + # push the false "develop" to coder123's fork (to the PR) + git push coder123 develop + cd .. + # delete the tmp space (done! \o/) + rm -rf /tmp/dataverse diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index bfbf9190357..3ea984923af 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -503,6 +503,41 @@ for `Fabrica `_ and their APIs. You need to provide the same credentials (``username``, ``password``) to Dataverse software to mint and manage DOIs for you. As noted above, you should use one of the more secure options for setting the password. +CrossRef-specific Settings +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +dataverse.pid.*.crossref.url +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +dataverse.pid.*.crossref.rest-api-url +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +dataverse.pid.*.crossref.username +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +dataverse.pid.*.crossref.password +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +dataverse.pid.*.crossref.depositor +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +dataverse.pid.*.crossref.depositor-email +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +CrossRef is an experimental provider. +PID Providers of type ``crossref`` require six additional parameters that define how the provider connects to CrossRef. +CrossRef has two APIs that are used in Dataverse: + +The base URL of the `CrossRef `_, +used to mint and manage DOIs. Current valid values for ``dataverse.pid.*.crossref.url`` are "https://doi.crossref.org" and ``dataverse.pid.*.crossref.rest-api-url`` are "https://api.crossref.org" (production). +``dataverse.pid.*.crossref.username=crusername`` +``dataverse.pid.*.crossref.password=secret`` +``dataverse.pid.*.crossref.depositor=xyz`` +``dataverse.pid.*.crossref.depositor-email=xyz@example.com`` + +CrossRef uses `HTTP Basic authentication `_ +XML files can be POSTed to CrossRef where they are added to the submission queue to await processing +`Post URL `_ +REST API allows the search and reuse our members' metadata. +`Rest API `_ and their APIs. +You need to provide the same credentials (``username``, ``password``) to Dataverse software to mint and manage DOIs for you. +As noted above, you should use one of the more secure options for setting the password. +Depositor and Depositor Email are used for the generation and distribution of Depositor reports. .. _dataverse.pid.*.ezid: @@ -532,8 +567,13 @@ dataverse.pid.*.permalink.separator ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ PermaLinks are a simple PID option intended for intranet and catalog use cases. They can be used without an external service or -be configured with the ``base-url`` of a resolution service. PermaLinks also allow a custom ``separator`` to be used. (Note: when using multiple -PermaLink providers, you should avoid ambiguous authority/separator/shoulder combinations that would result in the same overall prefix.) +be configured with the ``base-url`` of a resolution service. PermaLinks also allow a custom ``separator`` to be used. + +Note: + +- If you configure ``base-url``, it should include a "/" after the hostname like this: ``https://demo.dataverse.org/``. +- When using multiple PermaLink providers, you should avoid ambiguous authority/separator/shoulder combinations that would result in the same overall prefix. +- In general, PermaLink authority/shoulder values should be alphanumeric. For other cases, admins may need to consider the potential impact of special characters in S3 storage identifiers, resolver URLs, exports, etc. .. _dataverse.pid.*.handlenet: @@ -3026,6 +3066,8 @@ If not set, the :ref:`systemEmail` is used for the feedback API/contact form ema Note that only the email address is required, which you can supply without the ``<`` and ``>`` signs, but if you include the text, it's the way to customize the name of your support team, which appears in the "from" address in emails as well as in help text in the UI. If you don't include the text, the installation name (see :ref:`Branding Your Installation`) will appear in the "from" address. +Also note that the support email address is used at the end of notification mails where it states; "contact us for support at", followed by the support mail address if configured and the system email otherwise. + Can also be set via any `supported MicroProfile Config API source`_, e.g. the environment variable ``DATAVERSE_MAIL_SUPPORT_EMAIL``. See also :ref:`smtp-config`. diff --git a/modules/container-base/README.md b/modules/container-base/README.md index dc4d185bbb5..0598d709eac 100644 --- a/modules/container-base/README.md +++ b/modules/container-base/README.md @@ -31,17 +31,23 @@ to ask for help and guidance. ## Supported Image Tags This image is sourced within the main upstream code [repository of the Dataverse software](https://github.com/IQSS/dataverse). -Development and maintenance of the [image's code](https://github.com/IQSS/dataverse/tree/develop/modules/container-base) -happens there (again, by the community). Community-supported image tags are based on the two most important branches: +Development and maintenance of the [image's code](https://github.com/IQSS/dataverse/tree/develop/modules/container-base) happens there (again, by the community). +Community-supported image tags are based on the two most important branches: -- The `unstable` tag corresponds to the `develop` branch, where pull requests are merged. - ([`Dockerfile`](https://github.com/IQSS/dataverse/tree/develop/modules/container-base/src/main/docker/Dockerfile)) -- The `alpha` tag corresponds to the `master` branch, where releases are cut from. - ([`Dockerfile`](https://github.com/IQSS/dataverse/tree/master/modules/container-base/src/main/docker/Dockerfile)) +Our tagging is inspired by [Bitnami](https://docs.vmware.com/en/VMware-Tanzu-Application-Catalog/services/tutorials/GUID-understand-rolling-tags-containers-index.html). +For more detailed information about our tagging policy, please read about our [base image tags](https://guides.dataverse.org/en/latest/container/base-image.html#supported-image-tags) in the Dataverse Containers Guide. + +For ease of use, here is a list of images that are currently maintained. + + + +All of them are rolling tags, except those ending with `-r`, which are the most recent immutable tags. +The `unstable` tags are the current development branch snapshot. +We strongly recommend using only immutable tags for production use cases. Within the main repository, you may find the base image files at `/modules/container-base`. This Maven module uses the [Maven Docker Plugin](https://dmp.fabric8.io) to build and ship the image. -You may use, extend, or alter this image to your liking and/or host in some different registry if you want to. +You may use, extend, or alter this image to your liking and/or host in some different registry if you want to under the terms of the Apache 2.0 license. **Supported architectures:** This image is created as a "multi-arch image", supporting the most common architectures Dataverse usually runs on: AMD64 (Windows/Linux/...) and ARM64 (Apple M1/M2). diff --git a/modules/container-base/pom.xml b/modules/container-base/pom.xml index fc672696df4..6417b5b91fa 100644 --- a/modules/container-base/pom.xml +++ b/modules/container-base/pom.xml @@ -39,9 +39,13 @@ ct docker-build + gdcc/base:${base.image.tag} - unstable - eclipse-temurin:${target.java.version}-jre + + ${base.image.version}-${java.image.flavor}${base.image.tag.suffix} + -p${payara.version}-j${target.java.version} + eclipse-temurin:${target.java.version}-jre-${java.image.flavor} + noble 1000 1000 linux/amd64,linux/arm64 @@ -95,7 +99,6 @@ - ${docker.platforms} ${project.build.directory}/buildx-state diff --git a/modules/container-base/src/backports/v6.1/001-pom.xml.patch b/modules/container-base/src/backports/v6.1/001-pom.xml.patch new file mode 100644 index 00000000000..6498f972889 --- /dev/null +++ b/modules/container-base/src/backports/v6.1/001-pom.xml.patch @@ -0,0 +1,26 @@ +--- a/modules/container-base/pom.xml 2024-08-26 21:53:55.985584815 +0200 ++++ b/modules/container-base/pom.xml 2024-08-26 21:38:09.925910314 +0200 +@@ -40,8 +42,11 @@ + + docker-build + gdcc/base:${base.image.tag} +- unstable +- eclipse-temurin:${target.java.version}-jre ++ gdcc/base:${base.image.tag} ++ ${base.image.version}-${java.image.flavor}${base.image.tag.suffix} ++ -p${payara.version}-j${target.java.version} ++ eclipse-temurin:${target.java.version}-jre-${java.image.flavor} ++ jammy + 1000 + 1000 + linux/amd64,linux/arm64 +@@ -110,6 +113,9 @@ + + assembly.xml + ++ ++ ${base.image.tag.revision} ++ + + + diff --git a/modules/container-base/src/backports/v6.1/002-Dockerfile.patch b/modules/container-base/src/backports/v6.1/002-Dockerfile.patch new file mode 100644 index 00000000000..4bb7a1eac91 --- /dev/null +++ b/modules/container-base/src/backports/v6.1/002-Dockerfile.patch @@ -0,0 +1,10 @@ +--- a/modules/container-base/src/main/docker/Dockerfile ++++ b/modules/container-base/src/main/docker/Dockerfile +@@ -233,4 +233,6 @@ LABEL org.opencontainers.image.created="@git.build.time@" \ + org.opencontainers.image.vendor="Global Dataverse Community Consortium" \ + org.opencontainers.image.licenses="Apache-2.0" \ + org.opencontainers.image.title="Dataverse Base Image" \ +- org.opencontainers.image.description="This container image provides an application server tuned for Dataverse software" ++ org.opencontainers.image.description="This container image provides an application server tuned for Dataverse software" \ ++ org.opencontainers.image.base.name="@java.image@" \ ++ org.dataverse.deps.payara.version="@payara.version@" diff --git a/modules/container-base/src/backports/v6.1/003-parent-pom.xml.patch b/modules/container-base/src/backports/v6.1/003-parent-pom.xml.patch new file mode 100644 index 00000000000..a69cfd43ea7 --- /dev/null +++ b/modules/container-base/src/backports/v6.1/003-parent-pom.xml.patch @@ -0,0 +1,11 @@ +--- a/modules/dataverse-parent/pom.xml ++++ b/modules/dataverse-parent/pom.xml +@@ -457,7 +457,8 @@ + + ++ ${revision} + + + diff --git a/modules/container-base/src/backports/v6.2/001-pom.xml.patch b/modules/container-base/src/backports/v6.2/001-pom.xml.patch new file mode 100644 index 00000000000..6498f972889 --- /dev/null +++ b/modules/container-base/src/backports/v6.2/001-pom.xml.patch @@ -0,0 +1,26 @@ +--- a/modules/container-base/pom.xml 2024-08-26 21:53:55.985584815 +0200 ++++ b/modules/container-base/pom.xml 2024-08-26 21:38:09.925910314 +0200 +@@ -40,8 +42,11 @@ + + docker-build + gdcc/base:${base.image.tag} +- unstable +- eclipse-temurin:${target.java.version}-jre ++ gdcc/base:${base.image.tag} ++ ${base.image.version}-${java.image.flavor}${base.image.tag.suffix} ++ -p${payara.version}-j${target.java.version} ++ eclipse-temurin:${target.java.version}-jre-${java.image.flavor} ++ jammy + 1000 + 1000 + linux/amd64,linux/arm64 +@@ -110,6 +113,9 @@ + + assembly.xml + ++ ++ ${base.image.tag.revision} ++ + + + diff --git a/modules/container-base/src/backports/v6.2/002-Dockerfile.labels.patch b/modules/container-base/src/backports/v6.2/002-Dockerfile.labels.patch new file mode 100644 index 00000000000..fbb7f80c4ce --- /dev/null +++ b/modules/container-base/src/backports/v6.2/002-Dockerfile.labels.patch @@ -0,0 +1,10 @@ +--- a/modules/container-base/src/main/docker/Dockerfile ++++ b/modules/container-base/src/main/docker/Dockerfile +@@ -242,4 +242,6 @@ LABEL org.opencontainers.image.created="@git.build.time@" \ + org.opencontainers.image.vendor="Global Dataverse Community Consortium" \ + org.opencontainers.image.licenses="Apache-2.0" \ + org.opencontainers.image.title="Dataverse Base Image" \ +- org.opencontainers.image.description="This container image provides an application server tuned for Dataverse software" ++ org.opencontainers.image.description="This container image provides an application server tuned for Dataverse software" \ ++ org.opencontainers.image.base.name="@java.image@" \ ++ org.dataverse.deps.payara.version="@payara.version@" diff --git a/modules/container-base/src/backports/v6.2/003-Dockerfile.security.patch b/modules/container-base/src/backports/v6.2/003-Dockerfile.security.patch new file mode 100644 index 00000000000..1ab4c3a980a --- /dev/null +++ b/modules/container-base/src/backports/v6.2/003-Dockerfile.security.patch @@ -0,0 +1,10 @@ +--- a/modules/container-base/src/main/docker/Dockerfile ++++ b/modules/container-base/src/main/docker/Dockerfile +@@ -226,6 +226,7 @@ USER root + RUN true && \ + chgrp -R 0 "${DOMAIN_DIR}" && \ + chmod -R g=u "${DOMAIN_DIR}" ++USER payara + + # Set the entrypoint to tini (as a process supervisor) + ENTRYPOINT ["/usr/bin/dumb-init", "--"] diff --git a/modules/container-base/src/backports/v6.2/004-parent-pom.xml.patch b/modules/container-base/src/backports/v6.2/004-parent-pom.xml.patch new file mode 100644 index 00000000000..a69cfd43ea7 --- /dev/null +++ b/modules/container-base/src/backports/v6.2/004-parent-pom.xml.patch @@ -0,0 +1,11 @@ +--- a/modules/dataverse-parent/pom.xml ++++ b/modules/dataverse-parent/pom.xml +@@ -457,7 +457,8 @@ + + ++ ${revision} + + + diff --git a/modules/container-base/src/backports/v6.3/001-pom.xml.patch b/modules/container-base/src/backports/v6.3/001-pom.xml.patch new file mode 100644 index 00000000000..6498f972889 --- /dev/null +++ b/modules/container-base/src/backports/v6.3/001-pom.xml.patch @@ -0,0 +1,26 @@ +--- a/modules/container-base/pom.xml 2024-08-26 21:53:55.985584815 +0200 ++++ b/modules/container-base/pom.xml 2024-08-26 21:38:09.925910314 +0200 +@@ -40,8 +42,11 @@ + + docker-build + gdcc/base:${base.image.tag} +- unstable +- eclipse-temurin:${target.java.version}-jre ++ gdcc/base:${base.image.tag} ++ ${base.image.version}-${java.image.flavor}${base.image.tag.suffix} ++ -p${payara.version}-j${target.java.version} ++ eclipse-temurin:${target.java.version}-jre-${java.image.flavor} ++ jammy + 1000 + 1000 + linux/amd64,linux/arm64 +@@ -110,6 +113,9 @@ + + assembly.xml + ++ ++ ${base.image.tag.revision} ++ + + + diff --git a/modules/container-base/src/backports/v6.3/002-Dockerfile.labels.patch b/modules/container-base/src/backports/v6.3/002-Dockerfile.labels.patch new file mode 100644 index 00000000000..c7744882153 --- /dev/null +++ b/modules/container-base/src/backports/v6.3/002-Dockerfile.labels.patch @@ -0,0 +1,10 @@ +--- a/modules/container-base/src/main/docker/Dockerfile ++++ b/modules/container-base/src/main/docker/Dockerfile +@@ -240,4 +241,6 @@ LABEL org.opencontainers.image.created="@git.build.time@" \ + org.opencontainers.image.vendor="Global Dataverse Community Consortium" \ + org.opencontainers.image.licenses="Apache-2.0" \ + org.opencontainers.image.title="Dataverse Base Image" \ +- org.opencontainers.image.description="This container image provides an application server tuned for Dataverse software" ++ org.opencontainers.image.description="This container image provides an application server tuned for Dataverse software" \ ++ org.opencontainers.image.base.name="@java.image@" \ ++ org.dataverse.deps.payara.version="@payara.version@" diff --git a/modules/container-base/src/backports/v6.3/003-Dockerfile.security.patch b/modules/container-base/src/backports/v6.3/003-Dockerfile.security.patch new file mode 100644 index 00000000000..d8487b3aacc --- /dev/null +++ b/modules/container-base/src/backports/v6.3/003-Dockerfile.security.patch @@ -0,0 +1,7 @@ +--- a/modules/container-base/src/main/docker/Dockerfile ++++ b/modules/container-base/src/main/docker/Dockerfile +@@ -224,6 +224,7 @@ USER root + RUN true && \ + chgrp -R 0 "${DOMAIN_DIR}" && \ + chmod -R g=u "${DOMAIN_DIR}" ++USER payara diff --git a/modules/container-base/src/backports/v6.3/004-parent-pom.xml.patch b/modules/container-base/src/backports/v6.3/004-parent-pom.xml.patch new file mode 100644 index 00000000000..a69cfd43ea7 --- /dev/null +++ b/modules/container-base/src/backports/v6.3/004-parent-pom.xml.patch @@ -0,0 +1,11 @@ +--- a/modules/dataverse-parent/pom.xml ++++ b/modules/dataverse-parent/pom.xml +@@ -457,7 +457,8 @@ + + ++ ${revision} + + + diff --git a/modules/container-base/src/main/docker/Dockerfile b/modules/container-base/src/main/docker/Dockerfile index 29078e6896c..802db62e5e4 100644 --- a/modules/container-base/src/main/docker/Dockerfile +++ b/modules/container-base/src/main/docker/Dockerfile @@ -106,7 +106,7 @@ EOF ARG JATTACH_VERSION="v2.2" ARG JATTACH_TGZ_CHECKSUM_AMD64="acd9e17f15749306be843df392063893e97bfecc5260eef73ee98f06e5cfe02f" ARG JATTACH_TGZ_CHECKSUM_ARM64="288ae5ed87ee7fe0e608c06db5a23a096a6217c9878ede53c4e33710bdcaab51" -ARG WAIT4X_VERSION="v2.14.0" +ARG WAIT4X_VERSION="v2.14.2" ARG PKGS="jq imagemagick curl unzip wget acl lsof procps netcat-openbsd dumb-init" # Installing the packages in an extra container layer for better caching @@ -222,7 +222,7 @@ RUN < ct - - + ${parsedVersion.majorVersion}.${parsedVersion.nextMinorVersion} + @@ -470,6 +473,21 @@ false + + + org.codehaus.mojo + build-helper-maven-plugin + 3.6.0 + + + parse-version + + parse-version + + initialize + + + diff --git a/pom.xml b/pom.xml index 76a8f61444f..edf72067976 100644 --- a/pom.xml +++ b/pom.xml @@ -999,7 +999,10 @@ unstable false gdcc/base:${base.image.tag} - unstable + + noble + + ${base.image.version}-${base.image.flavor}-p${payara.version}-j${target.java.version} gdcc/configbaker:${conf.image.tag} ${app.image.tag} diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java index 9331ec67d12..98ac8ff387f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java @@ -1405,4 +1405,16 @@ public UploadSessionQuotaLimit getUploadSessionQuotaLimit(DvObjectContainer pare return new UploadSessionQuotaLimit(quota.getAllocation(), currentSize); } + + public boolean isInReleasedVersion(Long id) { + Query query = em.createQuery("SELECT fm.id FROM FileMetadata fm, DvObject dvo WHERE fm.datasetVersion.id=(SELECT dv.id FROM DatasetVersion dv WHERE dv.dataset.id=dvo.owner.id and dv.versionState=edu.harvard.iq.dataverse.DatasetVersion.VersionState.RELEASED ORDER BY dv.versionNumber DESC, dv.minorVersionNumber DESC LIMIT 1) AND dvo.id=fm.dataFile.id AND fm.dataFile.id=:fid"); + query.setParameter("fid", id); + + try { + query.getSingleResult(); + return true; + } catch (Exception ex) { + return false; + } + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java index e8d498b41c1..187e8a20a31 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java @@ -1080,4 +1080,16 @@ public void deleteHarvestedDataset(Dataset dataset, DataverseRequest request, Lo } } + public List getVersionStates(long id) { + try { + Query query = em.createNativeQuery("SELECT dv.versionState FROM datasetversion dv WHERE dataset_id=? ORDER BY id"); + query.setParameter(1, id); + return (List) query.getResultList(); + + } catch (Exception ex) { + logger.log(Level.WARNING, "exception trying to get versionstates of dataset " + id + ": {0}", ex); + return null; + } + } + } diff --git a/src/main/java/edu/harvard/iq/dataverse/DvObject.java b/src/main/java/edu/harvard/iq/dataverse/DvObject.java index cc5d7620969..a4882f772d6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DvObject.java +++ b/src/main/java/edu/harvard/iq/dataverse/DvObject.java @@ -24,8 +24,6 @@ query = "SELECT o FROM DvObject o ORDER BY o.id"), @NamedQuery(name = "DvObject.findById", query = "SELECT o FROM DvObject o WHERE o.id=:id"), - @NamedQuery(name = "DvObject.checkExists", - query = "SELECT count(o) from DvObject o WHERE o.id=:id"), @NamedQuery(name = "DvObject.ownedObjectsById", query="SELECT COUNT(obj) FROM DvObject obj WHERE obj.owner.id=:id"), @NamedQuery(name = "DvObject.findByGlobalId", diff --git a/src/main/java/edu/harvard/iq/dataverse/DvObjectServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DvObjectServiceBean.java index bd7fbeaff10..dc3393d82f5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DvObjectServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DvObjectServiceBean.java @@ -82,12 +82,15 @@ public List findByAuthenticatedUserId(AuthenticatedUser user) { return query.getResultList(); } - public boolean checkExists(Long id) { - Query query = em.createNamedQuery("DvObject.checkExists"); - query.setParameter("id", id); - Long result =(Long)query.getSingleResult(); - return result > 0; - } + public String getDtype(Long id) { + Query query = em.createNativeQuery("SELECT dvo.dtype FROM dvobject dvo WHERE dvo.id=?"); + query.setParameter(1, id); + try { + return (String) query.getSingleResult(); + } catch (NoResultException e) { + return null; + } + } public DvObject findByGlobalId(String globalIdString, DvObject.DType dtype) { try { diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index 99a44058809..87997731642 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -2092,6 +2092,12 @@ public void handleFileUpload(FileUploadEvent event) throws IOException { errorMessages.add(cex.getMessage()); uploadComponentId = event.getComponent().getClientId(); return; + } finally { + try { + uFile.delete(); + } catch (IOException ioex) { + logger.warning("Failed to delete temp file uploaded via PrimeFaces " + uFile.getFileName()); + } } /*catch (FileExceedsMaxSizeException ex) { logger.warning("Failed to process and/or save the file " + uFile.getFileName() + "; " + ex.getMessage()); diff --git a/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java index d29649ad3a6..2995c0c5f47 100644 --- a/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java @@ -117,10 +117,11 @@ public boolean sendSystemEmail(String to, String subject, String messageText, bo return false; } InternetAddress systemAddress = optionalAddress.get(); + InternetAddress supportAddress = getSupportAddress().orElse(systemAddress); String body = messageText + BundleUtil.getStringFromBundle(isHtmlContent ? "notification.email.closing.html" : "notification.email.closing", - List.of(BrandingUtil.getSupportTeamEmailAddress(systemAddress), BrandingUtil.getSupportTeamName(systemAddress))); + List.of(BrandingUtil.getSupportTeamEmailAddress(supportAddress), BrandingUtil.getSupportTeamName(supportAddress))); logger.fine(() -> "Sending email to %s. Subject: <<<%s>>>. Body: %s".formatted(to, subject, body)); try { diff --git a/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java b/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java index d31fdd4e380..c1127079da4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java +++ b/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java @@ -58,6 +58,10 @@ public String getFileCardImageAsBase64Url(SolrSearchResult result) { if (result.isHarvested()) { return null; } + + if (result.getEntity() == null) { + return null; + } Long imageFileId = result.getEntity().getId(); diff --git a/src/main/java/edu/harvard/iq/dataverse/export/dublincore/DublinCoreExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/dublincore/DublinCoreExportUtil.java index 6b7cb844f3e..9a2c3085d2d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/dublincore/DublinCoreExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/dublincore/DublinCoreExportUtil.java @@ -7,6 +7,8 @@ import com.google.gson.Gson; import edu.harvard.iq.dataverse.DatasetFieldConstant; +import edu.harvard.iq.dataverse.DatasetFieldType; +import edu.harvard.iq.dataverse.DatasetServiceBean; import edu.harvard.iq.dataverse.GlobalId; import edu.harvard.iq.dataverse.api.dto.DatasetDTO; import edu.harvard.iq.dataverse.api.dto.DatasetVersionDTO; @@ -176,11 +178,24 @@ private static void createOAIDC(XMLStreamWriter xmlw, DatasetDTO datasetDto, Str writeFullElementList(xmlw, dcFlavor+":"+"language", dto2PrimitiveList(version, DatasetFieldConstant.language)); - String date = dto2Primitive(version, DatasetFieldConstant.productionDate); - if (date == null) { - date = datasetDto.getPublicationDate(); + /** + * dc:date. "I suggest changing the Dataverse / DC Element (oai_dc) + * mapping, so that dc:date is mapped with Publication Date. This is + * also in line with citation recommendations. The publication date is + * the preferred date when citing research data; see, e.g., page 12 in + * The Tromsø Recommendations for Citation of Research Data in + * Linguistics; https://doi.org/10.15497/rda00040 ." -- + * https://github.com/IQSS/dataverse/issues/8129 + * + * However, if the citation date field has been set, use that. + */ + String date = datasetDto.getPublicationDate(); + DatasetFieldType citationDataType = jakarta.enterprise.inject.spi.CDI.current().select(DatasetServiceBean.class).get().findByGlobalId(globalId.asString()).getCitationDateDatasetFieldType(); + if (citationDataType != null) { + date = dto2Primitive(version, citationDataType.getName()); } - writeFullElement(xmlw, dcFlavor+":"+"date", date); + + writeFullElement(xmlw, dcFlavor+":"+"date", date); writeFullElement(xmlw, dcFlavor+":"+"contributor", dto2Primitive(version, DatasetFieldConstant.depositor)); @@ -188,10 +203,16 @@ private static void createOAIDC(XMLStreamWriter xmlw, DatasetDTO datasetDto, Str writeFullElementList(xmlw, dcFlavor+":"+"relation", dto2PrimitiveList(version, DatasetFieldConstant.relatedDatasets)); - writeFullElementList(xmlw, dcFlavor+":"+"type", dto2PrimitiveList(version, DatasetFieldConstant.kindOfData)); + /** + * dc:type. "Dublin Core (see + * https://www.dublincore.org/specifications/dublin-core/dcmi-terms/#http://purl.org/dc/terms/type + * ) recommends “to use a controlled vocabulary such as the DCMI Type + * Vocabulary” for dc:type." So we hard-coded it to "Dataset". See + * https://github.com/IQSS/dataverse/issues/8129 + */ + writeFullElement(xmlw, dcFlavor+":"+"type", "Dataset"); writeFullElementList(xmlw, dcFlavor+":"+"source", dto2PrimitiveList(version, DatasetFieldConstant.dataSources)); - } diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/crossref/CrossRefDOIProvider.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/crossref/CrossRefDOIProvider.java new file mode 100644 index 00000000000..98a4f806836 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/crossref/CrossRefDOIProvider.java @@ -0,0 +1,125 @@ +package edu.harvard.iq.dataverse.pidproviders.doi.crossref; + +import edu.harvard.iq.dataverse.DvObject; +import edu.harvard.iq.dataverse.GlobalId; +import edu.harvard.iq.dataverse.pidproviders.doi.AbstractDOIProvider; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; + +public class CrossRefDOIProvider extends AbstractDOIProvider { + private static final Logger logger = Logger.getLogger(CrossRefDOIProvider.class.getCanonicalName()); + + public static final String TYPE = "crossref"; + + CrossRefDOIRegisterService crossRefDOIRegisterService; + + public CrossRefDOIProvider(String id, String label, String providerAuthority, String providerShoulder, String identifierGenerationStyle, String datafilePidFormat, String managedList, String excludedList, + String url, String apiUrl, String username, String password, String depositor, String depositorEmail) { + super(id, label, providerAuthority, providerShoulder, identifierGenerationStyle, datafilePidFormat, + managedList, excludedList); + + crossRefDOIRegisterService = new CrossRefDOIRegisterService(url, apiUrl, username, password, depositor, depositorEmail); + } + + @Override + public boolean alreadyRegistered(GlobalId pid, boolean noProviderDefault) throws Exception { + logger.info("CrossRef alreadyRegistered"); + if (pid == null || pid.asString().isEmpty()) { + logger.fine("No identifier sent."); + return false; + } + boolean alreadyExists; + String identifier = pid.asString(); + try { + alreadyExists = crossRefDOIRegisterService.testDOIExists(identifier); + } catch (Exception e) { + logger.log(Level.WARNING, "alreadyExists failed"); + return false; + } + return alreadyExists; + } + + @Override + public boolean registerWhenPublished() { + return true; + } + + @Override + public List getProviderInformation() { + return List.of("CrossRef", "https://status.crossref.org/"); + } + + @Override + public String createIdentifier(DvObject dvObject) throws Throwable { + logger.info("CrossRef createIdentifier"); + if (dvObject.getIdentifier() == null || dvObject.getIdentifier().isEmpty()) { + dvObject = generatePid(dvObject); + } + String identifier = getIdentifier(dvObject); + try { + String retString = crossRefDOIRegisterService.reserveIdentifier(identifier, dvObject); + logger.log(Level.FINE, "CrossRef create DOI identifier retString : " + retString); + return retString; + } catch (Exception e) { + logger.log(Level.WARNING, "CrossRef Identifier not created: create failed", e); + throw e; + } + } + + @Override + public Map getIdentifierMetadata(DvObject dvObject) { + logger.info("CrossRef getIdentifierMetadata"); + String identifier = getIdentifier(dvObject); + Map metadata = new HashMap<>(); + try { + metadata = crossRefDOIRegisterService.getMetadata(identifier); + } catch (Exception e) { + logger.log(Level.WARNING, "getIdentifierMetadata failed", e); + } + return metadata; + } + + @Override + public String modifyIdentifierTargetURL(DvObject dvObject) throws Exception { + logger.info("CrossRef modifyIdentifier"); + String identifier = getIdentifier(dvObject); + try { + crossRefDOIRegisterService.modifyIdentifier(identifier, dvObject); + } catch (Exception e) { + logger.log(Level.WARNING, "modifyMetadata failed", e); + throw e; + } + return identifier; + } + + @Override + public void deleteIdentifier(DvObject dvo) throws Exception { + logger.info("CrossRef deleteIdentifier"); + } + + @Override + public boolean publicizeIdentifier(DvObject dvObject) { + logger.info("CrossRef updateIdentifierStatus"); + if (dvObject.getIdentifier() == null || dvObject.getIdentifier().isEmpty()) { + dvObject = generatePid(dvObject); + } + String identifier = getIdentifier(dvObject); + + try { + crossRefDOIRegisterService.reserveIdentifier(identifier, dvObject); + return true; + } catch (Exception e) { + logger.log(Level.WARNING, "modifyMetadata failed: " + e.getMessage(), e); + return false; + } + } + + @Override + public String getProviderType() { + return TYPE; + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/crossref/CrossRefDOIProviderFactory.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/crossref/CrossRefDOIProviderFactory.java new file mode 100644 index 00000000000..545212cc43c --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/crossref/CrossRefDOIProviderFactory.java @@ -0,0 +1,43 @@ +package edu.harvard.iq.dataverse.pidproviders.doi.crossref; + +import com.google.auto.service.AutoService; +import edu.harvard.iq.dataverse.pidproviders.PidProvider; +import edu.harvard.iq.dataverse.pidproviders.PidProviderFactory; +import edu.harvard.iq.dataverse.settings.JvmSettings; +import edu.harvard.iq.dataverse.util.SystemConfig; + +@AutoService(PidProviderFactory.class) +public class CrossRefDOIProviderFactory implements PidProviderFactory { + + @Override + public PidProvider createPidProvider(String providerId) { + String providerType = JvmSettings.PID_PROVIDER_TYPE.lookup(providerId); + if (!providerType.equals(CrossRefDOIProvider.TYPE)) { + // Being asked to create a non-CrossRef provider + return null; + } + String providerLabel = JvmSettings.PID_PROVIDER_LABEL.lookup(providerId); + String providerAuthority = JvmSettings.PID_PROVIDER_AUTHORITY.lookup(providerId); + String providerShoulder = JvmSettings.PID_PROVIDER_SHOULDER.lookupOptional(providerId).orElse(""); + String identifierGenerationStyle = JvmSettings.PID_PROVIDER_IDENTIFIER_GENERATION_STYLE + .lookupOptional(providerId).orElse("randomString"); + String datafilePidFormat = JvmSettings.PID_PROVIDER_DATAFILE_PID_FORMAT.lookupOptional(providerId) + .orElse(SystemConfig.DataFilePIDFormat.DEPENDENT.toString()); + String managedList = JvmSettings.PID_PROVIDER_MANAGED_LIST.lookupOptional(providerId).orElse(""); + String excludedList = JvmSettings.PID_PROVIDER_EXCLUDED_LIST.lookupOptional(providerId).orElse(""); + + String baseUrl = JvmSettings.CROSSREF_URL.lookup(providerId); + String apiUrl = JvmSettings.CROSSREF_REST_API_URL.lookup(providerId); + String username = JvmSettings.CROSSREF_USERNAME.lookup(providerId); + String password = JvmSettings.CROSSREF_PASSWORD.lookup(providerId); + String depositor = JvmSettings.CROSSREF_DEPOSITOR.lookup(providerId); + String depositorEmail = JvmSettings.CROSSREF_DEPOSITOR_EMAIL.lookup(providerId); + + return new CrossRefDOIProvider(providerId, providerLabel, providerAuthority, providerShoulder, identifierGenerationStyle, + datafilePidFormat, managedList, excludedList, baseUrl, apiUrl, username, password, depositor, depositorEmail); + } + + public String getType() { + return CrossRefDOIProvider.TYPE; + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/crossref/CrossRefDOIRegisterService.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/crossref/CrossRefDOIRegisterService.java new file mode 100644 index 00000000000..825b18e7f4b --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/crossref/CrossRefDOIRegisterService.java @@ -0,0 +1,319 @@ +package edu.harvard.iq.dataverse.pidproviders.doi.crossref; + +import com.fasterxml.jackson.databind.ObjectMapper; +import edu.harvard.iq.dataverse.*; +import edu.harvard.iq.dataverse.pidproviders.AbstractPidProvider; +import edu.harvard.iq.dataverse.pidproviders.doi.XmlMetadataTemplate; +import jakarta.enterprise.inject.spi.CDI; +import org.apache.commons.text.StringEscapeUtils; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; + +import static edu.harvard.iq.dataverse.util.SystemConfig.getDataverseSiteUrlStatic; + +public class CrossRefDOIRegisterService implements java.io.Serializable { + private static final Logger logger = Logger.getLogger(CrossRefDOIRegisterService.class.getCanonicalName()); + + private final String url; + private final String apiUrl; + private final String username; + private final String password; + private final String depositor; + private final String depositorEmail; + + + private static DataverseServiceBean dataverseService = null; + + private CrossRefRESTfullClient client = null; + + public CrossRefDOIRegisterService(String url, String apiUrl, String username, String password, String depositor, String depositorEmail) { + this.url = url; + this.apiUrl = apiUrl; + this.username = username; + this.password = password; + this.depositor = depositor; + this.depositorEmail = depositorEmail; + } + + private CrossRefRESTfullClient getClient() { + if (client == null) { + client = new CrossRefRESTfullClient(url, apiUrl, username, password); + } + return client; + } + + public boolean testDOIExists(String identifier) { + boolean doiExists; + try { + CrossRefRESTfullClient client = getClient(); + doiExists = client.testDOIExists(identifier.substring(identifier.indexOf(":") + 1)); + } catch (Exception e) { + logger.log(Level.INFO, identifier, e); + return false; + } + return doiExists; + } + + public HashMap getMetadata(String identifier) throws IOException { + HashMap metadata = new HashMap<>(); + try { + CrossRefRESTfullClient client = getClient(); + String jsonMetadata = client.getMetadata(identifier.substring(identifier.indexOf(":") + 1)); + Map mappedJson = new ObjectMapper().readValue(jsonMetadata, HashMap.class); + logger.log(Level.FINE, jsonMetadata); + metadata.put("_status", mappedJson.get("status").toString()); + } catch (RuntimeException e) { + logger.log(Level.INFO, identifier, e); + } + return metadata; + } + + public String reserveIdentifier(String identifier, DvObject dvObject) throws IOException { + logger.fine("Crossref reserveIdentifier"); + String xmlMetadata = getMetadataFromDvObject(identifier, dvObject); + + CrossRefRESTfullClient client = getClient(); + return client.postMetadata(xmlMetadata); + } + + public void modifyIdentifier(String identifier, DvObject dvObject) throws IOException { + logger.fine("Crossref modifyIdentifier"); + String xmlMetadata = getMetadataFromDvObject(identifier, dvObject); + + CrossRefRESTfullClient client = getClient(); + client.postMetadata(xmlMetadata); + } + + public String getMetadataFromDvObject(String identifier, DvObject dvObject) { + Dataset dataset; + + if (dvObject instanceof Dataset) { + dataset = (Dataset) dvObject; + } else { + dataset = (Dataset) dvObject.getOwner(); + } + if (dataverseService == null) { + dataverseService = CDI.current().select(DataverseServiceBean.class).get(); + } + + CrossRefMetadataTemplate metadataTemplate = new CrossRefMetadataTemplate(); + metadataTemplate.setIdentifier(identifier.substring(identifier.indexOf(':') + 1)); + metadataTemplate.setAuthors(dataset.getLatestVersion().getDatasetAuthors()); + metadataTemplate.setDepositor(depositor); + metadataTemplate.setDepositorEmail(depositorEmail); + metadataTemplate.setInstitution(dataverseService.getRootDataverseName()); + + String title = dvObject.getCurrentName(); + if (dvObject.isInstanceofDataFile()) { + //Note file title is not currently escaped the way the dataset title is, so adding it here. + title = StringEscapeUtils.escapeXml10(title); + } + + if (title.isEmpty() || title.equals(DatasetField.NA_VALUE)) { + title = AbstractPidProvider.UNAVAILABLE; + } + + metadataTemplate.setTitle(title); + + return metadataTemplate.generateXML(); + } +} + +class CrossRefMetadataTemplate { + + private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.edu.harvard.iq.dataverse.CrossRefMetadataTemplate"); + private static String template; + + static { + try (InputStream in = XmlMetadataTemplate.class.getResourceAsStream("crossref_metadata_template.xml")) { + template = CrossRefFileUtil.readAndClose(in, "utf-8"); + } catch (Exception e) { + logger.log(Level.SEVERE, "crossref metadata template load error"); + logger.log(Level.SEVERE, "String " + e); + logger.log(Level.SEVERE, "localized message " + e.getLocalizedMessage()); + logger.log(Level.SEVERE, "cause " + e.getCause()); + logger.log(Level.SEVERE, "message " + e.getMessage()); + } + } + + private final String timestamp = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date()); + private String institution; + private String depositor; + private String depositorEmail; + private String databaseTitle; + private String identifier; + private String title; + private final String baseUrl = getDataverseSiteUrlStatic(); + private List authors; + + public List getAuthors() { + return authors; + } + + public void setAuthors(List authors) { + this.authors = authors; + } + + public CrossRefMetadataTemplate() { + } + + public String generateXML() { + String xmlMetadata = template.replace("${depositor}", depositor) + .replace("${depositorEmail}", depositorEmail) + .replace("${title}", title) + .replace("${institution}", institution) + .replace("${batchId}", identifier + " " + timestamp) + .replace("${timestamp}", timestamp); + + StringBuilder datasetElement = new StringBuilder(); + datasetElement.append(""); + + StringBuilder contributorsElement = new StringBuilder(); + if (authors != null && !authors.isEmpty()) { + contributorsElement.append(""); + for (DatasetAuthor author : authors) { + contributorsElement.append(""); + contributorsElement.append(author.getName().getDisplayValue()); + contributorsElement.append(""); + contributorsElement.append(author.getName().getDisplayValue()); + contributorsElement.append(""); + + if (author.getAffiliation() != null && !author.getAffiliation().getDisplayValue().isEmpty()) { + contributorsElement.append("") + .append(author.getAffiliation().getDisplayValue()) + .append(""); + } + + if (author.getIdType() != null && + author.getIdValue() != null && + !author.getIdType().isEmpty() && + !author.getIdValue().isEmpty() && + author.getAffiliation() != null && + !author.getAffiliation().getDisplayValue().isEmpty()) { + if (author.getIdType().equals("ORCID")) { + contributorsElement.append("").append("https://orcid.org/").append(author.getIdValue()).append(""); + } + if (author.getIdType().equals("ISNI")) { + contributorsElement.append("").append(author.getIdValue()).append(""); + } + if (author.getIdType().equals("LCNA")) { + contributorsElement.append("").append(author.getIdValue()).append(""); + } + } + + contributorsElement.append(""); + } + contributorsElement.append(""); + + } else { + contributorsElement.append("") + .append(AbstractPidProvider.UNAVAILABLE) + .append(""); + } + + datasetElement.append(contributorsElement); + + datasetElement.append("") + .append(this.title) + .append(""); + + datasetElement.append("") + .append(this.identifier) + .append("") + .append(this.baseUrl).append("/dataset.xhtml?persistentId=doi:").append(this.identifier) + .append(""); + + datasetElement.append(""); + xmlMetadata = xmlMetadata.replace("${datasets}", datasetElement.toString()); + return xmlMetadata; + } + + public static String getTemplate() { + return template; + } + + public static void setTemplate(String template) { + CrossRefMetadataTemplate.template = template; + } + + public String getIdentifier() { + return identifier; + } + + public String getDepositor() { + return depositor; + } + + public void setDepositor(String depositor) { + this.depositor = depositor; + } + + public void setIdentifier(String identifier) { + this.identifier = identifier; + } + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + public String getInstitution() { + return institution; + } + + public void setInstitution(String institution) { + this.institution = institution; + } + + public String getDepositorEmail() { + return depositorEmail; + } + + public void setDepositorEmail(String depositorEmail) { + this.depositorEmail = depositorEmail; + } + + public String getDatabaseTitle() { + return databaseTitle; + } + + public void setDatabaseTitle(String databaseTitle) { + this.databaseTitle = databaseTitle; + } +} + +class CrossRefFileUtil { + + public static void close(InputStream in) { + if (in != null) { + try { + in.close(); + } catch (IOException e) { + throw new RuntimeException("Fail to close InputStream"); + } + } + } + + public static String readAndClose(InputStream inStream, String encoding) throws IOException { + ByteArrayOutputStream outStream = new ByteArrayOutputStream(); + byte[] buf = new byte[128]; + int cnt; + while ((cnt = inStream.read(buf)) >= 0) { + outStream.write(buf, 0, cnt); + } + return outStream.toString(encoding); + } +} + diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/crossref/CrossRefRESTfullClient.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/crossref/CrossRefRESTfullClient.java new file mode 100644 index 00000000000..ebdf875e664 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/crossref/CrossRefRESTfullClient.java @@ -0,0 +1,119 @@ +package edu.harvard.iq.dataverse.pidproviders.doi.crossref; + +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.mime.HttpMultipartMode; +import org.apache.http.entity.mime.MultipartEntityBuilder; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; + +import java.io.Closeable; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.logging.Level; +import java.util.logging.Logger; + +public class CrossRefRESTfullClient implements Closeable { + + private static final Logger logger = Logger.getLogger(CrossRefRESTfullClient.class.getCanonicalName()); + + private final String url; + private final String apiUrl; + private final String username; + private final String password; + private final CloseableHttpClient httpClient; + private final HttpClientContext context; + private final String encoding = "utf-8"; + + public CrossRefRESTfullClient(String url, String apiUrl, String username, String password) { + this.url = url; + this.apiUrl = apiUrl; + this.username = username; + this.password = password; + try { + context = HttpClientContext.create(); + CredentialsProvider credsProvider = new BasicCredentialsProvider(); + credsProvider.setCredentials(new AuthScope(null, -1), + new UsernamePasswordCredentials(username, password)); + context.setCredentialsProvider(credsProvider); + + httpClient = HttpClients.createDefault(); + } catch (Exception ioe) { + close(); + logger.log(Level.SEVERE,"Fail to init Client",ioe); + throw new RuntimeException("Fail to init Client", ioe); + } + } + + public void close() { + if (this.httpClient != null) { + try { + httpClient.close(); + } catch (IOException io) { + logger.warning("IOException closing httpClient: " + io.getMessage()); + } + } + } + + public String getMetadata(String doi) { + HttpGet httpGet = new HttpGet(this.apiUrl + "/works/" + doi); + httpGet.setHeader("Accept", "application/json"); + try { + HttpResponse response = httpClient.execute(httpGet); + String data = EntityUtils.toString(response.getEntity(), encoding); + if (response.getStatusLine().getStatusCode() != 200) { + String errMsg = "Response from getMetadata: " + response.getStatusLine().getStatusCode() + ", " + data; + logger.warning(errMsg); + throw new RuntimeException(errMsg); + } + return data; + } catch (IOException ioe) { + logger.warning("IOException when get metadata"); + throw new RuntimeException("IOException when get metadata", ioe); + } + } + + public String postMetadata(String xml) throws IOException { + HttpEntity entity = MultipartEntityBuilder.create() + .addTextBody("operation", "doMDUpload") + .addTextBody("login_id", username) + .addTextBody("login_passwd", password) + .addBinaryBody("fname", xml.getBytes(StandardCharsets.UTF_8), ContentType.APPLICATION_XML, "metadata.xml") + .setMode(HttpMultipartMode.BROWSER_COMPATIBLE) + .build(); + + HttpPost httpPost = new HttpPost(url + "/servlet/deposit"); + httpPost.setHeader("Accept", "*/*"); + httpPost.setEntity(entity); + HttpResponse response = httpClient.execute(httpPost); + + String data = EntityUtils.toString(response.getEntity(), encoding); + if (response.getStatusLine().getStatusCode() != 200) { + String errMsg = "Response from postMetadata: " + response.getStatusLine().getStatusCode() + ", " + data; + logger.warning(errMsg); + throw new IOException(errMsg); + } + return data; + } + + public boolean testDOIExists(String doi) throws IOException { + HttpGet httpGet = new HttpGet(this.apiUrl + "/works/" + doi); + httpGet.setHeader("Accept", "application/json"); + HttpResponse response = httpClient.execute(httpGet); + if (response.getStatusLine().getStatusCode() != 200) { + EntityUtils.consumeQuietly(response.getEntity()); + return false; + } + EntityUtils.consumeQuietly(response.getEntity()); + return true; + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/perma/PermaLinkPidProvider.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/perma/PermaLinkPidProvider.java index 91c7f527c88..7b55292350f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/perma/PermaLinkPidProvider.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/perma/PermaLinkPidProvider.java @@ -182,7 +182,7 @@ public GlobalId parsePersistentId(String protocol, String authority, String iden @Override public String getUrlPrefix() { - return getBaseUrl() + "/citation?persistentId=" + PERMA_PROTOCOL + ":"; + return getBaseUrl(); } @Override diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/perma/PermaLinkProviderFactory.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/perma/PermaLinkProviderFactory.java index 32b89223062..ca75fbff3a8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/perma/PermaLinkProviderFactory.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/perma/PermaLinkProviderFactory.java @@ -28,7 +28,7 @@ public PidProvider createPidProvider(String providerId) { String excludedList = JvmSettings.PID_PROVIDER_EXCLUDED_LIST.lookupOptional(providerId).orElse(""); String baseUrl = JvmSettings.PERMALINK_BASE_URL.lookupOptional(providerId) - .orElse(SystemConfig.getDataverseSiteUrlStatic()); + .orElse(SystemConfig.getDataverseSiteUrlStatic() + "/citation?persistentId=" + PermaLinkPidProvider.PERMA_PROTOCOL + ":"); ; String separator = JvmSettings.PERMALINK_SEPARATOR.lookupOptional(providerId).orElse(""); diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexBatchServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexBatchServiceBean.java index 0eeb681514c..3f7a7bb3363 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexBatchServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexBatchServiceBean.java @@ -50,35 +50,31 @@ public class IndexBatchServiceBean { public Future indexStatus() { JsonObjectBuilder response = Json.createObjectBuilder(); logger.info("Beginning indexStatus()"); - JsonObject contentInDatabaseButStaleInOrMissingFromSolr = getContentInDatabaseButStaleInOrMissingFromSolr().build(); - JsonObject contentInSolrButNotDatabase = null; - JsonObject permissionsInSolrButNotDatabase = null; try { - contentInSolrButNotDatabase = getContentInSolrButNotDatabase().build(); - permissionsInSolrButNotDatabase = getPermissionsInSolrButNotDatabase().build(); - - } catch (SearchException ex) { + JsonObject contentInDatabaseButStaleInOrMissingFromSolr = getContentInDatabaseButStaleInOrMissingFromSolr().build(); + JsonObject contentInSolrButNotDatabase = getContentInSolrButNotDatabase().build(); + JsonObject permissionsInSolrButNotDatabase = getPermissionsInSolrButNotDatabase().build(); + JsonObject permissionsInDatabaseButStaleInOrMissingFromSolr = getPermissionsInDatabaseButStaleInOrMissingFromSolr().build(); + + response + .add("contentInDatabaseButStaleInOrMissingFromIndex", contentInDatabaseButStaleInOrMissingFromSolr) + .add("contentInIndexButNotDatabase", contentInSolrButNotDatabase) + .add("permissionsInDatabaseButStaleInOrMissingFromIndex", permissionsInDatabaseButStaleInOrMissingFromSolr) + .add("permissionsInIndexButNotDatabase", permissionsInSolrButNotDatabase); + + logger.log(Level.INFO, "contentInDatabaseButStaleInOrMissingFromIndex: {0}", contentInDatabaseButStaleInOrMissingFromSolr); + logger.log(Level.INFO, "contentInIndexButNotDatabase: {0}", contentInSolrButNotDatabase); + logger.log(Level.INFO, "permissionsInDatabaseButStaleInOrMissingFromIndex: {0}", permissionsInDatabaseButStaleInOrMissingFromSolr); + logger.log(Level.INFO, "permissionsInIndexButNotDatabase: {0}", permissionsInSolrButNotDatabase); + } catch (Exception ex) { String msg = "Can not determine index status. " + ex.getLocalizedMessage() + ". Is Solr down? Exception: " + ex.getCause().getLocalizedMessage(); logger.info(msg); + ex.printStackTrace(); response.add("SearchException ", msg); - return new AsyncResult<>(response); } - - JsonObject permissionsInDatabaseButStaleInOrMissingFromSolr = getPermissionsInDatabaseButStaleInOrMissingFromSolr().build(); - - JsonObjectBuilder data = Json.createObjectBuilder() - .add("contentInDatabaseButStaleInOrMissingFromIndex", contentInDatabaseButStaleInOrMissingFromSolr) - .add("contentInIndexButNotDatabase", contentInSolrButNotDatabase) - .add("permissionsInDatabaseButStaleInOrMissingFromIndex", permissionsInDatabaseButStaleInOrMissingFromSolr) - .add("permissionsInIndexButNotDatabase", permissionsInSolrButNotDatabase); - - logger.log(Level.INFO, "contentInDatabaseButStaleInOrMissingFromIndex: {0}", contentInDatabaseButStaleInOrMissingFromSolr); - logger.log(Level.INFO, "contentInIndexButNotDatabase: {0}", contentInSolrButNotDatabase); - logger.log(Level.INFO, "permissionsInDatabaseButStaleInOrMissingFromIndex: {0}", permissionsInDatabaseButStaleInOrMissingFromSolr); - logger.log(Level.INFO, "permissionsInIndexButNotDatabase: {0}", permissionsInSolrButNotDatabase); - - return new AsyncResult<>(data); + return new AsyncResult<>(response); } + @Asynchronous public Future clearOrphans() { JsonObjectBuilder response = Json.createObjectBuilder(); diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index fd769846490..fe8f1030f82 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -2,6 +2,7 @@ import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.DatasetVersion.VersionState; +import edu.harvard.iq.dataverse.DvObject.DType; import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; import edu.harvard.iq.dataverse.authorization.providers.builtin.BuiltinUserServiceBean; import edu.harvard.iq.dataverse.batch.util.LoggingUtil; @@ -278,6 +279,7 @@ public Future indexDataverse(Dataverse dataverse, boolean processPaths) if (dataverse.getOwner() != null) { solrInputDocument.addField(SearchFields.PARENT_ID, dataverse.getOwner().getId()); solrInputDocument.addField(SearchFields.PARENT_NAME, dataverse.getOwner().getName()); + solrInputDocument.addField(SearchFields.DATAVERSE_PARENT_ALIAS, dataverse.getOwner().getAlias()); } } List dataversePathSegmentsAccumulator = new ArrayList<>(); @@ -2209,9 +2211,10 @@ public List findFilesInSolrOnly() throws SearchException { * @throws SearchException */ public List findPermissionsInSolrOnly() throws SearchException { + logger.info("Checking for solr-only permissions"); List permissionInSolrOnly = new ArrayList<>(); try { - int rows = 100; + int rows = 1000; SolrQuery q = (new SolrQuery(SearchFields.DEFINITION_POINT_DVOBJECT_ID+":*")).setRows(rows).setSort(SortClause.asc(SearchFields.ID)); String cursorMark = CursorMarkParams.CURSOR_MARK_START; boolean done = false; @@ -2219,52 +2222,55 @@ public List findPermissionsInSolrOnly() throws SearchException { q.set(CursorMarkParams.CURSOR_MARK_PARAM, cursorMark); QueryResponse rsp = solrServer.query(q); String nextCursorMark = rsp.getNextCursorMark(); + logger.fine("Next cursor mark (1K entries): " + nextCursorMark); SolrDocumentList list = rsp.getResults(); for (SolrDocument doc: list) { long id = Long.parseLong((String) doc.getFieldValue(SearchFields.DEFINITION_POINT_DVOBJECT_ID)); - String docId = (String)doc.getFieldValue(SearchFields.ID); - if(!dvObjectService.checkExists(id)) { + String docId = (String) doc.getFieldValue(SearchFields.ID); + String dtype = dvObjectService.getDtype(id); + if (dtype == null) { permissionInSolrOnly.add(docId); - } else { - DvObject obj = dvObjectService.findDvObject(id); - if (obj instanceof Dataset d) { - DatasetVersion dv = d.getLatestVersion(); + } + if (dtype.equals(DType.Dataset.getDType())) { + List states = datasetService.getVersionStates(id); + if (states != null) { + String latestState = states.get(states.size() - 1); if (docId.endsWith("draft_permission")) { - if (!dv.isDraft()) { + if (!latestState.equals(VersionState.DRAFT.toString())) { permissionInSolrOnly.add(docId); } } else if (docId.endsWith("deaccessioned_permission")) { - if (!dv.isDeaccessioned()) { + if (!latestState.equals(VersionState.DEACCESSIONED.toString())) { permissionInSolrOnly.add(docId); } } else { - if (d.getReleasedVersion() == null) { + if (!states.contains(VersionState.RELEASED.toString())) { permissionInSolrOnly.add(docId); } } - } else if (obj instanceof DataFile f) { - List states = dataFileService.findVersionStates(f.getId()); - Set strings = states.stream().map(VersionState::toString).collect(Collectors.toSet()); - logger.fine("States for " + docId + ": " + String.join(", ", strings)); - if (docId.endsWith("draft_permission")) { - if (!states.contains(VersionState.DRAFT)) { - permissionInSolrOnly.add(docId); - } - } else if (docId.endsWith("deaccessioned_permission")) { - if (!states.contains(VersionState.DEACCESSIONED) && states.size() == 1) { - permissionInSolrOnly.add(docId); - } + } + } else if (dtype.equals(DType.DataFile.getDType())) { + List states = dataFileService.findVersionStates(id); + Set strings = states.stream().map(VersionState::toString).collect(Collectors.toSet()); + logger.fine("States for " + docId + ": " + String.join(", ", strings)); + if (docId.endsWith("draft_permission")) { + if (!states.contains(VersionState.DRAFT)) { + permissionInSolrOnly.add(docId); + } + } else if (docId.endsWith("deaccessioned_permission")) { + if (!states.contains(VersionState.DEACCESSIONED) && states.size() == 1) { + permissionInSolrOnly.add(docId); + } + } else { + if (!states.contains(VersionState.RELEASED)) { + permissionInSolrOnly.add(docId); } else { - if (!states.contains(VersionState.RELEASED)) { + if (!dataFileService.isInReleasedVersion(id)) { + logger.fine("Adding doc " + docId + " to list of permissions in Solr only"); permissionInSolrOnly.add(docId); - } else { - if(dataFileService.findFileMetadataByDatasetVersionIdAndDataFileId(f.getOwner().getReleasedVersion().getId(), f.getId()) == null) { - logger.fine("Adding doc " + docId + " to list of permissions in Solr only"); - permissionInSolrOnly.add(docId); - } } - } + } } } @@ -2276,6 +2282,9 @@ public List findPermissionsInSolrOnly() throws SearchException { } catch (SolrServerException | IOException ex) { throw new SearchException("Error searching Solr for permissions" , ex); + } catch (Exception e) { + logger.warning(e.getLocalizedMessage()); + e.printStackTrace(); } return permissionInSolrOnly; } @@ -2306,7 +2315,7 @@ private List findDvObjectInSolrOnly(String type) throws SearchException if (idObject != null) { try { long id = (Long) idObject; - if (!dvObjectService.checkExists(id)) { + if (dvObjectService.getDtype(id) == null) { dvObjectInSolrOnly.add((String)doc.getFieldValue(SearchFields.ID)); } } catch (ClassCastException ex) { diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchFields.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchFields.java index bf9a1c47541..ef27a5eefaf 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SearchFields.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchFields.java @@ -94,6 +94,7 @@ public class SearchFields { public static final String UNF = "unf"; public static final String DATAVERSE_NAME = "dvName"; public static final String DATAVERSE_ALIAS = "dvAlias"; + public static final String DATAVERSE_PARENT_ALIAS = "dvParentAlias"; public static final String DATAVERSE_AFFILIATION = "dvAffiliation"; public static final String DATAVERSE_DESCRIPTION = "dvDescription"; public static final String DATAVERSE_CATEGORY = "dvCategory"; diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java index 5dcbf0450ad..28676caeac5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java @@ -1,14 +1,6 @@ package edu.harvard.iq.dataverse.search; -import edu.harvard.iq.dataverse.DataFile; -import edu.harvard.iq.dataverse.DatasetFieldConstant; -import edu.harvard.iq.dataverse.DatasetFieldServiceBean; -import edu.harvard.iq.dataverse.DatasetFieldType; -import edu.harvard.iq.dataverse.DatasetVersionServiceBean; -import edu.harvard.iq.dataverse.Dataverse; -import edu.harvard.iq.dataverse.DataverseFacet; -import edu.harvard.iq.dataverse.DataverseMetadataBlockFacet; -import edu.harvard.iq.dataverse.DvObjectServiceBean; +import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.authorization.groups.Group; import edu.harvard.iq.dataverse.authorization.groups.GroupServiceBean; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; @@ -40,6 +32,7 @@ import jakarta.ejb.EJBTransactionRolledbackException; import jakarta.ejb.Stateless; import jakarta.ejb.TransactionRolledbackLocalException; +import jakarta.inject.Inject; import jakarta.inject.Named; import jakarta.persistence.NoResultException; import org.apache.solr.client.solrj.SolrQuery; @@ -78,6 +71,8 @@ public class SearchServiceBean { SystemConfig systemConfig; @EJB SolrClientService solrClientService; + @Inject + ThumbnailServiceWrapper thumbnailServiceWrapper; /** * Import note: "onlyDatatRelatedToMe" relies on filterQueries for providing @@ -501,11 +496,14 @@ public SolrQueryResponse search( String dvTree = (String) solrDocument.getFirstValue(SearchFields.SUBTREE); String identifierOfDataverse = (String) solrDocument.getFieldValue(SearchFields.IDENTIFIER_OF_DATAVERSE); String nameOfDataverse = (String) solrDocument.getFieldValue(SearchFields.DATAVERSE_NAME); + String dataverseAffiliation = (String) solrDocument.getFieldValue(SearchFields.DATAVERSE_AFFILIATION); + String dataverseParentAlias = (String) solrDocument.getFieldValue(SearchFields.DATAVERSE_PARENT_ALIAS); + String dataverseParentName = (String) solrDocument.getFieldValue(SearchFields.PARENT_NAME); Long embargoEndDate = (Long) solrDocument.getFieldValue(SearchFields.EMBARGO_END_DATE); Long retentionEndDate = (Long) solrDocument.getFieldValue(SearchFields.RETENTION_END_DATE); // Boolean datasetValid = (Boolean) solrDocument.getFieldValue(SearchFields.DATASET_VALID); - + List matchedFields = new ArrayList<>(); SolrSearchResult solrSearchResult = new SolrSearchResult(query, name); @@ -592,10 +590,10 @@ public SolrQueryResponse search( if (type.equals("dataverses")) { solrSearchResult.setName(name); solrSearchResult.setHtmlUrl(baseUrl + SystemConfig.DATAVERSE_PATH + identifier); - // Do not set the ImageUrl, let the search include fragment fill in - // the thumbnail, similarly to how the dataset and datafile cards - // are handled. - //solrSearchResult.setImageUrl(baseUrl + "/api/access/dvCardImage/" + entityid); + solrSearchResult.setDataverseAffiliation(dataverseAffiliation); + solrSearchResult.setDataverseParentAlias(dataverseParentAlias); + solrSearchResult.setDataverseParentName(dataverseParentName); + solrSearchResult.setImageUrl(thumbnailServiceWrapper.getDataverseCardImageAsBase64Url(solrSearchResult)); /** * @todo Expose this API URL after "dvs" is changed to * "dataverses". Also, is an API token required for published @@ -605,6 +603,7 @@ public SolrQueryResponse search( } else if (type.equals("datasets")) { solrSearchResult.setHtmlUrl(baseUrl + "/dataset.xhtml?globalId=" + identifier); solrSearchResult.setApiUrl(baseUrl + "/api/datasets/" + entityid); + solrSearchResult.setImageUrl(thumbnailServiceWrapper.getDatasetCardImageAsUrl(solrSearchResult)); //Image url now set via thumbnail api //solrSearchResult.setImageUrl(baseUrl + "/api/access/dsCardImage/" + datasetVersionId); // No, we don't want to set the base64 thumbnails here. @@ -653,6 +652,7 @@ public SolrQueryResponse search( } solrSearchResult.setHtmlUrl(baseUrl + "/dataset.xhtml?persistentId=" + parentGlobalId); solrSearchResult.setDownloadUrl(baseUrl + "/api/access/datafile/" + entityid); + solrSearchResult.setImageUrl(thumbnailServiceWrapper.getFileCardImageAsBase64Url(solrSearchResult)); /** * @todo We are not yet setting the API URL for files because * not all files have metadata. Only subsettable files (those diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java index 01cbf7c1055..27900bac63f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java @@ -95,6 +95,7 @@ public class SolrSearchResult { private String fileChecksumValue; private String dataverseAlias; private String dataverseParentAlias; + private String dataverseParentName; // private boolean statePublished; /** * @todo Investigate/remove this "unpublishedState" variable. For files that @@ -504,8 +505,11 @@ public JsonObjectBuilder json(boolean showRelevance, boolean showEntityIds, bool // displayName = null; // testing NullSafeJsonBuilder // because we are using NullSafeJsonBuilder key/value pairs will be dropped if the value is null - NullSafeJsonBuilder nullSafeJsonBuilder = jsonObjectBuilder().add("name", displayName) - .add("type", getDisplayType(getType())).add("url", preferredUrl).add("image_url", getImageUrl()) + NullSafeJsonBuilder nullSafeJsonBuilder = jsonObjectBuilder() + .add("name", displayName) + .add("type", getDisplayType(getType())) + .add("url", preferredUrl) + .add("image_url", getImageUrl()) // .add("persistent_url", this.persistentUrl) // .add("download_url", this.downloadUrl) /** @@ -536,7 +540,8 @@ public JsonObjectBuilder json(boolean showRelevance, boolean showEntityIds, bool * @todo Expose MIME Type: * https://github.com/IQSS/dataverse/issues/1595 */ - .add("file_type", this.filetype).add("file_content_type", this.fileContentType) + .add("file_type", this.filetype) + .add("file_content_type", this.fileContentType) .add("size_in_bytes", getFileSizeInBytes()) /** * "md5" was the only possible value so it's hard-coded here but @@ -545,12 +550,18 @@ public JsonObjectBuilder json(boolean showRelevance, boolean showEntityIds, bool */ .add("md5", getFileMd5()) .add("checksum", JsonPrinter.getChecksumTypeAndValue(getFileChecksumType(), getFileChecksumValue())) - .add("unf", getUnf()).add("file_persistent_id", this.filePersistentId).add("dataset_name", datasetName) - .add("dataset_id", datasetId).add("publisher", publisherName) - .add("dataset_persistent_id", datasetPersistentId).add("dataset_citation", datasetCitation) - .add("deaccession_reason", this.deaccessionReason).add("citationHtml", this.citationHtml) + .add("unf", getUnf()) + .add("file_persistent_id", this.filePersistentId) + .add("dataset_name", datasetName) + .add("dataset_id", datasetId) + .add("publisher", publisherName) + .add("dataset_persistent_id", datasetPersistentId) + .add("dataset_citation", datasetCitation) + .add("deaccession_reason", this.deaccessionReason) + .add("citationHtml", this.citationHtml) .add("identifier_of_dataverse", this.identifierOfDataverse) - .add("name_of_dataverse", this.nameOfDataverse).add("citation", this.citation); + .add("name_of_dataverse", this.nameOfDataverse) + .add("citation", this.citation); // Now that nullSafeJsonBuilder has been instatiated, check for null before adding to it! if (showRelevance) { nullSafeJsonBuilder.add("matches", getRelevance()); @@ -668,6 +679,15 @@ public JsonObjectBuilder json(boolean showRelevance, boolean showEntityIds, bool nullSafeJsonBuilder.add("metadataBlocks", metadataFieldBuilder); } + } else if (this.entity.isInstanceofDataverse()) { + nullSafeJsonBuilder.add("affiliation", dataverseAffiliation); + nullSafeJsonBuilder.add("parentDataverseName", dataverseParentName); + nullSafeJsonBuilder.add("parentDataverseIdentifier", dataverseParentAlias); + } else if (this.entity.isInstanceofDataFile()) { + // "published_at" field is only set when the version state is not draft. + // On the contrary, this field also takes into account DataFiles in draft version, + // returning the creation date if the DataFile is not published, or the publication date otherwise. + nullSafeJsonBuilder.add("releaseOrCreateDate", getFormattedReleaseOrCreateDate()); } } @@ -747,11 +767,15 @@ private Map> computeRequestedMetadataFieldMapNames(List + + + ${batchId} + ${timestamp} + + ${depositor} + ${depositorEmail} + + Crossref + + + + + + ${title} + + + ${institution} + + + ${datasets} + + + diff --git a/src/main/webapp/404static.xhtml b/src/main/webapp/404static.xhtml index 69ff17ebc0f..dabcb299aa5 100644 --- a/src/main/webapp/404static.xhtml +++ b/src/main/webapp/404static.xhtml @@ -93,7 +93,7 @@
-

Copyright © 2023, The President & Fellows of Harvard College | Privacy Policy +

Copyright ©, The President & Fellows of Harvard College | Privacy Policy

@@ -106,4 +106,13 @@
- + + + + \ No newline at end of file diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java index 1dec51cc3ef..e8426b638d7 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetTypesIT.java @@ -240,7 +240,8 @@ public void testAddAndDeleteDatasetType() { numbersOnly.prettyPrint(); numbersOnly.then().assertThat().statusCode(BAD_REQUEST.getStatusCode()); - String randomName = UUID.randomUUID().toString().substring(0, 8); + //Avoid all-numeric names (which are not allowed) + String randomName = "A" + UUID.randomUUID().toString().substring(0, 8); String jsonIn = Json.createObjectBuilder().add("name", randomName).build().toString(); System.out.println("adding type with name " + randomName); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 3ff580268a9..f52aa4fe9bd 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -630,8 +630,7 @@ public void testCreatePublishDestroyDataset() { Response exportDatasetAsDublinCore = UtilIT.exportDataset(datasetPersistentId, "oai_dc", apiToken); exportDatasetAsDublinCore.prettyPrint(); exportDatasetAsDublinCore.then().assertThat() - // FIXME: Get this working. See https://github.com/rest-assured/rest-assured/wiki/Usage#example-3---complex-parsing-and-validation - // .body("oai_dc:dc.find { it == 'dc:title' }.item", hasItems("Darwin's Finches")) + .body("oai_dc.title", is("Darwin's Finches")) .statusCode(OK.getStatusCode()); Response exportDatasetAsDdi = UtilIT.exportDataset(datasetPersistentId, "ddi", apiToken); @@ -1195,8 +1194,7 @@ public void testExport() { Response exportDatasetAsDublinCore = UtilIT.exportDataset(datasetPersistentId, "oai_dc", apiToken); exportDatasetAsDublinCore.prettyPrint(); exportDatasetAsDublinCore.then().assertThat() - // FIXME: Get this working. See https://github.com/rest-assured/rest-assured/wiki/Usage#example-3---complex-parsing-and-validation - // .body("oai_dc:dc.find { it == 'dc:title' }.item", hasItems("Darwin's Finches")) + .body("oai_dc.title", is("Dataset One")) .statusCode(OK.getStatusCode()); Response exportDatasetAsDdi = UtilIT.exportDataset(datasetPersistentId, "ddi", apiToken); @@ -4103,7 +4101,87 @@ public void getDatasetVersionCitation() { .assertThat().body("data.message", containsString(String.valueOf(persistentId))); } - + @Test + public void testCitationDate() throws IOException { + + Response createUser = UtilIT.createRandomUser(); + createUser.then().assertThat().statusCode(OK.getStatusCode()); + String username = UtilIT.getUsernameFromResponse(createUser); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + + Response createDataverse = UtilIT.createRandomDataverse(apiToken); + createDataverse.then().assertThat().statusCode(CREATED.getStatusCode()); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverse); + Integer dataverseId = UtilIT.getDataverseIdFromResponse(createDataverse); + Response createDataset = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); + createDataset.then().assertThat().statusCode(CREATED.getStatusCode()); + Integer datasetId = UtilIT.getDatasetIdFromResponse(createDataset); + String datasetPid = JsonPath.from(createDataset.getBody().asString()).getString("data.persistentId"); + + Path pathToAddDateOfDepositJson = Paths.get(java.nio.file.Files.createTempDirectory(null) + File.separator + "dateOfDeposit.json"); + String dateOfDeposit = """ +{ + "fields": [ + { + "typeName": "dateOfDeposit", + "value": "1999-12-31" + } + ] +} +"""; + java.nio.file.Files.write(pathToAddDateOfDepositJson, dateOfDeposit.getBytes()); + + Response addDateOfDeposit = UtilIT.addDatasetMetadataViaNative(datasetPid, pathToAddDateOfDepositJson.toString(), apiToken); + addDateOfDeposit.prettyPrint(); + addDateOfDeposit.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.metadataBlocks.citation.fields[5].value", equalTo("1999-12-31")); + + Response setCitationDate = UtilIT.setDatasetCitationDateField(datasetPid, "dateOfDeposit", apiToken); + setCitationDate.prettyPrint(); + setCitationDate.then().assertThat().statusCode(OK.getStatusCode()); + + UtilIT.publishDataverseViaNativeApi(dataverseAlias, apiToken); + UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken).then().assertThat().statusCode(OK.getStatusCode()); + + Response getCitationAfter = UtilIT.getDatasetVersionCitation(datasetId, DS_VERSION_LATEST_PUBLISHED, true, apiToken); + getCitationAfter.prettyPrint(); + + String doi = datasetPid.substring(4); + + // Note that the year 1999 appears in the citation because we + // set the citation date field to a field that has that year. + String expectedCitation = "Finch, Fiona, 1999, \"Darwin's Finches\", https://doi.org/" + doi + ", Root, V1"; + + getCitationAfter.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.message", is(expectedCitation)); + + Response exportDatasetAsDublinCore = UtilIT.exportDataset(datasetPid, "oai_dc", apiToken); + exportDatasetAsDublinCore.prettyPrint(); + exportDatasetAsDublinCore.then().assertThat() + .body("oai_dc.type", equalTo("Dataset")) + .body("oai_dc.date", equalTo("1999-12-31")) + .statusCode(OK.getStatusCode()); + + Response clearDateField = UtilIT.clearDatasetCitationDateField(datasetPid, apiToken); + clearDateField.prettyPrint(); + clearDateField.then().assertThat().statusCode(OK.getStatusCode()); + + // Clearing not enough. You have to reexport because the previous date is cached. + Response rexport = UtilIT.reexportDatasetAllFormats(datasetPid); + rexport.prettyPrint(); + rexport.then().assertThat().statusCode(OK.getStatusCode()); + + String todayDate = LocalDate.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd")); + Response exportPostClear = UtilIT.exportDataset(datasetPid, "oai_dc", apiToken); + exportPostClear.prettyPrint(); + exportPostClear.then().assertThat() + .body("oai_dc.type", equalTo("Dataset")) + .body("oai_dc.date", equalTo(todayDate)) + .statusCode(OK.getStatusCode()); + } + @Test public void getVersionFiles() throws IOException, InterruptedException { Response createUser = UtilIT.createRandomUser(); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java index 9fa13bb2939..c743c12c7de 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java @@ -18,21 +18,17 @@ import java.io.UnsupportedEncodingException; import java.util.Base64; import jakarta.json.JsonArray; -import static jakarta.ws.rs.core.Response.Status.BAD_REQUEST; -import static jakarta.ws.rs.core.Response.Status.OK; -import static jakarta.ws.rs.core.Response.Status.FORBIDDEN; import org.hamcrest.CoreMatchers; import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter; import java.awt.image.BufferedImage; import java.io.IOException; import javax.imageio.ImageIO; -import static jakarta.ws.rs.core.Response.Status.CREATED; -import static jakarta.ws.rs.core.Response.Status.NOT_FOUND; -import static jakarta.ws.rs.core.Response.Status.UNAUTHORIZED; + import org.hamcrest.Matchers; import jakarta.json.JsonObjectBuilder; +import static jakarta.ws.rs.core.Response.Status.*; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; @@ -54,12 +50,12 @@ public static void setUpClass() { } @Test - public void testSearchPermisions() throws InterruptedException { + public void testSearchPermisions() { Response createUser1 = UtilIT.createRandomUser(); - String username1 = UtilIT.getUsernameFromResponse(createUser1); String apiToken1 = UtilIT.getApiTokenFromResponse(createUser1); + String affiliation = "testAffiliation"; - Response createDataverse1Response = UtilIT.createRandomDataverse(apiToken1); + Response createDataverse1Response = UtilIT.createRandomDataverse(apiToken1, affiliation); createDataverse1Response.prettyPrint(); assertEquals(201, createDataverse1Response.getStatusCode()); String dataverseAlias = UtilIT.getAliasFromResponse(createDataverse1Response); @@ -69,9 +65,9 @@ public void testSearchPermisions() throws InterruptedException { createDataset1Response.then().assertThat() .statusCode(CREATED.getStatusCode()); - Integer datasetId1 = UtilIT.getDatasetIdFromResponse(createDataset1Response); + Integer datasetId = UtilIT.getDatasetIdFromResponse(createDataset1Response); - Response shouldBeVisibleToUser1 = UtilIT.search("id:dataset_" + datasetId1 + "_draft", apiToken1); + Response shouldBeVisibleToUser1 = UtilIT.search("id:dataset_" + datasetId + "_draft", apiToken1); shouldBeVisibleToUser1.prettyPrint(); shouldBeVisibleToUser1.then().assertThat() .body("data.total_count", CoreMatchers.is(1)) @@ -83,7 +79,7 @@ public void testSearchPermisions() throws InterruptedException { String username2 = UtilIT.getUsernameFromResponse(createUser2); String apiToken2 = UtilIT.getApiTokenFromResponse(createUser2); - Response shouldNotBeVisibleToUser2 = UtilIT.search("id:dataset_" + datasetId1 + "_draft", apiToken2); + Response shouldNotBeVisibleToUser2 = UtilIT.search("id:dataset_" + datasetId + "_draft", apiToken2); shouldNotBeVisibleToUser2.prettyPrint(); shouldNotBeVisibleToUser2.then().assertThat() .body("data.total_count", CoreMatchers.is(0)) @@ -91,7 +87,7 @@ public void testSearchPermisions() throws InterruptedException { String nullToken = null; - Response shouldNotBeVisibleToTokenLess = UtilIT.search("id:dataset_" + datasetId1 + "_draft", nullToken); + Response shouldNotBeVisibleToTokenLess = UtilIT.search("id:dataset_" + datasetId + "_draft", nullToken); shouldNotBeVisibleToTokenLess.prettyPrint(); shouldNotBeVisibleToTokenLess.then().assertThat() .body("data.total_count", CoreMatchers.is(0)) @@ -103,10 +99,10 @@ public void testSearchPermisions() throws InterruptedException { grantUser2AccessOnDataset.prettyPrint(); assertEquals(200, grantUser2AccessOnDataset.getStatusCode()); - String searchPart = "id:dataset_" + datasetId1 + "_draft"; + String searchPart = "id:dataset_" + datasetId + "_draft"; assertTrue(UtilIT.sleepForSearch(searchPart, apiToken2, "", 1, UtilIT.MAXIMUM_INGEST_LOCK_DURATION), "Failed test if search exceeds max duration " + searchPart); - Response shouldBeVisibleToUser2 = UtilIT.search("id:dataset_" + datasetId1 + "_draft", apiToken2); + Response shouldBeVisibleToUser2 = UtilIT.search("id:dataset_" + datasetId + "_draft", apiToken2); shouldBeVisibleToUser2.prettyPrint(); shouldBeVisibleToUser2.then().assertThat() .body("data.total_count", CoreMatchers.is(1)) @@ -120,17 +116,16 @@ public void testSearchPermisions() throws InterruptedException { publishDataverse.then().assertThat() .statusCode(OK.getStatusCode()); - Response publishDataset = UtilIT.publishDatasetViaNativeApi(datasetId1, "major", apiToken1); + Response publishDataset = UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken1); publishDataset.prettyPrint(); publishDataset.then().assertThat() .statusCode(OK.getStatusCode()); - Response makeSureTokenlessSearchIsEnabled = UtilIT.deleteSetting(SettingsServiceBean.Key.SearchApiRequiresToken); makeSureTokenlessSearchIsEnabled.then().assertThat() .statusCode(OK.getStatusCode()); - Response publishedPublicDataShouldBeVisibleToTokenless = UtilIT.search("id:dataset_" + datasetId1, nullToken); + Response publishedPublicDataShouldBeVisibleToTokenless = UtilIT.search("id:dataset_" + datasetId, nullToken); publishedPublicDataShouldBeVisibleToTokenless.prettyPrint(); publishedPublicDataShouldBeVisibleToTokenless.then().assertThat() .body("data.total_count", CoreMatchers.is(1)) @@ -146,13 +141,15 @@ public void testSearchPermisions() throws InterruptedException { .body("data.count_in_response", CoreMatchers.is(1)) .body("data.items[0].name", CoreMatchers.is(dataverseAlias)) .body("data.items[0].type", CoreMatchers.is("dataverse")) - .body("data.items[0].identifier", CoreMatchers.is(dataverseAlias)); + .body("data.items[0].affiliation", CoreMatchers.is(affiliation)) + .body("data.items[0].parentDataverseName", CoreMatchers.is("Root")) + .body("data.items[0].parentDataverseIdentifier", CoreMatchers.is("root")); Response disableTokenlessSearch = UtilIT.setSetting(SettingsServiceBean.Key.SearchApiRequiresToken, "true"); disableTokenlessSearch.then().assertThat() .statusCode(OK.getStatusCode()); - Response dataverse47behaviorOfTokensBeingRequired = UtilIT.search("id:dataset_" + datasetId1, nullToken); + Response dataverse47behaviorOfTokensBeingRequired = UtilIT.search("id:dataset_" + datasetId, nullToken); dataverse47behaviorOfTokensBeingRequired.prettyPrint(); dataverse47behaviorOfTokensBeingRequired.then().assertThat() .body("message", CoreMatchers.equalTo(AbstractApiBean.RESPONSE_MESSAGE_AUTHENTICATED_USER_REQUIRED)) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 8b170ec5fce..4e20e8e4c33 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -360,11 +360,19 @@ static Response createDataverse(String alias, String category, String apiToken) return createSubDataverse(alias, category, apiToken, ":root"); } + static Response createDataverse(String alias, String category, String affiliation, String apiToken) { + return createSubDataverse(alias, category, apiToken, ":root", null, null, null, affiliation); + } + static Response createSubDataverse(String alias, String category, String apiToken, String parentDV) { - return createSubDataverse(alias, category, apiToken, parentDV, null, null, null); + return createSubDataverse(alias, category, apiToken, parentDV, null, null, null, null); } static Response createSubDataverse(String alias, String category, String apiToken, String parentDV, String[] inputLevelNames, String[] facetIds, String[] metadataBlockNames) { + return createSubDataverse(alias, category, apiToken, parentDV, inputLevelNames, facetIds, metadataBlockNames, null); + } + + static Response createSubDataverse(String alias, String category, String apiToken, String parentDV, String[] inputLevelNames, String[] facetIds, String[] metadataBlockNames, String affiliation) { JsonArrayBuilder contactArrayBuilder = Json.createArrayBuilder(); contactArrayBuilder.add(Json.createObjectBuilder().add("contactEmail", getEmailFromUserName(getRandomIdentifier()))); JsonArrayBuilder subjectArrayBuilder = Json.createArrayBuilder(); @@ -377,6 +385,10 @@ static Response createSubDataverse(String alias, String category, String apiToke // don't send "dataverseType" if category is null, must be a better way .add(category != null ? "dataverseType" : "notTheKeyDataverseType", category != null ? category : "whatever"); + if (affiliation != null) { + objectBuilder.add("affiliation", affiliation); + } + JsonObjectBuilder metadataBlocksObjectBuilder = Json.createObjectBuilder(); if (inputLevelNames != null) { @@ -430,6 +442,12 @@ static Response createRandomDataverse(String apiToken) { return createDataverse(alias, category, apiToken); } + static Response createRandomDataverse(String apiToken, String affiliation) { + String alias = "dv" + getRandomIdentifier(); + String category = null; + return createDataverse(alias, category, affiliation, apiToken); + } + /** * A convenience method for creating a random collection and getting its * alias in one step. @@ -3717,6 +3735,33 @@ static Response getDatasetVersionCitation(Integer datasetId, String version, boo return response; } + static Response setDatasetCitationDateField(String datasetIdOrPersistentId, String dateField, String apiToken) { + String idInPath = datasetIdOrPersistentId; // Assume it's a number. + String optionalQueryParam = ""; // If idOrPersistentId is a number we'll just put it in the path. + if (!NumberUtils.isCreatable(datasetIdOrPersistentId)) { + idInPath = ":persistentId"; + optionalQueryParam = "?persistentId=" + datasetIdOrPersistentId; + } + Response response = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .body(dateField) + .put("/api/datasets/" + idInPath + "/citationdate" + optionalQueryParam); + return response; + } + + static Response clearDatasetCitationDateField(String datasetIdOrPersistentId, String apiToken) { + String idInPath = datasetIdOrPersistentId; // Assume it's a number. + String optionalQueryParam = ""; // If idOrPersistentId is a number we'll just put it in the path. + if (!NumberUtils.isCreatable(datasetIdOrPersistentId)) { + idInPath = ":persistentId"; + optionalQueryParam = "?persistentId=" + datasetIdOrPersistentId; + } + Response response = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .delete("/api/datasets/" + idInPath + "/citationdate" + optionalQueryParam); + return response; + } + static Response getFileCitation(Integer fileId, String datasetVersion, String apiToken) { Boolean includeDeaccessioned = null; return getFileCitation(fileId, datasetVersion, includeDeaccessioned, apiToken); diff --git a/src/test/java/edu/harvard/iq/dataverse/pidproviders/PidUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/pidproviders/PidUtilTest.java index cffac741c78..58d69da743b 100644 --- a/src/test/java/edu/harvard/iq/dataverse/pidproviders/PidUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/pidproviders/PidUtilTest.java @@ -5,6 +5,8 @@ import edu.harvard.iq.dataverse.GlobalId; import edu.harvard.iq.dataverse.pidproviders.doi.AbstractDOIProvider; import edu.harvard.iq.dataverse.pidproviders.doi.UnmanagedDOIProvider; +import edu.harvard.iq.dataverse.pidproviders.doi.crossref.CrossRefDOIProvider; +import edu.harvard.iq.dataverse.pidproviders.doi.crossref.CrossRefDOIProviderFactory; import edu.harvard.iq.dataverse.pidproviders.doi.datacite.DataCiteDOIProvider; import edu.harvard.iq.dataverse.pidproviders.doi.datacite.DataCiteProviderFactory; import edu.harvard.iq.dataverse.pidproviders.doi.ezid.EZIdDOIProvider; @@ -25,7 +27,6 @@ import edu.harvard.iq.dataverse.util.testing.LocalJvmSettings; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -41,7 +42,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.MockitoAnnotations; @@ -67,7 +67,7 @@ @JvmSetting(key = JvmSettings.PID_PROVIDER_SHOULDER, value = "QE", varArgs = "perma2") @JvmSetting(key = JvmSettings.PID_PROVIDER_MANAGED_LIST, value = "perma:LINKIT/FK2ABCDEF", varArgs ="perma2") @JvmSetting(key = JvmSettings.PERMALINK_SEPARATOR, value = "/", varArgs = "perma2") -@JvmSetting(key = JvmSettings.PERMALINK_BASE_URL, value = "https://example.org/123", varArgs = "perma2") +@JvmSetting(key = JvmSettings.PERMALINK_BASE_URL, value = "https://example.org/123/citation?persistentId=perma:", varArgs = "perma2") // Datacite 1 @JvmSetting(key = JvmSettings.PID_PROVIDER_LABEL, value = "dataCite 1", varArgs = "dc1") @JvmSetting(key = JvmSettings.PID_PROVIDER_TYPE, value = DataCiteDOIProvider.TYPE, varArgs = "dc1") @@ -114,8 +114,20 @@ @JvmSetting(key = JvmSettings.HANDLENET_KEY_PASSPHRASE, value = "passphrase", varArgs ="hdl1") @JvmSetting(key = JvmSettings.HANDLENET_KEY_PATH, value = "/tmp/cred", varArgs ="hdl1") +// CrossRef 1 +@JvmSetting(key = JvmSettings.PID_PROVIDER_LABEL, value = "CrossRef 1", varArgs = "crossref1") +@JvmSetting(key = JvmSettings.PID_PROVIDER_TYPE, value = CrossRefDOIProvider.TYPE, varArgs = "crossref1") +@JvmSetting(key = JvmSettings.PID_PROVIDER_AUTHORITY, value = "10.11111", varArgs = "crossref1") +@JvmSetting(key = JvmSettings.PID_PROVIDER_SHOULDER, value = "DVN/", varArgs = "crossref1") +@JvmSetting(key = JvmSettings.PID_PROVIDER_MANAGED_LIST, value = "", varArgs ="crossref1") +@JvmSetting(key = JvmSettings.CROSSREF_URL, value = "https://doi.crossref.org", varArgs ="crossref1") +@JvmSetting(key = JvmSettings.CROSSREF_REST_API_URL, value = "https://test.crossref.org", varArgs ="crossref1") +@JvmSetting(key = JvmSettings.CROSSREF_USERNAME, value = "crusername", varArgs ="crossref1") +@JvmSetting(key = JvmSettings.CROSSREF_PASSWORD, value = "secret", varArgs ="crossref1") +@JvmSetting(key = JvmSettings.CROSSREF_DEPOSITOR, value = "xyz", varArgs ="crossref1") +@JvmSetting(key = JvmSettings.CROSSREF_DEPOSITOR_EMAIL, value = "xyz@example.com", varArgs ="crossref1") //List to instantiate -@JvmSetting(key = JvmSettings.PID_PROVIDERS, value = "perma1, perma2, dc1, dc2, ez1, fake1, hdl1") +@JvmSetting(key = JvmSettings.PID_PROVIDERS, value = "perma1, perma2, dc1, dc2, ez1, fake1, hdl1, crossref1") public class PidUtilTest { @@ -133,6 +145,7 @@ public static void setUpClass() throws Exception { pidProviderFactoryMap.put(HandlePidProvider.TYPE, new HandleProviderFactory()); pidProviderFactoryMap.put(FakeDOIProvider.TYPE, new FakeProviderFactory()); pidProviderFactoryMap.put(EZIdDOIProvider.TYPE, new EZIdProviderFactory()); + pidProviderFactoryMap.put(CrossRefDOIProvider.TYPE, new CrossRefDOIProviderFactory()); PidUtil.clearPidProviders(); @@ -191,7 +204,7 @@ public void testFactories() throws IOException { assertEquals("-", p.getSeparator()); assertTrue(p.getUrlPrefix().startsWith(SystemConfig.getDataverseSiteUrlStatic())); p = PidUtil.getPidProvider("perma2"); - assertTrue(p.getUrlPrefix().startsWith("https://example.org/123")); + assertTrue(p.getUrlPrefix().startsWith("https://example.org/123/citation?persistentId=")); p = PidUtil.getPidProvider("dc2"); assertEquals("FK3", p.getShoulder()); @@ -252,7 +265,10 @@ public void testDOIParsing() throws IOException { assertEquals(pid6String, pid6.asString()); assertEquals("fake1", pid6.getProviderId()); - + String pid7String = "doi:10.11111/DVN/ABCDEF"; + GlobalId pid7 = PidUtil.parseAsGlobalID(pid7String); + assertEquals(pid7String, pid7.asString()); + assertEquals("crossref1", pid7.getProviderId()); } @Test