restore cleaned up files (#27432)

docker/dev-env/Dockerfile

@@ -0,0 +1,65 @@
+# ----------------------------------------------
+# Stage 1:  Build dotCMS from our builder image
+# ----------------------------------------------
+ARG DOTCMS_DOCKER_TAG="latest"
+
+FROM dotcms/dotcms:${DOTCMS_DOCKER_TAG} AS dotcms
+
+FROM ubuntu:22.04
+
+# Defining default non-root user UID, GID, and name 
+ARG USER_UID="65001"
+ARG USER_GID="65001"
+ARG USER_GROUP="dotcms"
+ARG USER_NAME="dotcms"
+
+RUN groupadd -f -g $USER_GID $USER_GROUP
+# Creating default non-user
+# the useradd
+RUN useradd -l -d /srv -g $USER_GID -u $USER_UID $USER_NAME
+
+COPY --from=dotcms --chown=$USER_NAME:$USER_GROUP /srv/ /srv/
+
+ARG DEBIAN_FRONTEND=noninteractive
+ARG UBUNTU_RELEASE=jammy
+ARG PG_VERSION=15
+ARG PGDATA=/data/postgres
+ARG DEBIAN_FRONTEND=noninteractive
+ARG DEBCONF_NONINTERACTIVE_SEEN=true
+RUN mkdir /data
+RUN chmod 777 /data
+
+# Installing basic packages
+RUN apt-get update && \
+    apt-get upgrade -y && \
+    apt-get install -y --no-install-recommends bash zip unzip wget libtcnative-1\
+    tzdata tini ca-certificates openssl libapr1 libpq-dev curl gnupg\
+    vim libarchive-tools
+
+
+RUN sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $UBUNTU_RELEASE-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
+
+RUN wget -qO- https://www.postgresql.org/media/keys/ACCC4CF8.asc | tee /etc/apt/trusted.gpg.d/pgdg.asc &>/dev/null \
+  && apt-get update -y \
+  && apt-get upgrade -y \
+  && apt-get install -y postgresql-$PG_VERSION
+
+
+# Cleanup
+RUN apt-get autoremove -y && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+
+COPY --from=opensearchproject/opensearch:1.3.11 --chown=$USER_NAME:$USER_GROUP /usr/share/opensearch /usr/share/opensearch
+
+RUN echo "discovery.type: single-node\nbootstrap.memory_lock: true\ncluster.routing.allocation.disk.threshold_enabled: true\ncluster.routing.allocation.disk.watermark.low: 1g\ncluster.routing.allocation.disk.watermark.high: 500m\ncluster.routing.allocation.disk.watermark.flood_stage: 400m\ncluster.info.update.interval: 5m" >> /usr/share/opensearch/config/opensearch.yml
+
+
+ENV PATH=$PATH:/usr/share/opensearch/bin
+RUN /usr/share/opensearch/opensearch-onetime-setup.sh
+RUN chown -R dotcms.dotcms /usr/share/opensearch/config
+COPY entrypoint.sh /
+RUN chmod 755 /entrypoint.sh
+
+ENTRYPOINT ["/usr/bin/tini", "--", "/entrypoint.sh"]

docker/dev-env/README.md

@@ -0,0 +1,114 @@
+# dotCMS Development Docker Image
+### All in one docker image including Postgres and Opensearch
+This image, intended for development, runs Ubuntu 22.04 and contains dotCMS, Postgres 15 and Opensearch 1.x. All dotCMS, db and es index data is stored in the `/data` directory, which should be mapped in if you want your environment to persist.  The beauty of this image that it can be used to CLONE an existing dotCMS instance.  
+
+
+## Running the image
+This image takes all the normal dotCMS docker config switches - keep in mind that the DB and ES come pre-wired, so no need to change those.  This image also takes the following env variables:
+
+- `DOTCMS_SOURCE_ENVIRONMENT` : the url for the environment you wish to clone, e.g. https://demo.dotcms.com .
+- `DOTCMS_API_TOKEN` : A valid dotCMS API Token from an admin user in the source environment.
+- `DOTCMS_USERNAME_PASSWORD` :  The username:password for an admin user in the source environment.
+- `DOTCMS_DEBUG` :  Run dotCMS in debug mode and listen for a remote debugger on port 8000, defaults to `false`.
+- `ALL_ASSETS` : Controls whether old versions of assets are included in the download, defaults to false, which means only the current live and working versions of assets will be downloaded.
+
+
+
+## Cloning a dotCMS Environment
+When running this image, if you specify a source environment and a valid means to authenticate, the image will attempt to pull the **assets** and **db** from the source environment.  To do this, you start the image up and pass it a `DOTCMS_SOURCE_ENVIRONMENT` and either an `DOTCMS_API_TOKEN` or `DOTCMS_USERNAME_PASSWORD` (e.g. `[email protected]:admin`).  On startup, the image will try to reach out and download the database and assets from the specified dotCMS instance, load the db and assets and start dotCMS in debug mode. Once the server starts, you need to run a full reindex.
+
+
+
+#### Clone demo with a dotCMS API Token
+```
+export TOK=XXXXXXX_YOUR_DOTCMS_TOKEN.eXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+
+docker run --rm \
+-p 8000:8000 \
+-p 8443:8443 \
+-v $PWD/data:/data \
+-e DOTCMS_SOURCE_ENVIRONMENT=https://demo.dotcms.com \
+-e DOTCMS_API_TOKEN=$TOK \
+ dotcms/dotcms-dev:latest
+```
+
+#### Clone demo with UserID/Password
+```
+docker run --rm \
+-p 8443:8443 \
+-v $PWD/data:/data \
+-e DOTCMS_SOURCE_ENVIRONMENT=https://demo.dotcms.com \
+-e DOTCMS_USERNAME_PASSWORD="[email protected]:admin" \
+ dotcms/dotcms-dev:latest
+```
+
+#### Run normal startup with Postgres port exposed, running debug
+```
+docker run --rm  \
+-p 8443:8443 \
+-p 5432:5432 \
+-p 8000:8000 \
+-v $PWD/data:/data \
+-e DOTCMS_DEBUG=true \
+ dotcms/dotcms-dev:latest
+```
+
+
+#### Troubleshooting the Download
+Due to a bug in docker, downloading large environments can time out. To get around this, you can download the assets and db yourself (outside of docker) to seed your installation and add them to the data volume you map into the image.  dotCMS will look for `/data/assets.zip` and/or `/data/dotcms_db.sql.gz` to import before running the normal starter import routine.
+
+#### Downloading your assets and db outside of Docker
+dotCMS offers two admin only endpoints to download your data and assets
+- `/api/v1/maintenance/_downloadAssets`
+- `/api/v1/maintenance/_downloadDb`
+
+When downloading assets, you can specify `?oldAssets=false`, and dotCMS will only include the assets for live and working versions of your content, thus hopefully generating a MUCH smaller download
+
+#### Example wget to download assets
+```
+wget --header="$AUTH_HEADER" -t 1 -O assets.zip  $DOTCMS_SOURCE_ENVIRONMENT/api/v1/maintenance/_downloadAssets?oldAssets=false
+```
+
+#### Example wget to download your DB
+```
+wget --header="$AUTH_HEADER" -t 1 -O dotcms_db.sql.gz $DOTCMS_SOURCE_ENVIRONMENT/api/v1/maintenance/_downloadDb 
+
+```
+
+#### Starting from a clean slate
+Your development instance can be deleted and reset by deleting the ./data directory that is mapped in. 
+
+
+
+
+## Building this Image
+By default, this image is built from the `dotcms/dotcms:latest` tagged version of dotCMS.  You can specify another dotCMS version you want use for your dev instance by passing the build-arg `DOTCMS_DOCKER_TAG` to indicate which dotCMS image tag to use to build,  e.g.
+`--build-arg DOTCMS_DOCKER_TAG=latest` or `--build-arg DOTCMS_DOCKER_TAG=23.07`
+
+```
+docker build --pull --build-arg DOTCMS_DOCKER_TAG=latest  . -t dotcms/dotcms-dev
+```
+or
+```
+docker buildx build --build-arg DOTCMS_DOCKER_TAG=master_latest_SNAPSHOT --platform linux/amd64,linux/arm64 --pull --push -t dotcms/dotcms-dev:master_latest_SNAPSHOT .
+```
+
+### Included Database and Elasticsearch
+
+This image runs the following servers internally. 
+
+#### Opensearch 1.3.11
+Running https on 
+- https on port 9200 
+- basic auth (admin/admin)
+- data stored in /data/opensearch 
+
+
+#### Postgres 15
+Running on port 5432 and using the dotCMS defaults:
+- db: dotcms
+- user: dotcmsdbuser
+- pass: password
+- data stored in /data/postgres
+
+If you wish to connect to these instances remotely, you will need to expose their ports in docker when you run the image, e.g.

docker/dev-env/entrypoint.sh

@@ -0,0 +1,177 @@
+#!/bin/bash -e
+
+ASSETS_BACKUP_FILE=/data/assets.zip
+DB_BACKUP_FILE=/data/dotcms_db.sql.gz
+
+export JAVA_HOME=/usr/share/opensearch/jdk
+export PATH=$PATH:$JAVA_HOME/bin:/usr/local/pgsql/bin
+export ES_JAVA_OPTS=${ES_JAVA_OPTS:-"-Xmx512m"}
+
+
+setup_postgres () {
+    echo "Starting Postgres Database"
+    if [ ! -d "/data/postgres" ]; then
+        mv /var/lib/postgresql/$PG_VERSION/main /data/postgres
+    fi
+    rm -rf /var/lib/postgresql/$PG_VERSION/main
+    ln -sf  /data/postgres /var/lib/postgresql/$PG_VERSION/main
+
+
+    /etc/init.d/postgresql start
+
+    if su -c "psql -lqt" postgres | cut -d \| -f 1 | grep -qw dotcms; then
+        # database exists
+        echo "- dotCMS db exists, skipping import"
+        echo "- Delete the /data/postgres folder to force a re-import"
+        return 0
+    fi
+
+    # creating database
+    su -c "psql -c \"CREATE database dotcms;\" 1> /dev/null" postgres
+    su -c "psql -c \"CREATE USER dotcmsdbuser WITH PASSWORD 'password';\" 1> /dev/null" postgres
+    su -c "psql -c \"ALTER DATABASE dotcms OWNER TO dotcmsdbuser;\" 1> /dev/null" postgres
+
+    if [  -f "$DB_BACKUP_FILE" ]; then
+      echo "- Importing dotCMS db from backup"
+      # import database
+      cat $DB_BACKUP_FILE | gzip -d | PGPASSWORD=password psql -h 127.0.0.1 -Udotcmsdbuser dotcms
+    fi
+
+    return 0
+}
+
+
+setup_opensearch () {
+
+
+    if [ ! -d "/data/opensearch" ]; then
+        mv /usr/share/opensearch/data /data/opensearch
+        chown dotcms.dotcms /data/opensearch
+    fi
+
+    rm -rf /usr/share/opensearch/data
+    ln -sf  /data/opensearch /usr/share/opensearch/data
+    chown dotcms.dotcms /data/opensearch
+
+    echo "Starting OPENSEARCH"
+    # Start up Elasticsearch
+    su -c "/usr/share/opensearch/bin/opensearch 1> /dev/null" dotcms &
+}
+
+
+pull_dotcms_backups () {
+
+    # If these are 0 length files, delete them
+    if [ ! -s $ASSETS_BACKUP_FILE ] ; then
+        rm -rf $ASSETS_BACKUP_FILE
+    fi
+
+    if [ ! -s $DB_BACKUP_FILE ] ; then
+        rm -rf $DB_BACKUP_FILE
+    fi
+
+    if [ -f "$ASSETS_BACKUP_FILE" ] && [ -f $DB_BACKUP_FILE ]; then
+
+        echo "- DB and Assets backups exist, skipping"
+        echo "- Delete $ASSETS_BACKUP_FILE and $DB_BACKUP_FILE to force a re-download"
+        return 0
+    fi
+
+    if [ -z "$DOTCMS_SOURCE_ENVIRONMENT" ]; then
+        echo "- No dotCMS env to clone, starting normally"
+        return 0
+    fi
+    if [ -z "$DOTCMS_API_TOKEN" -a -z "$DOTCMS_USERNAME_PASSWORD" ]; then
+        echo "- Source environment specified, but no dotCMS auth available"
+        return 1
+    fi
+
+    echo "Pulling Environment from $DOTCMS_SOURCE_ENVIRONMENT"
+
+    if [ -n  "$DOTCMS_API_TOKEN"  ]; then
+        echo "- Using Authorization: Bearer"
+        AUTH_HEADER="Authorization: Bearer $DOTCMS_API_TOKEN"
+    else
+        echo "- Using Authorization: Basic"
+        AUTH_HEADER="Authorization: Basic $(echo -n $DOTCMS_USERNAME_PASSWORD | base64)"
+    fi
+
+    mkdir -p /data/shared/assets
+    chown -R dotcms.dotcms /data/shared
+
+    if [ ! -f "$ASSETS_BACKUP_FILE" ]; then
+        su -c "rm -rf $ASSETS_BACKUP_FILE.tmp"
+        echo "- Downloading ASSETS"
+        su -c "wget --no-check-certificate --header=\"$AUTH_HEADER\" -t 1 -O $ASSETS_BACKUP_FILE.tmp  $DOTCMS_SOURCE_ENVIRONMENT/api/v1/maintenance/_downloadAssets\?oldAssets=${ALL_ASSETS:-"false"} " dotcms
+        if [ -s $ASSETS_BACKUP_FILE.tmp ]; then
+          su -c "mv $ASSETS_BACKUP_FILE.tmp $ASSETS_BACKUP_FILE"
+        else
+          su -c "rm -rf $ASSETS_BACKUP_FILE.tmp"
+          echo "asset download failed, please check your credentials and try again"
+          exit 1
+        fi
+    fi
+
+    if [ ! -f "$DB_BACKUP_FILE" ]; then
+        echo "- Downloading database"
+        su -c "rm -rf $DB_BACKUP_FILE.tmp"
+        su -c "wget --no-check-certificate --header=\"$AUTH_HEADER\" -t 1 -O $DB_BACKUP_FILE.tmp $DOTCMS_SOURCE_ENVIRONMENT/api/v1/maintenance/_downloadDb" dotcms
+        if [ -s $DB_BACKUP_FILE.tmp ]; then
+          su -c "mv $DB_BACKUP_FILE.tmp $DB_BACKUP_FILE"
+        else
+          su -c "rm -rf $DB_BACKUP_FILE.tmp"
+          echo "database download failed, please check your credentials and try again"
+          exit 1
+        fi
+    fi
+
+}
+
+unpack_assets(){
+  if [ -d "/data/shared/assets/1" ]; then
+    echo "Assets Already Unpacked, skipping.  If you would like to unpack them again, please delete the /data/shared/assets folder"
+    return 0
+  fi
+  if [ ! -s "$ASSETS_BACKUP_FILE" ]; then
+    return 0
+  fi
+
+
+  echo "Unzipping assets.zip"
+  su -c "unzip -u $ASSETS_BACKUP_FILE -d /data/shared" || true
+}
+
+
+
+start_dotcms () {
+
+
+    if [ "$DOTCMS_DEBUG" == "true" ];then
+      echo "Setting java debug port to 8000. If you want to change the debug options,"
+      echo "pass in your options using the CMS_JAVA_OPTS variable instead"
+      export CMS_JAVA_OPTS="$CMS_JAVA_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=*:8000"
+    fi
+
+    export DB_BASE_URL=${DB_BASE_URL:-"jdbc:postgresql://127.0.0.1/dotcms"}
+    export DOT_ES_ENDPOINTS=${DOT_ES_ENDPOINTS:-"https://127.0.0.1:9200"}
+    export DOT_DOTCMS_CLUSTER_ID=${DOT_DOTCMS_CLUSTER_ID:-"dotcms_dev"}
+
+    echo "Starting dotCMS using"
+    echo " - CMS_JAVA_OPTS: $CMS_JAVA_OPTS"
+    echo " - ES_JAVA_OPTS: $ES_JAVA_OPTS"
+    echo " - DB_BASE_URL: $DB_BASE_URL"
+    echo " - DOT_ES_ENDPOINTS: $DOT_ES_ENDPOINTS"
+    echo " - DOT_DOTCMS_CLUSTER_ID: $DOT_DOTCMS_CLUSTER_ID"
+
+    . /srv/entrypoint.sh
+}
+
+
+
+
+
+pull_dotcms_backups && echo ""
+setup_postgres && echo ""
+unpack_assets && echo ""
+setup_opensearch && echo ""
+start_dotcms

docker/docker-compose-examples/README.md

@@ -0,0 +1,22 @@
+# Docker Compose Examples
+
+This directory contains docker-compose examples ready to use
+
+The following examples are provided:
+
+- **cluster-mode:** dotcms instance with 2 nodes in a cluster
+- **elasticsearch-with-kibana:** elasticsearch server with kibana
+- **oracle-database:** oracle database running on port 1521
+- **push-publish:** dotcms environment with a sender and a receiver
+- **single-node:** basic dotcms instance running with postgres database
+- **single-node-debug-mode:** basic dotcms instance running with postgres database and debug mode enabled
+- **single-node-demo-site:** basic dotcms instance running demo site
+- **with-kibana:** basic dotcms instance running with postgres database and kibana
+- **with-mssql:** basic dotcms instance running with MSSQL database
+- **with-oracle:** basic dotcms instance running with oracle database
+- **with-redis:** dotcms cluster with redis cache and pub/sub provider
+
+The following scripts are provided:
+
+- **dotcms-get-demo-site-starter-urls.sh:** prints demo site starter URL for each dotCMS version
+- **dotcms_properties_to_env_vars.py:** prints ENV variables based on dotCMS properties in a "binary" install - helpful for upgrades