diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml
index 61f287c15607e..97bab2902667d 100644
--- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml
@@ -305,7 +305,7 @@ stages:
             --network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64
             --build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-10/root
             --build-arg PREPEND_PATH=/opt/rh/devtoolset-10/root/usr/bin:
-            --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-10/root/usr/lib64:/opt/rh/devtoolset-10/root/usr/lib:/opt/rh/devtoolset-10/root/usr/lib64/dyninst:/opt/rh/devtoolset-10/root/usr/lib/dyninst:/usr/local/lib64
+            --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-10/root/usr/lib64:/opt/rh/devtoolset-10/root/usr/lib:/opt/rh/devtoolset-10/root/usr/lib64/dyninst:/opt/rh/devtoolset-10/root/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib
           Repository: onnxruntimetrainingrocmbuild-torch1.8.1
       - template: get-docker-image-steps.yml
         parameters:
@@ -318,7 +318,7 @@ stages:
             --network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64
             --build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-10/root
             --build-arg PREPEND_PATH=/opt/rh/devtoolset-10/root/usr/bin:
-            --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-10/root/usr/lib64:/opt/rh/devtoolset-10/root/usr/lib:/opt/rh/devtoolset-10/root/usr/lib64/dyninst:/opt/rh/devtoolset-10/root/usr/lib/dyninst:/usr/local/lib64
+            --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-10/root/usr/lib64:/opt/rh/devtoolset-10/root/usr/lib:/opt/rh/devtoolset-10/root/usr/lib64/dyninst:/opt/rh/devtoolset-10/root/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib
           Repository: onnxruntimetrainingrocmbuild-torch1.9.0
 
     - job: ROCM_training_wheels
diff --git a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm
index 6a1f7d251de71..e6fe3c24c74ea 100644
--- a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm
+++ b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm
@@ -16,6 +16,13 @@ LABEL maintainer="The ManyLinux project"
 
 RUN yum remove -y devtoolset\* git\* && conda remove -y cmake
 
+# remove protobuf 2.6.1 from rocm/pytorch:rocm4.2_centos7_py3.6_pytorch
+# it's too old to compile onnx 1.10
+RUN rm -fr /usr/local/bin/protoc \
+           /usr/local/libproto* \
+           /usr/local/include/google \
+           /usr/local/lib/pkgconfig/protobuf*
+
 ENV AUDITWHEEL_POLICY=${POLICY} AUDITWHEEL_ARCH=${PLATFORM} AUDITWHEEL_PLAT=${POLICY}_${PLATFORM}
 ENV LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 LANGUAGE=en_US.UTF-8
 ENV DEVTOOLSET_ROOTPATH=${DEVTOOLSET_ROOTPATH}
@@ -70,6 +77,12 @@ RUN export LIBXCRYPT_VERSION=4.4.23 && \
     export PERL_DOWNLOAD_URL=https://www.cpan.org/src/5.0 && \
     manylinux-entrypoint /build_scripts/install-libxcrypt.sh
 
+COPY build_scripts/install-protobuf.sh /build_scripts/
+RUN export PROTOBUF_VERSION=3.17.3 && \
+    export PROTOBUF_ROOT=protobuf-all-${PROTOBUF_VERSION} && \
+    export PROTOBUF_HASH=77ad26d3f65222fd96ccc18b055632b0bfedf295cb748b712a98ba1ac0b704b2 && \
+    export PROTOBUF_DOWNLOAD_URL=https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION} && \
+    manylinux-entrypoint /build_scripts/install-protobuf.sh
 
 FROM runtime_base AS build_base
 COPY build_scripts/install-build-packages.sh /build_scripts/
@@ -176,6 +189,12 @@ RUN cd /tmp/scripts && \
     /tmp/scripts/install_python_deps.sh -d gpu -p 3.9 -h ${TORCH_VERSION} $INSTALL_DEPS_EXTRA_ARGS && \
      rm -rf /tmp/scripts
 
+# remove protobuf to prevent ambiguity which is used for onnxruntime build
+RUN rm -fr /usr/local/bin/protoc \
+           /usr/local/libproto* \
+           /usr/local/include/google \
+           /usr/local/lib/pkgconfig/protobuf*
+
 ARG BUILD_UID=1001
 ARG BUILD_USER=onnxruntimedev
 RUN adduser --uid $BUILD_UID $BUILD_USER
diff --git a/tools/ci_build/github/linux/docker/build_scripts/install-protobuf.sh b/tools/ci_build/github/linux/docker/build_scripts/install-protobuf.sh
new file mode 100755
index 0000000000000..0433a5acf4855
--- /dev/null
+++ b/tools/ci_build/github/linux/docker/build_scripts/install-protobuf.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+# Top-level build script called from Dockerfile
+
+# Stop at any error, show all commands
+set -exuo pipefail
+
+# Get script directory
+MY_DIR=$(dirname "${BASH_SOURCE[0]}")
+
+# Get build utilities
+source $MY_DIR/build_utils.sh
+
+# Install newest libtool
+check_var ${PROTOBUF_ROOT}
+check_var ${PROTOBUF_HASH}
+check_var ${PROTOBUF_DOWNLOAD_URL}
+fetch_source ${PROTOBUF_ROOT}.tar.gz ${PROTOBUF_DOWNLOAD_URL}
+check_sha256sum ${PROTOBUF_ROOT}.tar.gz ${PROTOBUF_HASH}
+tar -zxf ${PROTOBUF_ROOT}.tar.gz
+pushd protobuf-${PROTOBUF_VERSION}
+DESTDIR=/manylinux-rootfs do_standard_install
+popd
+rm -rf ${PROTOBUF_ROOT} ${PROTOBUF_ROOT}.tar.gz
+
+# Strip what we can
+strip_ /manylinux-rootfs
+
+# Install
+cp -rlf /manylinux-rootfs/* /
+
+# Remove temporary rootfs
+rm -rf /manylinux-rootfs
+
+hash -r
+protoc --version