From cbd7b5d830a138aae92abdd634490907b8e00f12 Mon Sep 17 00:00:00 2001 From: Devedse Date: Sun, 11 Dec 2022 22:24:13 +0100 Subject: [PATCH 01/31] WIP --- Dockerfile | 52 +++++++++++++++++--------- buildding/waifu2x-converter-cpp | 1 + src/config_files/executable_paths.yaml | 2 +- 3 files changed, 37 insertions(+), 18 deletions(-) create mode 160000 buildding/waifu2x-converter-cpp diff --git a/Dockerfile b/Dockerfile index 7fbb064c..1da0baf4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,20 +1,27 @@ -FROM ubuntu:19.10 +FROM ubuntu:22.04 -# Since ubuntu 19.10 isn't LTS, use LTS sources for the packages we need. -RUN sed -i "s/archive/old-releases/" /etc/apt/sources.list \ - && sed -i "/security/d" /etc/apt/sources.list \ - && apt-get update +# # Since ubuntu 19.10 isn't LTS, use LTS sources for the packages we need. +# RUN sed -i "s/archive/old-releases/" /etc/apt/sources.list \ +# && sed -i "/security/d" /etc/apt/sources.list \ +# && apt-get update # We need Nvidia Drivers -RUN apt install -y --no-install-recommends apt-utils software-properties-common -RUN add-apt-repository -y ppa:graphics-drivers/ppa +RUN apt-get -y update +RUN apt install -y --no-install-recommends apt-utils software-properties-common gpg-agent +# RUN add-apt-repository -y ppa:graphics-drivers/ppa +RUN add-apt-repository universe RUN apt-get -y update # Set nvidia-driver installation to not ask for keyboard configeration -RUN export DEBIAN_FRONTEND="noninteractive" & apt-get install -y keyboard-configuration +RUN apt-get install -y keyboard-configuration +ENV DEBIAN_FRONTEND noninteractive # Needed Libraries for Dandere2x -RUN apt install -y --no-install-recommends ffmpeg nvidia-driver-440 python3.8 libvulkan1 libgtk2.0-dev pkg-config +RUN add-apt-repository ppa:deadsnakes/ppa +RUN apt-get update +RUN apt-get install python3.8 -y + +RUN apt install -y --no-install-recommends ffmpeg nvidia-driver-440 libvulkan1 libgtk2.0-dev pkg-config # Needed Library for Building Dandere2x (this will be removed later) RUN apt-get install -y cmake @@ -24,25 +31,36 @@ RUN apt-get install -y libgl1-mesa-glx RUN apt-get install -y ffmpeg RUN apt-get install -y wget RUN apt-get install -y zip +RUN apt-get install -y curl # Move Dandere2x's files to /dandere2x/ RUN mkdir /dandere2x/ RUN git clone --recurse-submodules --progress https://github.com/aka-katto/dandere2x.git /dandere2x/dandere2x # Begin the building process -RUN cd /dandere2x/dandere2x/src/ && bash /dandere2x/dandere2x/src/linux_setup.sh +RUN cd /dandere2x/dandere2x/src/ && bash /dandere2x/dandere2x/src/unix_setup.sh -# RUN wget https://github.com/nihui/waifu2x-ncnn-vulkan/releases/download/20200606/waifu2x-ncnn-vulkan-20200606-linux.zip -# RUN unzip waifu2x-ncnn-vulkan-20200606-linux.zip -# RUN mv waifu2x-ncnn-vulkan-20200606-linux /dandere2x/dandere2x/src/externals/waifu2x-ncnn-vulkan -# RUN rm waifu2x-ncnn-vulkan-20200606-linux.zip +RUN wget https://github.com/nihui/waifu2x-ncnn-vulkan/releases/download/20200606/waifu2x-ncnn-vulkan-20200606-linux.zip +RUN unzip waifu2x-ncnn-vulkan-20200606-linux.zip +RUN mv waifu2x-ncnn-vulkan-20200606-linux /dandere2x/dandere2x/src/externals/waifu2x-ncnn-vulkan +RUN rm waifu2x-ncnn-vulkan-20200606-linux.zip # Install Python Dependencies (note pyyaml has to be manually installed due to ubuntu:19.10 python3.8 restriction) # Ubuntu 19.10 will by default use python3.75 rather than the needed 3.8, so we have to manually get pip and refer to python + python as 3.8 -RUN wget 'https://bootstrap.pypa.io/get-pip.py' && python3.8 get-pip.py -RUN pip3.8 install -U pip -RUN pip3.8 install -r /dandere2x/dandere2x/src/requirements.txt +RUN apt-get install python3-distutils python3-apt python3-pip -y +RUN pip3 install -r /dandere2x/dandere2x/src/requirements.txt + +RUN apt -y install ocl-icd-opencl-dev +RUN apt -y install libopencv-dev libopencv-imgcodecs-dev libopencv-imgproc-dev libopencv-core-dev +RUN apt -y install nvidia-cuda-toolkit +WORKDIR /dandere2x +RUN git clone "https://github.com/DeadSix27/waifu2x-converter-cpp" +WORKDIR /dandere2x/waifu2x-converter-cpp +RUN mkdir out +WORKDIR /dandere2x/waifu2x-converter-cpp/out +RUN cmake .. +RUN make -j4 WORKDIR /dandere2x/dandere2x/src/ ENTRYPOINT ["python3.8", "/dandere2x/dandere2x/src/main.py"] diff --git a/buildding/waifu2x-converter-cpp b/buildding/waifu2x-converter-cpp new file mode 160000 index 00000000..57520b25 --- /dev/null +++ b/buildding/waifu2x-converter-cpp @@ -0,0 +1 @@ +Subproject commit 57520b25864e648c133ea0b2f4e7e07410ddc1f4 diff --git a/src/config_files/executable_paths.yaml b/src/config_files/executable_paths.yaml index b960b0b6..3e370347 100644 --- a/src/config_files/executable_paths.yaml +++ b/src/config_files/executable_paths.yaml @@ -18,6 +18,6 @@ ffmpeg: "externals/ffmpeg" dandere2x_cpp: "externals/dandere2x_cpp" waifu2x_vulkan: "externals/waifu2x-ncnn-vulkan/waifu2x-ncnn-vulkan" realsr-ncnn-vulkan: "externals/realsr-ncnn-vulkan/realsr-ncnn-vulkan" -waifu2x_converter_cpp: "externals/waifu2x-converter-cpp/waifu2x-converter-cpp" +waifu2x_converter_cpp: "/dandere2x/waifu2x-converter-cpp/out/waifu2x-converter-cpp" waifu2x_caffe: "externals/waifu2x-caffe/waifu2x-caffe-cui" realsr_ncnn_vulkan: "externals/waifu2x-ncnn-vulkan" From fdc77a2ebec83e8796c8f1d1c8f1ba2a0b89490e Mon Sep 17 00:00:00 2001 From: Devedse Date: Mon, 12 Dec 2022 01:08:31 +0100 Subject: [PATCH 02/31] WIP --- Dockerfile | 93 +++++++++++++++++--------- Dockerfile2 | 72 ++++++++++++++++++++ src/config_files/executable_paths.yaml | 2 +- 3 files changed, 134 insertions(+), 33 deletions(-) create mode 100644 Dockerfile2 diff --git a/Dockerfile b/Dockerfile index 1da0baf4..5f96ca2a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,27 +1,65 @@ -FROM ubuntu:22.04 +FROM ubuntu:19.10 as BASEWAIFUX2CPP -# # Since ubuntu 19.10 isn't LTS, use LTS sources for the packages we need. -# RUN sed -i "s/archive/old-releases/" /etc/apt/sources.list \ -# && sed -i "/security/d" /etc/apt/sources.list \ -# && apt-get update +# Since ubuntu 19.10 isn't LTS, use LTS sources for the packages we need. +RUN sed -i "s/archive/old-releases/" /etc/apt/sources.list \ + && sed -i "/security/d" /etc/apt/sources.list \ + && apt-get update # We need Nvidia Drivers -RUN apt-get -y update -RUN apt install -y --no-install-recommends apt-utils software-properties-common gpg-agent -# RUN add-apt-repository -y ppa:graphics-drivers/ppa -RUN add-apt-repository universe +RUN apt install -y --no-install-recommends apt-utils software-properties-common +RUN add-apt-repository -y ppa:graphics-drivers/ppa RUN apt-get -y update # Set nvidia-driver installation to not ask for keyboard configeration +ENV DEBIAN_FRONTEND noninteractive RUN apt-get install -y keyboard-configuration + + +RUN apt -y install ocl-icd-opencl-dev +RUN apt -y install libopencv-dev libopencv-imgcodecs-dev libopencv-imgproc-dev libopencv-core-dev +RUN apt -y install nvidia-cuda-toolkit + +RUN apt-get install -y git-core +RUN apt-get install -y cmake + +WORKDIR /dandere2x +RUN git clone "https://github.com/DeadSix27/waifu2x-converter-cpp" +WORKDIR /dandere2x/waifu2x-converter-cpp +RUN mkdir out +WORKDIR /dandere2x/waifu2x-converter-cpp/out +RUN cmake .. +RUN make -j4 + + + + + + + + + + + + + +FROM ubuntu:19.10 + +# Since ubuntu 19.10 isn't LTS, use LTS sources for the packages we need. +RUN sed -i "s/archive/old-releases/" /etc/apt/sources.list \ + && sed -i "/security/d" /etc/apt/sources.list \ + && apt-get update + +# We need Nvidia Drivers +RUN apt install -y --no-install-recommends apt-utils software-properties-common +RUN add-apt-repository -y ppa:graphics-drivers/ppa +RUN apt-get -y update + +# Set nvidia-driver installation to not ask for keyboard configeration ENV DEBIAN_FRONTEND noninteractive +RUN apt-get install -y keyboard-configuration # Needed Libraries for Dandere2x -RUN add-apt-repository ppa:deadsnakes/ppa -RUN apt-get update -RUN apt-get install python3.8 -y - -RUN apt install -y --no-install-recommends ffmpeg nvidia-driver-440 libvulkan1 libgtk2.0-dev pkg-config +RUN apt install -y --no-install-recommends ffmpeg nvidia-driver-440 python3.8 libvulkan1 libgtk2.0-dev pkg-config # Needed Library for Building Dandere2x (this will be removed later) RUN apt-get install -y cmake @@ -31,7 +69,6 @@ RUN apt-get install -y libgl1-mesa-glx RUN apt-get install -y ffmpeg RUN apt-get install -y wget RUN apt-get install -y zip -RUN apt-get install -y curl # Move Dandere2x's files to /dandere2x/ RUN mkdir /dandere2x/ @@ -40,30 +77,22 @@ RUN git clone --recurse-submodules --progress https://github.com/aka-katto/dande # Begin the building process RUN cd /dandere2x/dandere2x/src/ && bash /dandere2x/dandere2x/src/unix_setup.sh -RUN wget https://github.com/nihui/waifu2x-ncnn-vulkan/releases/download/20200606/waifu2x-ncnn-vulkan-20200606-linux.zip -RUN unzip waifu2x-ncnn-vulkan-20200606-linux.zip -RUN mv waifu2x-ncnn-vulkan-20200606-linux /dandere2x/dandere2x/src/externals/waifu2x-ncnn-vulkan -RUN rm waifu2x-ncnn-vulkan-20200606-linux.zip +# RUN wget https://github.com/nihui/waifu2x-ncnn-vulkan/releases/download/20200606/waifu2x-ncnn-vulkan-20200606-linux.zip +# RUN unzip waifu2x-ncnn-vulkan-20200606-linux.zip +# RUN mv waifu2x-ncnn-vulkan-20200606-linux /dandere2x/dandere2x/src/externals/waifu2x-ncnn-vulkan +# RUN rm waifu2x-ncnn-vulkan-20200606-linux.zip # Install Python Dependencies (note pyyaml has to be manually installed due to ubuntu:19.10 python3.8 restriction) # Ubuntu 19.10 will by default use python3.75 rather than the needed 3.8, so we have to manually get pip and refer to python + python as 3.8 -RUN apt-get install python3-distutils python3-apt python3-pip -y -RUN pip3 install -r /dandere2x/dandere2x/src/requirements.txt +RUN wget 'https://bootstrap.pypa.io/get-pip.py' && python3.8 get-pip.py +RUN pip3.8 install -U pip +RUN pip3.8 install -r /dandere2x/dandere2x/src/requirements.txt -RUN apt -y install ocl-icd-opencl-dev -RUN apt -y install libopencv-dev libopencv-imgcodecs-dev libopencv-imgproc-dev libopencv-core-dev -RUN apt -y install nvidia-cuda-toolkit -WORKDIR /dandere2x -RUN git clone "https://github.com/DeadSix27/waifu2x-converter-cpp" -WORKDIR /dandere2x/waifu2x-converter-cpp -RUN mkdir out -WORKDIR /dandere2x/waifu2x-converter-cpp/out -RUN cmake .. -RUN make -j4 +COPY --from=BASEWAIFUX2CPP /dandere2x/waifu2x-converter-cpp/out /dandere2x/dandere2x/src/externals WORKDIR /dandere2x/dandere2x/src/ ENTRYPOINT ["python3.8", "/dandere2x/dandere2x/src/main.py"] ENV NVIDIA_DRIVER_CAPABILITIES all -ENV DEBIAN_FRONTEND teletype +ENV DEBIAN_FRONTEND teletype \ No newline at end of file diff --git a/Dockerfile2 b/Dockerfile2 new file mode 100644 index 00000000..15a7d24b --- /dev/null +++ b/Dockerfile2 @@ -0,0 +1,72 @@ +FROM ubuntu:22.04 + +# # Since ubuntu 19.10 isn't LTS, use LTS sources for the packages we need. +# RUN sed -i "s/archive/old-releases/" /etc/apt/sources.list \ +# && sed -i "/security/d" /etc/apt/sources.list \ +# && apt-get update + +# We need Nvidia Drivers +RUN apt-get -y update +RUN apt install -y --no-install-recommends apt-utils software-properties-common gpg-agent +# RUN add-apt-repository -y ppa:graphics-drivers/ppa +RUN add-apt-repository universe +RUN apt-get -y update + +# Set nvidia-driver installation to not ask for keyboard configeration +RUN apt-get install -y keyboard-configuration +ENV DEBIAN_FRONTEND noninteractive + +# Needed Libraries for Dandere2x +RUN add-apt-repository ppa:deadsnakes/ppa +RUN apt-get update +RUN apt-get install python3.8 -y + +RUN apt install -y --no-install-recommends ffmpeg nvidia-driver-440 libvulkan1 libgtk2.0-dev pkg-config + +# Needed Library for Building Dandere2x (this will be removed later) +RUN apt-get install -y cmake +RUN apt-get install -y git-core +RUN apt-get install -y build-essential +RUN apt-get install -y libgl1-mesa-glx +RUN apt-get install -y ffmpeg +RUN apt-get install -y wget +RUN apt-get install -y zip +RUN apt-get install -y curl + +RUN apt -y install ocl-icd-opencl-dev +RUN apt -y install libopencv-dev libopencv-imgcodecs-dev libopencv-imgproc-dev libopencv-core-dev +RUN apt -y install nvidia-cuda-toolkit + +# Move Dandere2x's files to /dandere2x/ +RUN mkdir /dandere2x/ +RUN git clone --recurse-submodules --progress https://github.com/aka-katto/dandere2x.git /dandere2x/dandere2x + +# Begin the building process +RUN cd /dandere2x/dandere2x/src/ && bash /dandere2x/dandere2x/src/unix_setup.sh + +RUN wget https://github.com/nihui/waifu2x-ncnn-vulkan/releases/download/20200606/waifu2x-ncnn-vulkan-20200606-linux.zip +RUN unzip waifu2x-ncnn-vulkan-20200606-linux.zip +RUN mv waifu2x-ncnn-vulkan-20200606-linux /dandere2x/dandere2x/src/externals/waifu2x-ncnn-vulkan +RUN rm waifu2x-ncnn-vulkan-20200606-linux.zip + +# Install Python Dependencies (note pyyaml has to be manually installed due to ubuntu:19.10 python3.8 restriction) +# Ubuntu 19.10 will by default use python3.75 rather than the needed 3.8, so we have to manually get pip and refer to python + python as 3.8 + +RUN apt-get install python3-distutils python3-apt python3-pip -y +RUN pip3 install -r /dandere2x/dandere2x/src/requirements.txt +RUN pip3 install colorlog + + +WORKDIR /dandere2x +RUN git clone "https://github.com/DeadSix27/waifu2x-converter-cpp" +WORKDIR /dandere2x/waifu2x-converter-cpp +RUN mkdir out +WORKDIR /dandere2x/waifu2x-converter-cpp/out +RUN cmake .. +RUN make -j4 + +WORKDIR /dandere2x/dandere2x/src/ +ENTRYPOINT ["python3", "/dandere2x/dandere2x/src/main.py"] + +ENV NVIDIA_DRIVER_CAPABILITIES all +ENV DEBIAN_FRONTEND teletype diff --git a/src/config_files/executable_paths.yaml b/src/config_files/executable_paths.yaml index 3e370347..b960b0b6 100644 --- a/src/config_files/executable_paths.yaml +++ b/src/config_files/executable_paths.yaml @@ -18,6 +18,6 @@ ffmpeg: "externals/ffmpeg" dandere2x_cpp: "externals/dandere2x_cpp" waifu2x_vulkan: "externals/waifu2x-ncnn-vulkan/waifu2x-ncnn-vulkan" realsr-ncnn-vulkan: "externals/realsr-ncnn-vulkan/realsr-ncnn-vulkan" -waifu2x_converter_cpp: "/dandere2x/waifu2x-converter-cpp/out/waifu2x-converter-cpp" +waifu2x_converter_cpp: "externals/waifu2x-converter-cpp/waifu2x-converter-cpp" waifu2x_caffe: "externals/waifu2x-caffe/waifu2x-caffe-cui" realsr_ncnn_vulkan: "externals/waifu2x-ncnn-vulkan" From de9f5ddf9049056e2c8cf47a8365a9b7f5eec7f0 Mon Sep 17 00:00:00 2001 From: Devedse Date: Mon, 12 Dec 2022 01:51:53 +0100 Subject: [PATCH 03/31] Added dockerfile updates --- Dockerfile | 9 +------ Dockerfile2 | 67 ++++++++++++++++++++++++++++++++++++----------------- 2 files changed, 47 insertions(+), 29 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5f96ca2a..60685d0b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,14 +34,6 @@ RUN make -j4 - - - - - - - - FROM ubuntu:19.10 # Since ubuntu 19.10 isn't LTS, use LTS sources for the packages we need. @@ -89,6 +81,7 @@ RUN wget 'https://bootstrap.pypa.io/get-pip.py' && python3.8 get-pip.py RUN pip3.8 install -U pip RUN pip3.8 install -r /dandere2x/dandere2x/src/requirements.txt +RUN apt -y install libopencv-dev libopencv-imgcodecs-dev libopencv-imgproc-dev libopencv-core-dev COPY --from=BASEWAIFUX2CPP /dandere2x/waifu2x-converter-cpp/out /dandere2x/dandere2x/src/externals WORKDIR /dandere2x/dandere2x/src/ diff --git a/Dockerfile2 b/Dockerfile2 index 15a7d24b..5eb9b68d 100644 --- a/Dockerfile2 +++ b/Dockerfile2 @@ -1,9 +1,48 @@ -FROM ubuntu:22.04 +FROM ubuntu:22.04 as BASEWAIFUX2CPP -# # Since ubuntu 19.10 isn't LTS, use LTS sources for the packages we need. -# RUN sed -i "s/archive/old-releases/" /etc/apt/sources.list \ -# && sed -i "/security/d" /etc/apt/sources.list \ -# && apt-get update +ENV DEBIAN_FRONTEND noninteractive + +# We need Nvidia Drivers +RUN apt-get -y update +RUN apt -y install ocl-icd-opencl-dev +RUN apt -y install libopencv-dev libopencv-imgcodecs-dev libopencv-imgproc-dev libopencv-core-dev +RUN apt -y install nvidia-cuda-toolkit +RUN apt install -y --no-install-recommends apt-utils software-properties-common gpg-agent +# RUN add-apt-repository -y ppa:graphics-drivers/ppa +RUN add-apt-repository universe +RUN apt-get -y update + +# Set nvidia-driver installation to not ask for keyboard configeration +RUN apt-get install -y keyboard-configuration + + +# Needed Libraries for Dandere2x +# RUN add-apt-repository ppa:deadsnakes/ppa +# RUN apt-get update +# RUN apt-get install python3.8 -y + +# RUN apt install -y --no-install-recommends ffmpeg nvidia-driver-440 libvulkan1 libgtk2.0-dev pkg-config + +# Needed Library for Building Dandere2x (this will be removed later) +RUN apt-get install -y cmake +RUN apt-get install -y git-core +RUN apt-get install -y build-essential +# RUN apt-get install -y libgl1-mesa-glx +# RUN apt-get install -y ffmpeg +# RUN apt-get install -y wget +# RUN apt-get install -y zip +# RUN apt-get install -y curl + +WORKDIR /dandere2x +RUN git clone "https://github.com/DeadSix27/waifu2x-converter-cpp" +WORKDIR /dandere2x/waifu2x-converter-cpp +RUN mkdir out +WORKDIR /dandere2x/waifu2x-converter-cpp/out +RUN cmake .. +RUN make -j4 + + +FROM ubuntu:22.04 # We need Nvidia Drivers RUN apt-get -y update @@ -33,10 +72,6 @@ RUN apt-get install -y wget RUN apt-get install -y zip RUN apt-get install -y curl -RUN apt -y install ocl-icd-opencl-dev -RUN apt -y install libopencv-dev libopencv-imgcodecs-dev libopencv-imgproc-dev libopencv-core-dev -RUN apt -y install nvidia-cuda-toolkit - # Move Dandere2x's files to /dandere2x/ RUN mkdir /dandere2x/ RUN git clone --recurse-submodules --progress https://github.com/aka-katto/dandere2x.git /dandere2x/dandere2x @@ -49,21 +84,11 @@ RUN unzip waifu2x-ncnn-vulkan-20200606-linux.zip RUN mv waifu2x-ncnn-vulkan-20200606-linux /dandere2x/dandere2x/src/externals/waifu2x-ncnn-vulkan RUN rm waifu2x-ncnn-vulkan-20200606-linux.zip -# Install Python Dependencies (note pyyaml has to be manually installed due to ubuntu:19.10 python3.8 restriction) -# Ubuntu 19.10 will by default use python3.75 rather than the needed 3.8, so we have to manually get pip and refer to python + python as 3.8 - RUN apt-get install python3-distutils python3-apt python3-pip -y RUN pip3 install -r /dandere2x/dandere2x/src/requirements.txt -RUN pip3 install colorlog - -WORKDIR /dandere2x -RUN git clone "https://github.com/DeadSix27/waifu2x-converter-cpp" -WORKDIR /dandere2x/waifu2x-converter-cpp -RUN mkdir out -WORKDIR /dandere2x/waifu2x-converter-cpp/out -RUN cmake .. -RUN make -j4 +RUN apt -y install libopencv-dev libopencv-imgcodecs-dev libopencv-imgproc-dev libopencv-core-dev +COPY --from=BASEWAIFUX2CPP /dandere2x/waifu2x-converter-cpp/out /dandere2x/dandere2x/src/externals WORKDIR /dandere2x/dandere2x/src/ ENTRYPOINT ["python3", "/dandere2x/dandere2x/src/main.py"] From 8fa9c57048a6e865d0cf8a7e9692e102adb863c0 Mon Sep 17 00:00:00 2001 From: Devedse Date: Mon, 12 Dec 2022 01:56:43 +0100 Subject: [PATCH 04/31] Some cleanup --- Dockerfile | 20 +++++++------------- Dockerfile2 | 26 ++++---------------------- 2 files changed, 11 insertions(+), 35 deletions(-) diff --git a/Dockerfile b/Dockerfile index 60685d0b..575e67a6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,26 +1,20 @@ FROM ubuntu:19.10 as BASEWAIFUX2CPP -# Since ubuntu 19.10 isn't LTS, use LTS sources for the packages we need. -RUN sed -i "s/archive/old-releases/" /etc/apt/sources.list \ - && sed -i "/security/d" /etc/apt/sources.list \ - && apt-get update +ENV DEBIAN_FRONTEND noninteractive # We need Nvidia Drivers -RUN apt install -y --no-install-recommends apt-utils software-properties-common -RUN add-apt-repository -y ppa:graphics-drivers/ppa RUN apt-get -y update - -# Set nvidia-driver installation to not ask for keyboard configeration -ENV DEBIAN_FRONTEND noninteractive -RUN apt-get install -y keyboard-configuration - - RUN apt -y install ocl-icd-opencl-dev RUN apt -y install libopencv-dev libopencv-imgcodecs-dev libopencv-imgproc-dev libopencv-core-dev RUN apt -y install nvidia-cuda-toolkit +RUN apt install -y --no-install-recommends apt-utils software-properties-common gpg-agent +RUN add-apt-repository universe +RUN apt-get -y update -RUN apt-get install -y git-core +# Needed Library for Building Dandere2x (this will be removed later) RUN apt-get install -y cmake +RUN apt-get install -y git-core +RUN apt-get install -y build-essential WORKDIR /dandere2x RUN git clone "https://github.com/DeadSix27/waifu2x-converter-cpp" diff --git a/Dockerfile2 b/Dockerfile2 index 5eb9b68d..72561521 100644 --- a/Dockerfile2 +++ b/Dockerfile2 @@ -8,30 +8,13 @@ RUN apt -y install ocl-icd-opencl-dev RUN apt -y install libopencv-dev libopencv-imgcodecs-dev libopencv-imgproc-dev libopencv-core-dev RUN apt -y install nvidia-cuda-toolkit RUN apt install -y --no-install-recommends apt-utils software-properties-common gpg-agent -# RUN add-apt-repository -y ppa:graphics-drivers/ppa RUN add-apt-repository universe RUN apt-get -y update -# Set nvidia-driver installation to not ask for keyboard configeration -RUN apt-get install -y keyboard-configuration - - -# Needed Libraries for Dandere2x -# RUN add-apt-repository ppa:deadsnakes/ppa -# RUN apt-get update -# RUN apt-get install python3.8 -y - -# RUN apt install -y --no-install-recommends ffmpeg nvidia-driver-440 libvulkan1 libgtk2.0-dev pkg-config - # Needed Library for Building Dandere2x (this will be removed later) RUN apt-get install -y cmake RUN apt-get install -y git-core RUN apt-get install -y build-essential -# RUN apt-get install -y libgl1-mesa-glx -# RUN apt-get install -y ffmpeg -# RUN apt-get install -y wget -# RUN apt-get install -y zip -# RUN apt-get install -y curl WORKDIR /dandere2x RUN git clone "https://github.com/DeadSix27/waifu2x-converter-cpp" @@ -47,7 +30,6 @@ FROM ubuntu:22.04 # We need Nvidia Drivers RUN apt-get -y update RUN apt install -y --no-install-recommends apt-utils software-properties-common gpg-agent -# RUN add-apt-repository -y ppa:graphics-drivers/ppa RUN add-apt-repository universe RUN apt-get -y update @@ -79,10 +61,10 @@ RUN git clone --recurse-submodules --progress https://github.com/aka-katto/dande # Begin the building process RUN cd /dandere2x/dandere2x/src/ && bash /dandere2x/dandere2x/src/unix_setup.sh -RUN wget https://github.com/nihui/waifu2x-ncnn-vulkan/releases/download/20200606/waifu2x-ncnn-vulkan-20200606-linux.zip -RUN unzip waifu2x-ncnn-vulkan-20200606-linux.zip -RUN mv waifu2x-ncnn-vulkan-20200606-linux /dandere2x/dandere2x/src/externals/waifu2x-ncnn-vulkan -RUN rm waifu2x-ncnn-vulkan-20200606-linux.zip +# RUN wget https://github.com/nihui/waifu2x-ncnn-vulkan/releases/download/20200606/waifu2x-ncnn-vulkan-20200606-linux.zip +# RUN unzip waifu2x-ncnn-vulkan-20200606-linux.zip +# RUN mv waifu2x-ncnn-vulkan-20200606-linux /dandere2x/dandere2x/src/externals/waifu2x-ncnn-vulkan +# RUN rm waifu2x-ncnn-vulkan-20200606-linux.zip RUN apt-get install python3-distutils python3-apt python3-pip -y RUN pip3 install -r /dandere2x/dandere2x/src/requirements.txt From fde0a07b144c9aa8e4024ba1c21bfebf421a210d Mon Sep 17 00:00:00 2001 From: Devedse Date: Mon, 12 Dec 2022 02:23:25 +0100 Subject: [PATCH 05/31] Fix path and added test dockerfile3 --- Dockerfile | 2 +- Dockerfile2 | 2 +- Dockerfile3 | 79 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 2 deletions(-) create mode 100644 Dockerfile3 diff --git a/Dockerfile b/Dockerfile index 575e67a6..17765e92 100644 --- a/Dockerfile +++ b/Dockerfile @@ -76,7 +76,7 @@ RUN pip3.8 install -U pip RUN pip3.8 install -r /dandere2x/dandere2x/src/requirements.txt RUN apt -y install libopencv-dev libopencv-imgcodecs-dev libopencv-imgproc-dev libopencv-core-dev -COPY --from=BASEWAIFUX2CPP /dandere2x/waifu2x-converter-cpp/out /dandere2x/dandere2x/src/externals +COPY --from=BASEWAIFUX2CPP /dandere2x/waifu2x-converter-cpp/out /dandere2x/dandere2x/src/externals/waifu2x-converter-cpp WORKDIR /dandere2x/dandere2x/src/ ENTRYPOINT ["python3.8", "/dandere2x/dandere2x/src/main.py"] diff --git a/Dockerfile2 b/Dockerfile2 index 72561521..6af6ec3b 100644 --- a/Dockerfile2 +++ b/Dockerfile2 @@ -70,7 +70,7 @@ RUN apt-get install python3-distutils python3-apt python3-pip -y RUN pip3 install -r /dandere2x/dandere2x/src/requirements.txt RUN apt -y install libopencv-dev libopencv-imgcodecs-dev libopencv-imgproc-dev libopencv-core-dev -COPY --from=BASEWAIFUX2CPP /dandere2x/waifu2x-converter-cpp/out /dandere2x/dandere2x/src/externals +COPY --from=BASEWAIFUX2CPP /dandere2x/waifu2x-converter-cpp/out /dandere2x/dandere2x/src/externals/waifu2x-converter-cpp WORKDIR /dandere2x/dandere2x/src/ ENTRYPOINT ["python3", "/dandere2x/dandere2x/src/main.py"] diff --git a/Dockerfile3 b/Dockerfile3 new file mode 100644 index 00000000..0784d995 --- /dev/null +++ b/Dockerfile3 @@ -0,0 +1,79 @@ +FROM ubuntu:22.04 as BASEWAIFUX2CPP + +ENV DEBIAN_FRONTEND noninteractive + +# We need Nvidia Drivers +RUN apt-get -y update +RUN apt -y install ocl-icd-opencl-dev +RUN apt -y install libopencv-dev libopencv-imgcodecs-dev libopencv-imgproc-dev libopencv-core-dev +RUN apt -y install nvidia-cuda-toolkit +RUN apt install -y --no-install-recommends apt-utils software-properties-common gpg-agent +RUN add-apt-repository universe +RUN apt-get -y update + +# Needed Library for Building Dandere2x (this will be removed later) +RUN apt-get install -y cmake +RUN apt-get install -y git-core +RUN apt-get install -y build-essential + +WORKDIR /dandere2x +RUN git clone "https://github.com/DeadSix27/waifu2x-converter-cpp" +WORKDIR /dandere2x/waifu2x-converter-cpp +RUN mkdir out +WORKDIR /dandere2x/waifu2x-converter-cpp/out +RUN cmake .. +RUN make -j4 + + +# FROM ubuntu:22.04 + +# # We need Nvidia Drivers +# RUN apt-get -y update +# RUN apt install -y --no-install-recommends apt-utils software-properties-common gpg-agent +# RUN add-apt-repository universe +# RUN apt-get -y update + +# # Set nvidia-driver installation to not ask for keyboard configeration +# RUN apt-get install -y keyboard-configuration +# ENV DEBIAN_FRONTEND noninteractive + +# # Needed Libraries for Dandere2x +# RUN add-apt-repository ppa:deadsnakes/ppa +# RUN apt-get update +# RUN apt-get install python3.8 -y + +# RUN apt install -y --no-install-recommends ffmpeg nvidia-driver-440 libvulkan1 libgtk2.0-dev pkg-config + +# # Needed Library for Building Dandere2x (this will be removed later) +# RUN apt-get install -y cmake +# RUN apt-get install -y git-core +# RUN apt-get install -y build-essential +# RUN apt-get install -y libgl1-mesa-glx +# RUN apt-get install -y ffmpeg +# RUN apt-get install -y wget +# RUN apt-get install -y zip +# RUN apt-get install -y curl + +# # Move Dandere2x's files to /dandere2x/ +# RUN mkdir /dandere2x/ +# RUN git clone --recurse-submodules --progress https://github.com/aka-katto/dandere2x.git /dandere2x/dandere2x + +# # Begin the building process +# RUN cd /dandere2x/dandere2x/src/ && bash /dandere2x/dandere2x/src/unix_setup.sh + +# # RUN wget https://github.com/nihui/waifu2x-ncnn-vulkan/releases/download/20200606/waifu2x-ncnn-vulkan-20200606-linux.zip +# # RUN unzip waifu2x-ncnn-vulkan-20200606-linux.zip +# # RUN mv waifu2x-ncnn-vulkan-20200606-linux /dandere2x/dandere2x/src/externals/waifu2x-ncnn-vulkan +# # RUN rm waifu2x-ncnn-vulkan-20200606-linux.zip + +# RUN apt-get install python3-distutils python3-apt python3-pip -y +# RUN pip3 install -r /dandere2x/dandere2x/src/requirements.txt + +# RUN apt -y install libopencv-dev libopencv-imgcodecs-dev libopencv-imgproc-dev libopencv-core-dev +# COPY --from=BASEWAIFUX2CPP /dandere2x/waifu2x-converter-cpp/out /dandere2x/dandere2x/src/externals + +# WORKDIR /dandere2x/dandere2x/src/ +# ENTRYPOINT ["python3", "/dandere2x/dandere2x/src/main.py"] + +# ENV NVIDIA_DRIVER_CAPABILITIES all +# ENV DEBIAN_FRONTEND teletype From c96094f5d6513fe9eb64e7cb498f82804cb58c75 Mon Sep 17 00:00:00 2001 From: Devedse Date: Mon, 12 Dec 2022 14:29:40 +0100 Subject: [PATCH 06/31] Updated dockerfiles to include models in the right directory --- Dockerfile | 1 + Dockerfile2 | 1 + 2 files changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 17765e92..f45ce650 100644 --- a/Dockerfile +++ b/Dockerfile @@ -77,6 +77,7 @@ RUN pip3.8 install -r /dandere2x/dandere2x/src/requirements.txt RUN apt -y install libopencv-dev libopencv-imgcodecs-dev libopencv-imgproc-dev libopencv-core-dev COPY --from=BASEWAIFUX2CPP /dandere2x/waifu2x-converter-cpp/out /dandere2x/dandere2x/src/externals/waifu2x-converter-cpp +COPY --from=BASEWAIFUX2CPP /dandere2x/waifu2x-converter-cpp/models_rgb /usr/local/share/waifu2x-converter-cpp/ WORKDIR /dandere2x/dandere2x/src/ ENTRYPOINT ["python3.8", "/dandere2x/dandere2x/src/main.py"] diff --git a/Dockerfile2 b/Dockerfile2 index 6af6ec3b..63999dc7 100644 --- a/Dockerfile2 +++ b/Dockerfile2 @@ -71,6 +71,7 @@ RUN pip3 install -r /dandere2x/dandere2x/src/requirements.txt RUN apt -y install libopencv-dev libopencv-imgcodecs-dev libopencv-imgproc-dev libopencv-core-dev COPY --from=BASEWAIFUX2CPP /dandere2x/waifu2x-converter-cpp/out /dandere2x/dandere2x/src/externals/waifu2x-converter-cpp +COPY --from=BASEWAIFUX2CPP /dandere2x/waifu2x-converter-cpp/models_rgb /usr/local/share/waifu2x-converter-cpp/ WORKDIR /dandere2x/dandere2x/src/ ENTRYPOINT ["python3", "/dandere2x/dandere2x/src/main.py"] From f208bc5ca1f38ade90c90687f9c12bf49c168973 Mon Sep 17 00:00:00 2001 From: Devedse Date: Mon, 12 Dec 2022 19:03:52 +0100 Subject: [PATCH 07/31] Added support for globs --- buildding/waifu2x-converter-cpp | 1 - src/dandere2x/dandere2x_service/core/merge.py | 2 +- .../dandere2xlib/wrappers/frame/frame.py | 15 +++++++++++---- 3 files changed, 12 insertions(+), 6 deletions(-) delete mode 160000 buildding/waifu2x-converter-cpp diff --git a/buildding/waifu2x-converter-cpp b/buildding/waifu2x-converter-cpp deleted file mode 160000 index 57520b25..00000000 --- a/buildding/waifu2x-converter-cpp +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 57520b25864e648c133ea0b2f4e7e07410ddc1f4 diff --git a/src/dandere2x/dandere2x_service/core/merge.py b/src/dandere2x/dandere2x_service/core/merge.py index 7a028411..d155c6cd 100644 --- a/src/dandere2x/dandere2x_service/core/merge.py +++ b/src/dandere2x/dandere2x_service/core/merge.py @@ -148,7 +148,7 @@ def run(self): if not last_frame: # We need to wait until the next upscaled image exists before we move on. while not background_frame_load.load_complete: - wait_on_file(self.context.residual_upscaled_dir + "output_" + get_lexicon_value(6, x + 1) + ".png") + wait_on_file(self.context.residual_upscaled_dir + "output_" + get_lexicon_value(6, x + 1) + "*.png") # Added * here because waifu2x_converter_cpp outputs filenames as: output_000008_[L3][x2.00].png """ Now that we're all done with the current frame, the current `current_frame` is now the frame_previous (with respect to the next iteration). We could obviously manually load frame_previous = Frame(n-1) each diff --git a/src/dandere2x/dandere2xlib/wrappers/frame/frame.py b/src/dandere2x/dandere2xlib/wrappers/frame/frame.py index 3dc2a1da..1cf94686 100644 --- a/src/dandere2x/dandere2xlib/wrappers/frame/frame.py +++ b/src/dandere2x/dandere2xlib/wrappers/frame/frame.py @@ -3,6 +3,7 @@ import logging import os import time +from glob import glob from dataclasses import dataclass import imageio @@ -107,21 +108,27 @@ def load_from_string(self, input_string): from dandere2x.dandere2x_service.dandere2x_service_controller import Dandere2xController def load_from_string_controller(self, input_string, controller=Dandere2xController()): + + actual_file_name = '' logger = logging.getLogger(__name__) - exists = exists = os.path.isfile(input_string) count = 0 - while not exists: + while True: if count % 10000 == 0: logger.debug(input_string + " dne") - exists = os.path.isfile(input_string) + + potential_files = glob(input_string) + if len(potential_files) > 0: + actual_file_name = potential_files[0] + break + count += 1 time.sleep(.2) loaded = False while not loaded: try: - self.load_from_string(input_string) + self.load_from_string(actual_file_name) loaded = True except PermissionError: logger.debug("Permission Error - trying again ") From d3fb0d9ad1c8d2227a4af0d56718920d6946ae64 Mon Sep 17 00:00:00 2001 From: Devedse Date: Mon, 12 Dec 2022 19:25:43 +0100 Subject: [PATCH 08/31] Fix --- Dockerfile2 | 2 +- src/dandere2x/dandere2x_service/core/merge.py | 18 ++++++++++++------ .../dandere2xlib/utils/dandere2x_utils.py | 15 +++++++++++++++ 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/Dockerfile2 b/Dockerfile2 index 63999dc7..83e03e19 100644 --- a/Dockerfile2 +++ b/Dockerfile2 @@ -56,7 +56,7 @@ RUN apt-get install -y curl # Move Dandere2x's files to /dandere2x/ RUN mkdir /dandere2x/ -RUN git clone --recurse-submodules --progress https://github.com/aka-katto/dandere2x.git /dandere2x/dandere2x +RUN git clone --recurse-submodules --progress https://github.com/devedse/dandere2x.git /dandere2x/dandere2x # Begin the building process RUN cd /dandere2x/dandere2x/src/ && bash /dandere2x/dandere2x/src/unix_setup.sh diff --git a/src/dandere2x/dandere2x_service/core/merge.py b/src/dandere2x/dandere2x_service/core/merge.py index d155c6cd..791fafc1 100644 --- a/src/dandere2x/dandere2x_service/core/merge.py +++ b/src/dandere2x/dandere2x_service/core/merge.py @@ -38,7 +38,7 @@ from dandere2x.dandere2x_service.core.residual_plugins.fade import fade_image from dandere2x.dandere2x_service.dandere2x_service_context import Dandere2xServiceContext from dandere2x.dandere2x_service.dandere2x_service_controller import Dandere2xController -from dandere2x.dandere2xlib.utils.dandere2x_utils import get_lexicon_value, get_list_from_file_and_wait, wait_on_file +from dandere2x.dandere2xlib.utils.dandere2x_utils import get_lexicon_value, get_list_from_file_and_wait, wait_on_file_withglob from dandere2x.dandere2xlib.wrappers.ffmpeg.pipe_thread import Pipe from dandere2x.dandere2xlib.wrappers.frame.asyncframe import AsyncFrameRead, AsyncFrameWrite from dandere2x.dandere2xlib.wrappers.frame.frame import Frame @@ -90,8 +90,11 @@ def run(self): current_upscaled_residuals = Frame() current_upscaled_residuals.load_from_string_controller( - self.context.residual_upscaled_dir + "output_" + get_lexicon_value(6, 1) + ".png", + self.context.residual_upscaled_dir + "output_" + get_lexicon_value(6, 1) + "*.png", self.controller) + + logger = logging.getLogger(__name__) + logger.info("COOLLOG: Start runner") last_frame = False for x in range(1, self.context.frame_count): @@ -111,7 +114,7 @@ def run(self): it's well worth it. """ background_frame_load = AsyncFrameRead( - self.context.residual_upscaled_dir + "output_" + get_lexicon_value(6, x + 1) + ".png", + self.context.residual_upscaled_dir + "output_" + get_lexicon_value(6, x + 1) + "*.png", self.controller) background_frame_load.start() @@ -120,13 +123,15 @@ def run(self): ###################### # Load the needed vectors to create the merged image. - + logger.info("COOLLOG: Blah1") + prediction_data_list = get_list_from_file_and_wait( self.context.pframe_data_dir + "pframe_" + str(x) + ".txt") residual_data_list = get_list_from_file_and_wait( self.context.residual_data_dir + "residual_" + str(x) + ".txt") fade_data_list = get_list_from_file_and_wait(self.context.fade_data_dir + "fade_" + str(x) + ".txt") + logger.info("COOLLOG: Blah2") # Create the actual image itself. current_frame = self.make_merge_image(self.context, current_upscaled_residuals, frame_previous, prediction_data_list, residual_data_list, fade_data_list) @@ -141,14 +146,14 @@ def run(self): # if True: # output_file = self.context.merged_dir + "merged_" + str(x + 1) + ".png" # current_frame.save_image(output_file) - + logger.info("COOLLOG: Blah3") ####################################### # Assign variables for next iteration # ####################################### if not last_frame: # We need to wait until the next upscaled image exists before we move on. while not background_frame_load.load_complete: - wait_on_file(self.context.residual_upscaled_dir + "output_" + get_lexicon_value(6, x + 1) + "*.png") # Added * here because waifu2x_converter_cpp outputs filenames as: output_000008_[L3][x2.00].png + wait_on_file_withglob(self.context.residual_upscaled_dir + "output_" + get_lexicon_value(6, x + 1) + "*.png") # Added * here because waifu2x_converter_cpp outputs filenames as: output_000008_[L3][x2.00].png """ Now that we're all done with the current frame, the current `current_frame` is now the frame_previous (with respect to the next iteration). We could obviously manually load frame_previous = Frame(n-1) each @@ -157,6 +162,7 @@ def run(self): frame_previous = current_frame current_upscaled_residuals = background_frame_load.loaded_image self.controller.update_frame_count(x) + logger.info("COOLLOG: Blah4: " + str(x)) self.pipe.kill() diff --git a/src/dandere2x/dandere2xlib/utils/dandere2x_utils.py b/src/dandere2x/dandere2xlib/utils/dandere2x_utils.py index c0c7bbf3..73ee4610 100644 --- a/src/dandere2x/dandere2xlib/utils/dandere2x_utils.py +++ b/src/dandere2x/dandere2xlib/utils/dandere2x_utils.py @@ -12,6 +12,7 @@ import shutil import sys import time +from glob import glob from sys import platform from typing import Tuple @@ -96,6 +97,20 @@ def wait_on_file(file_string: str): count += 1 time.sleep(.001) +def wait_on_file_withglob(file_string: str): + logger = logging.getLogger(__name__) + count = 0 + while True: + if count / 500 == 0: + logger.debug(file_string + " does not exist, waiting") + + potential_files = glob(file_string) + if len(potential_files) > 0: + actual_file_name = potential_files[0] + break + count += 1 + time.sleep(.001) + # for renaming function, break when either file exists def wait_on_either_file(file_1: str, file_2: str): From 2771323ca8bf5d378aac1a194cc8f10a0c43d5dd Mon Sep 17 00:00:00 2001 From: Devedse Date: Mon, 12 Dec 2022 19:38:22 +0100 Subject: [PATCH 09/31] Some fixes --- Dockerfile2 | 1 - src/dandere2x/dandere2x_service/core/merge.py | 10 +++------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/Dockerfile2 b/Dockerfile2 index 83e03e19..3029f54e 100644 --- a/Dockerfile2 +++ b/Dockerfile2 @@ -52,7 +52,6 @@ RUN apt-get install -y libgl1-mesa-glx RUN apt-get install -y ffmpeg RUN apt-get install -y wget RUN apt-get install -y zip -RUN apt-get install -y curl # Move Dandere2x's files to /dandere2x/ RUN mkdir /dandere2x/ diff --git a/src/dandere2x/dandere2x_service/core/merge.py b/src/dandere2x/dandere2x_service/core/merge.py index 791fafc1..6afd8754 100644 --- a/src/dandere2x/dandere2x_service/core/merge.py +++ b/src/dandere2x/dandere2x_service/core/merge.py @@ -94,7 +94,6 @@ def run(self): self.controller) logger = logging.getLogger(__name__) - logger.info("COOLLOG: Start runner") last_frame = False for x in range(1, self.context.frame_count): @@ -122,16 +121,13 @@ def run(self): # Core Logic of Loop # ###################### - # Load the needed vectors to create the merged image. - logger.info("COOLLOG: Blah1") - + # Load the needed vectors to create the merged image. prediction_data_list = get_list_from_file_and_wait( self.context.pframe_data_dir + "pframe_" + str(x) + ".txt") residual_data_list = get_list_from_file_and_wait( self.context.residual_data_dir + "residual_" + str(x) + ".txt") fade_data_list = get_list_from_file_and_wait(self.context.fade_data_dir + "fade_" + str(x) + ".txt") - logger.info("COOLLOG: Blah2") # Create the actual image itself. current_frame = self.make_merge_image(self.context, current_upscaled_residuals, frame_previous, prediction_data_list, residual_data_list, fade_data_list) @@ -146,7 +142,7 @@ def run(self): # if True: # output_file = self.context.merged_dir + "merged_" + str(x + 1) + ".png" # current_frame.save_image(output_file) - logger.info("COOLLOG: Blah3") + ####################################### # Assign variables for next iteration # ####################################### @@ -162,7 +158,7 @@ def run(self): frame_previous = current_frame current_upscaled_residuals = background_frame_load.loaded_image self.controller.update_frame_count(x) - logger.info("COOLLOG: Blah4: " + str(x)) + logger.info("Completed frame: " + str(x)) self.pipe.kill() From e023ff3157bbad199bd9841abab1ce073e53a3be Mon Sep 17 00:00:00 2001 From: Devedse Date: Mon, 12 Dec 2022 19:46:30 +0100 Subject: [PATCH 10/31] Small logging improvement --- src/dandere2x/dandere2x_service/core/merge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dandere2x/dandere2x_service/core/merge.py b/src/dandere2x/dandere2x_service/core/merge.py index 6afd8754..7ef136ad 100644 --- a/src/dandere2x/dandere2x_service/core/merge.py +++ b/src/dandere2x/dandere2x_service/core/merge.py @@ -158,7 +158,7 @@ def run(self): frame_previous = current_frame current_upscaled_residuals = background_frame_load.loaded_image self.controller.update_frame_count(x) - logger.info("Completed frame: " + str(x)) + logger.info("Completed frame: " + str(x) + "/" + self.context.frame_count) self.pipe.kill() From 8b517c715de0f91a04228f4b85baedd978f6a241 Mon Sep 17 00:00:00 2001 From: Devedse Date: Mon, 12 Dec 2022 20:57:34 +0100 Subject: [PATCH 11/31] fix for str() --- src/dandere2x/dandere2x_service/core/merge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dandere2x/dandere2x_service/core/merge.py b/src/dandere2x/dandere2x_service/core/merge.py index 7ef136ad..b8342f40 100644 --- a/src/dandere2x/dandere2x_service/core/merge.py +++ b/src/dandere2x/dandere2x_service/core/merge.py @@ -158,7 +158,7 @@ def run(self): frame_previous = current_frame current_upscaled_residuals = background_frame_load.loaded_image self.controller.update_frame_count(x) - logger.info("Completed frame: " + str(x) + "/" + self.context.frame_count) + logger.info("Completed frame: " + str(x) + "/" + str(self.context.frame_count)) self.pipe.kill() From e73024f149b08e863607ac2d2b8aeadc0c93764e Mon Sep 17 00:00:00 2001 From: Devedse Date: Mon, 12 Dec 2022 20:58:51 +0100 Subject: [PATCH 12/31] Test fix 123 --- Dockerfile2 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile2 b/Dockerfile2 index 3029f54e..2dae5294 100644 --- a/Dockerfile2 +++ b/Dockerfile2 @@ -53,6 +53,8 @@ RUN apt-get install -y ffmpeg RUN apt-get install -y wget RUN apt-get install -y zip +RUN apt -y install libopencv-dev libopencv-imgcodecs-dev libopencv-imgproc-dev libopencv-core-dev + # Move Dandere2x's files to /dandere2x/ RUN mkdir /dandere2x/ RUN git clone --recurse-submodules --progress https://github.com/devedse/dandere2x.git /dandere2x/dandere2x @@ -68,7 +70,6 @@ RUN cd /dandere2x/dandere2x/src/ && bash /dandere2x/dandere2x/src/unix_setup.sh RUN apt-get install python3-distutils python3-apt python3-pip -y RUN pip3 install -r /dandere2x/dandere2x/src/requirements.txt -RUN apt -y install libopencv-dev libopencv-imgcodecs-dev libopencv-imgproc-dev libopencv-core-dev COPY --from=BASEWAIFUX2CPP /dandere2x/waifu2x-converter-cpp/out /dandere2x/dandere2x/src/externals/waifu2x-converter-cpp COPY --from=BASEWAIFUX2CPP /dandere2x/waifu2x-converter-cpp/models_rgb /usr/local/share/waifu2x-converter-cpp/ From 33adea189cdbd96647399f5f67cfc5acd18d6d2e Mon Sep 17 00:00:00 2001 From: Devedse Date: Mon, 12 Dec 2022 21:51:28 +0100 Subject: [PATCH 13/31] force exit after completion --- src/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main.py b/src/main.py index 83f2e8fe..a093a562 100644 --- a/src/main.py +++ b/src/main.py @@ -46,5 +46,6 @@ def main(): print("Total runtime duration:", time.time() - start) + sys.exit() main() From 412b1e49f9bdcc6c267def5f4f7b91271bd5595d Mon Sep 17 00:00:00 2001 From: Devedse Date: Mon, 12 Dec 2022 21:52:19 +0100 Subject: [PATCH 14/31] Add --- Dockerfile | 4 +++- Dockerfile2 | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index f45ce650..1ba3161b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -47,6 +47,9 @@ RUN apt-get install -y keyboard-configuration # Needed Libraries for Dandere2x RUN apt install -y --no-install-recommends ffmpeg nvidia-driver-440 python3.8 libvulkan1 libgtk2.0-dev pkg-config +# Dependencies for waifu2x-converter-cpp +RUN apt -y install libopencv-dev libopencv-imgcodecs-dev libopencv-imgproc-dev libopencv-core-dev + # Needed Library for Building Dandere2x (this will be removed later) RUN apt-get install -y cmake RUN apt-get install -y git-core @@ -75,7 +78,6 @@ RUN wget 'https://bootstrap.pypa.io/get-pip.py' && python3.8 get-pip.py RUN pip3.8 install -U pip RUN pip3.8 install -r /dandere2x/dandere2x/src/requirements.txt -RUN apt -y install libopencv-dev libopencv-imgcodecs-dev libopencv-imgproc-dev libopencv-core-dev COPY --from=BASEWAIFUX2CPP /dandere2x/waifu2x-converter-cpp/out /dandere2x/dandere2x/src/externals/waifu2x-converter-cpp COPY --from=BASEWAIFUX2CPP /dandere2x/waifu2x-converter-cpp/models_rgb /usr/local/share/waifu2x-converter-cpp/ diff --git a/Dockerfile2 b/Dockerfile2 index 2dae5294..2e19ff89 100644 --- a/Dockerfile2 +++ b/Dockerfile2 @@ -44,6 +44,9 @@ RUN apt-get install python3.8 -y RUN apt install -y --no-install-recommends ffmpeg nvidia-driver-440 libvulkan1 libgtk2.0-dev pkg-config +# Dependencies for waifu2x-converter-cpp +RUN apt -y install libopencv-dev libopencv-imgcodecs-dev libopencv-imgproc-dev libopencv-core-dev + # Needed Library for Building Dandere2x (this will be removed later) RUN apt-get install -y cmake RUN apt-get install -y git-core @@ -53,7 +56,6 @@ RUN apt-get install -y ffmpeg RUN apt-get install -y wget RUN apt-get install -y zip -RUN apt -y install libopencv-dev libopencv-imgcodecs-dev libopencv-imgproc-dev libopencv-core-dev # Move Dandere2x's files to /dandere2x/ RUN mkdir /dandere2x/ From 028e85ffdf91069c59b5b854a630771456241265 Mon Sep 17 00:00:00 2001 From: Devedse Date: Mon, 12 Dec 2022 23:08:25 +0100 Subject: [PATCH 15/31] test --- src/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main.py b/src/main.py index a093a562..aeecd052 100644 --- a/src/main.py +++ b/src/main.py @@ -46,6 +46,7 @@ def main(): print("Total runtime duration:", time.time() - start) + print("Calling sys.exit :)") sys.exit() main() From 76cce963d6ef0c04a51dc997ad1b160fb6713bb6 Mon Sep 17 00:00:00 2001 From: Devedse Date: Thu, 15 Dec 2022 13:20:46 +0100 Subject: [PATCH 16/31] Fixed container not exiting --- .../dandere2x_service/core/waifu2x/waifu2x_converter_cpp.py | 4 ++-- src/main.py | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/dandere2x/dandere2x_service/core/waifu2x/waifu2x_converter_cpp.py b/src/dandere2x/dandere2x_service/core/waifu2x/waifu2x_converter_cpp.py index d90d85c3..dd6ac7f5 100644 --- a/src/dandere2x/dandere2x_service/core/waifu2x/waifu2x_converter_cpp.py +++ b/src/dandere2x/dandere2x_service/core/waifu2x/waifu2x_converter_cpp.py @@ -117,9 +117,9 @@ def __fix_waifu2x_converter_cpp_names(self): file_names.append("output_" + get_lexicon_value(6, x)) for file in file_names: - dirty_name = self.context.residual_upscaled_dir + file + '_[NS-L' + str( + dirty_name = self.context.residual_upscaled_dir + file + '_[L' + str( self.context.service_request.denoise_level) + '][x' + str( - self.context.service_request.scale_factor) + '.000000]' + ".png" + self.context.service_request.scale_factor) + '.00]' + ".png" clean_name = self.context.residual_upscaled_dir + file + ".png" wait_on_either_file(clean_name, dirty_name) diff --git a/src/main.py b/src/main.py index aeecd052..85e65326 100644 --- a/src/main.py +++ b/src/main.py @@ -46,7 +46,4 @@ def main(): print("Total runtime duration:", time.time() - start) - print("Calling sys.exit :)") - sys.exit() - main() From 886cc3d3c2029ac59ef444f1269ed6a5ed8637a4 Mon Sep 17 00:00:00 2001 From: Devedse Date: Thu, 15 Dec 2022 13:38:12 +0100 Subject: [PATCH 17/31] Added builds and stuff --- .github/workflows/githubactionsbuilds.yml | 102 ++++++++++++++++++++++ Dockerfile | 40 ++++----- Dockerfile2 | 82 ----------------- Dockerfile3 | 79 ----------------- 4 files changed, 119 insertions(+), 184 deletions(-) create mode 100644 .github/workflows/githubactionsbuilds.yml delete mode 100644 Dockerfile2 delete mode 100644 Dockerfile3 diff --git a/.github/workflows/githubactionsbuilds.yml b/.github/workflows/githubactionsbuilds.yml new file mode 100644 index 00000000..aea6c6c4 --- /dev/null +++ b/.github/workflows/githubactionsbuilds.yml @@ -0,0 +1,102 @@ +name: GitHubActionsBuilds + +on: push + +jobs: + generate_version_number: + runs-on: ubuntu-latest + outputs: + build_number: ${{ steps.buildnumber.outputs.build_number }} + steps: + - name: Generate build number + id: buildnumber + uses: einaregilsson/build-number@v3 + with: + token: ${{secrets.github_token}} + + build_docker: + needs: generate_version_number + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: devedse + password: ${{ secrets.DOCKERHUBTOKEN }} + - name: Set up QEMU + id: qemu + uses: docker/setup-qemu-action@v1 + with: + image: tonistiigi/binfmt:latest + platforms: all + - name: Available platforms + run: echo ${{ steps.qemu.outputs.platforms }} + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 + - name: Builder instance name + run: echo ${{ steps.buildx.outputs.name }} + - name: Available platforms + run: echo ${{ steps.buildx.outputs.platforms }} + - name: Run Buildx + run: | + export VERSION=1.0.${{needs.generate_version_number.outputs.build_number}} + echo Version: $VERSION + #docker buildx build -f Dockerfile --platform linux/arm64,linux/amd64 -t devedse/dandere2x:${{needs.generate_version_number.outputs.build_number}} -t devedse/dandere2x:latest ${{ github.ref == 'refs/heads/master' && '--push' || '' }} . + docker buildx build -f Dockerfile --platform linux/amd64 -t devedse/dandere2x:${{needs.generate_version_number.outputs.build_number}} -t devedse/dandere2x:latest ${{ github.ref == 'refs/heads/master' && '--push' || '' }} . + + build_windows: + needs: generate_version_number + runs-on: windows-latest + steps: + - uses: actions/checkout@v2 + - name: Install PyInstaller + run: pip install pyinstaller + - name: Build to single exe + run: | + cd src + pyinstaller --onefile main.py + - name: find + run: find + - uses: actions/upload-artifact@v3 + with: + name: dandere2x_windows + path: src/main.exe + + release_github: + needs: [ + generate_version_number, + build_windows] + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/master' + steps: + - name: Download all artifacts + uses: actions/download-artifact@v2 + with: + path: ./artifacts/ + + - name: Display structure of downloaded files + run: | + cd ./artifacts/ + ls -R + - name: Create Release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token + with: + tag_name: 1.0.${{needs.generate_version_number.outputs.build_number}} + release_name: 1.0.${{needs.generate_version_number.outputs.build_number}} + body: | + ${{ github.event.head_commit.message }} + draft: false + prerelease: false + - name: Release + uses: softprops/action-gh-release@v1 + with: + tag_name: 1.0.${{needs.generate_version_number.outputs.build_number}} + files: | + ./artifacts/dandere2x_windows/main.exe + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 1ba3161b..2e19ff89 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:19.10 as BASEWAIFUX2CPP +FROM ubuntu:22.04 as BASEWAIFUX2CPP ENV DEBIAN_FRONTEND noninteractive @@ -25,27 +25,24 @@ RUN cmake .. RUN make -j4 - - - -FROM ubuntu:19.10 - -# Since ubuntu 19.10 isn't LTS, use LTS sources for the packages we need. -RUN sed -i "s/archive/old-releases/" /etc/apt/sources.list \ - && sed -i "/security/d" /etc/apt/sources.list \ - && apt-get update +FROM ubuntu:22.04 # We need Nvidia Drivers -RUN apt install -y --no-install-recommends apt-utils software-properties-common -RUN add-apt-repository -y ppa:graphics-drivers/ppa +RUN apt-get -y update +RUN apt install -y --no-install-recommends apt-utils software-properties-common gpg-agent +RUN add-apt-repository universe RUN apt-get -y update # Set nvidia-driver installation to not ask for keyboard configeration -ENV DEBIAN_FRONTEND noninteractive RUN apt-get install -y keyboard-configuration +ENV DEBIAN_FRONTEND noninteractive # Needed Libraries for Dandere2x -RUN apt install -y --no-install-recommends ffmpeg nvidia-driver-440 python3.8 libvulkan1 libgtk2.0-dev pkg-config +RUN add-apt-repository ppa:deadsnakes/ppa +RUN apt-get update +RUN apt-get install python3.8 -y + +RUN apt install -y --no-install-recommends ffmpeg nvidia-driver-440 libvulkan1 libgtk2.0-dev pkg-config # Dependencies for waifu2x-converter-cpp RUN apt -y install libopencv-dev libopencv-imgcodecs-dev libopencv-imgproc-dev libopencv-core-dev @@ -59,9 +56,10 @@ RUN apt-get install -y ffmpeg RUN apt-get install -y wget RUN apt-get install -y zip + # Move Dandere2x's files to /dandere2x/ RUN mkdir /dandere2x/ -RUN git clone --recurse-submodules --progress https://github.com/aka-katto/dandere2x.git /dandere2x/dandere2x +RUN git clone --recurse-submodules --progress https://github.com/devedse/dandere2x.git /dandere2x/dandere2x # Begin the building process RUN cd /dandere2x/dandere2x/src/ && bash /dandere2x/dandere2x/src/unix_setup.sh @@ -71,18 +69,14 @@ RUN cd /dandere2x/dandere2x/src/ && bash /dandere2x/dandere2x/src/unix_setup.sh # RUN mv waifu2x-ncnn-vulkan-20200606-linux /dandere2x/dandere2x/src/externals/waifu2x-ncnn-vulkan # RUN rm waifu2x-ncnn-vulkan-20200606-linux.zip -# Install Python Dependencies (note pyyaml has to be manually installed due to ubuntu:19.10 python3.8 restriction) -# Ubuntu 19.10 will by default use python3.75 rather than the needed 3.8, so we have to manually get pip and refer to python + python as 3.8 - -RUN wget 'https://bootstrap.pypa.io/get-pip.py' && python3.8 get-pip.py -RUN pip3.8 install -U pip -RUN pip3.8 install -r /dandere2x/dandere2x/src/requirements.txt +RUN apt-get install python3-distutils python3-apt python3-pip -y +RUN pip3 install -r /dandere2x/dandere2x/src/requirements.txt COPY --from=BASEWAIFUX2CPP /dandere2x/waifu2x-converter-cpp/out /dandere2x/dandere2x/src/externals/waifu2x-converter-cpp COPY --from=BASEWAIFUX2CPP /dandere2x/waifu2x-converter-cpp/models_rgb /usr/local/share/waifu2x-converter-cpp/ WORKDIR /dandere2x/dandere2x/src/ -ENTRYPOINT ["python3.8", "/dandere2x/dandere2x/src/main.py"] +ENTRYPOINT ["python3", "/dandere2x/dandere2x/src/main.py"] ENV NVIDIA_DRIVER_CAPABILITIES all -ENV DEBIAN_FRONTEND teletype \ No newline at end of file +ENV DEBIAN_FRONTEND teletype diff --git a/Dockerfile2 b/Dockerfile2 deleted file mode 100644 index 2e19ff89..00000000 --- a/Dockerfile2 +++ /dev/null @@ -1,82 +0,0 @@ -FROM ubuntu:22.04 as BASEWAIFUX2CPP - -ENV DEBIAN_FRONTEND noninteractive - -# We need Nvidia Drivers -RUN apt-get -y update -RUN apt -y install ocl-icd-opencl-dev -RUN apt -y install libopencv-dev libopencv-imgcodecs-dev libopencv-imgproc-dev libopencv-core-dev -RUN apt -y install nvidia-cuda-toolkit -RUN apt install -y --no-install-recommends apt-utils software-properties-common gpg-agent -RUN add-apt-repository universe -RUN apt-get -y update - -# Needed Library for Building Dandere2x (this will be removed later) -RUN apt-get install -y cmake -RUN apt-get install -y git-core -RUN apt-get install -y build-essential - -WORKDIR /dandere2x -RUN git clone "https://github.com/DeadSix27/waifu2x-converter-cpp" -WORKDIR /dandere2x/waifu2x-converter-cpp -RUN mkdir out -WORKDIR /dandere2x/waifu2x-converter-cpp/out -RUN cmake .. -RUN make -j4 - - -FROM ubuntu:22.04 - -# We need Nvidia Drivers -RUN apt-get -y update -RUN apt install -y --no-install-recommends apt-utils software-properties-common gpg-agent -RUN add-apt-repository universe -RUN apt-get -y update - -# Set nvidia-driver installation to not ask for keyboard configeration -RUN apt-get install -y keyboard-configuration -ENV DEBIAN_FRONTEND noninteractive - -# Needed Libraries for Dandere2x -RUN add-apt-repository ppa:deadsnakes/ppa -RUN apt-get update -RUN apt-get install python3.8 -y - -RUN apt install -y --no-install-recommends ffmpeg nvidia-driver-440 libvulkan1 libgtk2.0-dev pkg-config - -# Dependencies for waifu2x-converter-cpp -RUN apt -y install libopencv-dev libopencv-imgcodecs-dev libopencv-imgproc-dev libopencv-core-dev - -# Needed Library for Building Dandere2x (this will be removed later) -RUN apt-get install -y cmake -RUN apt-get install -y git-core -RUN apt-get install -y build-essential -RUN apt-get install -y libgl1-mesa-glx -RUN apt-get install -y ffmpeg -RUN apt-get install -y wget -RUN apt-get install -y zip - - -# Move Dandere2x's files to /dandere2x/ -RUN mkdir /dandere2x/ -RUN git clone --recurse-submodules --progress https://github.com/devedse/dandere2x.git /dandere2x/dandere2x - -# Begin the building process -RUN cd /dandere2x/dandere2x/src/ && bash /dandere2x/dandere2x/src/unix_setup.sh - -# RUN wget https://github.com/nihui/waifu2x-ncnn-vulkan/releases/download/20200606/waifu2x-ncnn-vulkan-20200606-linux.zip -# RUN unzip waifu2x-ncnn-vulkan-20200606-linux.zip -# RUN mv waifu2x-ncnn-vulkan-20200606-linux /dandere2x/dandere2x/src/externals/waifu2x-ncnn-vulkan -# RUN rm waifu2x-ncnn-vulkan-20200606-linux.zip - -RUN apt-get install python3-distutils python3-apt python3-pip -y -RUN pip3 install -r /dandere2x/dandere2x/src/requirements.txt - -COPY --from=BASEWAIFUX2CPP /dandere2x/waifu2x-converter-cpp/out /dandere2x/dandere2x/src/externals/waifu2x-converter-cpp -COPY --from=BASEWAIFUX2CPP /dandere2x/waifu2x-converter-cpp/models_rgb /usr/local/share/waifu2x-converter-cpp/ - -WORKDIR /dandere2x/dandere2x/src/ -ENTRYPOINT ["python3", "/dandere2x/dandere2x/src/main.py"] - -ENV NVIDIA_DRIVER_CAPABILITIES all -ENV DEBIAN_FRONTEND teletype diff --git a/Dockerfile3 b/Dockerfile3 deleted file mode 100644 index 0784d995..00000000 --- a/Dockerfile3 +++ /dev/null @@ -1,79 +0,0 @@ -FROM ubuntu:22.04 as BASEWAIFUX2CPP - -ENV DEBIAN_FRONTEND noninteractive - -# We need Nvidia Drivers -RUN apt-get -y update -RUN apt -y install ocl-icd-opencl-dev -RUN apt -y install libopencv-dev libopencv-imgcodecs-dev libopencv-imgproc-dev libopencv-core-dev -RUN apt -y install nvidia-cuda-toolkit -RUN apt install -y --no-install-recommends apt-utils software-properties-common gpg-agent -RUN add-apt-repository universe -RUN apt-get -y update - -# Needed Library for Building Dandere2x (this will be removed later) -RUN apt-get install -y cmake -RUN apt-get install -y git-core -RUN apt-get install -y build-essential - -WORKDIR /dandere2x -RUN git clone "https://github.com/DeadSix27/waifu2x-converter-cpp" -WORKDIR /dandere2x/waifu2x-converter-cpp -RUN mkdir out -WORKDIR /dandere2x/waifu2x-converter-cpp/out -RUN cmake .. -RUN make -j4 - - -# FROM ubuntu:22.04 - -# # We need Nvidia Drivers -# RUN apt-get -y update -# RUN apt install -y --no-install-recommends apt-utils software-properties-common gpg-agent -# RUN add-apt-repository universe -# RUN apt-get -y update - -# # Set nvidia-driver installation to not ask for keyboard configeration -# RUN apt-get install -y keyboard-configuration -# ENV DEBIAN_FRONTEND noninteractive - -# # Needed Libraries for Dandere2x -# RUN add-apt-repository ppa:deadsnakes/ppa -# RUN apt-get update -# RUN apt-get install python3.8 -y - -# RUN apt install -y --no-install-recommends ffmpeg nvidia-driver-440 libvulkan1 libgtk2.0-dev pkg-config - -# # Needed Library for Building Dandere2x (this will be removed later) -# RUN apt-get install -y cmake -# RUN apt-get install -y git-core -# RUN apt-get install -y build-essential -# RUN apt-get install -y libgl1-mesa-glx -# RUN apt-get install -y ffmpeg -# RUN apt-get install -y wget -# RUN apt-get install -y zip -# RUN apt-get install -y curl - -# # Move Dandere2x's files to /dandere2x/ -# RUN mkdir /dandere2x/ -# RUN git clone --recurse-submodules --progress https://github.com/aka-katto/dandere2x.git /dandere2x/dandere2x - -# # Begin the building process -# RUN cd /dandere2x/dandere2x/src/ && bash /dandere2x/dandere2x/src/unix_setup.sh - -# # RUN wget https://github.com/nihui/waifu2x-ncnn-vulkan/releases/download/20200606/waifu2x-ncnn-vulkan-20200606-linux.zip -# # RUN unzip waifu2x-ncnn-vulkan-20200606-linux.zip -# # RUN mv waifu2x-ncnn-vulkan-20200606-linux /dandere2x/dandere2x/src/externals/waifu2x-ncnn-vulkan -# # RUN rm waifu2x-ncnn-vulkan-20200606-linux.zip - -# RUN apt-get install python3-distutils python3-apt python3-pip -y -# RUN pip3 install -r /dandere2x/dandere2x/src/requirements.txt - -# RUN apt -y install libopencv-dev libopencv-imgcodecs-dev libopencv-imgproc-dev libopencv-core-dev -# COPY --from=BASEWAIFUX2CPP /dandere2x/waifu2x-converter-cpp/out /dandere2x/dandere2x/src/externals - -# WORKDIR /dandere2x/dandere2x/src/ -# ENTRYPOINT ["python3", "/dandere2x/dandere2x/src/main.py"] - -# ENV NVIDIA_DRIVER_CAPABILITIES all -# ENV DEBIAN_FRONTEND teletype From d33216c973ddee6f0db0d9c9fd13474e929b8718 Mon Sep 17 00:00:00 2001 From: Devedse Date: Thu, 15 Dec 2022 13:53:49 +0100 Subject: [PATCH 18/31] ls --- .github/workflows/githubactionsbuilds.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/githubactionsbuilds.yml b/.github/workflows/githubactionsbuilds.yml index aea6c6c4..313e7636 100644 --- a/.github/workflows/githubactionsbuilds.yml +++ b/.github/workflows/githubactionsbuilds.yml @@ -57,8 +57,8 @@ jobs: run: | cd src pyinstaller --onefile main.py - - name: find - run: find + - name: ls + run: ls -R - uses: actions/upload-artifact@v3 with: name: dandere2x_windows From 6bfa74c2bd759b2671925231d1fc0bb11c818bcb Mon Sep 17 00:00:00 2001 From: Devedse Date: Thu, 15 Dec 2022 13:56:03 +0100 Subject: [PATCH 19/31] This should be better --- Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2e19ff89..86a0c033 100644 --- a/Dockerfile +++ b/Dockerfile @@ -58,8 +58,9 @@ RUN apt-get install -y zip # Move Dandere2x's files to /dandere2x/ -RUN mkdir /dandere2x/ -RUN git clone --recurse-submodules --progress https://github.com/devedse/dandere2x.git /dandere2x/dandere2x +#RUN mkdir /dandere2x/ +#RUN git clone --recurse-submodules --progress https://github.com/devedse/dandere2x.git /dandere2x/dandere2x +COPY . /dandere/dandere2x # Begin the building process RUN cd /dandere2x/dandere2x/src/ && bash /dandere2x/dandere2x/src/unix_setup.sh From 23e730ba72cc44aa86579c7f0b49e0ca271a8bf5 Mon Sep 17 00:00:00 2001 From: Devedse Date: Thu, 15 Dec 2022 14:08:27 +0100 Subject: [PATCH 20/31] Made mistake --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 86a0c033..9eecf0f9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -60,7 +60,7 @@ RUN apt-get install -y zip # Move Dandere2x's files to /dandere2x/ #RUN mkdir /dandere2x/ #RUN git clone --recurse-submodules --progress https://github.com/devedse/dandere2x.git /dandere2x/dandere2x -COPY . /dandere/dandere2x +COPY . /dandere2x/dandere2x # Begin the building process RUN cd /dandere2x/dandere2x/src/ && bash /dandere2x/dandere2x/src/unix_setup.sh From 21f1bae6aa49ca165c86209b52d81f44d818a0f6 Mon Sep 17 00:00:00 2001 From: Devedse Date: Thu, 15 Dec 2022 14:10:05 +0100 Subject: [PATCH 21/31] Fix windows build --- .github/workflows/githubactionsbuilds.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/githubactionsbuilds.yml b/.github/workflows/githubactionsbuilds.yml index 313e7636..c0033fa6 100644 --- a/.github/workflows/githubactionsbuilds.yml +++ b/.github/workflows/githubactionsbuilds.yml @@ -62,7 +62,7 @@ jobs: - uses: actions/upload-artifact@v3 with: name: dandere2x_windows - path: src/main.exe + path: src/dist/main.exe release_github: needs: [ From eb7c1d6039dee4a193087ea50254f95a46744d71 Mon Sep 17 00:00:00 2001 From: Devedse Date: Thu, 15 Dec 2022 21:54:40 +0100 Subject: [PATCH 22/31] python 3.8 --- .github/workflows/githubactionsbuilds.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/githubactionsbuilds.yml b/.github/workflows/githubactionsbuilds.yml index c0033fa6..a9e12afe 100644 --- a/.github/workflows/githubactionsbuilds.yml +++ b/.github/workflows/githubactionsbuilds.yml @@ -51,6 +51,9 @@ jobs: runs-on: windows-latest steps: - uses: actions/checkout@v2 + - uses: actions/setup-python@v4 + with: + python-version: '3.8' - name: Install PyInstaller run: pip install pyinstaller - name: Build to single exe From 6423b2af8dd2fc60c497f4638733759b7b244393 Mon Sep 17 00:00:00 2001 From: Devedse Date: Thu, 15 Dec 2022 22:07:09 +0100 Subject: [PATCH 23/31] test new colorlog --- src/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/requirements.txt b/src/requirements.txt index 44a5a533..b34a03cc 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -6,5 +6,5 @@ dataclasses==0.6 pyyaml==5.4 psutil wget==3.2 -colorlog==4.1.0 +colorlog==6.7.0 future-annotations From c24bc985345918436fb028714668fcd61bcbd6bf Mon Sep 17 00:00:00 2001 From: Devedse Date: Thu, 15 Dec 2022 22:44:51 +0100 Subject: [PATCH 24/31] install requirements --- .github/workflows/githubactionsbuilds.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/githubactionsbuilds.yml b/.github/workflows/githubactionsbuilds.yml index a9e12afe..e146e143 100644 --- a/.github/workflows/githubactionsbuilds.yml +++ b/.github/workflows/githubactionsbuilds.yml @@ -56,6 +56,8 @@ jobs: python-version: '3.8' - name: Install PyInstaller run: pip install pyinstaller + - name: Pip install requirements + run: pip3 install -r src/requirements.txt - name: Build to single exe run: | cd src From 7e8b1cd0727a134e140b3fa7f8b1fa645b3a9545 Mon Sep 17 00:00:00 2001 From: Devedse Date: Thu, 15 Dec 2022 22:53:51 +0100 Subject: [PATCH 25/31] back to older colorlog --- src/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/requirements.txt b/src/requirements.txt index b34a03cc..44a5a533 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -6,5 +6,5 @@ dataclasses==0.6 pyyaml==5.4 psutil wget==3.2 -colorlog==6.7.0 +colorlog==4.1.0 future-annotations From ec5fab35d39b8348badc3ea9e4d892434c4f5962 Mon Sep 17 00:00:00 2001 From: Devedse Date: Fri, 16 Dec 2022 00:35:04 +0100 Subject: [PATCH 26/31] Reverted glob stuff --- src/dandere2x/dandere2x_service/core/merge.py | 15 ++++++++------- .../dandere2xlib/utils/dandere2x_utils.py | 16 ---------------- .../dandere2xlib/wrappers/frame/frame.py | 17 +++++------------ src/main.py | 3 ++- 4 files changed, 15 insertions(+), 36 deletions(-) diff --git a/src/dandere2x/dandere2x_service/core/merge.py b/src/dandere2x/dandere2x_service/core/merge.py index b8342f40..72a13606 100644 --- a/src/dandere2x/dandere2x_service/core/merge.py +++ b/src/dandere2x/dandere2x_service/core/merge.py @@ -38,7 +38,7 @@ from dandere2x.dandere2x_service.core.residual_plugins.fade import fade_image from dandere2x.dandere2x_service.dandere2x_service_context import Dandere2xServiceContext from dandere2x.dandere2x_service.dandere2x_service_controller import Dandere2xController -from dandere2x.dandere2xlib.utils.dandere2x_utils import get_lexicon_value, get_list_from_file_and_wait, wait_on_file_withglob +from dandere2x.dandere2xlib.utils.dandere2x_utils import get_lexicon_value, get_list_from_file_and_wait, wait_on_file from dandere2x.dandere2xlib.wrappers.ffmpeg.pipe_thread import Pipe from dandere2x.dandere2xlib.wrappers.frame.asyncframe import AsyncFrameRead, AsyncFrameWrite from dandere2x.dandere2xlib.wrappers.frame.frame import Frame @@ -90,9 +90,9 @@ def run(self): current_upscaled_residuals = Frame() current_upscaled_residuals.load_from_string_controller( - self.context.residual_upscaled_dir + "output_" + get_lexicon_value(6, 1) + "*.png", + self.context.residual_upscaled_dir + "output_" + get_lexicon_value(6, 1) + ".png", self.controller) - + logger = logging.getLogger(__name__) last_frame = False @@ -113,7 +113,7 @@ def run(self): it's well worth it. """ background_frame_load = AsyncFrameRead( - self.context.residual_upscaled_dir + "output_" + get_lexicon_value(6, x + 1) + "*.png", + self.context.residual_upscaled_dir + "output_" + get_lexicon_value(6, x + 1) + ".png", self.controller) background_frame_load.start() @@ -121,7 +121,8 @@ def run(self): # Core Logic of Loop # ###################### - # Load the needed vectors to create the merged image. + # Load the needed vectors to create the merged image. + prediction_data_list = get_list_from_file_and_wait( self.context.pframe_data_dir + "pframe_" + str(x) + ".txt") residual_data_list = get_list_from_file_and_wait( @@ -149,7 +150,7 @@ def run(self): if not last_frame: # We need to wait until the next upscaled image exists before we move on. while not background_frame_load.load_complete: - wait_on_file_withglob(self.context.residual_upscaled_dir + "output_" + get_lexicon_value(6, x + 1) + "*.png") # Added * here because waifu2x_converter_cpp outputs filenames as: output_000008_[L3][x2.00].png + wait_on_file(self.context.residual_upscaled_dir + "output_" + get_lexicon_value(6, x + 1) + ".png") """ Now that we're all done with the current frame, the current `current_frame` is now the frame_previous (with respect to the next iteration). We could obviously manually load frame_previous = Frame(n-1) each @@ -203,4 +204,4 @@ def make_merge_image(context: Dandere2xServiceContext, frame_residual: Frame, fr out_image = fade_image(context, out_image, list_fade) out_image = pframe_image(context, out_image, frame_previous, frame_residual, list_residual, list_predictive) - return out_image + return out_image \ No newline at end of file diff --git a/src/dandere2x/dandere2xlib/utils/dandere2x_utils.py b/src/dandere2x/dandere2xlib/utils/dandere2x_utils.py index 73ee4610..9e119f08 100644 --- a/src/dandere2x/dandere2xlib/utils/dandere2x_utils.py +++ b/src/dandere2x/dandere2xlib/utils/dandere2x_utils.py @@ -12,7 +12,6 @@ import shutil import sys import time -from glob import glob from sys import platform from typing import Tuple @@ -97,21 +96,6 @@ def wait_on_file(file_string: str): count += 1 time.sleep(.001) -def wait_on_file_withglob(file_string: str): - logger = logging.getLogger(__name__) - count = 0 - while True: - if count / 500 == 0: - logger.debug(file_string + " does not exist, waiting") - - potential_files = glob(file_string) - if len(potential_files) > 0: - actual_file_name = potential_files[0] - break - count += 1 - time.sleep(.001) - - # for renaming function, break when either file exists def wait_on_either_file(file_1: str, file_2: str): logger = logging.getLogger(__name__) diff --git a/src/dandere2x/dandere2xlib/wrappers/frame/frame.py b/src/dandere2x/dandere2xlib/wrappers/frame/frame.py index 1cf94686..675af47a 100644 --- a/src/dandere2x/dandere2xlib/wrappers/frame/frame.py +++ b/src/dandere2x/dandere2xlib/wrappers/frame/frame.py @@ -3,7 +3,6 @@ import logging import os import time -from glob import glob from dataclasses import dataclass import imageio @@ -108,27 +107,21 @@ def load_from_string(self, input_string): from dandere2x.dandere2x_service.dandere2x_service_controller import Dandere2xController def load_from_string_controller(self, input_string, controller=Dandere2xController()): - - actual_file_name = '' logger = logging.getLogger(__name__) + exists = exists = os.path.isfile(input_string) count = 0 - while True: + while not exists: if count % 10000 == 0: logger.debug(input_string + " dne") - - potential_files = glob(input_string) - if len(potential_files) > 0: - actual_file_name = potential_files[0] - break - + exists = os.path.isfile(input_string) count += 1 time.sleep(.2) loaded = False while not loaded: try: - self.load_from_string(actual_file_name) + self.load_from_string(input_string) loaded = True except PermissionError: logger.debug("Permission Error - trying again ") @@ -316,4 +309,4 @@ def create_bleeded_image(self, bleed): return im_out def mean(self, other): - return numpy.mean((self.frame - other.frame) ** 2) + return numpy.mean((self.frame - other.frame) ** 2) \ No newline at end of file diff --git a/src/main.py b/src/main.py index 85e65326..5df7bac4 100644 --- a/src/main.py +++ b/src/main.py @@ -46,4 +46,5 @@ def main(): print("Total runtime duration:", time.time() - start) -main() + +main() \ No newline at end of file From 52390feaacbac346d259d48ac9358305e3abc90a Mon Sep 17 00:00:00 2001 From: Devedse Date: Fri, 16 Dec 2022 01:46:35 +0100 Subject: [PATCH 27/31] test nuitka --- .github/workflows/githubactionsbuilds.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/githubactionsbuilds.yml b/.github/workflows/githubactionsbuilds.yml index e146e143..c132c3e3 100644 --- a/.github/workflows/githubactionsbuilds.yml +++ b/.github/workflows/githubactionsbuilds.yml @@ -54,14 +54,17 @@ jobs: - uses: actions/setup-python@v4 with: python-version: '3.8' - - name: Install PyInstaller - run: pip install pyinstaller + # - name: Install PyInstaller + # run: pip install pyinstaller + - name: In + run: python -m pip install nuitka - name: Pip install requirements run: pip3 install -r src/requirements.txt - name: Build to single exe run: | cd src - pyinstaller --onefile main.py + #pyinstaller --onefile main.py + python -m nuitka --follow-imports --onefile main.py - name: ls run: ls -R - uses: actions/upload-artifact@v3 From 5db8962197095b25bb40dab3171318fc170a259c Mon Sep 17 00:00:00 2001 From: Devedse Date: Fri, 16 Dec 2022 02:06:16 +0100 Subject: [PATCH 28/31] Added linux build too --- .github/workflows/githubactionsbuilds.yml | 36 ++++++++++++++++++++--- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/.github/workflows/githubactionsbuilds.yml b/.github/workflows/githubactionsbuilds.yml index c132c3e3..d4699ccd 100644 --- a/.github/workflows/githubactionsbuilds.yml +++ b/.github/workflows/githubactionsbuilds.yml @@ -56,11 +56,11 @@ jobs: python-version: '3.8' # - name: Install PyInstaller # run: pip install pyinstaller - - name: In - run: python -m pip install nuitka + - name: pip install nuitka + run: pip install nuitka - name: Pip install requirements run: pip3 install -r src/requirements.txt - - name: Build to single exe + - name: Build to single executable run: | cd src #pyinstaller --onefile main.py @@ -72,10 +72,37 @@ jobs: name: dandere2x_windows path: src/dist/main.exe + build_linux: + needs: generate_version_number + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v4 + with: + python-version: '3.8' + # - name: Install PyInstaller + # run: pip install pyinstaller + - name: pip install nuitka + run: pip install nuitka + - name: Pip install requirements + run: pip3 install -r src/requirements.txt + - name: Build to single executable + run: | + cd src + #pyinstaller --onefile main.py + python -m nuitka --follow-imports --onefile main.py + - name: ls + run: ls -R + - uses: actions/upload-artifact@v3 + with: + name: dandere2x_linux + path: src/dist/main.bin + release_github: needs: [ generate_version_number, - build_windows] + build_windows, + build_linux] runs-on: ubuntu-latest if: github.ref == 'refs/heads/master' steps: @@ -106,5 +133,6 @@ jobs: tag_name: 1.0.${{needs.generate_version_number.outputs.build_number}} files: | ./artifacts/dandere2x_windows/main.exe + ./artifacts/dandere2x_linux/main.bin env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file From 4bfaf79e6d46bb8a85fb484227d538c99bfe989a Mon Sep 17 00:00:00 2001 From: Devedse Date: Fri, 16 Dec 2022 02:23:13 +0100 Subject: [PATCH 29/31] Fixed some errors --- .github/workflows/githubactionsbuilds.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/githubactionsbuilds.yml b/.github/workflows/githubactionsbuilds.yml index d4699ccd..0c12b813 100644 --- a/.github/workflows/githubactionsbuilds.yml +++ b/.github/workflows/githubactionsbuilds.yml @@ -64,7 +64,7 @@ jobs: run: | cd src #pyinstaller --onefile main.py - python -m nuitka --follow-imports --onefile main.py + python -m nuitka --onefile --enable-plugin=numpy --assume-yes-for-downloads main.py - name: ls run: ls -R - uses: actions/upload-artifact@v3 @@ -90,7 +90,8 @@ jobs: run: | cd src #pyinstaller --onefile main.py - python -m nuitka --follow-imports --onefile main.py + pip install orderedset + python -m nuitka --onefile --enable-plugin=numpy --assume-yes-for-downloads main.py - name: ls run: ls -R - uses: actions/upload-artifact@v3 From e86626e44be5ed9a1f7087a26bcd6d973b39d8f7 Mon Sep 17 00:00:00 2001 From: Devedse <2350015+devedse@users.noreply.github.com> Date: Fri, 16 Dec 2022 03:01:45 +0100 Subject: [PATCH 30/31] Update githubactionsbuilds.yml --- .github/workflows/githubactionsbuilds.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/githubactionsbuilds.yml b/.github/workflows/githubactionsbuilds.yml index 0c12b813..e601b331 100644 --- a/.github/workflows/githubactionsbuilds.yml +++ b/.github/workflows/githubactionsbuilds.yml @@ -64,6 +64,7 @@ jobs: run: | cd src #pyinstaller --onefile main.py + pip install orderedset python -m nuitka --onefile --enable-plugin=numpy --assume-yes-for-downloads main.py - name: ls run: ls -R @@ -136,4 +137,4 @@ jobs: ./artifacts/dandere2x_windows/main.exe ./artifacts/dandere2x_linux/main.bin env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 94ab7660facef2fd6a63f767a547fbc5323d9caf Mon Sep 17 00:00:00 2001 From: Devedse <2350015+devedse@users.noreply.github.com> Date: Fri, 16 Dec 2022 03:06:46 +0100 Subject: [PATCH 31/31] Update githubactionsbuilds.yml --- .github/workflows/githubactionsbuilds.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/githubactionsbuilds.yml b/.github/workflows/githubactionsbuilds.yml index e601b331..517e0e59 100644 --- a/.github/workflows/githubactionsbuilds.yml +++ b/.github/workflows/githubactionsbuilds.yml @@ -71,7 +71,7 @@ jobs: - uses: actions/upload-artifact@v3 with: name: dandere2x_windows - path: src/dist/main.exe + path: src/main.exe build_linux: needs: generate_version_number @@ -98,7 +98,7 @@ jobs: - uses: actions/upload-artifact@v3 with: name: dandere2x_linux - path: src/dist/main.bin + path: src/main.bin release_github: needs: [