test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/anaconda3/lib/python3.7/site-packages/sklearn/externals/joblib/__init__.py:15: DeprecationWarning: sklearn.externals.joblib is deprecated in 0.21 and will be removed in 0.23. Please import this functionality directly from joblib, which can be installed with: pip install joblib. If this warning is raised when loading pickled models, you may need to re-serialize those models with scikit-learn 0.21+.\n",
      "  warnings.warn(msg, category=DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import numba as nb\n",
    "import os\n",
    "import cv2\n",
    "import warnings\n",
    "import pickle\n",
    "import time\n",
    "from math import floor\n",
    "from skimage.measure import compare_psnr\n",
    "from scipy.misc import imresize\n",
    "from Functions import *\n",
    "import matplotlib.image as mpimg\n",
    "from multiprocessing import Pool\n",
    "\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "testPath = 'set 14'\n",
    "isKmeans = True\n",
    "\n",
    "R = 3  # Upscaling factor=2 R = [ 2 3 4 ]\n",
    "patchSize = 11  # Pacth Size=11\n",
    "gradientSize = 9\n",
    "Qangle = 24  # Quantization factor of angle =24\n",
    "Qstrength = 3  # Quantization factor of strength =3\n",
    "Qcoherence = 3  # Quantization factor of coherence =3\n",
    "\n",
    "if isKmeans:\n",
    "    # Kmeans-RAISR\n",
    "    with open(\"Filters/Ckmeans_filter\"+str(R), \"rb\") as fp:\n",
    "        h = pickle.load(fp)\n",
    "\n",
    "    with open(\"Filters/Ckmeans_Qfactor_str\"+str(R), \"rb\") as sp:\n",
    "        stre = pickle.load(sp)\n",
    "\n",
    "    with open(\"Filters/Ckmeans_Qfactor_coh\"+str(R), \"rb\") as cp:\n",
    "        cohe = pickle.load(cp)\n",
    "else:   \n",
    "    # RAISR   \n",
    "    with open(\"Filters/8x_filter\"+str(R), \"rb\") as fp:\n",
    "        h = pickle.load(fp)\n",
    "\n",
    "    with open(\"Filters/8x_Qfactor_str\"+str(R), \"rb\") as sp:\n",
    "        stre = pickle.load(sp)\n",
    "\n",
    "    with open(\"Filters/8x_Qfactor_coh\"+str(R), \"rb\") as cp:\n",
    "        cohe = pickle.load(cp)\n",
    "\n",
    "kmeans = joblib.load('x'+str(R)+'_'+'1_3000_kmeans.model')\n",
    "pca_matrix = io.loadmat('x'+str(R)+'_'+'1_3000_patch_PCA_matrix.mat')['p']\n",
    "mean = io.loadmat('x'+str(R)+'_'+'1_3000_patch_mean_matrix.mat')['m']\n",
    "\n",
    "\n",
    "filelist = make_dataset(testPath)\n",
    "\n",
    "wGaussian = Gaussian2d([patchSize, patchSize], 2)\n",
    "wGaussian = wGaussian/wGaussian.max()\n",
    "wGaussian = np.diag(wGaussian.ravel())\n",
    "imagecount = 1\n",
    "patchMargin = floor(patchSize/2)\n",
    "psnrRAISR = []\n",
    "psnrBicubic = []\n",
    "psnrBlending = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def TestProcess(i):\n",
    "    if (i < iteration - 1):\n",
    "        offset_i = offset[i * batch:i * batch + batch]\n",
    "        offset2_i = offset2[i * batch:i * batch + batch]\n",
    "        grid = np.tile(gridon[..., None], [1, 1, batch]) + np.tile(offset_i, [patchSize, patchSize, 1])\n",
    "    else:\n",
    "        offset_i = offset[i * batch:im.size]\n",
    "        offset2_i = offset2[i * batch:im.size]\n",
    "        grid = np.tile(gridon[..., None], [1, 1, im.size - (iteration - 1) * batch]) + np.tile(offset_i,[patchSize, patchSize,1])\n",
    "    f = im_LR.ravel()[grid]\n",
    "    gx = im_GX.ravel()[grid]\n",
    "    gy = im_GY.ravel()[grid]\n",
    "    gx = gx.reshape((1, patchSize * patchSize, gx.shape[2]))\n",
    "    gy = gy.reshape((1, patchSize * patchSize, gy.shape[2]))\n",
    "    G = np.vstack((gx, gy))\n",
    "    g1 = np.transpose(G, (2, 0, 1))\n",
    "    g2 = np.transpose(G, (2, 1, 0))\n",
    "    x = Gaussian_Mul(g1, g2, wGaussian)\n",
    "    w, v = np.linalg.eig(x)\n",
    "    idx = (-w).argsort()\n",
    "    w = w[np.arange(np.shape(w)[0])[:, np.newaxis], idx]\n",
    "    v = v[np.arange(np.shape(v)[0])[:, np.newaxis, np.newaxis], np.arange(np.shape(v)[1])[np.newaxis, :, np.newaxis], idx[:,np.newaxis,:]]\n",
    "    thelta = np.arctan(v[:, 1, 0] / v[:, 0, 0])\n",
    "    thelta[thelta < 0] = thelta[thelta < 0] + pi\n",
    "    thelta = np.floor(thelta / (pi / Qangle))\n",
    "    thelta[thelta > Qangle - 1] = Qangle - 1\n",
    "    thelta[thelta < 0] = 0\n",
    "    lamda = w[:, 0]\n",
    "    u = (np.sqrt(w[:, 0]) - np.sqrt(w[:, 1])) / (np.sqrt(w[:, 0]) + np.sqrt(w[:, 1]) + 0.00000000000000001)\n",
    "    lamda = np.searchsorted(stre, lamda)\n",
    "    u = np.searchsorted(cohe, u)\n",
    "    j = thelta * Qstrength * Qcoherence + lamda * Qcoherence + u\n",
    "    j = j.astype('int')\n",
    "    offset2_i = np.unravel_index(offset2_i, (H, W))\n",
    "    \n",
    "    if isKmeans:\n",
    "        f_vec = np.reshape(f, (patchSize * patchSize, -1)) - mean\n",
    "        f_vec_reduced = np.dot(pca_matrix, f_vec)\n",
    "        t = kmeans.predict(f_vec_reduced.reshape(-1, 1))\n",
    "        if i ==10:\n",
    "            print(np.sum(t==0)/t.shape[0])\n",
    "            print(np.sum(t==1)/t.shape[0])\n",
    "            print(np.sum(t==2)/t.shape[0])\n",
    "            print(np.sum(t==3)/t.shape[0])\n",
    "    else:\n",
    "        t = ((offset2_i[0] - patchMargin) % R) * R + ((offset2_i[1] - patchMargin) % R)\n",
    "        if i ==10:\n",
    "            print(np.sum(t==0)/t.shape[0])\n",
    "            print(np.sum(t==1)/t.shape[0])\n",
    "            print(np.sum(t==2)/t.shape[0])\n",
    "            print(np.sum(t==3)/t.shape[0])\n",
    "    \n",
    "    filtertj = h[t, j]\n",
    "    filtertj = filtertj[:, :, np.newaxis]\n",
    "    patch = f.reshape((1, patchSize * patchSize, gx.shape[2]))\n",
    "    patch = np.transpose(patch, (2, 0, 1))\n",
    "    result = np.matmul(patch, filtertj)\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Begin to process images:\n",
      " Processing 1/14 image (set 14/foreman.bmp)\n",
      "0.007\n",
      "0.103\n",
      "0.003\n",
      "0.026\n",
      "1.831017017364502\n",
      " Processing 2/14 image (set 14/man.bmp)\n",
      "0.201\n",
      "0.0\n",
      "0.1285\n",
      "0.1545\n",
      "2.272819995880127\n",
      " Processing 3/14 image (set 14/pepper.bmp)\n",
      "0.0795\n",
      "0.199\n",
      "0.0105\n",
      "0.098\n",
      "2.2575268745422363\n",
      " Processing 4/14 image (set 14/comic.bmp)\n",
      "0.1235\n",
      "0.1765\n",
      "0.0075\n",
      "0.144\n",
      "1.6734352111816406\n",
      " Processing 5/14 image (set 14/coastguard.bmp)\n",
      "0.002\n",
      "0.0\n",
      "0.396\n",
      "0.0\n",
      "1.744534969329834\n",
      " Processing 6/14 image (set 14/baboon.bmp)\n",
      "0.262\n",
      "0.0\n",
      "0.178\n",
      "0.07\n",
      "2.1803929805755615\n",
      " Processing 7/14 image (set 14/monarch.bmp)\n",
      "0.2015\n",
      "0.0425\n",
      "0.002\n",
      "0.092\n",
      "2.756720781326294\n",
      " Processing 8/14 image (set 14/flowers.bmp)\n",
      "0.144\n",
      "0.0305\n",
      "0.0625\n",
      "0.0625\n",
      "1.9593002796173096\n",
      " Processing 9/14 image (set 14/face.bmp)\n",
      "0.0\n",
      "0.0\n",
      "0.2825\n",
      "0.0\n",
      "1.7156181335449219\n",
      " Processing 10/14 image (set 14/ppt3.bmp)\n",
      "0.002\n",
      "0.442\n",
      "0.0\n",
      "0.006\n",
      "2.7992820739746094\n",
      " Processing 11/14 image (set 14/zebra.bmp)\n",
      "0.0045\n",
      "0.8675\n",
      "0.0\n",
      "0.1205\n",
      "2.5149271488189697\n",
      " Processing 12/14 image (set 14/bridge.bmp)\n",
      "0.189\n",
      "0.099\n",
      "0.0005\n",
      "0.1165\n",
      "2.4983890056610107\n",
      " Processing 13/14 image (set 14/barbara.bmp)\n",
      "0.2255\n",
      "0.128\n",
      "0.0245\n",
      "0.2385\n",
      "3.076923131942749\n",
      " Processing 14/14 image (set 14/lenna.bmp)\n",
      "0.41\n",
      "0.206\n",
      "0.061\n",
      "0.204\n",
      "3.32521390914917\n",
      " RAISR PSNR of set 14 is 28.63918137404164\n",
      " Bicubic PSNR of set 14 is 27.550055549338857\n"
     ]
    }
   ],
   "source": [
    "print('Begin to process images:')\n",
    "for image in filelist:\n",
    "    print('\\r', end='')\n",
    "    print('' * 60, end='')\n",
    "    print('\\r Processing ' + str(imagecount) + '/' + str(len(filelist)) + ' image (' + image + ')')\n",
    "    im_uint8 = cv2.imread(image)\n",
    "    im_mp = mpimg.imread(image)\n",
    "    if len(im_mp.shape) == 2:\n",
    "        im_uint8 = im_uint8[:,:,0]\n",
    "    im_uint8 = modcrop(im_uint8, R)\n",
    "    if len(im_uint8.shape) > 2:\n",
    "        im_ycbcr = BGR2YCbCr(im_uint8)\n",
    "        im = im_ycbcr[:, :, 0]\n",
    "    else:\n",
    "        im = im_uint8\n",
    "    im_double = im2double(im)\n",
    "    H, W = im.shape\n",
    "    region = (slice(patchMargin, H - patchMargin), slice(patchMargin, W - patchMargin))\n",
    "    start = time.time()\n",
    "    imL = imresize(im_double, 1 / R, interp='bicubic', mode='F')\n",
    "    im_bicubic = imresize(imL, (H, W), interp='bicubic', mode='F')\n",
    "    im_bicubic = im_bicubic.astype('float64')\n",
    "    im_bicubic = np.clip(im_bicubic, 0, 1)\n",
    "    im_LR = np.zeros((H+patchSize-1,W+patchSize-1))\n",
    "    im_LR[(patchMargin):(H+patchMargin),(patchMargin):(W+patchMargin)] = im_bicubic\n",
    "    im_result = np.zeros((H, W))\n",
    "    im_GX, im_GY = np.gradient(im_LR)\n",
    "    index = np.array(range(im_LR.size)).reshape(im_LR.shape)\n",
    "    offset = np.array(index[0:H, 0:W].ravel())\n",
    "    offset2 = np.array(range(im.size))\n",
    "    gridon = index[0:patchSize, 0:patchSize]\n",
    "    batch = 2000\n",
    "    iteration = ceil(im.size / batch + 0.000000000000001)\n",
    "\n",
    "    im_result = np.array([])\n",
    "\n",
    "    i = range(iteration)\n",
    "    p = Pool()\n",
    "    im_in = p.map(TestProcess, i)\n",
    "\n",
    "    for i in range(iteration):\n",
    "        im_result = np.append(im_result, im_in[i])\n",
    "\n",
    "    im_result = im_result.reshape(H, W)\n",
    "    im_result = np.clip(im_result, 0, 1)\n",
    "\n",
    "    end = time.time()\n",
    "    print(end - start)\n",
    "\n",
    "    im_blending = Blending2(im_bicubic, im_result)\n",
    "    im_blending = Backprojection(imL, im_blending, 1) #Optional: Backprojection, which can slightly improve PSNR, especilly for large upscaling factor.\n",
    "    im_blending = np.clip(im_blending, 0, 1)\n",
    "\n",
    "    if len(im_uint8.shape) > 2:\n",
    "        result_ycbcr = np.zeros((H, W, 3))\n",
    "        result_ycbcr[:, :, 0] = im_blending * 255\n",
    "        result_ycbcr[:, :, 1] = Prepare(im_ycbcr[:, :, 1], patchSize, R) * 255\n",
    "        result_ycbcr[:, :, 2] = Prepare(im_ycbcr[:, :, 2], patchSize, R) * 255\n",
    "        result_ycbcr = result_ycbcr[region].astype('uint8')\n",
    "        result_RAISR = YCbCr2BGR(result_ycbcr)\n",
    "    else:\n",
    "        result_RAISR = im_result[region] * 255\n",
    "        result_RAISR = result_RAISR.astype('uint8')\n",
    "\n",
    "    im_result = im_result*255\n",
    "    im_result = np.rint(im_result).astype('uint8')\n",
    "    im_bicubic = im_bicubic*255\n",
    "    im_bicubic = np.rint(im_bicubic).astype('uint8')\n",
    "    im_blending = im_blending * 255\n",
    "    im_blending = np.rint(im_blending).astype('uint8')\n",
    "\n",
    "    PSNR_bicubic = compare_psnr(im[region], im_bicubic[region])\n",
    "    PSNR_result = compare_psnr(im[region], im_result[region])\n",
    "    PSNR_blending = compare_psnr(im[region], im_blending[region])\n",
    "    PSNR_blending = max(PSNR_result, PSNR_blending)\n",
    "\n",
    "    createFolder('./results/')\n",
    "    cv2.imwrite('results/' + os.path.splitext(os.path.basename(image))[0] + '_result.bmp', result_RAISR)\n",
    "    psnrRAISR.append(PSNR_result)\n",
    "    psnrBicubic.append(PSNR_bicubic)\n",
    "    psnrBlending.append(PSNR_blending)\n",
    "\n",
    "    imagecount += 1\n",
    "\n",
    "\n",
    "RAISR_psnrmean = np.array(psnrBlending).mean()\n",
    "Bicubic_psnrmean = np.array(psnrBicubic).mean()\n",
    "\n",
    "\n",
    "print('\\r', end='')\n",
    "print('' * 60, end='')\n",
    "print('\\r RAISR PSNR of '+ testPath +' is ' + str(RAISR_psnrmean))\n",
    "print('\\r Bicubic PSNR of '+ testPath +' is ' + str(Bicubic_psnrmean))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Playground"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-3.44721766],\n",
       "       [ 2.60396842],\n",
       "       [-1.12103449],\n",
       "       [ 0.65496382]])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kmeans.cluster_centers_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.07130166666666667"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum(kmeans.labels_==8) / kmeans.labels_.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}