silhouette-cv.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "---\n",
    "\n",
    "# Silhouette masking with OpenCV\n",
    "\n",
    "---\n",
    "\n",
    "One component of the art/tech project I'm working on for the Dec 2019 exhibition requires creating silhouette masks of people moving around a webcam. This notebook explores different ways to do that."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## Setup\n",
    "\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "import cv2 as cv\n",
    "import imutils\n",
    "import numpy as np\n",
    "\n",
    "from imutils.video import FPS"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## Detection via background subtraction\n",
    "\n",
    "---\n",
    "\n",
    "One possibility for silhouette masking is _background subtraction_ - using motion and color changes in a video feed to find static background regions vs. dynamic foreground regions OpenCV has multiple background subtraction methods."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## Webcam\n",
    "\n",
    "Press 'Q' to quit."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Stream resolution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "width_stream = 160\n",
    "height_stream = 90"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Display resolution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "width_display = 960\n",
    "height_display = 540\n",
    "\n",
    "# Plus some useful auxiliary stuff\n",
    "aspect_ratio = height_display / width_display\n",
    "screen_inverse = (1 / width_display, 1 / height_display)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Visualization parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Decay constant. Each frame, the fx field's values get multiplied\n",
    "# by this\n",
    "c_decay = 0.999\n",
    "# Mask emission strength constant.\n",
    "c_emit = 0.0002\n",
    "# Sorta hard to describe overall intensity constant\n",
    "c_viz = 20"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Visualization function utilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cos1(x):\n",
    "    return 0.5 * (np.cos(x) + 1)\n",
    "\n",
    "def mod1(x):\n",
    "    return ((x % 1) + 1) % 1\n",
    "\n",
    "def sigmoid(x, c, m):\n",
    "    return 1. / (1. + np.exp(-m * (x - c)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Visualization functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def visualize(field, mask, count_on, t):\n",
    "    \"\"\"\n",
    "    \n",
    "    \"\"\"\n",
    "    h = mod1(0.004 * t + 0.26 * field)\n",
    "    s = np.maximum(0, 1 - 0.1 * count_on)\n",
    "#     s = np.maximum(0, 1 - 0.1 * cv.GaussianBlur(count_on, (21, 21), 0))\n",
    "    b = np.minimum(1, 0.6 * np.ones_like(s) + 0.025 * mask + 0.175 * count_on)\n",
    "    hsb = (255 * np.stack((h, s, b), axis=-1)).astype(np.uint8)\n",
    "    bgr = cv.cvtColor(hsb, cv.COLOR_HSV2BGR)\n",
    "    return bgr\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# cap = cv.VideoCapture(1)\n",
    "# cap.set(3, width_stream)\n",
    "# cap.set(4, height_stream)\n",
    "\n",
    "# fgbg = cv.createBackgroundSubtractorKNN(100, 100, True)\n",
    "\n",
    "# field = np.zeros((height_display, width_display), dtype=np.float32)\n",
    "# count_on = np.zeros((height_display, width_display), dtype=np.float32)\n",
    "# # cv.namedWindow('frame', cv.WINDOW_NORMAL)\n",
    "# # cv.namedWindow('mask', cv.WINDOW_NORMAL)\n",
    "\n",
    "# fps = FPS().start()\n",
    "\n",
    "# t = 0\n",
    "\n",
    "# while(1):\n",
    "#     ret, frame = cap.read()\n",
    "    \n",
    "#     fgmask = 255 * (fgbg.apply(frame) > 0).astype(np.uint8)\n",
    "#     fgmask = cv.medianBlur(fgmask, 11)\n",
    "#     fgmask = cv.GaussianBlur(fgmask, (7, 7), 0)\n",
    "#     fgmask_display = (imutils.resize(fgmask, width=width_display, inter=cv.INTER_CUBIC)).astype(np.float32) / 255\n",
    "    \n",
    "#     count_on = np.maximum(0, count_on - 1 + 1.1 * fgmask_display)\n",
    "    \n",
    "#     field = cv.GaussianBlur(c_decay * field + c_emit * fgmask_display, (31, 31), 0)\n",
    "#     field = np.minimum(1, field)\n",
    "    \n",
    "#     fx = visualize(field, fgmask_display, count_on, t)\n",
    "#     t += 0.06667\n",
    "    \n",
    "# #     cv.imshow('frame', frame)\n",
    "#     cv.imshow('mask', fgmask)\n",
    "# #     cv.imshow('field', field)\n",
    "#     cv.imshow('fx', fx)\n",
    "#     if cv.waitKey(1) & 0xFF == ord('q'):\n",
    "#         break\n",
    "#     fps.update()\n",
    "    \n",
    "# fps.stop()\n",
    "# cap.release()\n",
    "# cv.destroyAllWindows()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## Game of Life integration\n",
    "\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "width_gol = 320\n",
    "height_gol = 180"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def life_step(X, born, stay):\n",
    "    \"\"\"Game of life step using generator expressions\"\"\"\n",
    "    nbrs_count = sum(np.roll(np.roll(X, i, 0), j, 1)\n",
    "                     for i in (-1, 0, 1) for j in (-1, 0, 1)\n",
    "                     if (i != 0 or j != 0))[1:-1, 1:-1]\n",
    "    out = np.isin(nbrs_count, born) | (X[1:-1, 1:-1] & np.isin(nbrs_count, stay))\n",
    "    return out\n",
    "#     return np.isin(nbrs_count, [3, 6]) | (X & np.isin(nbrs_count, [2, 3]))\n",
    "#     return (nbrs_count == 3) | (X & (nbrs_count == 2))\n",
    "\n",
    "def game_of_life(field, mask, ruleset):\n",
    "    thresholds = ruleset['thresholds']\n",
    "    born_rules = ruleset['born']\n",
    "    stay_rules = ruleset['stay']\n",
    "#     born_rules = [[2,3,4,5], [3], [3,6,8]]\n",
    "#     stay_rules = [[4,5,6,7,8], [2, 3], [2,4,5]]\n",
    "#     born_rules = [[6], [3], [3,6,8]]\n",
    "#     stay_rules = [[2, 3, 5], [2, 3], [2,4,5]]\n",
    "#     born_rules = [[3], [3], [5]]\n",
    "#     stay_rules = [[1,2,3,4], [2, 3], [2,3]]\n",
    "#     born_rules = [[5], [3], [3]]\n",
    "#     stay_rules = [[2,3], [2, 3], [1,2,3,4]]\n",
    "\n",
    "    \n",
    "    thresholds = [0] + thresholds + [1]\n",
    "    \n",
    "    for i, t in enumerate(thresholds[1:]):\n",
    "        t0 = thresholds[i]\n",
    "        update_region = (mask >= t0) & (mask <= t)\n",
    "        field[1:-1, 1:-1][update_region] = \\\n",
    "            life_step(field, born_rules[i], stay_rules[i])[update_region]\n",
    "    return field\n",
    "                          \n",
    "def visualize_gol(field, field_history, t, mask):\n",
    "    h = (((2 * t) % 256) * np.ones_like(field)).astype(np.uint8)\n",
    "    s = np.zeros_like(field).astype(np.float32)\n",
    "#     s = (240 * np.ones_like(field)).astype(np.float32)\n",
    "    s[field] = np.divide(\n",
    "        230, \n",
    "        np.maximum(1, 0.1 * field_history[field]))\n",
    "    s = s.astype(np.uint8)\n",
    "    b = (255 * field).astype(np.float)\n",
    "    b[1:-1, 1:-1] = np.minimum(255, b[1:-1, 1:-1] + 10 * mask)\n",
    "    b = b.astype(np.uint8)\n",
    "    hsb = np.stack((h, s, b), axis=-1).astype(np.uint8)\n",
    "    bgr = cv.cvtColor(hsb, cv.COLOR_HSV2BGR)\n",
    "    bgr_out= imutils.resize(bgr, width=width_display)\n",
    "#     bgr = (245 * big_field).astype(np.uint8)\n",
    "    return bgr_out[1:-1, 1:-1]\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "cap = cv.VideoCapture(0)\n",
    "cap.set(3, width_stream)\n",
    "cap.set(4, height_stream)\n",
    "\n",
    "fgbg = cv.createBackgroundSubtractorKNN(100, 100, True)\n",
    "\n",
    "field = np.zeros((height_gol + 2, width_gol + 2), dtype=np.bool)\n",
    "field[1:-1, 1:-1] = np.random.random((height_gol, width_gol)) < 0.75\n",
    "field_history = np.zeros_like(field).astype(np.int)\n",
    "memory = np.zeros((height_gol, width_gol), dtype=np.float32)\n",
    "\n",
    "# ruleset_blackmaze = {\n",
    "#     'thresholds': [0.2, 0.7],\n",
    "#     'born': [[3], [3], [5]],\n",
    "#     'stay': [[1,2,3,4], [2, 3], [2,3]]}\n",
    "\n",
    "\n",
    "ruleset = {\n",
    "    'thresholds': [0.01, 0.7],\n",
    "    'born': [[4, 7], [3], [3]],\n",
    "    'stay': [[3, 5, 6, 7, 8], [2, 3], [1,2,3,4]]}\n",
    "# cv.namedWindow('frame', cv.WINDOW_NORMAL)\n",
    "# cv.namedWindow('mask', cv.WINDOW_NORMAL)\n",
    "# cv.namedWindow('fx', cv.WND_PROP_FULLSCREEN)\n",
    "# cv.moveWindow('fx', 0, 0)\n",
    "# cv.setWindowProperty('fx', cv.WND_PROP_FULLSCREEN, cv.WINDOW_FULLSCREEN)\n",
    "\n",
    "fps = FPS().start()\n",
    "\n",
    "t = 0\n",
    "while(1):\n",
    "    ret, frame = cap.read()\n",
    "    frame = frame[:, ::-1, :]\n",
    "    \n",
    "    fgmask = 255 * (fgbg.apply(frame) > 0).astype(np.uint8)\n",
    "    fgmask = cv.medianBlur(fgmask, 11)\n",
    "    fgmask = cv.GaussianBlur(fgmask, (7, 7), 0)\n",
    "\n",
    "    fgmask_gol = (imutils.resize(fgmask, width=width_gol, inter=cv.INTER_CUBIC)).astype(np.float32) / 255\n",
    "    \n",
    "#     memory = np.maximum(0, np.minimum(1, 0.8 * memory + 0.5 * fgmask_gol))\n",
    "#     memory = cv.GaussianBlur(memory, (45, 45), 0)\n",
    "    \n",
    "    field = game_of_life(field, fgmask_gol, ruleset)\n",
    "    field_history[field] += 1\n",
    "    field_history[np.logical_not(field)] = 0\n",
    "    fx = visualize_gol(field, field_history, t, fgmask_gol)\n",
    "    t += 0.06667\n",
    "    \n",
    "    pip = np.copy(fx)\n",
    "    \n",
    "    pip[(-10-height_stream):-10, 10:(10+width_stream), :] = np.stack([fgmask]*3, axis=-1)\n",
    "    \n",
    "#     cv.imshow('frame', frame)\n",
    "#     cv.imshow('mask', imutils.resize(fgmask, width=width_gol))\n",
    "#     cv.imshow('memory', memory)\n",
    "#     cv.imshow('field', field)\n",
    "    cv.imshow('fx', pip)\n",
    "    if cv.waitKey(1) & 0xFF == ord('q'):\n",
    "        break\n",
    "    fps.update()\n",
    "    \n",
    "fps.stop()\n",
    "cap.release()\n",
    "cv.destroyAllWindows()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Prerecorded video"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def background_subtraction_mosaic(\n",
    "    video_file,\n",
    "    save_file,\n",
    "    subtractors,\n",
    "    median_blur_size,\n",
    "    gaussian_blur_size,\n",
    "    scale,\n",
    "    spacing,\n",
    "    show_current_frame=False):\n",
    "    \"\"\"Run a background subtraction configuration on a source\n",
    "    video file. Save the output to another video.\n",
    "    \"\"\"\n",
    "    # Create a VideoCapture from a video file\n",
    "    cap = cv.VideoCapture(video_file)\n",
    "    # Get video file attributes for creating the output video\n",
    "    n_frames = int(cap.get(cv.CAP_PROP_FRAME_COUNT))\n",
    "    width = int(cap.get(cv.CAP_PROP_FRAME_WIDTH))\n",
    "    height = int(cap.get(cv.CAP_PROP_FRAME_HEIGHT))\n",
    "    fps = int(cap.get(cv.CAP_PROP_FPS))\n",
    "    \n",
    "    # Codec for the VideoWriter\n",
    "    fourcc = cv.VideoWriter_fourcc(*'X264')\n",
    "    # Create a VideoWriter\n",
    "    w_each = int(scale * width)\n",
    "    h_each = int(scale * height)\n",
    "    ncols = len(subtractors) + 1\n",
    "    nrows = max([len(s) for s in subtractors])\n",
    "    w = w_each * ncols + spacing * (ncols - 1)\n",
    "    h = h_each * nrows + spacing * (nrows - 1)\n",
    "    vid_out_size = (w, h)\n",
    "    vid_out = cv.VideoWriter(\n",
    "        save_file, \n",
    "        fourcc, \n",
    "        fps, \n",
    "        vid_out_size,\n",
    "        True)\n",
    "    \n",
    "    # background is off-white\n",
    "    frame_out = 235 * np.ones((h, w, 3)).astype(np.uint8)\n",
    "    for _ in range(n_frames):\n",
    "        ret, frame = cap.read()\n",
    "        if show_current_frame:\n",
    "            cv.imshow('frame', frame)\n",
    "            cv.waitKey(1)\n",
    "        fresized = imutils.resize(frame, height=h_each, width=w_each)\n",
    "        frame_out[:h_each, :w_each, :] = fresized\n",
    "        for x, col in enumerate(subtractors):\n",
    "            x0 = (x + 1) * (w_each + spacing) \n",
    "            x1 = x0 + w_each\n",
    "            for y, subtractor in enumerate(col):\n",
    "                y0 = y * (h_each + spacing)\n",
    "                y1 = y0 + h_each\n",
    "                mask = subtractor.apply(frame)\n",
    "                if median_blur_size > 0:\n",
    "                    mask = cv.medianBlur(mask, median_blur_size)\n",
    "                if gaussian_blur_size > 0:\n",
    "                    shape = (gaussian_blur_size, gaussian_blur_size)\n",
    "                    mask = cv.GaussianBlur(mask, shape, 0)\n",
    "                \n",
    "                mresized = imutils.resize(\n",
    "                    mask, \n",
    "                    height=h_each,\n",
    "                    width=w_each)\n",
    "                frame_out[y0:y1, x0:x1, :] = np.stack(\n",
    "                    [mresized]*3,\n",
    "                    axis=-1)\n",
    "        vid_out.write(frame_out)\n",
    "        \n",
    "    cap.release()\n",
    "    vid_out.release()\n",
    "    cv.destroyAllWindows()\n",
    "    pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define some subtraction algorithm configs to test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_dir = 'masks'\n",
    "os.makedirs(save_dir, exist_ok=True)\n",
    "\n",
    "median_value = 11\n",
    "gaussian_value = 5\n",
    "\n",
    "subtractors = [\n",
    "    [\n",
    "        cv.createBackgroundSubtractorKNN(500, 400, True),\n",
    "        cv.createBackgroundSubtractorKNN(200, 400, True),\n",
    "        cv.createBackgroundSubtractorKNN(200, 1000, True),\n",
    "        cv.createBackgroundSubtractorKNN(500, 200, True),\n",
    "        cv.createBackgroundSubtractorKNN(500, 1000, True),\n",
    "        cv.createBackgroundSubtractorKNN(500, 400, False)\n",
    "    ],\n",
    "    \n",
    "    [\n",
    "        cv.createBackgroundSubtractorMOG2(500, 16, True),\n",
    "        cv.createBackgroundSubtractorMOG2(200, 16, True),\n",
    "        cv.createBackgroundSubtractorMOG2(200, 32, True),\n",
    "        cv.createBackgroundSubtractorMOG2(500, 10, True),\n",
    "        cv.createBackgroundSubtractorMOG2(500, 32, True),\n",
    "        cv.createBackgroundSubtractorMOG2(500, 16, False)\n",
    "    ],\n",
    "    \n",
    "    [\n",
    "        cv.bgsegm.createBackgroundSubtractorCNT(15, True, 900),\n",
    "        cv.bgsegm.createBackgroundSubtractorCNT(15, True, 400),\n",
    "        cv.bgsegm.createBackgroundSubtractorCNT(5, True, 900),\n",
    "        cv.bgsegm.createBackgroundSubtractorCNT(45, True, 900),\n",
    "        cv.bgsegm.createBackgroundSubtractorCNT(5, True, 400),\n",
    "        cv.bgsegm.createBackgroundSubtractorCNT(45, True, 400)\n",
    "    ],\n",
    "    \n",
    "    [\n",
    "        cv.bgsegm.createBackgroundSubtractorGMG(45, 0.8),\n",
    "        cv.bgsegm.createBackgroundSubtractorGMG(45, 0.9),\n",
    "        cv.bgsegm.createBackgroundSubtractorGMG(45, 0.95),\n",
    "        cv.bgsegm.createBackgroundSubtractorGMG(45, 0.7),\n",
    "        cv.bgsegm.createBackgroundSubtractorGMG(45, 0.6),\n",
    "        cv.bgsegm.createBackgroundSubtractorGMG(20, 0.8)\n",
    "    ],\n",
    "    \n",
    "    [\n",
    "        cv.bgsegm.createBackgroundSubtractorGSOC(replaceRate=0.003, propagationRate=0.01),\n",
    "        cv.bgsegm.createBackgroundSubtractorGSOC(replaceRate=0.003, propagationRate=0.025),\n",
    "        cv.bgsegm.createBackgroundSubtractorGSOC(replaceRate=0.006, propagationRate=0.01),\n",
    "        cv.bgsegm.createBackgroundSubtractorGSOC(replaceRate=0.0015, propagationRate=0.01),\n",
    "        cv.bgsegm.createBackgroundSubtractorGSOC(replaceRate=0.006, propagationRate=0.025),\n",
    "        cv.bgsegm.createBackgroundSubtractorGSOC(replaceRate=0.0015, propagationRate=0.025)\n",
    "    ],\n",
    "    \n",
    "    [\n",
    "        cv.bgsegm.createBackgroundSubtractorLSBP(noiseRemovalThresholdFacBG=0.0004, noiseRemovalThresholdFacFG=0.0008),\n",
    "        cv.bgsegm.createBackgroundSubtractorLSBP(noiseRemovalThresholdFacBG=0.0004, noiseRemovalThresholdFacFG=0.00016),\n",
    "        cv.bgsegm.createBackgroundSubtractorLSBP(noiseRemovalThresholdFacBG=0.0002, noiseRemovalThresholdFacFG=0.0004),\n",
    "        cv.bgsegm.createBackgroundSubtractorLSBP(noiseRemovalThresholdFacBG=0.0008, noiseRemovalThresholdFacFG=0.0004),\n",
    "        cv.bgsegm.createBackgroundSubtractorLSBP(noiseRemovalThresholdFacBG=0.0002, noiseRemovalThresholdFacFG=0.0016),\n",
    "        cv.bgsegm.createBackgroundSubtractorLSBP(noiseRemovalThresholdFacBG=0.0008, noiseRemovalThresholdFacFG=0.0016)\n",
    "    ],\n",
    "    \n",
    "    [\n",
    "        cv.bgsegm.createBackgroundSubtractorMOG(200, 5, 0.7),\n",
    "        cv.bgsegm.createBackgroundSubtractorMOG(200, 3, 0.7),\n",
    "        cv.bgsegm.createBackgroundSubtractorMOG(200, 7, 0.7),\n",
    "        cv.bgsegm.createBackgroundSubtractorMOG(200, 5, 0.55),\n",
    "        cv.bgsegm.createBackgroundSubtractorMOG(200, 5, 0.85),\n",
    "        cv.bgsegm.createBackgroundSubtractorMOG(200, 7, 0.85)\n",
    "    ]\n",
    "]\n",
    "\n",
    "background_subtraction_mosaic(\n",
    "    'test.mp4',\n",
    "    os.path.join(save_dir, 'mosaic1013.mp4'),\n",
    "    subtractors,\n",
    "    median_value,\n",
    "    gaussian_value,\n",
    "    0.37,\n",
    "    2,\n",
    "    True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "artech",
   "language": "python",
   "name": "artech"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}