camera.js

/**
 * @license
 * Copyright 2020 Google Inc. All Rights Reserved.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * https://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * =============================================================================
 */

import * as posenet_module from '@tensorflow-models/posenet';
import * as facemesh_module from '@tensorflow-models/facemesh';
import * as tf from '@tensorflow/tfjs';
import * as paper from 'paper';
import dat from 'dat.gui';
import Stats from 'stats.js';
import "babel-polyfill";

import {drawKeypoints, drawPoint, drawSkeleton, isMobile, toggleLoadingUI, setStatusText} from './utils/demoUtils';
import {SVGUtils} from './utils/svgUtils'
import {PoseIllustration} from './illustrationGen/illustration';
import {Skeleton, facePartName2Index} from './illustrationGen/skeleton';
import {FileUtils} from './utils/fileUtils';

import * as girlSVG from './resources/illustration/girl.svg';
import * as boySVG from './resources/illustration/boy.svg';
import * as abstractSVG from './resources/illustration/abstract.svg';
import * as blathersSVG from './resources/illustration/blathers.svg';
import * as tomNookSVG from './resources/illustration/tom-nook.svg';

// Camera stream video element
let video;
let videoWidth = 400;
let videoHeight = 400;

// Canvas
let faceDetection = null;
let illustration = null;
let canvasScope;
let canvasWidth = 400;
let canvasHeight = 400;

// ML models
let facemesh;
let posenet;
let minPoseConfidence = 0.15;
let minPartConfidence = 0.1;
let nmsRadius = 30.0;

// Misc
let mobile = false;
const stats = new Stats();
const avatarSvgs = {
  'girl': girlSVG.default,
  'boy': boySVG.default,
  'abstract': abstractSVG.default,
  'blathers': blathersSVG.default,
  'tom-nook': tomNookSVG.default,
};



//peerjs

var peer = new Peer(undefined, {
  host: 'login.davidvelho.tech',
  port: '3000'
});
var con = null;
var id = null;
var myId;

peer.on('open', function(id) {
  console.log('My peer ID is: ' + id);
  myId = id;
  myIdSpan.innerHTML = myId;
});

var myIdSpan = document.getElementById("myId");
var sub = document.getElementById("peerSub");
var Send = document.getElementById("send");
var peerText = document.getElementById("peerId");

sub.addEventListener("click",()=>{

  var peerId = peerText.value;
  id =peerId;
  if(id!=null){
    con = peer.connect(id);
    if(con!=null){
      console.log("connected to" ,id);
      
    }
  }
  


});


// Send.addEventListener("click",()=>{
//   if(con){
//     console.log("send")
//     SendMessage();
//   }
// });

  


peer.on('connection', function(conn) 
{ 

    console.log('peer connected');
    con = conn;
    conn.on('open', function() {
        console.log('conn open');
        
    });
   
});



// function SendMessage()
// {
//     con.send('Hello!');
// };







/**
 * Loads a the camera to be used in the demo
 *
 */
async function setupCamera() {
  if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
    throw new Error(
        'Browser API navigator.mediaDevices.getUserMedia not available');
  }

  const video = document.getElementById('video');
  video.width = videoWidth;
  video.height = videoHeight;

  const stream = await navigator.mediaDevices.getUserMedia({
    'audio': false,
    'video': {
      facingMode: 'user',
      width: videoWidth,
      height: videoHeight,
    },
  });
  video.srcObject = stream;

  return new Promise((resolve) => {
    video.onloadedmetadata = () => {
      resolve(video);
    };
  });
}

async function loadVideo() {
  const video = await setupCamera();
  video.play();

  return video;
}

const defaultPoseNetArchitecture = 'MobileNetV1';
const defaultQuantBytes = 2;
const defaultMultiplier = 1.0;
const defaultStride = 16;
const defaultInputResolution = 200;

const guiState = {
  avatarSVG: Object.keys(avatarSvgs)[0],
  debug: {
    showDetectionDebug: true,
    showIllustrationDebug: false,
  },
};

/**
 * Sets up dat.gui controller on the top-right of the window
 */
function setupGui(cameras) {

  if (cameras.length > 0) {
    guiState.camera = cameras[0].deviceId;
  }

  const gui = new dat.GUI({width: 300});

  let multi = gui.addFolder('Image');
  gui.add(guiState, 'avatarSVG', Object.keys(avatarSvgs)).onChange(() => parseSVG(avatarSvgs[guiState.avatarSVG]));
  multi.open();

  let output = gui.addFolder('Debug control');
  output.add(guiState.debug, 'showDetectionDebug');
  output.add(guiState.debug, 'showIllustrationDebug');
  output.open();
}

/**
 * Sets up a frames per second panel on the top-left of the window
 */
function setupFPS() {
  stats.showPanel(0);  // 0: fps, 1: ms, 2: mb, 3+: custom
  document.getElementById('main').appendChild(stats.dom);
}

/**
 * Feeds an image to posenet to estimate poses - this is where the magic
 * happens. This function loops with a requestAnimationFrame method.
 */

async function drawPoseInRealTime(dataPoints){
  // console.log(dataPoints);
  
  if (dataPoints[0].length >= 1 && illustration) {
    // console.log(illustration);
    Skeleton.flipPose(dataPoints[0][0]);
    // console.log(dataPoints);
    

    if (dataPoints[1] && dataPoints[1].length > 0) {
      let face = Skeleton.toFaceFrame(dataPoints[1][0]);
      illustration.updateSkeleton(dataPoints[0][0], face);
    } else {
      illustration.updateSkeleton(dataPoints[0][0], null);
    }
    illustration.draw(canvasScope, canvasWidth, canvasHeight);

    if (guiState.debug.showIllustrationDebug) {
      illustration.debugDraw(canvasScope);
    }
  }

}




async function detectPoseInRealTime(video) {

  const canvas = document.getElementById('output');
  const videoCtx = canvas.getContext('2d');
  
  canvas.width = videoWidth;
  canvas.height = videoHeight;
  console.log(videoHeight,videoWidth);

  async function poseDetectionFrame() {

    
    // Begin monitoring code for frames per second
    // stats.begin();
  
    let poses = [];
   
    videoCtx.clearRect(0, 0, videoWidth, videoHeight);
    // Draw video
    videoCtx.save();
    videoCtx.scale(-1, 1);
    videoCtx.translate(-videoWidth, 0);
    videoCtx.drawImage(video, 0, 0, videoWidth, videoHeight);
    videoCtx.restore();
  
    // Creates a tensor from an image
    const input = tf.browser.fromPixels(canvas);
  
    faceDetection = await facemesh.estimateFaces(input, false, false);
    let all_poses = await posenet.estimatePoses(video, {
      flipHorizontal: true,
      decodingMethod: 'multi-person',
      maxDetections: 1,
      scoreThreshold: minPartConfidence,
      nmsRadius: nmsRadius
    });
  
    poses = poses.concat(all_poses);
    let dataPoints = [poses,faceDetection];
    input.dispose();
  
  
    // return dataPoints
    canvasScope.project.clear();
  
    
      if(con!=null){
        con.send(dataPoints);
        console.log("sent");
      }
      
    

    
    if(con!=null){
      con.on('data', function(data) {
        
        if(data){
         drawPoseInRealTime(data);
        }
          
      });
    }

    
  
    requestAnimationFrame(poseDetectionFrame);


  }

  poseDetectionFrame();
  

}

function setupCanvas() {
  mobile = isMobile();
  if (mobile) {
    canvasWidth = Math.min(window.innerWidth, window.innerHeight);
    canvasHeight = canvasWidth;
    videoWidth *= 0.7;
    videoHeight *= 0.7;
  }  



  canvasScope = paper.default;
  let canvas = document.querySelector('.illustration-canvas');
  canvas.width = canvasWidth;
  canvas.height = canvasHeight;
  canvasScope.setup(canvas);




 
}

/**
 * Kicks off the demo by loading the posenet model, finding and loading
 * available camera devices, and setting off the detectPoseInRealTime function.
 */
export async function bindPage() {
  setupCanvas();

  toggleLoadingUI(true);
  setStatusText('Loading PoseNet model...');
  posenet = await posenet_module.load({
    architecture: defaultPoseNetArchitecture,
    outputStride: defaultStride,
    inputResolution: defaultInputResolution,
    multiplier: defaultMultiplier,
    quantBytes: defaultQuantBytes
  });
  setStatusText('Loading FaceMesh model...');
  facemesh = await facemesh_module.load();

  setStatusText('Loading Avatar file...');
  let t0 = new Date();
  await parseSVG(Object.values(avatarSvgs)[0]);

  setStatusText('Setting up camera...');
  try {
    video = await loadVideo();
  } catch (e) {
    let info = document.getElementById('info');
    info.textContent = 'this device type is not supported yet, ' +
      'or this browser does not support video capture: ' + e.toString();
    info.style.display = 'block';
    throw e;
  }

  setupGui([], posenet);
  // setupFPS();
  
  toggleLoadingUI(false);
  detectPoseInRealTime(video, posenet);
}

navigator.getUserMedia = navigator.getUserMedia ||
    navigator.webkitGetUserMedia || navigator.mozGetUserMedia;
FileUtils.setDragDropHandler((result) => {parseSVG(result)});

async function parseSVG(target) {
  let svgScope = await SVGUtils.importSVG(target /* SVG string or file path */);
  let skeleton = new Skeleton(svgScope);
  illustration = new PoseIllustration(canvasScope);
  illustration.bindSkeleton(skeleton, svgScope);
 
}
    
bindPage();