Working Extension

README.md

@@ -29,7 +29,7 @@ Nothing to compile if you want to use the extension - Follow the instructions to
 ##### <img src="https://upload.wikimedia.org/wikipedia/commons/thumb/a/a0/Firefox_logo%2C_2019.svg/68px-Firefox_logo%2C_2019.svg.png" width=20> Firefox 
 * download or clone the repo
 * go to about:debugging#/runtime/this-firefox
-* Click _"Load Temporary Add-on..."_
+* Click "Load Temporary Add-on..."
 * Browse to the `dist` folder with the extension and select the `manifest.json` file
 
 #### Usage without Extension
@@ -42,3 +42,5 @@ This humble _hack-speriment_ would not be possible without the following project
 * [Virtual Browser Camera shaders](https://github.com/spite/virtual-webcam) by [spite](https://github.com/spite)
 * [Pose Animator](https://github.com/yemount/pose-animator) by [yemount](https://github.com/yemount)
 * [Meething](https://us.meething.space) by [Team Meething](https://github.com/meething/meething/graphs/contributors)
+
+![image](https://user-images.githubusercontent.com/1423657/82818656-561dbe80-9e9f-11ea-90a1-5436fdcb84e5.png)

js/filter-stream.js

@@ -7,58 +7,65 @@ class FilterStream {
     this.stream = stream;
     const video = document.createElement("video");
     const canvas = document.createElement("canvas");
-    //const svg = document.querySelector(".illustration-canvas");
     this.canvas = canvas;
-    //this.svg = svg;
-
-    video.srcObject = stream;
-    video.autoplay = true;
-    this.video = video;
-    //this.ctx = this.svg.getContext("2d");
-    this.outputStream = this.canvas.captureStream();
-    this.canvasActiveLayer = null;
-    this.poseEmitter = new PoseEmitter(this.video, this.video.videoWidth, this.video.videoHeight, false)
+
     this.addedCanvas = false;
-    video.addEventListener("playing", () => {
+    this.svgCanvas;
+
+    video.addEventListener("playing", async () => {
+      console.log('video is playing',this.video.videoWidth, this.video.videoHeight);
       // Use a 2D Canvas.
       this.canvas.width = this.video.videoWidth;
       this.canvas.height = this.video.videoHeight;
+      if (!this.poseEmitter) this.poseEmitter = new PoseEmitter(this.video, this.video.videoWidth, this.video.videoHeight)
       this.update();
     });
+
+    video.srcObject = stream;
+    video.autoplay = true;
+    this.video = video;
+
+    this.ctx = this.canvas.getContext("2d");
+    this.outputStream = this.canvas.captureStream();
+
   }
 
-  update() {
+  async update() {
+    if(!this.poseEmitter) return;
     // Use a 2D Canvas
-    // this.ctx.filter = 'invert(1)';
-    // this.canvas.width = this.video.videoWidth;
-    // this.canvas.height = this.video.videoHeight;
-    // this.svg.width = this.video.videoWidth;
-    // this.svg.height = this.video.videoHeight;
-
-    // this.ctx.drawImage(this.video, 0, 0);
-    // this.ctx.fillStyle = "#ff00ff";
-    // this.ctx.textBaseline = "top";
-    // this.ctx.fillText("Virtual", 10, 10);
-    this.drawOnCanvas();
-
+    //if (this.svgCanvas instanceof HTMLCanvasElement) {
+    if (this.svgCanvas) {
+     	this.canvas.width = this.video.videoWidth;
+     	this.canvas.height = this.video.videoHeight;
+	this.ctx.drawImage(this.svgCanvas, 0, 0, this.video.videoHeight, this.video.videoWidth);
+        //this.ctx.fillStyle = "#ffff00";
+        //this.ctx.textBaseline = "top";
+        //this.ctx.fillText("Virtual Camera", 10, 10);
+    } else {
+     	//this.canvas.width = this.video.videoWidth;
+     	//this.canvas.height = this.video.videoHeight;
+        //this.ctx.drawImage(this.video, 0, 0);
+        //this.ctx.fillStyle = "#ff00ff";
+        //this.ctx.textBaseline = "top";
+        //this.ctx.fillText("Loading...", 10, 10);
+    }
+    await this.drawOnCanvas();
     requestAnimationFrame(() => this.update());
   }
-
+
+
   async drawOnCanvas()
   {
-    let svgCanvas = await this.poseEmitter.sampleAndDetect();
-    if(svgCanvas instanceof HTMLCanvasElement){
+    this.svgCanvas = await this.poseEmitter.sampleAndDetect();
+    if(this.svgCanvas instanceof HTMLCanvasElement){
+      this.svgCanvas.width = this.video.videoWidth;
+      this.svgCanvas.height = this.video.videoHeight;
       if(!this.addedCanvas){
-        document.body.appendChild(svgCanvas);
+        this.outputStream = this.svgCanvas.captureStream();
         this.addedCanvas = true;
       }
-      // console.log("SVG invisible canvas - ", svgCanvas);
-      // let svgImage = svgCanvas.getContext("2d").createImageData(svgCanvas.width, svgCanvas.height);
-      // this.ctx.drawImage(svgImage, 0, 0);
-    // TODO: REPLACE INPUT WITH DRIVER VIDEO AND OUTPUT CANVAS WITH SVG CANVAS
     }
   }
-
 }
 
 export { FilterStream };

js/main.js

@@ -1,21 +1,14 @@
 import { monkeyPatchMediaDevices } from "./media-devices.js";
+
 monkeyPatchMediaDevices();
 
-//import PoseDetector from "./poseDetection.js";
 async function init() {
   const res = await navigator.mediaDevices.enumerateDevices();
   console.log(res);
   const stream = await navigator.mediaDevices.getUserMedia({
-    video: { deviceId: "virtual", width: 320, height: 240 },
+    video: { deviceId: "virtual" },
     audio: false
   });
-  // const video = document.createElement("video");
-  // video.setAttribute("id", "local");
-  // video.setAttribute("class", "video-local");
-  // video.srcObject = stream;
-  // video.autoplay = true;
-  // document.body.append(video);
-  //new PoseDetector();
 }
 
 init();

js/media-devices.js

@@ -3,6 +3,7 @@ import { FilterStream } from "./filter-stream.js";
 function monkeyPatchMediaDevices() {
   const enumerateDevicesFn = MediaDevices.prototype.enumerateDevices;
   const getUserMediaFn = MediaDevices.prototype.getUserMedia;
+  var filter;
 
   MediaDevices.prototype.enumerateDevices = async function() {
     const res = await enumerateDevicesFn.call(navigator.mediaDevices);
@@ -24,15 +25,14 @@ function monkeyPatchMediaDevices() {
         args[0].video.deviceId === "virtual" ||
         args[0].video.deviceId.exact === "virtual"
       ) {
+        if(filter && filter.outputStream) return filter.outputStream;
         // This constraints could mimick closely the request.
         // Also, there could be a preferred webcam on the options.
         // Right now it defaults to the predefined input.
         const constraints = {
           video: {
-            facingMode: args[0].facingMode,
-            advanced: args[0].video.advanced,
-            width: args[0].video.width,
-            height: args[0].video.height
+            width: args[0].video.width || 640,
+            height: args[0].video.height || 480
           },
           audio: false
         };
@@ -42,7 +42,7 @@ function monkeyPatchMediaDevices() {
         );
         if (res) {
           var shader = false;
-          const filter = new FilterStream(res, shader);
+          filter = new FilterStream(res, shader);
           return filter.outputStream;
         }
       }

js/poseDetection.js

@@ -14,7 +14,7 @@ import * as paper from "paper";
 import "babel-polyfill";
 
 import * as boySVG from "../svg/boy.svg";
-import * as girlSVG from "../svg/girl.svg";
+//import * as girlSVG from "../svg/girl.svg";
 
 export default class PoseDetector {
   constructor(video,videoWidth,videoHeight) {

js/poseEmitter.js

@@ -1,8 +1,8 @@
+import "babel-polyfill";
 import * as posenet from '@tensorflow-models/posenet';
 import * as facemesh from '@tensorflow-models/facemesh';
 import * as tf from '@tensorflow/tfjs';
 import * as paper from "paper";
-import "babel-polyfill";
 
 import {
   drawKeypoints,
@@ -19,15 +19,14 @@ import { FileUtils } from "./utils/fileUtils.js";
 
 
 import * as boySVG from "../svg/boy.svg";
-import * as girlSVG from "../svg/girl.svg";
 
 export default class PoseEmitter {
 
-  constructor(video, videoWidth, videoHeight, callback) {
+  constructor(video, videoWidth, videoHeight) {
     this.video = video ? video : document.getElementById("local");
     // Camera stream video element
-    this.videoWidth = videoWidth ? videoWidth : 320;
-    this.videoHeight = videoHeight ? videoHeight : 240;
+    this.videoWidth = videoWidth ? videoWidth : 640;
+    this.videoHeight = videoHeight ? videoHeight : 480;
 
     // Canvas
     this.faceDetection = null;
@@ -36,13 +35,13 @@ export default class PoseEmitter {
     // let canvas = document.querySelector(".illustration-canvas");
     // TODO: use an invisible canvas we return at the end, do not render it
     this.canvas = document.createElement('canvas');
-    this.canvas.width = videoWidth ? videoWidth : 320;
-    this.canvas.height = videoHeight ? videoHeight : 240;
+    this.canvas.width = videoWidth ? videoWidth : 640;
+    this.canvas.height = videoHeight ? videoHeight : 480;
     this.canvasScope.setup(this.canvas);
 
     this.canvasWidth = this.canvas.width;
     this.canvasHeight = this.canvas.height;
-    console.log("Canvas scope = ", this.canvasScope);
+    // console.log("Canvas scope = ", this.canvasScope);
 
 
     // ML models
@@ -122,24 +121,15 @@ export default class PoseEmitter {
         scoreThreshold: self.minPartConfidence,
         nmsRadius: self.nmsRadius
       });
-      console.log("pose detected : ", all_poses);
-
-      //Dispatch event
-      // var event = new CustomEvent("poseDetected", {
-      //   detail: {
-      //     faceMesh: self.faceDetection,
-      //     pose: all_poses
-      //   }
-      // });
-      // self.video.dispatchEvent(event);
+      // console.log("pose detected : ", all_poses);
 
       poses = poses.concat(all_poses);
       input.dispose();
 
       self.canvasScope.project.clear();
 
       if (poses.length >= 1 && self.illustration) {
-        //Skeleton.flipPose(poses[0]);
+        Skeleton.flipPose(poses[0]);
 
         if (self.faceDetection && self.faceDetection.length > 0) {
           let face = Skeleton.toFaceFrame(self.faceDetection[0]);
@@ -165,11 +155,6 @@ export default class PoseEmitter {
         console.log("ERROR! Paper project undefined", self.canvasScope);
       }
 
-      // Send the activelayer to the callback function
-      // if(callback){ 
-      //   callback(self.canvasScope.project.activeLayer);
-      // }
-
     } catch (err) {
       // input.dispose();
       console.log(err);