Spout and videoExporter Enabled Kinect Masker

Over the last two weeks I have gone from not knowing much about processing to having a final product, thank you to @GoToLoop, @hamoid and others. Your sketches and contributions to this forum are invaluable!

I give you Body Mapper !!

Attached is a sketch that interfaces with a KinectV1 - The depth image is used to create a mask overlay. User videos are used as textures for projection mapping, or more specifically body mapping. The sketch looks for .mp4 and .mov files in your data directory and allows you to cycle forward and backwards through these videos. When you are ready, you can either export the video via the onboard videoExporter to .mp4 (saved in a directory that you need to create called "savedVideo"), or you can share the frames via Spout (i really wish there was a spout recorder, similar to syphon recorder and there is, cause i made one using Max MSP).

Let me know what you think! I'm sure I have made some strange code, but then again I dont really know what I'm doing and this is very much a learning experience for me.

Enjoy!

N

//    BODY MAPPER

//Cobbled together by Nicolas de Cosson 2016


//
//            SpoutSender
//
//      Send to a Spout receiver
//
//           spout.zeal.co
//
//       http://spout.zeal.co/download-spout/
//
/**
 * Movie Player (v1.21)
 * by GoToLoop  (2014/Oct/31)
 *
 * forum.processing.org/two/discussion/7852/
 * problem-with-toggling-between-multiple-videos-on-processing-2-2-1
 */
/*
  This sketch shows how you can record different takes.
 */

import com.hamoid.*;
import processing.video.Movie;
import spout.*;
import org.openkinect.freenect.*;
import org.openkinect.processing.*;
import org.gstreamer.elements.PlayBin2;
import java.io.FilenameFilter;

static final PlayBin2.ABOUT_TO_FINISH FINISHING = new PlayBin2.ABOUT_TO_FINISH() {
  @ Override public void aboutToFinish(PlayBin2 elt) {
  }
};

//usefull so that we do not overwrite movie files in save directory
int ye = year();
int mo = month();
int da = day();
int ho = hour();
int mi = minute();
int se = second();
//global frames per second
static final float FPS = 30.0;
//index
int idx;
//string array for films located in data directory
String[] FILMS;
//string for weather or not we are exporting to .mp4 using ffmpeg
String record;
boolean isPaused;
boolean recording = false;
// Depth image
PImage depthImg;
// Which pixels do we care about?
int minDepth =  60;
int maxDepth = 800;
//max depth 2048

//declare a kinect object
Kinect kinect;
//declare videoExport
VideoExport videoExport;
//movie array
Movie[] movies;
//movie
Movie m;
// DECLARE A SPOUT OBJECT
Spout spout;


void setup() {
  //I have to call resize becasue for some reason P2D does not
  //seem to to actually size to display width/height on first call
  size(displayWidth, displayHeight, P2D);
  surface.setResizable(true);
  surface.setSize(displayWidth, displayHeight);
  surface.setLocation(0, 0);
  noSmooth();
  frameRate(FPS);
  background(0);

  kinect = new Kinect(this);
  kinect.initDepth();

  // Blank image with alpha channel
  depthImg = new PImage(kinect.width, kinect.height, ARGB);

  // CREATE A NEW SPOUT OBJECT
  spout = new Spout(this);
  //CREATE A NAMED SENDER
  spout.createSender("BodyMapper Spout");

  println("Press R to toggle recording");
  //.mp4 is created with year month date hour minute and second data so we never save over a video
  videoExport = new VideoExport(this, "savedVideo/Video" + ye + mo + da + ho + mi + se + ".mp4");

  videoExport.setFrameRate(15);

  //videoExport.forgetFfmpegPath();
  //videoExport.dontSaveDebugInfo();

  java.io.File folder = new java.io.File(dataPath(""));

  // this is the filter (returns true if file's extension is .mov or .mp4)
  java.io.FilenameFilter movFilter = new java.io.FilenameFilter() {
    String[] exts = {
      ".mov", ".mp4"
    };
    public boolean accept(File dir, String name) {
      name = name.toLowerCase();
      for (String ext : exts) if (name.endsWith(ext)) return true;
      return false;
    }
  };
  //create an array of strings comprised of .mov/.mp4 in data directory
  FILMS = folder.list(movFilter);
  //using the number of videos in data directory we can create array of videos
  movies = new Movie[FILMS.length];

  for (String s : FILMS)  (movies[idx++] = new Movie(this, s))
    .playbin.connect(FINISHING);
  //start us off by playing the first movie in the array
  (m = movies[idx = 0]).loop();
}

void draw() {
  // Threshold the depth image
  int[] rawDepth = kinect.getRawDepth();
  for (int i=0; i < rawDepth.length; i++) {
    if (rawDepth[i] >= minDepth && rawDepth[i] <= maxDepth) {
      //if pixels are in range then turn them to alpha transparency
      depthImg.pixels[i] = color(0, 0);
    } else {
      //otherwise turn them black
      depthImg.pixels[i] = color(0);
    }
  }
  //update pixels from depth map to reflect change of pixel colour
  depthImg.updatePixels();
  //blur the edges of depth map
  depthImg.filter(BLUR, 1);


  //draw movie to size of current display
  image(m, 0, 0, displayWidth, displayHeight);
  //draw depth map mask to size of current display
  image(depthImg, 0, 0, displayWidth, displayHeight);
  //share image through Spout
  // Sends at the size of the window
  spout.sendTexture();
  //if key r is pressed begin export of .mp4 to save directory
  if (recording) {
    videoExport.saveFrame();
  }
  //TODO - create second window for preferences and instructions
  fill(255);
  text("Recording is " + (recording ? "ON" : "OFF"), 30, 100);
  text("Press r to toggle recording ON/OFF", 30, 60);
  text("Video saved to file after application is closed", 30, 80);
}

void movieEvent(Movie m) {
  m.read();
}

void keyPressed() {
  int k = keyCode;
  if (k == RIGHT) {
    // Cycle forwards
    if (idx >= movies.length - 1) {
      idx = 0;
    } else {
      idx += 1;
    }
  } else if (k == LEFT) {
    // Cycle backwards
    if (idx <= 0) {
      idx = movies.length - 1;
    } else {
      idx -= 1;
    }
  }

  if (k == LEFT || k == RIGHT) {
    m.stop();
    (m = movies[idx]).loop();
    isPaused = false;
    background(0);
  }

  if (key == 'r' || key == 'R') {
    recording = !recording;
    println("Recording is " + (recording ? "ON" : "OFF"));
  }
}

@ Override public void exit() {
  for (Movie m : movies)  m.stop();
  super.exit();
}