import java.io.BufferedReader; import java.io.InputStreamReader; import java.net.URL; import java.util.Collection; import java.util.Date; import java.util.List; import processing.video.*; import processing.opengl.*; import vrpn.TrackerRemote; // Overall goal: provide an interactive and engaging experience for // the fishnet stockings video created by Joellyn Rock. // // Ideas: // // 1) Water shimmer - When users stand in front of video and poke at // the screen the system generates a water ripple on the video at that // location. Using the Kinect, we can determine when the user's left // or right hands break a plane (in the Z dimension) in front of the // screen. When this happens, we can map the (X,Y) coordinate into the // video and issue a water ripple in the fragment shader system. // // 2) Shadow Puppet - Using the Kinect have the viewers hold a shadow // puppet generated by the system in the location of their hands as // projected onto the video screen. // // 3) Twitter - Use twitter feed and access a #nordicmermaid or some // other hashtag and post to the video. // // 4) Implicit Shadow Puppet - rather than using a real-shadow, have the // Kinect detect the user's hand and place a shadow puppet on it. // // Other ideas? // Initial brainstorming ideas that Joellyn expressed to me: // // Internet connection - twitter feed -- infuse text with the video // // break the story into 9 vinettes ... 9 categories of text - time // based progression of data presentation, // // get shadow processing working // ; // practive multiple video streams // // and colored text // // more "watery" effect // Try to play mermaid 4 and mermaid 7 // smooth motion // fix math for projection // add new silhouettes // // and maybe change size of puppets // different themes for each segment... structure for the overall show // scissors! // 6/10/2015 // option to remove other hashtags from twitter feed // // Add calibration // // dynamic twitter features // text wave shader // stop movement with hand // // Sound effects and music // // Part 1 // more subtle ripples, smaller // // Part 2 // more turbulent waves/ripples // localized color changes // jagged movement based on harsh "rows" of video manipulation... // multiple people pulling on a net // // Part 3 // gesture-based puppet duplication // // font options // larger text size // // TODO for next week: // wave shader for tweet text // how to tweet out from program // reduce query time // putting a line on the screen for the scissor-cutting gesture // 6/17/2015 // TODO: // Show them at different layers depending on their age. // Fix the wave shader so the video isn't upside-down. // // 6/24/2015 // Clean up calibration // Add tweets from sample text // // 6/29/2014 // TODO: // Add offset to calibration // Other Thoughts // // 7/1/2015 // TODO: // Calibration serialization // Gestures, how they start and end // // 7/8/2015 // add third user // why aren't the tweets showing up // puppet/image states, picking random points to go to, waiting // "fix" shaders // maybe don't follow hand // make it not upside down // fix how the images and tweets show up witht he shader // gestures // // 7/15/2015 // TODO: // Keep puppets off-screen until tracked user walks in // add transparency to the white puppet images // have puppets move off screen again when not tracked // change idle movement to more like swimming // make movements slower rather than snapping // // Add support for puppets following multiple users with different tracking points // stretching net with two people // // transparency to all puppet images // // Scene 1 // Flocking // flock 1: 3 jellyfish // flock 2: flat fish, tiny, lots // flock 3: mermaids // // Scene 2 // net grabbing with two people // // Scene 3 // image duplication with slashing String fontFile = "Calibri-255.vlw"; String vrpnServer = "localhost"; String formUrl = "https://docs.google.com/spreadsheets/d/1MJNVnrWfzfi9QGWh9ZSqQCI2RSd13K0mvfnNIs5797Y/pub?output=tsv"; int startTime; int frameTime; int texSize = 20; int texSizeDir = 1; SceneManager sceneManager; boolean calibrating = false; final static int CALIBRATION_POINTS_COUNT = 100; PVector calibrationPoints; int calibrationIndex; boolean calibratePointOne = false; boolean calibratePointTwo = false; boolean userDebug = false; void setup() { noCursor(); // shut off mouse size(1920, 1080, P3D); Util.getInstance(this); Util.scaleAmount = 0f; calibrationPoints = new PVector(); calibrationIndex = 0; PShader waveShader = loadShader("fragWave.glsl", "vert.glsl"); // load and set the font PFont font = loadFont(fontFile); textFont(font); // user 1 info final TrackedUser user1 = new TrackedUser("Tracker0", vrpnServer) { public float handAngle; public void update(int deltaTime) { // Compute an angle for user 1 of the angle PVector hand = trackData[10]; PVector finger = trackData[11]; // calculate this angle float xD = finger.x - hand.x; float yD = finger.y - hand.y; float mag = sqrt(xD * xD + yD * yD); handAngle = 180.0/PI * acos( xD/mag ); // other components are 0 since we're comparing with X axis - so cos theta is simple } }; for (int i = 0; i < user1.trackData.length; i++) { user1.trackData[i].set(0.0f, 0.0f, 1.0f); } user1.trackData[11].set(0.0f, 0.0f, 2.0f); // connect user1 to its tracker user1.connect(); // user2 info final TrackedUser user2 = new TrackedUser("Tracker1", vrpnServer); for (int i = 0; i < user2.trackData.length; i++) { user2.trackData[i].set(0.0f, 0.0f, 1.0f); } // connect user2 to its tracker user2.connect(); // user3 final TrackedUser user3 = new TrackedUser("Tracker2", vrpnServer); for (int i = 0; i < user3.trackData.length; i++) { user3.trackData[i].set(0.0f, 0.0f, 1.0f); } // connect user3 to its tracker user3.connect(); // create the scene manager! sceneManager = new SceneManager(formUrl); sceneManager.users.add(user1); sceneManager.users.add(user2); sceneManager.users.add(user3); Flock jellyFlock = new Flock(user1, 11); jellyFlock.addTracker(user1, 7); jellyFlock.lastTrackedTimer = 50000; jellyFlock.defaultIdlePosition = new PVector(-400, -400); DynamicSceneImage jellyBoid1 = new DynamicSceneImage(loadImage("jellylegs/JellyLegs__0000.png")); jellyBoid1.baseScale = 0.25f; jellyBoid1.updateScale(); jellyBoid1.position.set(jellyFlock.defaultIdlePosition.get()); jellyBoid1.useForceMovement = true; jellyBoid1.alpha = 191; jellyFlock.boids.add(jellyBoid1); Animation jellylegs = new Animation("fishnet1/jellylegs/JellyLegs__", ".png", 8, 4, jellyBoid1); jellylegs.baseScale = 0.5f; jellylegs.updateScale(); jellylegs.imageTime = 1000; jellyFlock.boids.add(jellylegs); jellyFlock.boids.add(new DynamicSceneImage(jellyBoid1, loadImage("jellylegs/JellyLegs__0001.png"))); jellyFlock.boids.add(new DynamicSceneImage(jellyBoid1, loadImage("jellylegs/JellyLegs__0002.png"))); jellyFlock.boids.add(new DynamicSceneImage(jellyBoid1, loadImage("jellylegs/JellyLegs__0003.png"))); jellyFlock.boids.add(new DynamicSceneImage(jellyBoid1, loadImage("jellylegs/JellyLegs__0004.png"))); jellyFlock.boids.add(new DynamicSceneImage(jellyBoid1, loadImage("jellylegs/JellyLegs__0005.png"))); jellyFlock.boids.add(new DynamicSceneImage(jellyBoid1, loadImage("jellylegs/JellyLegs__0006.png"))); jellyFlock.boids.add(new DynamicSceneImage(jellyBoid1, loadImage("jellylegs/JellyLegs__0007.png"))); // make a few in the flock a little larger jellyFlock.boids.get(2).baseScale = 0.45f; jellyFlock.boids.get(2).updateScale(); jellyFlock.boids.get(4).baseScale = 0.45f; jellyFlock.boids.get(4).updateScale(); jellyFlock.boids.get(6).baseScale = 0.45f; jellyFlock.boids.get(6).updateScale(); Flock mermaidFlock = new Flock(user2, 11); mermaidFlock.addTracker(user2, 7); mermaidFlock.lastTrackedTimer = 50000; mermaidFlock.defaultIdlePosition = new PVector(width + 400, -400); DynamicSceneImage merBoid1 = new DynamicSceneImage(loadImage("fishnet1/merFlock/mer1_flock.png")); merBoid1.baseScale = 0.4f; merBoid1.updateScale(); merBoid1.position.set(mermaidFlock.defaultIdlePosition.get()); merBoid1.useForceMovement = true; merBoid1.alpha = 230; mermaidFlock.boids.add(merBoid1); Animation mermaidColors = new Animation("mermaid_colors/float_", "_mer_colors.png", 6, 4, merBoid1); mermaidColors.baseScale = 0.8f; mermaidColors.updateScale(); mermaidColors.imageTime = 1000; mermaidFlock.boids.add(mermaidColors); mermaidFlock.boids.add(new DynamicSceneImage(merBoid1, loadImage("fishnet1/merFlock/AuneCutout1_White.png"))); mermaidFlock.boids.add(new DynamicSceneImage(merBoid1, loadImage("fishnet1/merFlock/ColinNeptune.png"))); mermaidFlock.boids.add(new DynamicSceneImage(merBoid1, loadImage("fishnet1/merFlock/HannahMermaid.png"))); mermaidFlock.boids.add(new DynamicSceneImage(merBoid1, loadImage("fishnet1/merFlock/mer_monster01.png"))); mermaidFlock.boids.add(new DynamicSceneImage(merBoid1, loadImage("fishnet1/merFlock/mer_monster02.png"))); mermaidFlock.boids.add(new DynamicSceneImage(merBoid1, loadImage("fishnet1/merFlock/mer2_pink_flock.png"))); mermaidFlock.boids.add(new DynamicSceneImage(merBoid1, loadImage("fishnet1/merFlock/mer3_mama_flip_flock.png"))); mermaidFlock.boids.add(new DynamicSceneImage(merBoid1, loadImage("fishnet1/merFlock/monster_exq2_side.png"))); mermaidFlock.boids.add(new DynamicSceneImage(merBoid1, loadImage("fishnet1/merFlock/merlady_green.png"))); mermaidFlock.boids.add(new DynamicSceneImage(merBoid1, loadImage("fishnet1/merFlock/net_monster2.png"))); mermaidFlock.boids.add(new DynamicSceneImage(merBoid1, loadImage("fishnet1/merFlock/monster_flock_orange.png"))); // make a few in the flock a little larger mermaidFlock.boids.get(2).baseScale = 0.45f; mermaidFlock.boids.get(2).updateScale(); mermaidFlock.boids.get(4).baseScale = 0.45f; mermaidFlock.boids.get(4).updateScale(); mermaidFlock.boids.get(6).baseScale = 0.45f; mermaidFlock.boids.get(6).updateScale(); Scene scene1 = new Scene(); scene1.add(new Movie(this, "fishnet1/Fishnet1_Music1.mkv")); scene1.add(new Movie(this, "fishnet1/Fishnet1_Music2_GreaterSilence.mkv")); scene1.add(new Movie(this, "fishnet1/Fishnet1_Music3_Beauty.mkv")); scene1.hashtag = "#fishnet1"; scene1.shader = waveShader; scene1.imageOptions.add(loadImage("fishnet1/Alison_Mermaid_dive.png")); scene1.imageOptions.add(loadImage("fishnet1/Alison_mermaid1.png")); scene1.imageOptions.add(loadImage("fishnet1/AuneCutout1_White.png")); scene1.imageOptions.add(loadImage("fishnet1/Mermaid_pattern_colors.png")); scene1.imageOptions.add(loadImage("fishnet1/Mermaid_scales_white.png")); scene1.specialPuppets.add(jellyFlock); scene1.specialPuppets.add(mermaidFlock); sceneManager.add(scene1); Puppet net = new Puppet(loadImage("fishnet2/net_violet_warped.png"), user1, 11) { PVector[] corners = new PVector[4]; PVector[] desiredCorners = new PVector[4]; PVector[] defaultCorners = new PVector[4]; PVector[] hiddenCorners = new PVector[2]; PVector[] lastTrackedCorners = new PVector[4]; int[] lastTrackedTimers = new int[4]; final PVector screenCenter = new PVector(Util.applet.width * 0.5f, Util.applet.height * 0.5f, 0f); DynamicSceneImage caughtThing; PShader projective = loadShader("fragProjective.glsl", "vertProjective.glsl"); CircularList caughtThings = new CircularList(); CircularList netImages = new CircularList(); final int imageTime = 30000; final float grabbingRadius = 300f; { defaultCorners[0] = new PVector(480, 950); defaultCorners[1] = new PVector(1440, 950); defaultCorners[2] = new PVector(1440, 1180); defaultCorners[3] = new PVector(480, 1180); hiddenCorners[0] = new PVector(480, 1080); hiddenCorners[1] = new PVector(1440, 1080); for (int i = 0; i < 4; i++) { corners[i] = defaultCorners[i].get(); desiredCorners[i] = defaultCorners[i].get(); lastTrackedCorners[i] = screenCenter.get(); lastTrackedTimers[i] = 1001; } for (int i = 0; i < 2; i++) { corners[i] = hiddenCorners[i].get(); desiredCorners[i] = hiddenCorners[i].get(); } imageTimer = 0; lastTrackedThreshhold = 1000; addTracker(user1, 7); addTracker(user2, 11); addTracker(user2, 7); alpha = 153; caughtThing = new DynamicSceneImage(loadImage("fishnet2/Ship_Mermaid_fishnet2.png")); caughtThing.baseScale = 1.5f; caughtThing.updateScale(); caughtThing.position.set(PVector.add(defaultCorners[3], defaultCorners[2])); caughtThing.position.mult(0.5f); caughtThing.desiredPosition.set(PVector.add(defaultCorners[3], defaultCorners[2])); caughtThing.desiredPosition.mult(0.5f); caughtThing.movementSpeed = 500f; netImages.add(loadImage("fishnet2/net_green_warped.png")); caughtThings.add(new DynamicSceneImage(caughtThing, loadImage("fishnet2/Alison_mermaid1_black.png"))); netImages.add(loadImage("fishnet2/net_orange_warped.png")); caughtThings.add(new DynamicSceneImage(caughtThing, loadImage("fishnet2/net_monster.png"))); caughtThings.getList().get(1).baseScale = 3f; caughtThings.getList().get(1).updateScale(); netImages.add(loadImage("fishnet2/net_violet_warped.png")); caughtThings.add(new DynamicSceneImage(caughtThing, loadImage("fishnet2/Ship_Mermaid_fishnet2.png"))); image = netImages.current(); caughtThing = caughtThings.current(); } @Override public void update(int deltaTime) { imageTimer += deltaTime; if (imageTimer > imageTime) { imageTimer = 0; image = netImages.next(); caughtThing = caughtThings.next(); } // get all the tracked points int size = 0; for (Collection ids : users.values()) { size += ids.size(); } PVector[] points = new PVector[size]; int index = 0; for (TrackedUser user : users.keySet()) { int trackPoints = users.get(user).size(); points[index] = new PVector(); for (Integer id : users.get(user)) { points[index].add(Util.transformToScreen(user.weightedAverage[id].get())); } points[index].div((float) trackPoints); index++; } // organize the points so if (points[0].x > points[1].x) { Util.swap(points[0], points[1]); } for (int i = 0; i < 2; i++) { // check last tracked if (Util.areLike(lastTrackedCorners[i], points[i])) { lastTrackedTimers[i] += deltaTime; } else { lastTrackedTimers[i] = 0; lastTrackedCorners[i].set(points[i].get()); } } if (lastTrackedTimers[0] > lastTrackedThreshhold && lastTrackedTimers[1] > lastTrackedThreshhold) { // hide both top corners if neither timer is good desiredCorners[0].set(hiddenCorners[0].get()); desiredCorners[1].set(hiddenCorners[1].get()); caughtThing.desiredPosition.set(width * 0.5f, height + (caughtThing.image.height * caughtThing.scale.y)); move(deltaTime); } else { for (int i = 0; i < 2; i++) { if (Util.areLike(desiredCorners[i], points[i], grabbingRadius)) { // we're grabbing a corner desiredCorners[i].set(points[i]); } else { // not grabbing a corner desiredCorners[i].set(defaultCorners[i].get()); } } move(deltaTime); caughtThing.desiredPosition.set(0f,0f,0f); for (int i = 0; i < 4; i++) { caughtThing.desiredPosition.add(corners[i]); } caughtThing.desiredPosition.mult(0.25f); caughtThing.desiredPosition.y -= (0.25f * caughtThing.image.height * caughtThing.scale.y); caughtThing.desiredPosition.z = 0f; } caughtThing.move(deltaTime); // update the z components of each vertex for the projective transform // initial it to 0 for (int i = 0; i < 4; i++) { corners[i].z = 0; } float[] diagonals = new float[4]; // http://stackoverflow.com/a/565282 PVector p = corners[3]; PVector q = corners[2]; PVector r = PVector.sub(corners[1], corners[3]); PVector s = PVector.sub(corners[0], corners[2]); // t = (q - p) � s / (r � s) float subCrossS = floatCross(PVector.sub(q, p), s); float rCrossS = floatCross(r, s); float t = subCrossS / rCrossS; // u = (q - p) � r / (r � s) float subCrossR = floatCross(PVector.sub(q, p), r); float u = subCrossR / rCrossS; // intersection: p + t r = q + u s PVector intersection = PVector.add(p, PVector.mult(r, t)); // now that we have the intersection we can build the diagonals for (int i = 0; i < 4; i++) { diagonals[i] = PVector.dist(intersection, corners[i]); } for (int i = 0; i < 4; i++) { // I don't think this is being used anymore, // was intended to be used for the texture warping corners[i].z = (diagonals[i] + diagonals[(i + 2) % 4]) / diagonals[(i + 2) % 4]; } } @Override public void move(int deltaTime) { for (int i = 0; i < 4; i++) { PVector direction = PVector.sub(desiredCorners[i], corners[i]); PVector newPosition = PVector.mult(direction, (deltaTime * 0.001f) * movementSpeed); corners[i].add(newPosition); } } @Override public void render() { caughtThing.render(); // render each corner at the four tracked points pushMatrix(); beginShape(); textureMode(NORMAL); textureWrap(CLAMP); texture(image); tint(255, alpha); vertex(corners[0].x, corners[0].y, 0, 0, 0); vertex(corners[1].x, corners[1].y, 0, 1, 0); vertex(corners[2].x, corners[2].y, 0, 1, 1); vertex(corners[3].x, corners[3].y, 0, 0, 1); endShape(); popMatrix(); } // this could probably be moved to Util class float floatCross(PVector a, PVector b) { return (a.x * b.y) - (a.y * b.x); } }; Puppet ship = new Puppet(loadImage("fishnet2/Ship_Mermaid_fishnet2.png"), user3, 11); ship.addTracker(user3, 7); ship.scale.set(0.4f, 0.4f); ship.defaultIdlePosition.set(960, -500); ship.position.set(ship.defaultIdlePosition.get()); ship.desiredPosition.set(ship.defaultIdlePosition.get()); ship.lastTrackedTimer = 50000; Scene scene2 = new Scene(); scene2.add(new Movie(this, "fishnet2/Fishnet2_NoMoreBlood_Music1.mkv")); scene2.add(new Movie(this, "fishnet2/fishnet2_ship_music2recut.mkv")); scene2.hashtag = "#fishnet2"; scene2.shader = waveShader; scene2.specialPuppets.add(net); scene2.specialPuppets.add(ship); sceneManager.add(scene2); Puppet splittingFish = new Puppet(loadImage("fishnet3/fishnet_stage1_white.png"), user1, 11) { final float fishOffset = 30f; int splitStage = 0; /* Total number of stages for mirroring. */ int MAX_SPLIT_STAGE = 6; public PVector startingPosition; Collection moreFish = new ArrayList(); int slashTimer = 0; int MAX_SLASH_TIME = 750; boolean enteringCollision = false; boolean alreadyTouchingFish = false; PVector[] startingPositions = new PVector[3]; int endingTimer = 0; final int MAX_ENDING_TIME = 4000; int initialAlpha = 217; PVector initialScale = new PVector(); CircularList imageOptions = new CircularList(); { startingPositions[0] = new PVector(width / 6f, height / 2f); startingPositions[1] = new PVector(width / 2f, height / 2f); startingPositions[2] = new PVector((width * 5) / 6f, height / 2f); imageOptions.add(loadImage("fishnet3/fishnet_stage1_green.png")); imageOptions.add(loadImage("fishnet3/mermaid_morph2_red.png")); imageOptions.add(loadImage("fishnet3/merlady_net_gold.png")); image = imageOptions.current(); startingPosition = startingPositions[1].get(); updateScale(); scale.mult(0.5f); position.set(startingPosition.x + fishOffset + (image.width * scale.x * 0.5f), startingPosition.y); desiredPosition.set(position); alpha = initialAlpha; initialScale.set(scale.get()); lastTrackedTimer = 500000; } @Override public void update(int deltaTime) { TrackedUser firstUser = (TrackedUser) users.keySet().toArray()[0]; PVector rightHand = Util.transformToScreen(firstUser.weightedAverage[11].get()); PVector leftHand = Util.transformToScreen(firstUser.weightedAverage[7].get()); // update last tracked timer newUserPosition.set(PVector.add(rightHand, leftHand)); newUserPosition.mult(0.5f); if (Util.areLike(lastTrackedPosition, newUserPosition)) { lastTrackedTimer += deltaTime; } else { lastTrackedTimer = 0; lastTrackedPosition.set(newUserPosition); } // check hand collisions with fish boolean collision = contains(rightHand) || contains(leftHand); for (SceneImage fish : moreFish) { if (fish.contains(rightHand) || fish.contains(leftHand)) { collision = true; break; } } if (collision) { // we're touching a fish if (alreadyTouchingFish) { // we were already touching a fish enteringCollision = false; slashTimer = 0; } else { // touching fish new enteringCollision = true; } } else { if (alreadyTouchingFish) { // no collision, was touching fish before alreadyTouchingFish = false; } enteringCollision = false; slashTimer += deltaTime; } if (splitStage == MAX_SPLIT_STAGE - 1) { // last stage make things bigger and fade away endingTimer += deltaTime; if (endingTimer > MAX_ENDING_TIME) { endingTimer = 0; nextStage(); } else { move(deltaTime); for (DynamicSceneImage fish : moreFish) { fish.move(deltaTime); } } slashTimer = 0; } else if (enteringCollision && !alreadyTouchingFish && slashTimer > MAX_SLASH_TIME) { slashTimer = 0; alreadyTouchingFish = true; enteringCollision = false; nextStage(); } } @Override public void render() { if (lastTrackedTimer < lastTrackedThreshhold) { super.render(); for (DynamicSceneImage fish : moreFish) { fish.render(); } } } void nextStage() { splitStage = (splitStage + 1) % MAX_SPLIT_STAGE; switch (splitStage) { case 0: { // starting stage with one fish moreFish.clear(); // change image imageOptions.next(); image = imageOptions.current(); // reset the scale updateScale(); scale.mult(0.5f); // pick a new starting position startingPosition = startingPositions[1].get(); position.set(startingPosition.x + fishOffset + (image.width * scale.x * 0.5f), startingPosition.y); desiredPosition.set(position); alpha = initialAlpha; break; } case 1: { // second stage with 2 fish // left fish DynamicSceneImage fish = new DynamicSceneImage(image); fish.scale.set(scale); fish.position.set(startingPosition.x - fishOffset - (fish.image.width * fish.scale.x * 0.5f), startingPosition.y); fish.rotation = PI; fish.desiredPosition.set(fish.position); fish.alpha = alpha; moreFish.add(fish); break; } case 2: { // four fish now // top fish DynamicSceneImage fish1 = new DynamicSceneImage(image); fish1.scale.set(scale); // use width and x scale because of rotation fish1.position.set(startingPosition.x, startingPosition.y - fishOffset - (fish1.image.width * fish1.scale.x * 0.5f)); fish1.rotation = PI * 1.5f; fish1.desiredPosition.set(fish1.position); fish1.alpha = alpha; moreFish.add(fish1); // bottom fish DynamicSceneImage fish2 = new DynamicSceneImage(image); fish2.scale.set(scale); fish2.position.set(startingPosition.x, startingPosition.y + fishOffset + (fish2.image.width * fish2.scale.x * 0.5f)); fish2.rotation = PI * 0.5f; fish2.desiredPosition.set(fish2.position); fish2.alpha = alpha; moreFish.add(fish2); break; } case 3: { // eight total fish // top right fish float offset = sqrt(2) * 0.5f; DynamicSceneImage fish1 = new DynamicSceneImage(image); fish1.scale.set(scale); fish1.position.set(startingPosition.x + offset * (fishOffset + (fish1.image.width * fish1.scale.x * 0.5f)), startingPosition.y - offset * (fishOffset + (fish1.image.width * fish1.scale.x * 0.5f))); fish1.rotation = PI * 1.75f; fish1.desiredPosition.set(fish1.position); fish1.alpha = alpha; moreFish.add(fish1); // top left fish DynamicSceneImage fish2 = new DynamicSceneImage(image); fish2.scale.set(scale); fish2.position.set(startingPosition.x - offset * (fishOffset + (fish2.image.width * fish2.scale.x * 0.5f)), startingPosition.y - offset * (fishOffset + (fish2.image.width * fish2.scale.x * 0.5f))); fish2.rotation = PI * 1.25f; fish2.desiredPosition.set(fish2.position); fish2.alpha = alpha; moreFish.add(fish2); // bottom right fish DynamicSceneImage fish3 = new DynamicSceneImage(image); fish3.scale.set(scale); fish3.position.set(startingPosition.x - offset * (fishOffset + (fish3.image.width * fish3.scale.x * 0.5f)), startingPosition.y + offset * (fishOffset + (fish3.image.width * fish3.scale.x * 0.5f))); fish3.rotation = PI * 0.75f; fish3.desiredPosition.set(fish3.position); fish3.alpha = alpha; moreFish.add(fish3); // bottom left fish DynamicSceneImage fish4 = new DynamicSceneImage(image); fish4.scale.set(scale); fish4.position.set(startingPosition.x + offset * (fishOffset + (fish4.image.width * fish4.scale.x * 0.5f)), startingPosition.y + offset * (fishOffset + (fish4.image.width * fish4.scale.x * 0.5f))); fish4.rotation = PI * 0.25f; fish4.desiredPosition.set(fish4.position); fish4.alpha = alpha; moreFish.add(fish4); break; } case 4: { Collection baseGroup = new ArrayList(); baseGroup.add(this); baseGroup.addAll(moreFish); // the new fish should be about 1/5 the original size // group1 positioned at PI/8 float bigOffset = sqrt(2 + sqrt(2)) * 0.5f; float smallOffset = sqrt(2 - sqrt(2)) * 0.5f; createMandala(baseGroup, new PVector(bigOffset, -1 * smallOffset)); // group2 positioned at 3PI/8 createMandala(baseGroup, new PVector(smallOffset, -1 * bigOffset)); // group3 at 5PI/8 createMandala(baseGroup, new PVector(-1 * smallOffset, -1 * bigOffset)); // group4 at 7PI/8 createMandala(baseGroup, new PVector(-1 * bigOffset, -1 * smallOffset)); // group5 at 9PI/8 createMandala(baseGroup, new PVector(-1 * bigOffset, smallOffset)); // group6 at 11PI/8 createMandala(baseGroup, new PVector(-1 * smallOffset, bigOffset)); // group7 at 13PI/8 createMandala(baseGroup, new PVector(smallOffset, bigOffset)); // group8 at 15PI/8 createMandala(baseGroup, new PVector(bigOffset, smallOffset)); break; } case 5: { // everything flies away movementSpeed = 1100f; desiredPosition.set(PVector.mult(PVector.sub(position, startingPosition), 1000f)); for (DynamicSceneImage fish : moreFish) { fish.movementSpeed = 1100f; fish.desiredPosition.set(PVector.mult(PVector.sub(fish.position, startingPosition), 1000f)); } break; } default: break; } } void createMandala(Collection baseGroup, PVector newPosition) { newPosition.mult(250f); newPosition.add(startingPosition); Collection newGroup = new ArrayList(); for (DynamicSceneImage fish : baseGroup) { PVector difference = PVector.sub(fish.position, startingPosition); difference.mult(0.2f); // the scaled distance from the original fish to the starting position DynamicSceneImage newFish = new DynamicSceneImage(fish); newFish.scale.mult(0.2f); newFish.position.set(PVector.add(newPosition, difference)); newFish.desiredPosition.set(newFish.position); newGroup.add(newFish); } moreFish.addAll(newGroup); } }; Flock monsterFlock = new Flock(user2, 11); monsterFlock.addTracker(user2, 7); monsterFlock.lastTrackedTimer = 50000; monsterFlock.defaultIdlePosition = new PVector(-400, -400); DynamicSceneImage monsterBoid1 = new DynamicSceneImage(loadImage("fishnet3/monsterFlock/mer_monster01.png")); monsterBoid1.baseScale = 0.25f; monsterBoid1.updateScale(); monsterBoid1.position.set(monsterFlock.defaultIdlePosition.get()); monsterBoid1.useForceMovement = true; monsterBoid1.alpha = 191; monsterFlock.boids.add(monsterBoid1); monsterFlock.boids.add(new DynamicSceneImage(monsterBoid1, loadImage("fishnet3/monsterFlock/mer_monster02.png"))); monsterFlock.boids.add(new DynamicSceneImage(monsterBoid1, loadImage("fishnet3/monsterFlock/mer_monster03.png"))); monsterFlock.boids.add(new DynamicSceneImage(monsterBoid1, loadImage("fishnet3/monsterFlock/mer_monster10.png"))); monsterFlock.boids.add(new DynamicSceneImage(monsterBoid1, loadImage("fishnet3/monsterFlock/mer_monster11.png"))); monsterFlock.boids.add(new DynamicSceneImage(monsterBoid1, loadImage("fishnet3/monsterFlock/monster_exq2_side.png"))); monsterFlock.boids.add(new DynamicSceneImage(monsterBoid1, loadImage("fishnet3/monsterFlock/monster_exq_1.png"))); monsterFlock.boids.add(new DynamicSceneImage(monsterBoid1, loadImage("fishnet3/monsterFlock/monster_exq_flip.png"))); monsterFlock.boids.add(new DynamicSceneImage(monsterBoid1, loadImage("fishnet3/monsterFlock/monster_flock1.png"))); monsterFlock.boids.add(new DynamicSceneImage(monsterBoid1, loadImage("fishnet3/monsterFlock/monster_flock2.png"))); monsterFlock.boids.add(new DynamicSceneImage(monsterBoid1, loadImage("fishnet3/monsterFlock/monster_flock_blue2.png"))); monsterFlock.boids.add(new DynamicSceneImage(monsterBoid1, loadImage("fishnet3/monsterFlock/monster_flock_green.png"))); monsterFlock.boids.add(new DynamicSceneImage(monsterBoid1, loadImage("fishnet3/monsterFlock/monster_flock_orange.png"))); monsterFlock.boids.add(new DynamicSceneImage(monsterBoid1, loadImage("fishnet3/monsterFlock/monster_flock_red.png"))); Scene scene3 = new Scene(); //scene3.add(new Movie(this, "fishnet3/Fishnet3_NEW_Merged_Music1.mkv")); //scene3.add(new Movie(this, "fishnet3/Fishnet3_Music2_TobinDack1.mkv")); scene3.add(new Movie(this, "fishnet3/Fishnet3_Aarhus_TobinDack_Music1 2-HD 1080p.mov")); scene3.hashtag = "#fishnet3"; scene3.shader = waveShader; scene3.imageOptions.add(loadImage("fishnet3/Fish1_black.png")); scene3.imageOptions.add(loadImage("fishnet3/Fish2_black.png")); scene3.specialPuppets.add(splittingFish); scene3.specialPuppets.add(monsterFlock); sceneManager.add(scene3); sceneManager.play(); startTime = millis(); frameTime = millis(); } void draw() { int currentTime = millis(); int elapsedFrameTime = currentTime - frameTime; frameTime = currentTime; sceneManager.update(elapsedFrameTime, currentTime); Scene currentScene = sceneManager.current(); background(Util.background.x, Util.background.y, Util.background.z); clear(); ortho(0, width, 0, height); if (calibrating) { TrackedUser user1 = sceneManager.users.get(0); // show the right hand PVector rightHand = user1.trackData[11].get(); PVector leftHand = user1.trackData[7].get(); Util.background.set(255, 255, 255); if (calibratePointOne) { calibrationPoints.add(leftHand); calibrationIndex++; if (calibrationIndex >= CALIBRATION_POINTS_COUNT) { Util.background.set(255, 50, 50); // we're done calibrating calibratePointOne = false; // get the average of the calibration points calibrationPoints.div((float) CALIBRATION_POINTS_COUNT); Util.calibration.topLeft.set(calibrationPoints); Util.calibration.recalculate(); calibrationPoints.set(0f, 0f, 0f); calibrationIndex = 0; } } else if (calibratePointTwo) { calibrationPoints.add(rightHand); calibrationIndex++; if (calibrationIndex >= CALIBRATION_POINTS_COUNT) { Util.background.set(50, 255, 50); // we're done calibrating calibratePointTwo = false; // get the average of the calibration points calibrationPoints.div((float) CALIBRATION_POINTS_COUNT); Util.calibration.bottomRight.set(calibrationPoints); Util.calibration.recalculate(); calibrationPoints.set(0f, 0f, 0f); calibrationIndex = 0; } } background(Util.background.x, Util.background.y, Util.background.z); text("veclocity: " + user1.currentHandVelocity.x + ", " + user1.currentHandVelocity.y, 20, 20); text("acceleration: " + user1.currentHandAcceleration.x + ", " + user1.currentHandAcceleration.y, 20, 120); textSize(14); strokeWeight(8); fill(0, 0, 0, 255); stroke(255, 0, 255); for (TrackedUser user : sceneManager.users) { for (int i = 0; i < user.trackData.length; i++) { PVector point = Util.transformToScreen(user.weightedAverage[i]); text(i, point.x + 0.1, point.y + 0.1, 2.0); point(point.x, point.y, 1.0); } } return; } if (userDebug) { textSize(14); strokeWeight(8); fill(0, 0, 0, 255); stroke(255, 0, 255); for (TrackedUser user : sceneManager.users) { for (int i = 0; i < user.trackData.length; i++) { PVector point = Util.transformToScreen(user.weightedAverage[i]); text(i, point.x + 0.1, point.y + 0.1, 2.0); point(point.x, point.y, 1.0); } } } currentScene.updateShader(frameTime); PShader currentShader = currentScene.shader; shader(currentShader); currentScene.render(); // render the scene images for (Puppet sceneImage : sceneManager.getAllSceneImages()) { sceneImage.render(); } } void stop() { Util.closeLog(); for (TrackedUser user : sceneManager.users) { try { user.tracker.finalize(); } catch (Throwable t) { } } } void movieEvent(Movie m) { if (m.available() && !calibrating) { m.read(); } } void renderSceneImage(SceneImage image) { /* pushMatrix(); translate(image.position.x, image.position.y); scale(image.scale.x, image.scale.y); beginShape(); textureMode(NORMAL); textureWrap(CLAMP); texture(image.image); float shadowAspectRatio = image.image.width/(float)image.image.height; float h = 100f; float w = h * shadowAspectRatio; vertex(-w, -h, 0, 0, 0); vertex(w, -h, 0, 1.0, 0); vertex(w, h, 0, 1.0, 1.0); vertex(-w, h, 0, 0, 1.0); endShape(); popMatrix();*/ image(image.image, image.position.x, image.position.y, image.image.width * image.scale.x, image.image.height *image.scale.y); } void renderScreenQuad(PImage tex) { pushMatrix(); translate(width/2, height/2, -300); beginShape(); texture(tex); vertex(-width/2, -height/2, 0, 0, 0); vertex( width/2, -height/2, 0, 1, 0); vertex( width/2, height/2, 0, 1, 1); vertex(-width/2, height/2, 0, 0, 1); endShape(); popMatrix(); } void keyPressed() { if (key == 'X') { Util.calibration.offset.x += 0.01; Util.logInfo("Offset = " + Util.calibration.offset); } else if (key == 'x') { Util.calibration.offset.x -= 0.01; Util.logInfo("Offset = " + Util.calibration.offset); } if (key == 'Y') { Util.calibration.offset.y += 0.01; Util.logInfo("Offset = " + Util.calibration.offset); } else if (key == 'y') { Util.calibration.offset.y -= 0.01; Util.logInfo("Offset = " + Util.calibration.offset); } if (key == 'i') { // in appropriate tweet // no censorship for now } // we can use this to quickly test scenes rather than waiting for the next one if (key == 'n') { // play the next scene sceneManager.next(); } if (key == 'c') { calibrating = true; userDebug = true; Util.logInfo("Calibrating"); Util.background.set(255,255,255); } if (key == 'C') { calibrating = false; userDebug = false; Util.logInfo("Done Calibrating"); Util.logInfo("TopLeft: ", Util.calibration.topLeft); Util.logInfo("BottomRight: ", Util.calibration.bottomRight); Util.logInfo("Screen Dimensions: ", Util.calibration.screenDimensions); Util.logInfo("Offset = " + Util.calibration.offset); Util.background.set(50, 50, 50); Util.saveCalibration(); } if (key == '1') { calibratePointOne = true; calibrationIndex = 0; } if (key == '2') { calibratePointTwo = true; calibrationIndex = 0; } if (key == 'd') { userDebug = !userDebug; } }