Sync Sprite sheet texture with video textue

[effinrich]

Hi,

I'm relatively new to Pixi, but so far it's great.  In short, I'm cropping a segment of video, then exporting each frame of that video into a png sequence, which is then converted into sprite sheets and exported via Texture Packer.  These sprite sheets are then animated in a layer above the video and match up perfectly.  The idea is to be able to change the color of the overlaying sprite sheet as if it were happening in the video.  Make sense?  I understand if not.  

 

I have all of this working except the syncing of sprite sheets and video.  I initially started with sprite sheets of both the video and the cropped out piece, which matched perfectly, but the sprite sheets were quite large and I'm trying to reduce load time.

 

Is there a way to sync the video and sprite sheets?  They have the same number of frames, I've tried setting the frame rate to that of the original video with no luck.

 

Here's a bit of code to illustrate what I've tried:

// for the videoloader.add('video', './video/video.mp4').load(function(loader, resources) {            var video = document.getElementById('video');            video.src = resources.video.url;            var texture = PIXI.Texture.fromVideo(video);            var videoSprite = new PIXI.Sprite(texture);            stage.addChild(videoSprite);            var source = texture.baseTexture.source;            source.play();});// for the overlay to be syncedloader.add('./assets/assets.json').load(loader, resources) {// create a MovieClip (brings back memories from the days of Flash, right ?)            var mc = new PIXI.extras.MovieClip(array of frames);                      mc.position.x = 0;            mc.position.y = 0;            mc.anchor.set(0);            mc.animationSpeed = 0.333;            mc.play();            stage.addChild(mc);});

[xerver]

Syncing the js animation loop (which you have little control over) with a video play rate (which you have no control over) in javascript is an impossible task. The browser controls all the things that you would need to control to make this happen.

 

Alternatively to what you are doing, you can use a video file directly as a texture using PIXI.VideoBaseTexture if you wanted to. That would avoid all the spritesheet work, but syncing two videos may still prove challenging.

[effinrich]

Thanks for the quick reply.  Yeah, there's no way to sync the two, so I'm going with sprite sheets for both, which works perfectly except for loading so many json and sprite files.  Working on optimizing that now, but I'm not completely comfortable with the loader class just yet.  Maybe you can make a suggestion as to more efficient coding.  Currently the following code is called on click of a thumbnail of which there are three, each loading a pair of sprite sheets overlaying each other.  It works, but I feel there must be a cleaner way of doing this without clearing the loader and reloading every time the user goes between the thumbnail options.

 

Is there a way to load all of the assets on page load, then access them via user click so as not to wait for the loader?  I assumed once they were loaded I wouldn't need to worry about it again, but got an error about reloading existing assets, hence the inclusion of "loader.reset();".

 

Finally, is there a method of taking care of all of the frames in one load event rather than calling two for each set of sprite sheets and JSON I'm loading?  I'm guessing adding an ID to the "add" methods on the loader would be involved.

var loader = new PIXI.loaders.Loader();// draw spritesheets to canvasvar animate = function() {    requestAnimationFrame(animate);    renderer.render(stage);};function loadModels(modelName, spriteNumber, frameNumber) {    var faceAssetsToLoad = [];    var lipsAssetsToLoad = [];    for (var i = 0; i < spriteNumber; i++) {        faceAssetsToLoad[i] = './img/spritesheets/' + modelName + '/face/' + modelName + '_' + i + '.json';        lipsAssetsToLoad[i] = './img/spritesheets/' + modelName + '/lips/' + modelName + '_' + i + '.json';    }    loader.reset();    loader.add(faceAssetsToLoad).load(onFaceAssetsLoaded);    loader.add(lipsAssetsToLoad).load(onLipsAssetsLoaded);    function onFaceAssetsLoaded() {        var faceFrames = [];        for (var i = 0; i < frameNumber; i++) {            var val = i < 10 ? '0' + i : i;            if (val > 99) {                faceFrames.push(PIXI.Texture.fromFrame('' + modelName + '_00' + val + '.jpg'));            } else {                faceFrames.push(PIXI.Texture.fromFrame('' + modelName + '_000' + val + '.jpg'));            }        }        faceMovie = new PIXI.extras.MovieClip(faceFrames);        faceMovie.position.x = 0;        faceMovie.position.y = 0;        faceMovie.anchor.set(0);        faceMovie.animationSpeed = 0.417;        stage.addChild(faceMovie);        faceMovie.play();        animate();    }    function onLipsAssetsLoaded() {        var lipsFrames = [];        for (var i = 0; i < frameNumber; i++) {            var val = i < 10 ? '0' + i : i;            if (val > 99) {                lipsFrames.push(PIXI.Texture.fromFrame('' + modelName + '_00' + val + '.png'));            } else {                lipsFrames.push(PIXI.Texture.fromFrame('' + modelName + '_000' + val + '.png'));            }        }        lipsMovie = new PIXI.extras.MovieClip(lipsFrames);             lipsMovie.alpha = 0.6;        lipsMovie.position.x = 0;        lipsMovie.position.y = 0;        lipsMovie.anchor.set(0);        lipsMovie.animationSpeed = 0.417;        lipsMovie.tint = currentShadeHex;        stage.addChild(lipsMovie);        lipsMovie.play();        animate();    }}// click event which starts the loader$('#modelThumbs li').on("click", function() {        var ID = this.id;        switch (ID) {            case "1":                loadModels("1", 58, 521);                break;            case "2":                loadModels("2", 26, 233);                break;            case "3":                loadModels("3", 34, 304);                break;        }    }); 

I know that's a lot to digest, but any help would be great.  This framework is awesome, btw.  For the sake of dispelling any talk from my team about using createjs for this project, I built a createjs version for comparison.  The Pixi version blew the doors off the createjs version.  

 

thanks,

Rich

[xerver]

The code you have now will actually have 2 render loops, so you don't want that. I went through and cleaned it up a bit, hope this helps:

var assetData = {    1: {        sprites: 58,        frames: 521    },    2: {        sprites: 26,        frames: 233    },    3: {        sprites: 34,        frames: 304    }};var movieClips = {    1: { face: null, lips: null },    2: { face: null, lips: null },    3: { face: null, lips: null }};// draw spritesheets to canvasfunction animate() {    requestAnimationFrame(animate);    renderer.render(stage);};function loadModels() {    for (var i = 1; i < 4; ++i) {        var num = assetData[i].sprites;        for (var j = 0; j < num; ++j) {            PIXI.loader.add('./img/spritesheets/' + i + '/face/' + i + '_' + j + '.json');            PIXI.loader.add('./img/spritesheets/' + i + '/lips/' + i + '_' + j + '.json');        }    }}function prepareFrames() {    for (var i = 1; i < 4; ++i) {        var num = assetData[i].frames;        var faceFrames = [];        var lipsFrames = [];        for (var j = 0; j < num; ++j) {            var val = j < 10 ? '0' + j : j;            if (val > 99) {                faceFrames.push(PIXI.Texture.fromFrame('' + i + '_00' + val + '.jpg'));                lipsFrames.push(PIXI.Texture.fromFrame('' + i + '_00' + val + '.png'));            } else {                faceFrames.push(PIXI.Texture.fromFrame('' + i + '_000' + val + '.jpg'));                lipsFrames.push(PIXI.Texture.fromFrame('' + i + '_000' + val + '.png'));            }        }        // create and add face movie-clip        var faceMovie = new PIXI.extras.MovieClip(faceFrames);        faceMovie.visible = false;        faceMovie.position.x = 0;        faceMovie.position.y = 0;        faceMovie.anchor.set(0);        faceMovie.animationSpeed = 0.417;        stage.addChild(faceMovie);        faceMovie.stop();        movieClips[i].face = faceMovie;        // create and add lips movie-clip        var lipsMovie = new PIXI.extras.MovieClip(lipsFrames);        lipsMovie.visible = false;        lipsMovie.alpha = 0.6;        lipsMovie.position.x = 0;        lipsMovie.position.y = 0;        lipsMovie.anchor.set(0);        lipsMovie.animationSpeed = 0.417;        lipsMovie.tint = currentShadeHex;        stage.addChild(lipsMovie);        lipsMovie.stop();        movieClips[i].lips = lipsMovie;    }}loadModels();PIXI.loader.load(function () {    prepareFrames();    animate();});// click event which starts the loader$('#modelThumbs li').on("click", function() {    var ID = this.id;    var clips = movieClips[ID];    if (!clips) {        // this will mean it hasn't loaded yet, or you have a weird button.        // you may decide instead to only enable the buttons after the loading completes.        return;     }    clips.face.visible = true;    clips.face.gotoAndPlay(0);    clips.lips.visible = true;    clips.lips.gotoAndPlay(0);});