extract image array from video

[askerpro]

I have an mp4 video file (H264 encoded).

I want to create a PIXI.extras.AnimatedSprite from this video to be able not lagging reversed animation play;

If I splice this video to image sequence the size becoming huge (about 8Mb vs 500kb of mp4).

I want to load low size mp4 file and extract image array from it to create an Animated Sprite.

Reverse playing directly from <video> by changing video.currentTime is lagging. I want to create something like hover/unhover animation.

[ivan.popelyshev]

Possible workaround: what is zipped size? You can just enable static-gzip on server for this thing. You can try several things that are not as good as video but work and have small enough size. Don't forget that in videomemory it will have full-RGBA size (4 bytes on pixel), you can use compressed textures (16 bytes per 4x4 pixels).

The things PIXI can do to video are very limited, but if you find something interesting that can be added, we'll be happy to accept your PR.

Good luck! https://github.com/pixijs/pixi.js/blob/dev/src/core/textures/VideoBaseTexture.js

[askerpro]

I still don't fully understand the structure of PIXI's backend.

I'm working with next code.

What am I trying to do:

1) load video;

2) play it on background and extract rendered image from each step to a texture

3)push every texture to array and then use it as argument for animated sprite constructor.

where am I going wrong?

var renderer =  new PIXI.WebGLRenderer(width, height);
animationContainer.append(renderer.view);
var stage = new PIXI.Stage();
var baseTexture = PIXI.VideoBaseTexture.fromVideo(video);
baseTexture.autoPlay=false;
var texture = new PIXI.Texture(baseTexture);
var sprite = new PIXI.Sprite(texture);
baseTexture.on('loaded', function(){
  	stage.addChild(sprite )
    renderer.render(stage);
	requestAnimationFrame(extractFrames);
	var rt = PIXI.RenderTexture.create(width, height);
	function extractFrames(){
	if(video.currentTime<(video.duration-0.0001)){
		video.currentTime+=(1/24);
		renderer.render(stage);
	    rt=renderer.extract.pixels(renderer._lastObjectRendered);
		extractFrames();
	}
}

Edited November 20, 2017 by askerpro
tryed to wrap code into snippet

[askerpro]

Quote

var renderer =  new PIXI.WebGLRenderer(width, height);
		
		var stage = new PIXI.Container();
		
		
		
		
		var videoBaseTexture = PIXI.VideoBaseTexture.fromVideo(video);
		var videoTexture = new PIXI.Texture(videoBaseTexture);
		var videoSprite = new PIXI.Sprite(videoTexture);
		videoBaseTexture.autoPlay=false;
		
		videoSprite.width = renderer.width;
		videoSprite.height = renderer.height;
	
		stage.addChild(videoSprite); 
		renderer.render(stage);
		var extractedTextures=[];
		
		var ticker = PIXI.ticker.shared;
		ticker.autoStart=false;
		ticker.stop();
		var extracted=false;
		var anim;
		ticker.add(function (time) {
			
			
			renderer.render(stage);
			
			
			if(video.currentTime<(video.duration-0.0001)){
				var baseRenderTexture = new PIXI.BaseRenderTexture(width, height);
				var renderTexture = new PIXI.RenderTexture(baseRenderTexture);
				var transforms=new PIXI.Transform();
				videoSprite.setTransform();
				renderer.render(videoSprite, renderTexture);
				extractedTextures.push(new PIXI.Texture(renderTexture));
				
			}else{
				
				if(!extracted){
					extracted=true;
					anim = new PIXI.extras.AnimatedSprite(extractedTextures);
					anim.height=height;
					anim.width=width;
					var animspeed=0.5;
					anim.loop=false;
					stage.removeChildren();
					
					stage.addChild(anim);
				
					
					$(el).hover(function(){
						anim.animationSpeed=animspeed;
						anim.play();
						},
						function(){
							anim.stop();
							anim.animationSpeed=-animspeed;
							anim.play();
					});
					animationContainer.append(renderer.view);
				}
				
				
			}
			
			
		});
		videoBaseTexture.on('loaded', function(){
			
			videoBaseTexture.source.play();
		
			ticker.start();
		
		
		})
		
	
	})

 

Finally, some progress with work. Code is 100% working, but on video load, extracting images from video  takes as much time as video length in this version.

Optimization time has come

Example here

http://askerpro.ddns.net

[ivan.popelyshev]

Congratulations! Of course there's better way if you go on lower level, but this is sufficient.

By the way:
 

extractedTextures.push(new PIXI.Texture(renderTexture));

//same as

extractedTextures.push(renderTexture);

Also I think I know how to make it easier by hacking. This thing creates simple Texture instead of RenderTexture, thus, saving us some FrameBuffers and memory.

//1. turn off pixi gc, so it doesnt just remove our texture from videomemory
renderer.textureGC.mode = PIXI.GC_MODES.MANUAL;

//2. we just need one temporary canvas, nothing special, just for pixi to think that we use it as source for base textures.

var myCanvas = document.createElement('canvas');
myCanvas.width = videoBaseTexture.width;
myCanvas.height= videoBaseTexture.height;

...
...
...

ticker.add((time) => {
    if (video.currentTime<...) {
        var newBaseTex = new BaseTexture(myCanvas);
        renderer.bindTexture(videoBaseTexture);
        //HIJACK THE RESULTING WEBGL TEXTURE!!!
        newBaseTex._glTextures = videoBaseTexture._glTextures;
        videoBaseTexture._glTextures = {};
 
        extracterTextures.push(new PIXI.Texture(newBaseTex);
    } else {
        if (!extracted) {
              ...
        }
    }
});

That's first time Im doing that hack  I didnt try it, it just came to me.

[askerpro]

is there a way to extract images without playing a source in realtime?

I'm sure, that GPU can process video faster than 30 fps.