What I want to happen:
For testing a game art style I thought of, I want to render a 3D world in pixel-art form. So for example, take a scene like this (but rendered with certain coloring / style so as to look good once pixelated):
And make it look something like this:
By playing with different ways of styling the 3D source I think the pixelated output could look nice. Of course to get this effect one just sizes the image down to ~80p and upscales it to 1080p with nearest neighbor resampling. But it's more efficient to render straight to an 80p canvas to begin with and just do the upscaling.
This is not typically how one would use a shader, to resize a bitmap in nearest neighbor format, but the performance on it is better than any other way I've found to make such a conversion in real time.
My code:
My buffer for the bitmap is stored in row major, as r1, g1, b1, a1, r2, g2, b2, a2... and I'm using gpu.js which essentially converts this JS func into a shader. My goal is to take one bitmap and return one at larger scale with nearest-neighbor scaling, so each pixel becomes a 2x2 square or 3x3 and so on. Assume inputBuffer is a scaled fraction of size of the output determined by the setOutput method.
var pixelateMatrix = gpu.createKernel(function(inputBuffer, width, height, scale) { var y = Math.floor((this.thread.x / (width[0] * 4)) / scale[0]); var x = Math.floor((this.thread.x % (width[0] * 4)) / scale[0]); var remainder = this.thread.x % 4; return inputBuffer[(x * y) + remainder]; }).setOutput([width * height * 4]); Keep in mind it's iterating over a new buffer of the full size output, so I have to find the correct coordinates that will exist in the smaller
sourceBufferbased on the current index in theoutputBuffer(index is exposed by the lib asthis.thread.x).
What's happening instead:
This, instead of making a nearest neighbor upscale, is making a nice little rainbow (above is the small normal render, below is the result of the shader, and to the right you can see some debug logging with stats about the input and output buffers):
What am I doing wrong?
Note: I asked a related question here, Is there a simpler (and still performant) way to upscale a canvas render with nearest neighbor resampling?
2 Answers
Answers 1
Update 1 - 25th May 2018
I was able to resolve most of the issues. There were quite a few
The logic of transformation was wrong, also the data was coming flipped for some reason, so I flipped cols and rows to start from bottom right
var pixelateMatrix = gpu.createKernel(function(inputBuffer, width, height, scale) { var size = width[0] * height[0] * 4; var current_index = Math.floor((size - this.thread.x)/4); var row = Math.floor(current_index / (width[0] * scale[0]) ); var col = Math.floor((current_index % width[0])/scale[0]); var index_old = Math.floor(row * (width[0] / scale[0])) + width[0] - col; var remainder = this.thread.x % 4; return inputBuffer[index_old * 4 + remainder]; }).setOutput([width * height * 4]);You were using width and height in floats, which I have changed to be calculated first and then scaled
var smallWidth = Math.floor(window.innerWidth / scale); var smallHeight = Math.floor(window.innerHeight / scale); var width = smallWidth * scale; var height = smallHeight * scale; var rt = new THREE.WebGLRenderTarget(smallWidth, smallHeight); var frameBuffer = new Uint8Array(smallHeight * smallHeight * 4); var outputBuffer = new Uint8ClampedArray(width * height * 4);The canvas size was set to while inner width and height, you need to set it to just the image width and height
context = canvas.getContext('2d'); canvas.width = width; canvas.height = height;
Below is the final JSFiddle for the same
https://jsfiddle.net/are5Lbw8/6/
Results:
Final Code for reference
var container; var camera, scene, renderer; var mouseX = 0; var mouseY = 0; var scale = 4; var windowHalfX = window.innerWidth / 2; var windowHalfY = window.innerHeight / 2; var smallWidth = Math.floor(window.innerWidth / scale); var smallHeight = Math.floor(window.innerHeight / scale); var width = smallWidth * scale; var height = smallHeight * scale; var rt = new THREE.WebGLRenderTarget(smallWidth, smallHeight); var frameBuffer = new Uint8Array(smallHeight * smallHeight * 4); var outputBuffer = new Uint8ClampedArray(width * height * 4); var output; var divisor = 2; var divisorHalf = divisor / 2; var negativeDivisorHalf = -1 * divisorHalf; var canvas; var context; var gpu = new GPU(); var pixelateMatrix = gpu.createKernel(function(inputBuffer, width, height, scale) { /* var y = Math.floor((this.thread.x / (width[0] * 4)) / scale[0]); var x = Math.floor((this.thread.x % (width[0] * 4)) / scale[0]); var remainder = this.thread.x % 4; return inputBuffer[(x * y) + remainder]; */ var size = width[0] * height[0] * 4; var current_index = Math.floor((size - this.thread.x)/4); var row = Math.floor(current_index / (width[0] * scale[0]) ); var col = Math.floor((current_index % width[0])/scale[0]); var index_old = Math.floor(row * (width[0] / scale[0])) + width[0] - col; var remainder = this.thread.x % 4; return inputBuffer[index_old * 4 + remainder]; }).setOutput([width * height * 4]); console.log(window.innerWidth); console.log(window.innerHeight); init(); animate(); function init() { container = document.createElement('div'); document.body.appendChild(container); canvas = document.createElement('canvas'); document.body.appendChild(canvas); context = canvas.getContext('2d'); canvas.width = width; canvas.height = height; camera = new THREE.PerspectiveCamera(45, window.innerWidth / window.innerHeight, 1, 2000); camera.position.z = 100; // scene scene = new THREE.Scene(); var ambient = new THREE.AmbientLight(0xbbbbbb); scene.add(ambient); var directionalLight = new THREE.DirectionalLight(0xdddddd); directionalLight.position.set(0, 0, 1); scene.add(directionalLight); // texture var manager = new THREE.LoadingManager(); manager.onProgress = function(item, loaded, total) { console.log(item, loaded, total); }; var texture = new THREE.Texture(); var onProgress = function(xhr) { if (xhr.lengthComputable) { var percentComplete = xhr.loaded / xhr.total * 100; console.log(Math.round(percentComplete, 2) + '% downloaded'); } }; var onError = function(xhr) {}; var imgLoader = new THREE.ImageLoader(manager); imgLoader.load('https://i.imgur.com/P6158Su.jpg', function(image) { texture.image = image; texture.needsUpdate = true; }); // model var objLoader = new THREE.OBJLoader(manager); objLoader.load('https://s3-us-west-2.amazonaws.com/s.cdpn.io/286022/Bulbasaur.obj', function(object) { object.traverse(function(child) { if (child instanceof THREE.Mesh) { child.material.map = texture; } }); object.scale.x = 45; object.scale.y = 45; object.scale.z = 45; object.rotation.y = 3; object.position.y = -10.5; scene.add(object); }, onProgress, onError); renderer = new THREE.WebGLRenderer({ alpha: true, antialias: false }); renderer.setPixelRatio(window.devicePixelRatio); renderer.setSize(smallWidth, smallHeight); container.appendChild(renderer.domElement); renderer.context.webkitImageSmoothingEnabled = false; renderer.context.mozImageSmoothingEnabled = false; renderer.context.imageSmoothingEnabled = false; document.addEventListener('mousemove', onDocumentMouseMove, false); window.addEventListener('resize', onWindowResize, false); } function onWindowResize() { windowHalfX = (window.innerWidth / 2) / scale; windowHalfY = (window.innerHeight / 2) / scale; camera.aspect = (window.innerWidth / window.innerHeight) / scale; camera.updateProjectionMatrix(); renderer.setSize(smallWidth, smallHeight); } function onDocumentMouseMove(event) { mouseX = (event.clientX - windowHalfX) / scale; mouseY = (event.clientY - windowHalfY) / scale; } function animate() { requestAnimationFrame(animate); render(); } var flag = 0; function render() { camera.position.x += (mouseX - camera.position.x) * .05; camera.position.y += (-mouseY - camera.position.y) * .05; camera.lookAt(scene.position); renderer.render(scene, camera); renderer.render(scene, camera, rt); renderer.readRenderTargetPixels(rt, 0, 0, smallWidth, smallHeight, frameBuffer); //console.time('gpu'); console.log(frameBuffer, [width], [height], [scale]); var outputBufferRaw = pixelateMatrix(frameBuffer, [width], [height], [scale]); //console.timeEnd('gpu'); if (flag < 15) { console.log('source', frameBuffer); console.log('output', outputBufferRaw); var count = 0; for (let i = 0; i < frameBuffer.length; i++) { if (frameBuffer[i] != 0) { count++; } } console.log('source buffer length', frameBuffer.length) console.log('source non zero', count); var count = 0; for (let i = 0; i < outputBufferRaw.length; i++) { if (outputBufferRaw[i] != 0) { count++; } } console.log('output buffer length', outputBufferRaw.length) console.log('output non zero', count); } outputBuffer = new Uint8ClampedArray(outputBufferRaw); output = new ImageData(outputBuffer, width, height); context.putImageData(output, 0, 0); flag++; } Original Answer
I have gotten close but two issues are left
- The image is getting inverted
- Sometimes your
inputBuffersize is not a multiple of 4, which causes it to misbehave.
Below is code I used
var pixelateMatrix = gpu.createKernel(function(inputBuffer, width, height, scale) { var current_index = Math.floor(this.thread.x/4); var row = Math.floor(current_index / (width[0] * scale[0]) ); var col = Math.floor((current_index % width[0])/scale[0]); var index_old = Math.floor(row * (width[0] / scale[0])) + col; var remainder = this.thread.x % 4; return inputBuffer[index_old * 4 + remainder]; }).setOutput([width * height * 4]); Below is the JSFiddle
https://jsfiddle.net/are5Lbw8/
And below is the current output
Answers 2
I think the function should look like:
var pixelateMatrix = gpu.createKernel(function(inputBuffer, width, height, scale) { var x = Math.floor((this.thread.x / (width[0] * 4)) / scale[0]); var y = Math.floor((this.thread.x % (width[0] * 4)) / scale[0]); var finalval = y * (Math.floor(width[0]/scale[0]) * 4) + (x * 4); var remainder = this.thread.x % 4; return inputBuffer[finalval + remainder]; }).setOutput([width * height * 4]); Basically, get x and y in similar manner as you, scale x and y, then converting back from (x,y) you multiple the y value by the new scaled width and add the x value. Not sure how you got x*y for that part of it.
0 comments:
Post a Comment