Tuesday, May 29, 2018

How to adjust brushing on a scatterplot matrix in d3?

Leave a Comment

I am using Mike Bostok's Block for brushing a scatterplot matrix.

I want to brush the diagonal plots of the matrix using an opacity number; middle of brush range opacity = 1 and opacity value = low on the sides of the brush range.

I couldn't find a hint online, besides this question, which does an identical action but using dc.js (with help of pretransition attribute).

Can this be achieved in d3?

Running fiddle

1 Answers

Answers 1

Interesting question.

Here's a code snippet that'll set opacity on the selected circles by their distance from the center of the selection. It does this on the brush end with a nice transition. Here's the relevant method:

function brushend(p) {      // reset opacity     svg.selectAll("circle").style('opacity', 1);      // if no brush then bail     if (brush.empty()) {       svg.selectAll(".hidden").classed("hidden", false);       return;     }      // some calculations     var e = brush.extent(),         xC = (e[1][0] + e[0][0]) / 2,         yC = (e[1][1] + e[0][1]) / 2,         // maxD is the maximum of the distance to the two edges         maxD = Math.max( Math.sqrt((xC - e[0][0])**2), Math.sqrt ((yC - e[0][1])**2) );      // those circles not hidden     svg.selectAll("circle:not(.hidden)")       .transition()       .duration(1000)       .style("opacity", function(d) {         // distance from furthest edge         var xD = Math.sqrt( (xC - d[p.x])**2 + (yC - d[p.y])**2 );         // set opacity as percent of distance         return (maxD - xD) / maxD;     });    } 

Running Code:

<!DOCTYPE html>  <meta charset="utf-8">  <style>    svg {    font: 10px sans-serif;    padding: 10px;  }    .axis,  .frame {    shape-rendering: crispEdges;  }    .axis line {    stroke: #ddd;  }    .axis path {    display: none;  }    .cell text {    font-weight: bold;    text-transform: capitalize;  }    .frame {    fill: none;    stroke: #aaa;  }    circle {    fill-opacity: .7;  }    circle.hidden {    fill: #ccc !important;  }    .extent {    fill: #000;    fill-opacity: .125;    stroke: #fff;  }    </style>  <body>  <script src="//d3js.org/d3.v3.min.js"></script>  <script>    var width = 960,      size = 230,      padding = 20;    var x = d3.scale.linear()      .range([padding / 2, size - padding / 2]);    var y = d3.scale.linear()      .range([size - padding / 2, padding / 2]);    var xAxis = d3.svg.axis()      .scale(x)      .orient("bottom")      .ticks(6);    var yAxis = d3.svg.axis()      .scale(y)      .orient("left")      .ticks(6);    var color = d3.scale.category10();    d3.csv("https://gist.githubusercontent.com/mbostock/4063663/raw/13276d1a3d8e99e74c9fe33685da3afa6ed3a926/flowers.csv", function(error, data) {    if (error) throw error;      var domainByTrait = {},        traits = d3.keys(data[0]).filter(function(d) { return d !== "species"; }),        n = traits.length;      traits.forEach(function(trait) {      domainByTrait[trait] = d3.extent(data, function(d) { return d[trait]; });    });      xAxis.tickSize(size * n);    yAxis.tickSize(-size * n);      var brush = d3.svg.brush()        .x(x)        .y(y)        .on("brushstart", brushstart)        .on("brush", brushmove)        .on("brushend", brushend);      var svg = d3.select("body").append("svg")        .attr("width", size * n + padding)        .attr("height", size * n + padding)      .append("g")        .attr("transform", "translate(" + padding + "," + padding / 2 + ")");      svg.selectAll(".x.axis")        .data(traits)      .enter().append("g")        .attr("class", "x axis")        .attr("transform", function(d, i) { return "translate(" + (n - i - 1) * size + ",0)"; })        .each(function(d) { x.domain(domainByTrait[d]); d3.select(this).call(xAxis); });      svg.selectAll(".y.axis")        .data(traits)      .enter().append("g")        .attr("class", "y axis")        .attr("transform", function(d, i) { return "translate(0," + i * size + ")"; })        .each(function(d) { y.domain(domainByTrait[d]); d3.select(this).call(yAxis); });      var cell = svg.selectAll(".cell")        .data(cross(traits, traits))      .enter().append("g")        .attr("class", "cell")        .attr("transform", function(d) { return "translate(" + (n - d.i - 1) * size + "," + d.j * size + ")"; })        .each(plot);      // Titles for the diagonal.    cell.filter(function(d) { return d.i === d.j; }).append("text")        .attr("x", padding)        .attr("y", padding)        .attr("dy", ".71em")        .text(function(d) { return d.x; });      cell.call(brush);      function plot(p) {      var cell = d3.select(this);        x.domain(domainByTrait[p.x]);      y.domain(domainByTrait[p.y]);        cell.append("rect")          .attr("class", "frame")          .attr("x", padding / 2)          .attr("y", padding / 2)          .attr("width", size - padding)          .attr("height", size - padding);        cell.selectAll("circle")          .data(data)        .enter().append("circle")          .attr("cx", function(d) { return x(d[p.x]); })          .attr("cy", function(d) { return y(d[p.y]); })          .attr("r", 4)          .style("fill", function(d) { return "#4682b4"; });    }      var brushCell;      // Clear the previously-active brush, if any.    function brushstart(p) {      if (brushCell !== this) {        d3.select(brushCell).call(brush.clear());        x.domain(domainByTrait[p.x]);        y.domain(domainByTrait[p.y]);        brushCell = this;      }    }      // Highlight the selected circles.    function brushmove(p) {      var e = brush.extent();      svg.selectAll("circle").classed("hidden", function(d) {        return e[0][0] > d[p.x] || d[p.x] > e[1][0]            || e[0][1] > d[p.y] || d[p.y] > e[1][1];      });    }      function brushend(p) {            // reset opacity      svg.selectAll("circle").style('opacity', 1);            // if no brush then bail      if (brush.empty()) {        svg.selectAll(".hidden").classed("hidden", false);        return;      }            // some calculations      var e = brush.extent(),          xC = (e[1][0] + e[0][0]) / 2,          yC = (e[1][1] + e[0][1]) / 2,          // maxD is the maximum of the distance to the two edges          maxD = Math.max( Math.sqrt((xC - e[0][0])**2), Math.sqrt ((yC - e[0][1])**2) );            // those circles not hidden      svg.selectAll("circle:not(.hidden)")        .transition()        .duration(1000)        .style("opacity", function(d) {          // distance from furthest edge          var xD = Math.sqrt( (xC - d[p.x])**2 + (yC - d[p.y])**2 );          // set opacity as percent of distance          return (maxD - xD) / maxD;      });          }  });    function cross(a, b) {    var c = [], n = a.length, m = b.length, i, j;    for (i = -1; ++i < n;) for (j = -1; ++j < m;) c.push({x: a[i], i: i, y: b[j], j: j});    return c;  }    </script>

Read More

How can I properly write this shader function in JS?

Leave a Comment

What I want to happen:

For testing a game art style I thought of, I want to render a 3D world in pixel-art form. So for example, take a scene like this (but rendered with certain coloring / style so as to look good once pixelated):

Original Image

And make it look something like this:

Pixelated Image

By playing with different ways of styling the 3D source I think the pixelated output could look nice. Of course to get this effect one just sizes the image down to ~80p and upscales it to 1080p with nearest neighbor resampling. But it's more efficient to render straight to an 80p canvas to begin with and just do the upscaling.

This is not typically how one would use a shader, to resize a bitmap in nearest neighbor format, but the performance on it is better than any other way I've found to make such a conversion in real time.

My code:

My buffer for the bitmap is stored in row major, as r1, g1, b1, a1, r2, g2, b2, a2... and I'm using gpu.js which essentially converts this JS func into a shader. My goal is to take one bitmap and return one at larger scale with nearest-neighbor scaling, so each pixel becomes a 2x2 square or 3x3 and so on. Assume inputBuffer is a scaled fraction of size of the output determined by the setOutput method.

var pixelateMatrix = gpu.createKernel(function(inputBuffer, width, height, scale) {   var y = Math.floor((this.thread.x / (width[0] * 4)) / scale[0]);   var x = Math.floor((this.thread.x % (width[0] * 4)) / scale[0]);   var remainder = this.thread.x % 4;   return inputBuffer[(x * y) + remainder];  }).setOutput([width * height * 4]); 

JSFiddle

Keep in mind it's iterating over a new buffer of the full size output, so I have to find the correct coordinates that will exist in the smaller sourceBuffer based on the current index in the outputBuffer (index is exposed by the lib as this.thread.x).

What's happening instead:

This, instead of making a nearest neighbor upscale, is making a nice little rainbow (above is the small normal render, below is the result of the shader, and to the right you can see some debug logging with stats about the input and output buffers):

Result

What am I doing wrong?

Note: I asked a related question here, Is there a simpler (and still performant) way to upscale a canvas render with nearest neighbor resampling?

2 Answers

Answers 1

Update 1 - 25th May 2018

I was able to resolve most of the issues. There were quite a few

  1. The logic of transformation was wrong, also the data was coming flipped for some reason, so I flipped cols and rows to start from bottom right

    var pixelateMatrix = gpu.createKernel(function(inputBuffer, width, height, scale) { var size = width[0] * height[0] * 4; var current_index = Math.floor((size - this.thread.x)/4);  var row = Math.floor(current_index / (width[0] * scale[0]) ); var col = Math.floor((current_index % width[0])/scale[0]); var index_old = Math.floor(row * (width[0] / scale[0])) + width[0] - col; var remainder = this.thread.x % 4; return inputBuffer[index_old * 4 + remainder];  }).setOutput([width * height * 4]); 
  2. You were using width and height in floats, which I have changed to be calculated first and then scaled

    var smallWidth = Math.floor(window.innerWidth / scale); var smallHeight = Math.floor(window.innerHeight / scale);  var width = smallWidth * scale; var height = smallHeight * scale;   var rt = new THREE.WebGLRenderTarget(smallWidth, smallHeight); var frameBuffer = new Uint8Array(smallHeight * smallHeight * 4); var outputBuffer = new Uint8ClampedArray(width * height * 4); 
  3. The canvas size was set to while inner width and height, you need to set it to just the image width and height

    context = canvas.getContext('2d'); canvas.width = width; canvas.height = height; 

Below is the final JSFiddle for the same

https://jsfiddle.net/are5Lbw8/6/

Results:

Working Upscale

Final Code for reference

var container; var camera, scene, renderer; var mouseX = 0; var mouseY = 0; var scale = 4; var windowHalfX = window.innerWidth / 2; var windowHalfY = window.innerHeight / 2; var smallWidth = Math.floor(window.innerWidth / scale); var smallHeight = Math.floor(window.innerHeight / scale);  var width = smallWidth * scale; var height = smallHeight * scale;   var rt = new THREE.WebGLRenderTarget(smallWidth, smallHeight); var frameBuffer = new Uint8Array(smallHeight * smallHeight * 4); var outputBuffer = new Uint8ClampedArray(width * height * 4); var output; var divisor = 2; var divisorHalf = divisor / 2; var negativeDivisorHalf = -1 * divisorHalf; var canvas; var context; var gpu = new GPU();  var pixelateMatrix = gpu.createKernel(function(inputBuffer, width, height, scale) { /*   var y = Math.floor((this.thread.x / (width[0] * 4)) / scale[0]);   var x = Math.floor((this.thread.x % (width[0] * 4)) / scale[0]);   var remainder = this.thread.x % 4;   return inputBuffer[(x * y) + remainder];    */    var size = width[0] * height[0] * 4;    var current_index = Math.floor((size - this.thread.x)/4);     var row = Math.floor(current_index / (width[0] * scale[0]) );    var col = Math.floor((current_index % width[0])/scale[0]);    var index_old = Math.floor(row * (width[0] / scale[0])) + width[0] - col;    var remainder = this.thread.x % 4;    return inputBuffer[index_old * 4 + remainder];  }).setOutput([width * height * 4]); console.log(window.innerWidth); console.log(window.innerHeight); init(); animate();  function init() {   container = document.createElement('div');   document.body.appendChild(container);   canvas = document.createElement('canvas');   document.body.appendChild(canvas);   context = canvas.getContext('2d');   canvas.width = width;   canvas.height = height;   camera = new THREE.PerspectiveCamera(45, window.innerWidth / window.innerHeight, 1, 2000);   camera.position.z = 100;    // scene   scene = new THREE.Scene();   var ambient = new THREE.AmbientLight(0xbbbbbb);   scene.add(ambient);   var directionalLight = new THREE.DirectionalLight(0xdddddd);   directionalLight.position.set(0, 0, 1);   scene.add(directionalLight);    // texture   var manager = new THREE.LoadingManager();   manager.onProgress = function(item, loaded, total) {     console.log(item, loaded, total);   };   var texture = new THREE.Texture();   var onProgress = function(xhr) {     if (xhr.lengthComputable) {       var percentComplete = xhr.loaded / xhr.total * 100;       console.log(Math.round(percentComplete, 2) + '% downloaded');     }   };   var onError = function(xhr) {};   var imgLoader = new THREE.ImageLoader(manager);   imgLoader.load('https://i.imgur.com/P6158Su.jpg', function(image) {     texture.image = image;     texture.needsUpdate = true;   });    // model   var objLoader = new THREE.OBJLoader(manager);   objLoader.load('https://s3-us-west-2.amazonaws.com/s.cdpn.io/286022/Bulbasaur.obj', function(object) {     object.traverse(function(child) {       if (child instanceof THREE.Mesh) {         child.material.map = texture;       }     });     object.scale.x = 45;     object.scale.y = 45;     object.scale.z = 45;     object.rotation.y = 3;     object.position.y = -10.5;     scene.add(object);   }, onProgress, onError);   renderer = new THREE.WebGLRenderer({     alpha: true,     antialias: false   });   renderer.setPixelRatio(window.devicePixelRatio);   renderer.setSize(smallWidth, smallHeight);   container.appendChild(renderer.domElement);   renderer.context.webkitImageSmoothingEnabled = false;   renderer.context.mozImageSmoothingEnabled = false;   renderer.context.imageSmoothingEnabled = false;   document.addEventListener('mousemove', onDocumentMouseMove, false);   window.addEventListener('resize', onWindowResize, false); }  function onWindowResize() {   windowHalfX = (window.innerWidth / 2) / scale;   windowHalfY = (window.innerHeight / 2) / scale;   camera.aspect = (window.innerWidth / window.innerHeight) / scale;   camera.updateProjectionMatrix();   renderer.setSize(smallWidth, smallHeight); }  function onDocumentMouseMove(event) {    mouseX = (event.clientX - windowHalfX) / scale;   mouseY = (event.clientY - windowHalfY) / scale;  }  function animate() {   requestAnimationFrame(animate);   render(); }  var flag = 0;  function render() {   camera.position.x += (mouseX - camera.position.x) * .05;   camera.position.y += (-mouseY - camera.position.y) * .05;   camera.lookAt(scene.position);   renderer.render(scene, camera);   renderer.render(scene, camera, rt);   renderer.readRenderTargetPixels(rt, 0, 0, smallWidth, smallHeight, frameBuffer);   //console.time('gpu');   console.log(frameBuffer, [width], [height], [scale]);   var outputBufferRaw = pixelateMatrix(frameBuffer, [width], [height], [scale]);   //console.timeEnd('gpu');   if (flag < 15) {     console.log('source', frameBuffer);     console.log('output', outputBufferRaw);      var count = 0;     for (let i = 0; i < frameBuffer.length; i++) {       if (frameBuffer[i] != 0) {         count++;       }     }     console.log('source buffer length', frameBuffer.length)     console.log('source non zero', count);      var count = 0;     for (let i = 0; i < outputBufferRaw.length; i++) {       if (outputBufferRaw[i] != 0) {         count++;       }     }     console.log('output buffer length', outputBufferRaw.length)     console.log('output non zero', count);   }   outputBuffer = new Uint8ClampedArray(outputBufferRaw);   output = new ImageData(outputBuffer, width, height);   context.putImageData(output, 0, 0);   flag++; } 

Original Answer

I have gotten close but two issues are left

  1. The image is getting inverted
  2. Sometimes your inputBuffer size is not a multiple of 4, which causes it to misbehave.

Below is code I used

var pixelateMatrix = gpu.createKernel(function(inputBuffer, width, height, scale) {    var current_index = Math.floor(this.thread.x/4);     var row = Math.floor(current_index / (width[0] * scale[0]) );    var col = Math.floor((current_index % width[0])/scale[0]);    var index_old = Math.floor(row * (width[0] / scale[0])) + col;    var remainder = this.thread.x % 4;    return inputBuffer[index_old * 4 + remainder];  }).setOutput([width * height * 4]); 

Below is the JSFiddle

https://jsfiddle.net/are5Lbw8/

And below is the current output

Current State

Answers 2

I think the function should look like:

var pixelateMatrix = gpu.createKernel(function(inputBuffer, width, height, scale) {     var x = Math.floor((this.thread.x / (width[0] * 4)) / scale[0]);     var y = Math.floor((this.thread.x % (width[0] * 4)) / scale[0]);     var finalval = y * (Math.floor(width[0]/scale[0]) * 4) + (x * 4);     var remainder = this.thread.x % 4;     return inputBuffer[finalval + remainder]; }).setOutput([width * height * 4]); 

Basically, get x and y in similar manner as you, scale x and y, then converting back from (x,y) you multiple the y value by the new scaled width and add the x value. Not sure how you got x*y for that part of it.

Read More

The jquery dialogbox didn't close when it is opened on usercontrol page

Leave a Comment

I used the jQuery dialog. The parent has user control which has an image to open the popup page when it is on click. The aspx page has cancel button to close jquery method and it is worked. I added the my jquery files on the parent page. I put the dialog div on parent page.

The problem is for closing the jquery dialog box and reloading the parent. If I added my jquery files on the header on the popup page, the function is called but the error is:

JavaScript runtime error: Object doesn't support property or method 'dialog'. Also the Cancel button didn't close the jquery.

However, when I uncomment my jquery files on the popup page, the cancel button works. But the another button which close the popup and reload parent page, the jquery method is not called and the popup page reload and not closed.

There is my code in jQuery

function openmodel(url, name, width, height) {   var maxHeight = dialogMaxHeight(height);   var dialogHeight = height;   if (height > maxHeight)     dialogHeight = maxHeight;       $('#dialog-model').dialog({     my: "center",     at: "center",     of: window,     autoOpen: false,     resizable: true,     max_height:'auto',           height: 'auto',     width: width,     title: name,     modal: true,     draggable: true,             open: function( ) {          $(this).load(url);                },   });  $('#dialog-model').dialog('open'); return false; }  function CloseDialogmodel() {         $('#dialog-model').dialog({         autoOpen: false,         resizable: true,              title: name,         modal: true,      });      $('#dialog-model').dialog('close');  }   function CloseDialogModelAndReloadParent() {        CloseDialogmodel(); } 

The code behind on aspx page:

 Private Sub btnDone_Click(sender As Object, e As EventArgs) Handles btnDone.Click  'do something on server   Dim cs As ClientScriptManager = Page.ClientScript   cs.RegisterStartupScript(Page.GetType (), "closeandload", "CloseDialogBoxAndReloadParent();", True) End Sub 

Hope someone tell me how to resolve the problem, so I can close the popup page and reload it. Thanks in advance.

3 Answers

Answers 1

I have same issue and I handled it like below:

Child.aspx-Markup

<div class="button">        <asp:Button ID="btnDone" Text="Done" runat="server" />      <a ID="lnkClose" onclick="DecideForParent(false,true);return false;">         Close     </a> </div>  <script type="text/javascript">     function DecideForParent(AllowReferesh, AllowClose) {         try {             if (AllowReferesh)                 window.opener.HandlePopupResult();         }         catch (ex) {         }         // Close dialog         if (AllowClose)             $('#dialog-model').dialog('close');     } </script> 

Child.aspx-Code behind (C#)

private void btnDone_Click(object sender, EventArgs e)  {     // Do something ...     Page.ClientScript.RegisterStartupScript(this.GetType(), "CallreturnToParent", "$(window).load(function () { DecideForParent(true,true);return false;});", true); } 

Parent.aspx-Markup

<script type="text/javascript">     function HandlePopupResultLURow() {         document.getElementById('<%=btnRefresh.ClientID%>').click();     } </script> 

Parent.aspx-Code behind (C#)

private void btnRefresh_Click(object sender, EventArgs e)  {     // Refresh your data         } 

Answers 2

if i guess right, you create the dialog like this

$( "#dialog-confirm" ).dialog({   //....   modal: true,   buttons: {     "Do Action": function() {               CloseDialogBoxAndReloadParent();     },     "Cancel" : function() {      CloseDialogBox();      }   } }); 

} );

if you add the element to the parameters

 "Cancel" : function() {      CloseDialogBox(this);      } 

you can access the right element

function CloseDialogBox(element) { $(element).dialog('close'); 

}

Answers 3

function CloseDialogmodel () {     $('#dialog-model').dialog({ autoOpen: false, resizable: true, title: name, modal: true, });     $('#dialog-model').modal('hide'); } 
Read More

Run MySQL a prefilled docker container as random (non-root) linux user?

Leave a Comment

I am trying to create an OpenShift compliant prefilled MySQL container image.

Running the container with a specified user is (sadly) not an option for us.

This is a problem since OpenShift simply creates some random UID without a username so setting a username at runtime with a script before starting the MySQL service is not an option.

Is there any way to get MySQL to run with any random UID in a docker container?

edit: The idea behind this question is being able to start a MySQL container like this Dockerfile for randomusermysql:example

FROM mysql:5.7.22  #IMPORTANT: MySQL Container runs init in alphanumerical order! COPY src/some.sql /docker-entrypoint-initdb.d/  ENV MYSQL_ROOT_PASSWORD='somepw'  RUN mkdir -p /var/lib/mysql2 && \     chown -R mysql:mysql /var/lib/mysql2 && \     chmod -R 777 /var/lib/mysql2 && \     sed -i 's|/var/lib/mysql|/var/lib/mysql2|g' /etc/mysql/mysql.conf.d/mysqld.cnf && \     sed -i 's|exec "$@"||g' /entrypoint.sh && \     /entrypoint.sh mysqld && \     chmod -R 777 /var/lib/mysql2/ && \     chown -R mysql:mysql /var/lib/mysql2 && \     find /var/lib/mysql2/ -name "*.cnf" -exec chmod 775 {} \; && \     echo 'exec "$@"' >> /entrypoint.sh 

Then starting it like this

docker run -u 123456789 randomusermysql:example 

Results in the following error when starting the container

2018-05-22T11:39:35.084034Z 0 [ERROR] Fatal error: Can't open and lock privilege tables: Table storage engine for 'user' doesn't have this option 2018-05-22T11:39:35.084235Z 0 [ERROR] Aborting 

There is no possibility of passing the user as docker ENV when starting the container

edit2: Bounty text is incorrect.
Corrected bounty statement:
A solution is needed with a prefilled MySQL database without just copying the dump files into /docker-entrypoint-initdb.d directory!

1 Answers

Answers 1

The problem is that if you pre-create the database files as part of the image in the required location, is that they will have user the same as the Dockerfile created them. You will not know in advance what the user is and so can't match what the database may be started as, causing MySQL to fail on startup because the directory owning the database files is not the same as what it is being started as.

The only solution I have seen to this is to add the database files into the image in a tar file at some location. In the startup command for the database, create the directory for the database and unpack the tar file into it. This way the directory and the files will be the user that MySQL runs as.

Note that you will want to make the parent directory of where the database directory is to be created, group root and writable by group so you can create the database directory when image run as arbitrary user ID for which there is no passwd file entry. In that case, the group ID will fallback to being root group and so that will allow the database directory to be created.

Read More

What's the proper way to escape URL variables with Spring's RestTemplate when calling a Spring RestController?

Leave a Comment

When calling RestTemplate.exchange to do a get request, such as:

String foo = "fo+o"; String bar = "ba r"; restTemplate.exchange("http://example.com/?foo={foo}&bar={bar}", HttpMethod.GET, null, foo, bar) 

what's the proper to have the URL variables correctly escaped for the get request?

Specifically, how do I get pluses (+) correctly escaped because Spring is interpreting as spaces, so, I need to encode them.

I tried using UriComponentsBuilder like this:

String foo = "fo+o"; String bar = "ba r"; UriComponentsBuilder ucb = UriComponentsBuilder.fromUriString("http://example.com/?foo={foo}&bar={bar}"); System.out.println(ucb.build().expand(foo, bar).toUri()); System.out.println(ucb.build().expand(foo, bar).toString()); System.out.println(ucb.build().expand(foo, bar).toUriString()); System.out.println(ucb.build().expand(foo, bar).encode().toUri()); System.out.println(ucb.build().expand(foo, bar).encode().toString()); System.out.println(ucb.build().expand(foo, bar).encode().toUriString()); System.out.println(ucb.buildAndExpand(foo, bar).toUri()); System.out.println(ucb.buildAndExpand(foo, bar).toString()); System.out.println(ucb.buildAndExpand(foo, bar).toUriString()); System.out.println(ucb.buildAndExpand(foo, bar).encode().toUri()); System.out.println(ucb.buildAndExpand(foo, bar).encode().toString()); System.out.println(ucb.buildAndExpand(foo, bar).encode().toUriString()); 

and that printed:

http://example.com/?foo=fo+o&bar=ba%20r http://example.com/?foo=fo+o&bar=ba r http://example.com/?foo=fo+o&bar=ba r http://example.com/?foo=fo+o&bar=ba%20r http://example.com/?foo=fo+o&bar=ba%20r http://example.com/?foo=fo+o&bar=ba%20r http://example.com/?foo=fo+o&bar=ba%20r http://example.com/?foo=fo+o&bar=ba r http://example.com/?foo=fo+o&bar=ba r http://example.com/?foo=fo+o&bar=ba%20r http://example.com/?foo=fo+o&bar=ba%20r http://example.com/?foo=fo+o&bar=ba%20r 

The space is correctly escaped in some instances, but the plus is never escaped.

I also tried UriTemplate like this:

String foo = "fo+o"; String bar = "ba r"; UriTemplate uriTemplate = new UriTemplate("http://example.com/?foo={foo}&bar={bar}"); Map<String, String> vars = new HashMap<>(); vars.put("foo", foo); vars.put("bar", bar); URI uri = uriTemplate.expand(vars); System.out.println(uri); 

with the exact same result:

http://example.com/?foo=fo+o&bar=ba%20r 

5 Answers

Answers 1

Apparently, the correct way of doing this is by defining a factory and changing the encoding mode:

String foo = "fo+o"; String bar = "ba r"; DefaultUriBuilderFactory factory = new DefaultUriBuilderFactory(); factory.setEncodingMode(DefaultUriBuilderFactory.EncodingMode.VALUES_ONLY); URI uri = factory.uriString("http://example.com/?foo={foo}&bar={bar}").build(foo, bar); System.out.println(uri); 

That prints out:

http://example.com/?foo=fo%2Bo&bar=ba%20r 

This is documented here: https://docs.spring.io/spring/docs/current/spring-framework-reference/web.html#web-uri-encoding

Answers 2

I think your problem here is that RFC 3986, on which UriComponents and by extension UriTemplate are based, does not mandate the escaping of + in a query string.

The spec's view on this is simply:

sub-delims  = "!" / "$" / "&" / "'" / "(" / ")"                   / "*" / "+" / "," / ";" / "="  pchar       = unreserved / pct-encoded / sub-delims / ":" / "@"  query       = *( pchar / "/" / "?" )  URI         = scheme ":" hier-part [ "?" query ] [ "#" fragment ] 

If your web framework (Spring MVC, for example!) is interpreting + as a space, then that is its decision and not required under the URI spec.

With reference to the above, you will also see that !$'()*+,; are not escaped by UriTemplate. = and & are escaped because Spring has taken an "opinionated" view of what a query string looks like -- a sequence of key=value pairs.

Likewise, #[] and whitespace are escaped because they are illegal in a query string under the spec.

Granted, none of this is likely to be any consolation to you if you just quite reasonably want your query parameters escaped!

To actually encode the query params so your web framework can tolerate them, you could use something like org.springframework.web.util.UriUtils.encode(foo, charset).

Answers 3

I'm starting to believe this is a bug and I reported here: https://jira.spring.io/browse/SPR-16860

Currently, my workaround is this:

String foo = "fo+o"; String bar = "ba r"; String uri = UriComponentsBuilder.     fromUriString("http://example.com/?foo={foo}&bar={bar}").     buildAndExpand(vars).toUriString(); uri = uri.replace("+", "%2B"); // This is the horrible hack. try {     return new URI(uriString); } catch (URISyntaxException e) {     throw new RuntimeException("UriComponentsBuilder generated an invalid URI.", e); } 

which is a horrible hack that might fail in some situations.

Answers 4

For this one I would still prefer the encoding to be resolved using a proper method instead of using a Hack like you did. I would just use something like below

String foo = "fo+o"; String bar = "ba r"; MyUriComponentsBuilder ucb = MyUriComponentsBuilder.fromUriString("http://example.com/?foo={foo}&bar={bar}");  UriComponents uriString = ucb.buildAndExpand(foo, bar); // http://example.com/?foo=fo%252Bo&bar=ba+r URI x = uriString.toUri();  // http://example.com/?foo=fo%2Bo&bar=ba+r String y = uriString.toUriString();  // http://example.com/?foo=fo%2Bo&bar=ba+r String z = uriString.toString(); 

And of course the class is like below

class MyUriComponentsBuilder extends UriComponentsBuilder {     protected UriComponentsBuilder originalBuilder;       public MyUriComponentsBuilder(UriComponentsBuilder builder) {         // TODO Auto-generated constructor stub         originalBuilder = builder;     }       public static MyUriComponentsBuilder fromUriString(String uri) {         return new MyUriComponentsBuilder(UriComponentsBuilder.fromUriString(uri));     }       @Override     public UriComponents buildAndExpand(Object... values) {         // TODO Auto-generated method stub         for (int i = 0; i< values.length; i ++) {             try {                 values[i] = URLEncoder.encode((String) values[i], StandardCharsets.UTF_8.toString());             } catch (UnsupportedEncodingException e) {                 // TODO Auto-generated catch block                 e.printStackTrace();             }         }         return originalBuilder.buildAndExpand(values);     }  } 

Still not a cleanest possible way but better then doing a hardcoded replace approach

Answers 5

You could use UriComponentsBuilder in Spring (org.springframework.web.util.UriComponentsBuilder)

String url = UriComponentsBuilder     .fromUriString("http://example.com/")     .queryParam("foo", "fo+o")     .queryParam("bar", "ba r")     .build().toUriString();  restTemplate.exchange(url , HttpMethod.GET, httpEntity); 
Read More

webpack plugin that adds other plugins

Leave a Comment

I'm trying to create a plugin that acts as a plugin wrapper, loading configuration, files, etc. and adding more plugins to the current webpack compilation process.

In the apply function I create the plugins and then apply them against the original compiler, something like this:

apply(compiler) {   const plugins = [     new HtmlWebpackPlugin(options1),     new HtmlWebpackPlugin(options2),     ...   ];    plugins.forEach((plugin) => {     plugin.apply(compiler);   }); }); 

My plugin doesn't add any hook to the webpack compiler but let other plugins with add them. It works nice when the container project uses webpack 4... but I found an error when working with different webpack versions.

One of the injected plugins is the html-webpack-plugin, which is a dependency of my plugin:

package.json

"dependencies": {   "copy-webpack-plugin": "^4.5.1",   "html-webpack-plugin": "^3.2.0",   "webpack-bundle-analyzer": "^2.11.3" } 

So as you can imagine, I inject instances of those 3 plugins.

The problem comes when the project using my plugin uses webpack 3, because at some point, inside html-webpack-plugin requires NodeTemplatePlugin which is under html-webpack-plugin/node_modules/webpack. So as far as I understand, html-webpack-plugin has its own webpack which is webpack 4. Since the original passed compiler comes from webpack 3, and the one expected in NodeTemplatePlugin is webpack 4 I get an error when the latest tries to access compiler.hooks.thisCompilation because it doesn't exist in the compiler from webpack 3.

Funny thing is, in my package json webpack is defined as a peerDependency (same as in html-webpack-plugin) so it shouldn't be installed under the node_modules of my plugin.

"peerDependencies": {   "webpack": "^2.0.0 || ^3.0.0 || ^4.0.0" }, 

Any idea of why is this happening, or an alternative on how to inject plugins from another plugin?

If you need more context/code, you can check it here:
https://github.com/danikaze/generate-examples-index-webpack-plugin/blob/dev/src/ExamplesGenerator.js#L61

0 Answers

Read More

Java SecurityManager - how to ensure a method is run only by another method?

Leave a Comment

I want B to be run only by the private method A#getSensitiveData() that uses or does some processing on sensitive data (example: cryptographic keys, national id, whatever).

public final class A{     private transient final B sensitiveHolder; //set at Constructor     public A(B sensitiveHolder){         this.sensitiveHolder = sensitiveHolder;     }     private final byte[] getSensitiveData(){         return sensitiveHolder.getSensitiveData();     } }  public final class B{     private transient final byte[] sensitiveData;//encrypt and set at Constructor     public final byte[] getSensitiveData(){         //check if it is run by A#getSensitiveData(); if it is, decrypt by DEK and give plaintext.     } } 

Please take into account that the code would be obfuscated, so please refrain from putting in any package names as String.

What must I write with SecurityManager#checkPrivilege() and AccessController.doPrivileged() before I can achieve such an effect?

EDIT: Obviously this is different because the so called "answer" does not contain any CODE. WORKING CODE is worth infinitely more than "oh, just do this and that".

2 Answers

Answers 1

You could do something like this:

private boolean verify(final StackTraceElement e[]) {     boolean doNext = false;     for (final StackTraceElement s : e) {         if (doNext && s.getClassName().equals("A") && s.getMethodName().equals("getSensitiveData"))             return true;         doNext = s.getMethodName().equals("getStackTrace");     }     return false; } 

And to call the method:

public final byte[] getSensitiveData(StackTraceElement e[]){     if (verify(e)) {         // Do something good     } } 

In your A class call your B class like this:

return sensitiveHolder.getSensitiveData(Thread.currentThread().getStackTrace()); 

I don't know if this is what you need or it is near that. You could play around the values in the equals section of the if. I got and modified the example from this site.

Answers 2

If you're able to use JDK 9+, which introduces StackWalker, this sort of thing might work for you. This technique seems to supersede use of sun.reflect.Reflection#getCallerClass(int). (I hope you weren't counting on a SecurityManager-related answer.)

package asdf;  import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test;  import java.util.EnumSet; import java.util.List; import java.util.stream.Collectors;  public class Asdf {      @Test     public void what() {         get();     }      void get() {         StackWalker.StackFrame stackFrame =                 StackWalker.getInstance(EnumSet.of(StackWalker.Option.RETAIN_CLASS_REFERENCE))                         .walk(stream -> {                             List<StackWalker.StackFrame> stackFrames = stream.collect(Collectors.toList());                             return stackFrames.get(1);                         });         Assertions.assertEquals(Asdf.class, stackFrame.getDeclaringClass());         Assertions.assertEquals("what", stackFrame.getMethodName());         Assertions.assertEquals(0, stackFrame.getMethodType().parameterCount());     } } 
Read More