refineEstimate = ({initialModelViewTransform, projectionTransform, worldCoords: inWorldCoords, screenCoords: inScreenCoords}) => {
  const {normalizedCoords: worldCoords, param: worldParam} = normalizePoints(inWorldCoords);
  const {normalizedCoords: screenCoords, param: screenParam} = normalizePoints(inScreenCoords);

  const modelViewProjectionTransform = buildModelViewProjectionTransform(projectionTransform, initialModelViewTransform);

  const normModelViewProjectionTransform = _getNormalizedModelViewTransform(modelViewProjectionTransform, worldParam, screenParam);
  /*
   * porjection matrix
   *     [k00,   0, k02]
   * K = [  0, k11, k12]
   *     [  0,   0,   1]
   *
   *          [1/k00,      0,  -k02/k00]
   * inv(K) = [    0,  1/k11,  -k12/k11]
   * 	      [    0,      0,         1]
   *
   *
   * denote modelViewProjectionTransform as A,
   * since A = K * M,   M = inv(K) * A
   *
   *     [a00 / k00 - a20 * k02/k00,  a01 / k00 - k02/k00 * a21,  a02 / k00 - k02/k00 * a22,  a03 / k00 - k02/k00 * a23]
   * M = [a10 / k11 - a20 * k12/k11,  a11 / k11 - k12/k11 * a21,  a13 / k11 - k12/k11 * a22,  a13 / k11 - k12/111 * a23]
   *     [            a20          ,  			    a21,			a22, 		            a23] 
   */
  const a = normModelViewProjectionTransform;
  const k = projectionTransform;
  const normModelViewTransform  = [
    [a[0][0]/k[0][0] - a[2][0]*k[0][2]/k[0][0],  a[0][1]/k[0][0] - a[2][1]*k[0][2]/k[0][0],  a[0][2]/k[0][0] - a[2][2]*k[0][2]/k[0][0],  a[0][3]/k[0][0] - a[2][3]*k[0][2]/k[0][0]],
    [a[1][0]/k[1][1] - a[2][0]*k[1][2]/k[1][1],  a[1][1]/k[1][1] - a[2][1]*k[1][2]/k[1][1],  a[1][2]/k[1][1] - a[2][2]*k[1][2]/k[1][1],  a[1][3]/k[1][1] - a[2][3]*k[1][2]/k[1][1]],
    [		       a[2][0], 				   a[2][1], 				       a[2][2], 			           a[2][3]                ]
  ];

  const inlierProbs = [1.0, 0.8, 0.6, 0.4, 0.0];
  let updatedModelViewTransform = normModelViewTransform;
  let finalModelViewTransform = null;
  for (let i = 0; i < inlierProbs.length; i++) {
    const ret = _doICP({initialModelViewTransform: updatedModelViewTransform, projectionTransform, worldCoords, screenCoords, inlierProb: inlierProbs[i]});

    updatedModelViewTransform = ret.modelViewTransform;

    if (ret.err < TRACKING_THRESH) {
      finalModelViewTransform = updatedModelViewTransform;
      break;
    }
  }

  if (finalModelViewTransform === null) return null;

  const denormModelViewTransform = _getDenormalizedModelViewTransform(finalModelViewTransform, worldParam, screenParam);

  return denormModelViewTransform;
}
Trying to do normalization before running ICP
  i.e. make coodinates centroid at origin and avg distance from origin is sqrt(2)

can we get rid of projectionTransform, and just do ICP on modelViewTransform?

but couldn't make it work yet. Can someone with theoretical knowledge on ICP reach out to help?, particularly Multiview Levenberg-Marquardt ICP
  I have problem understanding the jacobian and things like that