def _compute_hessian(self, inp: _torch.Tensor, out: _torch.Tensor):
        """
        Compute Hessian of the L2 loss between the original output
        of the layer and the output computed using compressed weights.
        """
        self._inp1 = inp
        self._out1 = out
        if len(inp.shape) == 2:
            inp = inp.unsqueeze(0)
        tmp = inp.shape[0]
        if isinstance(self._layer, _nn.Linear):
            if len(inp.shape) == 3:
                inp = inp.reshape((-1, inp.shape[-1]))
            inp = inp.t()
        if isinstance(self._layer, _nn.Conv2d):
            unfold = _nn.Unfold(
                self._layer.kernel_size,
                dilation=self._layer.dilation,
                padding=self._layer.padding,
                stride=self._layer.stride,
            )
            inp = unfold(inp)
            inp = inp.permute([1, 0, 2])
            inp = inp.flatten(1)
        self._hessian *= self._nsamples / (self._nsamples + tmp)
        self._nsamples += tmp
        inp = _math.sqrt(2 / self._nsamples) * inp.float()
        self._hessian += inp.matmul(inp.t())

        Compute Hessian of the L2 loss between the original output
        of the layer and the output computed using compressed weights.
        
coremltools/optimize/torch/layerwise_compression/algorithms.py