def forward(self, position, normals, view_dirs, feature_vectors, img_index,
                pts_pixel_color, pts_pixel_mask, pts_patch_color=None, pts_patch_mask=None):
        """

        :param position: can be 3d coord or interpolated volume latent
        :param normals:
        :param view_dirs:
        :param feature_vectors:
        :param img_index: [N_views], used to extract corresponding weights
        :param pts_pixel_color: [N_pts, N_views, 3]
        :param pts_pixel_mask: [N_pts, N_views]
        :param pts_patch_color: [N_pts, N_views, Npx, 3]
        :return:
        """
        if self.embedder is not None:
            view_dirs = self.embedder(view_dirs)

        rendering_input = None

        if self.mode == 'idr':
            rendering_input = torch.cat([position, view_dirs, normals, feature_vectors], dim=-1)
        elif self.mode == 'no_view_dir':
            rendering_input = torch.cat([position, normals, feature_vectors], dim=-1)
        elif self.mode == 'no_normal':
            rendering_input = torch.cat([position, view_dirs, feature_vectors], dim=-1)
        elif self.mode == 'no_points':
            rendering_input = torch.cat([view_dirs, normals, feature_vectors], dim=-1)
        elif self.mode == 'no_points_no_view_dir':
            rendering_input = torch.cat([normals, feature_vectors], dim=-1)

        x = rendering_input

        for l in range(0, self.num_layers - 1):
            lin = getattr(self, "lin" + str(l))

            x = lin(x)

            if l < self.num_layers - 2:
                x = self.relu(x)  # [n_pts, d_out]

        ## extract value based on img_index
        x_extracted = torch.index_select(x, 1, img_index.long())

        weights_pixel = self.softmax(x_extracted)  # [n_pts, N_views]
        weights_pixel = weights_pixel * pts_pixel_mask
        weights_pixel = weights_pixel / (
                torch.sum(weights_pixel.float(), dim=1, keepdim=True) + 1e-8)  # [n_pts, N_views]
        final_pixel_color = torch.sum(pts_pixel_color * weights_pixel[:, :, None], dim=1,
                                      keepdim=False)  # [N_pts, 3]

        final_pixel_mask = torch.sum(pts_pixel_mask.float(), dim=1, keepdim=True) > 0  # [N_pts, 1]

        final_patch_color, final_patch_mask = None, None
        # pts_patch_color  [N_pts, N_views, Npx, 3]; pts_patch_mask  [N_pts, N_views, Npx]
        if pts_patch_color is not None:
            N_pts, N_views, Npx, _ = pts_patch_color.shape
            patch_mask = torch.sum(pts_patch_mask, dim=-1, keepdim=False) > Npx - 1  # [N_pts, N_views]

            weights_patch = self.softmax(x_extracted)  # [N_pts, N_views]
            weights_patch = weights_patch * patch_mask
            weights_patch = weights_patch / (
                    torch.sum(weights_patch.float(), dim=1, keepdim=True) + 1e-8)  # [n_pts, N_views]

            final_patch_color = torch.sum(pts_patch_color * weights_patch[:, :, None, None], dim=1,
                                          keepdim=False)  # [N_pts, Npx, 3]
            final_patch_mask = torch.sum(patch_mask, dim=1, keepdim=True) > 0  # [N_pts, 1]  at least one image sees

        return final_pixel_color, final_pixel_mask, final_patch_color, final_patch_mask


        :param position: can be 3d coord or interpolated volume latent
        :param normals:
        :param view_dirs:
        :param feature_vectors:
        :param img_index: [N_views], used to extract corresponding weights
        :param pts_pixel_color: [N_pts, N_views, 3]
        :param pts_pixel_mask: [N_pts, N_views]
        :param pts_patch_color: [N_pts, N_views, Npx, 3]
        :return:
        
mv-diffusion/One-2-3-45/reconstruction/models/sparse_sdf_network.py