def annToRLE(self, ann):
        """
        Convert annotation which can be polygons, uncompressed RLE to RLE.
        :return: binary mask (numpy 2D array)
        """
        t = self.imgs[ann['image_id']]
        h, w = t['height'], t['width']
        segm = ann['segmentation']
        if type(segm) == list:
            # polygon -- a single object might consist of multiple parts
            # we merge all parts into one mask rle code
            rles = mask.frPyObjects(segm, h, w)
            rle = mask.merge(rles)
        elif type(segm['counts']) == list:
            # uncompressed RLE
            rle = mask.frPyObjects(segm, h, w)
        else:
            # rle
            rle = ann['segmentation']
        return rle

        Convert annotation which can be polygons, uncompressed RLE to RLE.
        :return: binary mask (numpy 2D array)
        
lib/dataset/pycocotools/coco.py