def test_layernorm_with_cycles(self):
        """
        The test basically forms the following subgraph:

        layernorm_sigmoid_mul_1 = layernorm_sigmoid_mul(...)
        gemm_rcr_2 = gemm_rcr(layernorm_sigmoid_mul_1)
        layernorm_3 = layernorm(gemm_rcr_2)
        layernorm_4 = layernorm(...)
        gemm_rcr_5 = gemm_rcr(layernorm_4)
        layernorm_sigmoid_mul_6 = layernorm_sigmoid_mul(gemm_rcr_5)

        For example, grouping (layernorm_sigmoid_mul_1, layernorm_sigmoid_mul_6)
        and (gemm_rcr_2, gemm_rcr_5) at the same time would introduce a cycle
        between the fused group ops, because we have the following dependency:
            layernorm_sigmoid_mul_1 -> gemm_rcr_2
            gemm_rcr_5 -> layernrom_sigmoid_mul_6
        """
        torch.manual_seed(0)
        testname = "layernorm_with_cycles_0"
        dtype = "float16"
        batch_sizes = [1, 2048]
        eps = 1e-5

        Input0 = Tensor(
            shape=[IntVar(values=batch_sizes, name="batch"), IntImm(value=1024)],
            dtype=dtype,
            name="input0",
            is_input=True,
        )
        reshape_to_shape_0 = [-1, 32, 32]
        reshape_0 = ops.reshape()(Input0, reshape_to_shape_0)

        W0 = Tensor(shape=[IntImm(16), IntImm(32)], name="w0", is_input=True)
        gemm_rcr_0 = ops.gemm_rcr()(reshape_0, W0)

        reshape_to_shape_1 = [-1, 512]
        reshape_1 = ops.reshape()(gemm_rcr_0, reshape_to_shape_1)

        Input1 = Tensor(
            shape=[IntVar(values=batch_sizes, name="batch"), IntImm(value=512)],
            dtype=dtype,
            name="input1",
            is_input=True,
        )
        elementwise_0 = ops.elementwise(func_enum=FuncEnum.MUL)(reshape_1, Input1)

        W1 = Tensor(shape=[IntImm(3821), IntImm(512)], name="w1", is_input=True)
        gemm_rcr_1 = ops.gemm_rcr()(elementwise_0, W1)

        concat_dim = 1
        concatenate_0 = ops.concatenate()([Input1, gemm_rcr_1], concat_dim)

        Gamma0 = Tensor(shape=[IntImm(4333)], name="gamma0", is_input=True)
        Beta0 = Tensor(shape=[IntImm(4333)], name="beta0", is_input=True)
        layernorm_0 = ops.layernorm(normalized_shape=None)(
            concatenate_0, Gamma0, Beta0, [IntImm(4333)], eps
        )

        Input2 = Tensor(
            shape=[IntVar(values=batch_sizes, name="batch"), IntImm(value=256)],
            dtype=dtype,
            name="input2",
            is_input=True,
        )
        W2 = Tensor(shape=[IntImm(256), IntImm(256)], name="w2", is_input=True)
        gemm_rcr_2 = ops.gemm_rcr()(Input2, W2)

        Gamma1 = Tensor(shape=[IntImm(256)], name="gamma1", is_input=True)
        Beta1 = Tensor(shape=[IntImm(256)], name="beta1", is_input=True)
        layernorm_1 = ops.layernorm(normalized_shape=None)(
            gemm_rcr_2, Gamma1, Beta1, [IntImm(256)], eps
        )
        elementwise_1 = ops.elementwise(func_enum=FuncEnum.SIGMOID)(layernorm_1)
        elementwise_2 = ops.elementwise(func_enum=FuncEnum.MUL)(
            gemm_rcr_2, elementwise_1
        )

        W3 = Tensor(shape=[IntImm(2048), IntImm(256)], name="w3", is_input=True)
        gemm_rcr_3 = ops.gemm_rcr()(elementwise_2, W3)

        Gamma2 = Tensor(shape=[IntImm(2048)], name="gamma2", is_input=True)
        Beta2 = Tensor(shape=[IntImm(2048)], name="beta2", is_input=True)
        layernorm_2 = ops.layernorm(normalized_shape=None)(
            gemm_rcr_3, Gamma2, Beta2, [IntImm(2048)], eps
        )

        Input3 = Tensor(
            shape=[IntVar(values=batch_sizes, name="batch"), IntImm(value=1320)],
            dtype=dtype,
            name="input3",
            is_input=True,
        )
        Gamma3 = Tensor(shape=[IntImm(1320)], name="gamma3", is_input=True)
        Beta3 = Tensor(shape=[IntImm(1320)], name="beta3", is_input=True)
        layernorm_3 = ops.layernorm(normalized_shape=None)(
            Input3, Gamma3, Beta3, [IntImm(1320)], eps
        )

        W4 = Tensor(shape=[IntImm(128), IntImm(1320)], name="w4", is_input=True)
        gemm_rcr_4 = ops.gemm_rcr()(layernorm_3, W4)

        Gamma4 = Tensor(shape=[IntImm(128)], name="gamma4", is_input=True)
        Beta4 = Tensor(shape=[IntImm(128)], name="beta4", is_input=True)
        layernorm_4 = ops.layernorm(normalized_shape=None)(
            gemm_rcr_4, Gamma4, Beta4, [IntImm(128)], eps
        )
        elementwise_3 = ops.elementwise(func_enum=FuncEnum.SIGMOID)(layernorm_4)
        elementwise_4 = ops.elementwise(func_enum=FuncEnum.MUL)(
            gemm_rcr_4, elementwise_3
        )

        output_0 = ops.concatenate()(
            [elementwise_4, layernorm_3, layernorm_0, layernorm_2], concat_dim
        )
        output_0._attrs["name"] = "output_0"
        output_0._attrs["is_output"] = True

        target = detect_target()
        module = compile_model(
            [output_0],
            target,
            "./tmp",
            testname,
        )

        for batch in batch_sizes:
            input0_pt = get_random_torch_tensor([batch, 1024], dtype)
            reshape_0_pt = torch.reshape(input0_pt, reshape_to_shape_0)

            w0_pt = get_random_torch_tensor([16, 32], dtype)
            gemm_rcr_0_pt = torch.nn.functional.linear(reshape_0_pt, w0_pt)

            reshape_1_pt = torch.reshape(gemm_rcr_0_pt, reshape_to_shape_1)

            input1_pt = get_random_torch_tensor([batch, 512], dtype)
            elementwise_0_pt = reshape_1_pt * input1_pt

            w1_pt = get_random_torch_tensor([3821, 512], dtype)
            gemm_rcr_1_pt = torch.nn.functional.linear(elementwise_0_pt, w1_pt)

            concatenate_0_pt = torch.cat([input1_pt, gemm_rcr_1_pt], concat_dim)

            gamma0_pt = get_random_torch_tensor([4333], dtype)
            beta0_pt = get_random_torch_tensor([4333], dtype)
            layernorm_0_pt = torch.nn.functional.layer_norm(
                concatenate_0_pt,
                concatenate_0_pt.size()[1:],
                gamma0_pt,
                beta0_pt,
                eps=eps,
            )

            input2_pt = get_random_torch_tensor([batch, 256], dtype)
            w2_pt = get_random_torch_tensor([256, 256], dtype)
            gemm_rcr_2_pt = torch.nn.functional.linear(input2_pt, w2_pt)

            gamma1_pt = get_random_torch_tensor([256], dtype)
            beta1_pt = get_random_torch_tensor([256], dtype)
            layernorm_1_pt = torch.nn.functional.layer_norm(
                gemm_rcr_2_pt, gemm_rcr_2_pt.size()[1:], gamma1_pt, beta1_pt, eps=eps
            )
            elementwise_1_pt = torch.sigmoid(layernorm_1_pt)
            elementwise_2_pt = torch.mul(gemm_rcr_2_pt, elementwise_1_pt)

            w3_pt = get_random_torch_tensor([2048, 256], dtype)
            gemm_rcr_3_pt = torch.nn.functional.linear(elementwise_2_pt, w3_pt)

            gamma2_pt = get_random_torch_tensor([2048], dtype)
            beta2_pt = get_random_torch_tensor([2048], dtype)
            layernorm_2_pt = torch.nn.functional.layer_norm(
                gemm_rcr_3_pt, gemm_rcr_3_pt.size()[1:], gamma2_pt, beta2_pt, eps=eps
            )

            input3_pt = get_random_torch_tensor([batch, 1320], dtype)
            gamma3_pt = get_random_torch_tensor([1320], dtype)
            beta3_pt = get_random_torch_tensor([1320], dtype)
            layernorm_3_pt = torch.nn.functional.layer_norm(
                input3_pt, input3_pt.size()[1:], gamma3_pt, beta3_pt, eps=eps
            )

            w4_pt = get_random_torch_tensor([128, 1320], dtype)
            gemm_rcr_4_pt = torch.nn.functional.linear(layernorm_3_pt, w4_pt)

            gamma4_pt = get_random_torch_tensor([128], dtype)
            beta4_pt = get_random_torch_tensor([128], dtype)
            layernorm_4_pt = torch.nn.functional.layer_norm(
                gemm_rcr_4_pt, gemm_rcr_4_pt.size()[1:], gamma4_pt, beta4_pt, eps=eps
            )
            elementwise_3_pt = torch.sigmoid(layernorm_4_pt)
            elementwise_4_pt = torch.mul(gemm_rcr_4_pt, elementwise_3_pt)

            output_0_pt = torch.cat(
                [elementwise_4_pt, layernorm_3_pt, layernorm_0_pt, layernorm_2_pt],
                concat_dim,
            )

            inputs = {
                "input0": input0_pt,
                "input1": input1_pt,
                "input2": input2_pt,
                "input3": input3_pt,
                "w0": w0_pt,
                "w1": w1_pt,
                "w2": w2_pt,
                "w3": w3_pt,
                "w4": w4_pt,
                "gamma0": gamma0_pt,
                "beta0": beta0_pt,
                "gamma1": gamma1_pt,
                "beta1": beta1_pt,
                "gamma2": gamma2_pt,
                "beta2": beta2_pt,
                "gamma3": gamma3_pt,
                "beta3": beta3_pt,
                "gamma4": gamma4_pt,
                "beta4": beta4_pt,
            }
            y = torch.empty_like(output_0_pt)
            module.run_with_tensors(inputs, [y])
            self.assertTrue(torch.allclose(output_0_pt, y, atol=0.03, rtol=0.03))

        The test basically forms the following subgraph:

        layernorm_sigmoid_mul_1 = layernorm_sigmoid_mul(...)
        gemm_rcr_2 = gemm_rcr(layernorm_sigmoid_mul_1)
        layernorm_3 = layernorm(gemm_rcr_2)
        layernorm_4 = layernorm(...)
        gemm_rcr_5 = gemm_rcr(layernorm_4)
        layernorm_sigmoid_mul_6 = layernorm_sigmoid_mul(gemm_rcr_5)

        For example, grouping (layernorm_sigmoid_mul_1, layernorm_sigmoid_mul_6)
        and (gemm_rcr_2, gemm_rcr_5) at the same time would introduce a cycle
        between the fused group ops, because we have the following dependency:
            layernorm_sigmoid_mul_1 -> gemm_rcr_2
            gemm_rcr_5 -> layernrom_sigmoid_mul_6
        
tests/unittest/compiler/test_group_fusions.py