def addtoken(self, type: int, value: str, context: Context) -> bool:
        """Add a token; return True iff this is the end of the program."""
        # Map from token to label
        ilabels = self.classify(type, value, context)
        assert len(ilabels) >= 1

        # If we have only one state to advance, we'll directly
        # take it as is.
        if len(ilabels) == 1:
            [ilabel] = ilabels
            return self._addtoken(ilabel, type, value, context)

        # If there are multiple states which we can advance (only
        # happen under soft-keywords), then we will try all of them
        # in parallel and as soon as one state can reach further than
        # the rest, we'll choose that one. This is a pretty hacky
        # and hopefully temporary algorithm.
        #
        # For a more detailed explanation, check out this post:
        # https://tree.science/what-the-backtracking.html

        with self.proxy.release() as proxy:
            counter, force = 0, False
            recorder = Recorder(self, ilabels, context)
            recorder.add_token(type, value, raw=True)

            next_token_value = value
            while recorder.determine_route(next_token_value) is None:
                if not proxy.can_advance(counter):
                    force = True
                    break

                next_token_type, next_token_value, *_ = proxy.eat(counter)
                if next_token_type in (tokenize.COMMENT, tokenize.NL):
                    counter += 1
                    continue

                if next_token_type == tokenize.OP:
                    next_token_type = grammar.opmap[next_token_value]

                recorder.add_token(next_token_type, next_token_value)
                counter += 1

            ilabel = cast(int, recorder.determine_route(next_token_value, force=force))
            assert ilabel is not None

        return self._addtoken(ilabel, type, value, context)
Add a token; return True iff this is the end of the program.
src/blib2to3/pgen2/parse.py