def tokenize(s):
    '''return a list of token tuples: (token, value, start)'''
    l=[]                                  # list to hold tokens
    start=0                               # used to check for bad tokens
    sc=lexre.scanner(s)
    while 1:
        tg=sc.search()
        if tg:
            if tg.start()!=start:       # we should start at end of last token
                raise BadToken(start)
            l.append( [ (token, value, (tg.start(), tg.end())) 
                     for (token, value) in tg.groupdict().items() if value][0])
            start=tg.end()                    # prepare start for next pass
        else:
            if start!=len(s):
                raise BadToken(start)
            break
    return l
return a list of token tuples: (token, value, start)
recipes/Python/577333_Numerical_type_units/recipe-577333.py