def get_tokens(doc):
    """Get the tokens and save them to .csv

    Parameters
    ----------
    doc : spacy.doc
        A doc object.

    """

    data_list = [["text", "text_lower", "lemma", "lemma_lower",
                  "part_of_speech", "is_alphabet", "is_stopword"]]

    for token in doc:
        data_list.append([
            token.text, token.lower_, token.lemma_, token.lemma_.lower(),
            token.pos_, token.is_alpha, token.is_stop
        ])

    with open("./tokens.csv", "w", encoding="utf-8", newline="") as tokens_file:
        csv.writer(tokens_file).writerows(data_list)
Get the tokens and save them to .csv

    Parameters
    ----------
    doc : spacy.doc
        A doc object.

    
scripts/step2.py