/**
 * Copyright (c) 2008, SnakeYAML
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */
package org.yaml.snakeyaml.scanner;

import java.nio.ByteBuffer;
import java.nio.charset.CharacterCodingException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
import org.yaml.snakeyaml.DumperOptions;
import org.yaml.snakeyaml.LoaderOptions;
import org.yaml.snakeyaml.comments.CommentType;
import org.yaml.snakeyaml.error.Mark;
import org.yaml.snakeyaml.error.YAMLException;
import org.yaml.snakeyaml.reader.StreamReader;
import org.yaml.snakeyaml.tokens.AliasToken;
import org.yaml.snakeyaml.tokens.AnchorToken;
import org.yaml.snakeyaml.tokens.BlockEndToken;
import org.yaml.snakeyaml.tokens.BlockEntryToken;
import org.yaml.snakeyaml.tokens.BlockMappingStartToken;
import org.yaml.snakeyaml.tokens.BlockSequenceStartToken;
import org.yaml.snakeyaml.tokens.CommentToken;
import org.yaml.snakeyaml.tokens.DirectiveToken;
import org.yaml.snakeyaml.tokens.DocumentEndToken;
import org.yaml.snakeyaml.tokens.DocumentStartToken;
import org.yaml.snakeyaml.tokens.FlowEntryToken;
import org.yaml.snakeyaml.tokens.FlowMappingEndToken;
import org.yaml.snakeyaml.tokens.FlowMappingStartToken;
import org.yaml.snakeyaml.tokens.FlowSequenceEndToken;
import org.yaml.snakeyaml.tokens.FlowSequenceStartToken;
import org.yaml.snakeyaml.tokens.KeyToken;
import org.yaml.snakeyaml.tokens.ScalarToken;
import org.yaml.snakeyaml.tokens.StreamEndToken;
import org.yaml.snakeyaml.tokens.StreamStartToken;
import org.yaml.snakeyaml.tokens.TagToken;
import org.yaml.snakeyaml.tokens.TagTuple;
import org.yaml.snakeyaml.tokens.Token;
import org.yaml.snakeyaml.tokens.ValueToken;
import org.yaml.snakeyaml.util.ArrayStack;
import org.yaml.snakeyaml.util.UriEncoder;

/**
 * <pre>
 * Scanner produces tokens of the following types:
 * STREAM-START
 * STREAM-END
 * COMMENT
 * DIRECTIVE(name, value)
 * DOCUMENT-START
 * DOCUMENT-END
 * BLOCK-SEQUENCE-START
 * BLOCK-MAPPING-START
 * BLOCK-END
 * FLOW-SEQUENCE-START
 * FLOW-MAPPING-START
 * FLOW-SEQUENCE-END
 * FLOW-MAPPING-END
 * BLOCK-ENTRY
 * FLOW-ENTRY
 * KEY
 * VALUE
 * ALIAS(value)
 * ANCHOR(value)
 * TAG(value)
 * SCALAR(value, plain, style)
 * Read comments in the Scanner code for more details.
 * </pre>
 */
public final class ScannerImpl implements Scanner {

  /**
   * A regular expression matching characters which are not in the hexadecimal set (0-9, A-F, a-f).
   */
  private static final Pattern NOT_HEXA = Pattern.compile("[^0-9A-Fa-f]");

  /**
   * A mapping from an escaped character in the input stream to the string representation that they
   * should be replaced with.
   *
   * YAML defines several common and a few uncommon escape sequences.
   *
   * @see <a href="http://www.yaml.org/spec/current.html#id2517668">4.1.6. Escape Sequences</a>
   */
  public static final Map<Character, String> ESCAPE_REPLACEMENTS = new HashMap<Character, String>();

  /**
   * A mapping from a character to a number of bytes to read-ahead for that escape sequence. These
   * escape sequences are used to handle unicode escaping in the following formats, where H is a
   * hexadecimal character:
   *
   * <pre>
   * &#92;xHH         : escaped 8-bit Unicode character
   * &#92;uHHHH       : escaped 16-bit Unicode character
   * &#92;UHHHHHHHH   : escaped 32-bit Unicode character
   * </pre>
   *
   * @see <a href="http://yaml.org/spec/1.1/current.html#id872840">5.6. Escape Sequences</a>
   */
  public static final Map<Character, Integer> ESCAPE_CODES = new HashMap<Character, Integer>();

  static {
    // ASCII null
    ESCAPE_REPLACEMENTS.put(Character.valueOf('0'), "\0");
    // ASCII bell
    ESCAPE_REPLACEMENTS.put(Character.valueOf('a'), "\u0007");
    // ASCII backspace
    ESCAPE_REPLACEMENTS.put(Character.valueOf('b'), "\u0008");
    // ASCII horizontal tab
    ESCAPE_REPLACEMENTS.put(Character.valueOf('t'), "\u0009");
    // ASCII newline (line feed; &#92;n maps to 0x0A)
    ESCAPE_REPLACEMENTS.put(Character.valueOf('n'), "\n");
    // ASCII vertical tab
    ESCAPE_REPLACEMENTS.put(Character.valueOf('v'), "\u000B");
    // ASCII form-feed
    ESCAPE_REPLACEMENTS.put(Character.valueOf('f'), "\u000C");
    // carriage-return (&#92;r maps to 0x0D)
    ESCAPE_REPLACEMENTS.put(Character.valueOf('r'), "\r");
    // ASCII escape character (Esc)
    ESCAPE_REPLACEMENTS.put(Character.valueOf('e'), "\u001B");
    // ASCII space
    ESCAPE_REPLACEMENTS.put(Character.valueOf(' '), "\u0020");
    // ASCII double-quote
    ESCAPE_REPLACEMENTS.put(Character.valueOf('"'), "\"");
    // ASCII backslash
    ESCAPE_REPLACEMENTS.put(Character.valueOf('\\'), "\\");
    // Unicode next line
    ESCAPE_REPLACEMENTS.put(Character.valueOf('N'), "\u0085");
    // Unicode non-breaking-space
    ESCAPE_REPLACEMENTS.put(Character.valueOf('_'), "\u00A0");
    // Unicode line-separator
    ESCAPE_REPLACEMENTS.put(Character.valueOf('L'), "\u2028");
    // Unicode paragraph separator
    ESCAPE_REPLACEMENTS.put(Character.valueOf('P'), "\u2029");

    // 8-bit Unicode
    ESCAPE_CODES.put(Character.valueOf('x'), 2);
    // 16-bit Unicode
    ESCAPE_CODES.put(Character.valueOf('u'), 4);
    // 32-bit Unicode (Supplementary characters are supported)
    ESCAPE_CODES.put(Character.valueOf('U'), 8);
  }

  private final StreamReader reader;
  // Had we reached the end of the stream?
  private boolean done = false;

  // The number of unclosed '{' and '['. `flow_level == 0` means block context.
  private int flowLevel = 0;

  // List of processed tokens that are not yet emitted.
  private final List<Token> tokens;

  // The last added token
  private Token lastToken;

  // Number of tokens that were emitted through the `getToken()` method.
  private int tokensTaken = 0;

  // The current indentation level.
  private int indent = -1;

  // Past indentation levels.
  private final ArrayStack<Integer> indents;

  // A flag that indicates if comments should be parsed
  private final boolean parseComments;

  private final LoaderOptions loaderOptions;

  // Variables related to simple keys treatment. See PyYAML.

  /**
   * <pre>
   * A simple key is a key that is not denoted by the '?' indicator.
   * Example of simple keys:
   *   ---
   *   block simple key: value
   *   ? not a simple key:
   *   : { flow simple key: value }
   * We emit the KEY token before all keys, so when we find a potential
   * simple key, we try to locate the corresponding ':' indicator.
   * Simple keys should be limited to a single line and 1024 characters.
   *
   * Can a simple key start at the current position? A simple key may
   * start:
   * - at the beginning of the line, not counting indentation spaces
   *       (in block context),
   * - after '{', '[', ',' (in the flow context),
   * - after '?', ':', '-' (in the block context).
   * In the block context, this flag also signifies if a block collection
   * may start at the current position.
   * </pre>
   */
  private boolean allowSimpleKey = true;

  /*
   * Keep track of possible simple keys. This is a dictionary. The key is `flow_level`; there can be
   * no more than one possible simple key for each level. The value is a SimpleKey record:
   * (token_number, required, index, line, column, mark) A simple key may start with ALIAS, ANCHOR,
   * TAG, SCALAR(flow), '[', or '{' tokens.
   */
  private final Map<Integer, SimpleKey> possibleSimpleKeys;

  public ScannerImpl(StreamReader reader, LoaderOptions options) {
    if (options == null) {
      throw new NullPointerException("LoaderOptions must be provided.");
    }
    this.parseComments = options.isProcessComments();
    this.reader = reader;
    this.tokens = new ArrayList<Token>(100);
    this.indents = new ArrayStack<Integer>(10);
    // The order in possibleSimpleKeys is kept for nextPossibleSimpleKey()
    this.possibleSimpleKeys = new LinkedHashMap<Integer, SimpleKey>();
    this.loaderOptions = options;
    fetchStreamStart();// Add the STREAM-START token.
  }

  /**
   * Check whether the next token is the given type.
   */
  @Override
  public boolean checkToken(Token.ID choice) {
    while (needMoreTokens()) {
      fetchMoreTokens();
    }
    if (!this.tokens.isEmpty()) {
      return this.tokens.get(0).getTokenId() == choice;
    }
    return false;
  }

  /**
   * Check whether the next token is one of the given types.
   */
  @Override
  public boolean checkToken(Token.ID... choices) {
    while (needMoreTokens()) {
      fetchMoreTokens();
    }
    if (!this.tokens.isEmpty()) {
      if (choices.length == 0) {
        return true;
      }
      // since profiler puts this method on top (it is used a lot), we
      // should not use 'foreach' here because of the performance reasons
      Token.ID first = this.tokens.get(0).getTokenId();
      for (int i = 0; i < choices.length; i++) {
        if (first == choices[i]) {
          return true;
        }
      }
    }
    return false;
  }

  /**
   * Return the next token, but do not delete it from the queue.
   */
  public Token peekToken() {
    while (needMoreTokens()) {
      fetchMoreTokens();
    }
    return this.tokens.get(0);
  }

  /**
   * Return the next token, removing it from the queue.
   */
  public Token getToken() {
    this.tokensTaken++;
    return this.tokens.remove(0);
  }

  // Private methods.

  private void addToken(Token token) {
    lastToken = token;
    this.tokens.add(token);
  }

  private void addToken(int index, Token token) {
    if (index == this.tokens.size()) {
      lastToken = token;
    }
    this.tokens.add(index, token);
  }

  private void addAllTokens(List<Token> tokens) {
    lastToken = tokens.get(tokens.size() - 1);
    this.tokens.addAll(tokens);
  }

  /**
   * Returns true if more tokens should be scanned.
   */
  private boolean needMoreTokens() {
    // If we are done, we do not require more tokens.
    if (this.done) {
      return false;
    }
    // If we aren't done, but we have no tokens, we need to scan more.
    if (this.tokens.isEmpty()) {
      return true;
    }
    // The current token may be a potential simple key, so we
    // need to look further.
    stalePossibleSimpleKeys();
    return nextPossibleSimpleKey() == this.tokensTaken;
  }

  /**
   * Fetch one or more tokens from the StreamReader.
   */
  private void fetchMoreTokens() {
    if (reader.getDocumentIndex() > loaderOptions.getCodePointLimit()) {
      throw new YAMLException("The incoming YAML document exceeds the limit: "
          + loaderOptions.getCodePointLimit() + " code points.");
    }
    // Eat whitespaces and process comments until we reach the next token.
    scanToNextToken();
    // Remove obsolete possible simple keys.
    stalePossibleSimpleKeys();
    // Compare the current indentation and column. It may add some tokens
    // and decrease the current indentation level.
    unwindIndent(reader.getColumn());
    // Peek the next code point, to decide what the next group of tokens
    // will look like.
    int c = reader.peek();
    switch (c) {
      case '\0':
        // Is it the end of stream?
        fetchStreamEnd();
        return;
      case '%':
        // Is it a directive?
        if (checkDirective()) {
          fetchDirective();
          return;
        }
        break;
      case '-':
        // Is it the document start?
        if (checkDocumentStart()) {
          fetchDocumentStart();
          return;
          // Is it the block entry indicator?
        } else if (checkBlockEntry()) {
          fetchBlockEntry();
          return;
        }
        break;
      case '.':
        // Is it the document end?
        if (checkDocumentEnd()) {
          fetchDocumentEnd();
          return;
        }
        break;
      // TODO support for BOM within a stream. (also not implemented in PyYAML)
      case '[':
        // Is it the flow sequence start indicator?
        fetchFlowSequenceStart();
        return;
      case '{':
        // Is it the flow mapping start indicator?
        fetchFlowMappingStart();
        return;
      case ']':
        // Is it the flow sequence end indicator?
        fetchFlowSequenceEnd();
        return;
      case '}':
        // Is it the flow mapping end indicator?
        fetchFlowMappingEnd();
        return;
      case ',':
        // Is it the flow entry indicator?
        fetchFlowEntry();
        return;
      // see block entry indicator above
      case '?':
        // Is it the key indicator?
        if (checkKey()) {
          fetchKey();
          return;
        }
        break;
      case ':':
        // Is it the value indicator?
        if (checkValue()) {
          fetchValue();
          return;
        }
        break;
      case '*':
        // Is it an alias?
        fetchAlias();
        return;
      case '&':
        // Is it an anchor?
        fetchAnchor();
        return;
      case '!':
        // Is it a tag?
        fetchTag();
        return;
      case '|':
        // Is it a literal scalar?
        if (this.flowLevel == 0) {
          fetchLiteral();
          return;
        }
        break;
      case '>':
        // Is it a folded scalar?
        if (this.flowLevel == 0) {
          fetchFolded();
          return;
        }
        break;
      case '\'':
        // Is it a single quoted scalar?
        fetchSingle();
        return;
      case '"':
        // Is it a double quoted scalar?
        fetchDouble();
        return;
    }
    // It must be a plain scalar then.
    if (checkPlain()) {
      fetchPlain();
      return;
    }
    // No? It's an error. Let's produce a nice error message.We do this by
    // converting escaped characters into their escape sequences. This is a
    // backwards use of the ESCAPE_REPLACEMENTS map.
    String chRepresentation = escapeChar(String.valueOf(Character.toChars(c)));
    if (c == '\t') {
      chRepresentation += "(TAB)";
    }
    String text = String.format(
        "found character '%s' that cannot start any token. (Do not use %s for indentation)",
        chRepresentation, chRepresentation);
    throw new ScannerException("while scanning for the next token", null, text, reader.getMark());
  }

  /**
   * This is implemented in CharConstants in SnakeYAML Engine
   */
  private String escapeChar(String chRepresentation) {
    for (Character s : ESCAPE_REPLACEMENTS.keySet()) {
      String v = ESCAPE_REPLACEMENTS.get(s);
      if (v.equals(chRepresentation)) {
        return "\\" + s;// ' ' -> '\t'
      }
    }
    return chRepresentation;
  }

  // Simple keys treatment.

  /**
   * Return the number of the nearest possible simple key. Actually we don't need to loop through
   * the whole dictionary.
   */
  private int nextPossibleSimpleKey() {
    /*
     * the implementation is not as in PyYAML. Because this.possibleSimpleKeys is ordered we can
     * simply take the first key
     */
    if (!this.possibleSimpleKeys.isEmpty()) {
      return this.possibleSimpleKeys.values().iterator().next().getTokenNumber();
    }
    return -1;
  }

  /**
   * <pre>
   * Remove entries that are no longer possible simple keys. According to
   * the YAML specification, simple keys
   * - should be limited to a single line,
   * - should be no longer than 1024 characters.
   * Disabling this procedure will allow simple keys of any length and
   * height (may cause problems if indentation is broken though).
   * </pre>
   */
  private void stalePossibleSimpleKeys() {
    if (!this.possibleSimpleKeys.isEmpty()) {
      for (Iterator<SimpleKey> iterator = this.possibleSimpleKeys.values().iterator(); iterator
          .hasNext();) {
        SimpleKey key = iterator.next();
        if ((key.getLine() != reader.getLine()) || (reader.getIndex() - key.getIndex() > 1024)) {
          // If the key is not on the same line as the current
          // position OR the difference in column between the token
          // start and the current position is more than the maximum
          // simple key length, then this cannot be a simple key.
          if (key.isRequired()) {
            // If the key was required, this implies an error
            // condition.
            throw new ScannerException("while scanning a simple key", key.getMark(),
                "could not find expected ':'", reader.getMark());
          }
          iterator.remove();
        }
      }
    }
  }

  /**
   * The next token may start a simple key. We check if it's possible and save its position. This
   * function is called for ALIAS, ANCHOR, TAG, SCALAR(flow), '[', and '{'.
   */
  private void savePossibleSimpleKey() {
    // The next token may start a simple key. We check if it's possible
    // and save its position. This function is called for
    // ALIAS, ANCHOR, TAG, SCALAR(flow), '[', and '{'.

    // Check if a simple key is required at the current position.
    // A simple key is required if this position is the root flowLevel, AND
    // the current indentation level is the same as the last indent-level.
    boolean required = (this.flowLevel == 0) && (this.indent == this.reader.getColumn());

    if (allowSimpleKey || !required) {
      // A simple key is required only if it is the first token in the
      // current line. Therefore it is always allowed.
    } else {
      throw new YAMLException(
          "A simple key is required only if it is the first token in the current line");
    }

    // The next token might be a simple key. Let's save it's number and
    // position.
    if (this.allowSimpleKey) {
      removePossibleSimpleKey();
      int tokenNumber = this.tokensTaken + this.tokens.size();
      SimpleKey key = new SimpleKey(tokenNumber, required, reader.getIndex(), reader.getLine(),
          this.reader.getColumn(), this.reader.getMark());
      this.possibleSimpleKeys.put(this.flowLevel, key);
    }
  }

  /**
   * Remove the saved possible key position at the current flow level.
   */
  private void removePossibleSimpleKey() {
    SimpleKey key = possibleSimpleKeys.remove(flowLevel);
    if (key != null && key.isRequired()) {
      throw new ScannerException("while scanning a simple key", key.getMark(),
          "could not find expected ':'", reader.getMark());
    }
  }

  // Indentation functions.

  /**
   * * Handle implicitly ending multiple levels of block nodes by decreased indentation. This
   * function becomes important on lines 4 and 7 of this example:
   *
   * <pre>
   * 1) book one:
   * 2)   part one:
   * 3)     chapter one
   * 4)   part two:
   * 5)     chapter one
   * 6)     chapter two
   * 7) book two:
   * </pre>
   *
   * In flow context, tokens should respect indentation. Actually the condition should be
   * `self.indent &gt;= column` according to the spec. But this condition will prohibit intuitively
   * correct constructions such as key : { }
   * </pre>
   */
  private void unwindIndent(int col) {
    // In the flow context, indentation is ignored. We make the scanner less
    // restrictive than specification requires.
    if (this.flowLevel != 0) {
      return;
    }

    // In block context, we may need to issue the BLOCK-END tokens.
    while (this.indent > col) {
      Mark mark = reader.getMark();
      this.indent = this.indents.pop();
      addToken(new BlockEndToken(mark, mark));
    }
  }

  /**
   * Check if we need to increase indentation.
   */
  private boolean addIndent(int column) {
    if (this.indent < column) {
      this.indents.push(this.indent);
      this.indent = column;
      return true;
    }
    return false;
  }

  // Fetchers.

  /**
   * We always add STREAM-START as the first token and STREAM-END as the last token.
   */
  private void fetchStreamStart() {
    // Read the token.
    Mark mark = reader.getMark();

    // Add STREAM-START.
    Token token = new StreamStartToken(mark, mark);
    addToken(token);
  }

  private void fetchStreamEnd() {
    // Set the current indentation to -1.
    unwindIndent(-1);

    // Reset simple keys.
    removePossibleSimpleKey();
    this.allowSimpleKey = false;
    this.possibleSimpleKeys.clear();

    // Read the token.
    Mark mark = reader.getMark();

    // Add STREAM-END.
    Token token = new StreamEndToken(mark, mark);
    addToken(token);

    // The stream is finished.
    this.done = true;
  }

  /**
   * Fetch a YAML directive. Directives are presentation details that are interpreted as
   * instructions to the processor. YAML defines two kinds of directives, YAML and TAG; all other
   * types are reserved for future use.
   *
   * @see <a href="http://www.yaml.org/spec/1.1/#id864824">3.2.3.4. Directives</a>
   */
  private void fetchDirective() {
    // Set the current indentation to -1.
    unwindIndent(-1);

    // Reset simple keys.
    removePossibleSimpleKey();
    this.allowSimpleKey = false;

    // Scan and add DIRECTIVE.
    List<Token> tok = scanDirective();
    addAllTokens(tok);
  }

  /**
   * Fetch a document-start token ("---").
   */
  private void fetchDocumentStart() {
    fetchDocumentIndicator(true);
  }

  /**
   * Fetch a document-end token ("...").
   */
  private void fetchDocumentEnd() {
    fetchDocumentIndicator(false);
  }

  /**
   * Fetch a document indicator, either "---" for "document-start", or else "..." for "document-end.
   * The type is chosen by the given boolean.
   */
  private void fetchDocumentIndicator(boolean isDocumentStart) {
    // Set the current indentation to -1.
    unwindIndent(-1);

    // Reset simple keys. Note that there could not be a block collection
    // after '---'.
    removePossibleSimpleKey();
    this.allowSimpleKey = false;

    // Add DOCUMENT-START or DOCUMENT-END.
    Mark startMark = reader.getMark();
    reader.forward(3);
    Mark endMark = reader.getMark();
    Token token;
    if (isDocumentStart) {
      token = new DocumentStartToken(startMark, endMark);
    } else {
      token = new DocumentEndToken(startMark, endMark);
    }
    addToken(token);
  }

  private void fetchFlowSequenceStart() {
    fetchFlowCollectionStart(false);
  }

  private void fetchFlowMappingStart() {
    fetchFlowCollectionStart(true);
  }

  /**
   * Fetch a flow-style collection start, which is either a sequence or a mapping. The type is
   * determined by the given boolean.
   *
   * A flow-style collection is in a format similar to JSON. Sequences are started by '[' and ended
   * by ']'; mappings are started by '{' and ended by '}'.
   *
   * @see <a href="http://www.yaml.org/spec/1.1/#id863975">3.2.3.1. Node Styles</a>
   *
   * @param isMappingStart
   */
  private void fetchFlowCollectionStart(boolean isMappingStart) {
    // '[' and '{' may start a simple key.
    savePossibleSimpleKey();

    // Increase the flow level.
    this.flowLevel++;

    // Simple keys are allowed after '[' and '{'.
    this.allowSimpleKey = true;

    // Add FLOW-SEQUENCE-START or FLOW-MAPPING-START.
    Mark startMark = reader.getMark();
    reader.forward(1);
    Mark endMark = reader.getMark();
    Token token;
    if (isMappingStart) {
      token = new FlowMappingStartToken(startMark, endMark);
    } else {
      token = new FlowSequenceStartToken(startMark, endMark);
    }
    addToken(token);
  }

  private void fetchFlowSequenceEnd() {
    fetchFlowCollectionEnd(false);
  }

  private void fetchFlowMappingEnd() {
    fetchFlowCollectionEnd(true);
  }

  /**
   * Fetch a flow-style collection end, which is either a sequence or a mapping. The type is
   * determined by the given boolean.
   *
   * A flow-style collection is in a format similar to JSON. Sequences are started by '[' and ended
   * by ']'; mappings are started by '{' and ended by '}'.
   *
   * @see <a href="http://www.yaml.org/spec/1.1/#id863975">3.2.3.1. Node Styles</a>
   */
  private void fetchFlowCollectionEnd(boolean isMappingEnd) {
    // Reset possible simple key on the current level.
    removePossibleSimpleKey();

    // Decrease the flow level.
    this.flowLevel--;

    // No simple keys after ']' or '}'.
    this.allowSimpleKey = false;

    // Add FLOW-SEQUENCE-END or FLOW-MAPPING-END.
    Mark startMark = reader.getMark();
    reader.forward();
    Mark endMark = reader.getMark();
    Token token;
    if (isMappingEnd) {
      token = new FlowMappingEndToken(startMark, endMark);
    } else {
      token = new FlowSequenceEndToken(startMark, endMark);
    }
    addToken(token);
  }

  /**
   * Fetch an entry in the flow style. Flow-style entries occur either immediately after the start
   * of a collection, or else after a comma.
   *
   * @see <a href="http://www.yaml.org/spec/1.1/#id863975">3.2.3.1. Node Styles</a>
   */
  private void fetchFlowEntry() {
    // Simple keys are allowed after ','.
    this.allowSimpleKey = true;

    // Reset possible simple key on the current level.
    removePossibleSimpleKey();

    // Add FLOW-ENTRY.
    Mark startMark = reader.getMark();
    reader.forward();
    Mark endMark = reader.getMark();
    Token token = new FlowEntryToken(startMark, endMark);
    addToken(token);
  }

  /**
   * Fetch an entry in the block style.
   *
   * @see <a href="http://www.yaml.org/spec/1.1/#id863975">3.2.3.1. Node Styles</a>
   */
  private void fetchBlockEntry() {
    // Block context needs additional checks.
    if (this.flowLevel == 0) {
      // Are we allowed to start a new entry?
      if (!this.allowSimpleKey) {
        throw new ScannerException(null, null, "sequence entries are not allowed here",
            reader.getMark());
      }

      // We may need to add BLOCK-SEQUENCE-START.
      if (addIndent(this.reader.getColumn())) {
        Mark mark = reader.getMark();
        addToken(new BlockSequenceStartToken(mark, mark));
      }
    } else {
      // It's an error for the block entry to occur in the flow
      // context,but we let the parser detect this.
    }
    // Simple keys are allowed after '-'.
    this.allowSimpleKey = true;

    // Reset possible simple key on the current level.
    removePossibleSimpleKey();

    // Add BLOCK-ENTRY.
    Mark startMark = reader.getMark();
    reader.forward();
    Mark endMark = reader.getMark();
    Token token = new BlockEntryToken(startMark, endMark);
    addToken(token);
  }

  /**
   * Fetch a key in a block-style mapping.
   *
   * @see <a href="http://www.yaml.org/spec/1.1/#id863975">3.2.3.1. Node Styles</a>
   */
  private void fetchKey() {
    // Block context needs additional checks.
    if (this.flowLevel == 0) {
      // Are we allowed to start a key (not necessary a simple)?
      if (!this.allowSimpleKey) {
        throw new ScannerException(null, null, "mapping keys are not allowed here",
            reader.getMark());
      }
      // We may need to add BLOCK-MAPPING-START.
      if (addIndent(this.reader.getColumn())) {
        Mark mark = reader.getMark();
        addToken(new BlockMappingStartToken(mark, mark));
      }
    }
    // Simple keys are allowed after '?' in the block context.
    this.allowSimpleKey = this.flowLevel == 0;

    // Reset possible simple key on the current level.
    removePossibleSimpleKey();

    // Add KEY.
    Mark startMark = reader.getMark();
    reader.forward();
    Mark endMark = reader.getMark();
    Token token = new KeyToken(startMark, endMark);
    addToken(token);
  }

  /**
   * Fetch a value in a block-style mapping.
   *
   * @see <a href="http://www.yaml.org/spec/1.1/#id863975">3.2.3.1. Node Styles</a>
   */
  private void fetchValue() {
    // Do we determine a simple key?
    SimpleKey key = this.possibleSimpleKeys.remove(this.flowLevel);
    if (key != null) {
      // Add KEY.
      addToken(key.getTokenNumber() - this.tokensTaken, new KeyToken(key.getMark(), key.getMark()));

      // If this key starts a new block mapping, we need to add
      // BLOCK-MAPPING-START.
      if (this.flowLevel == 0) {
        if (addIndent(key.getColumn())) {
          addToken(key.getTokenNumber() - this.tokensTaken,
              new BlockMappingStartToken(key.getMark(), key.getMark()));
        }
      }
      // There cannot be two simple keys one after another.
      this.allowSimpleKey = false;

    } else {
      // It must be a part of a complex key.
      // Block context needs additional checks. Do we really need them?
      // They will be caught by the parser anyway.
      if (this.flowLevel == 0) {

        // We are allowed to start a complex value if and only if we can
        // start a simple key.
        if (!this.allowSimpleKey) {
          throw new ScannerException(null, null, "mapping values are not allowed here",
              reader.getMark());
        }
      }

      // If this value starts a new block mapping, we need to add
      // BLOCK-MAPPING-START. It will be detected as an error later by
      // the parser.
      if (flowLevel == 0) {
        if (addIndent(reader.getColumn())) {
          Mark mark = reader.getMark();
          addToken(new BlockMappingStartToken(mark, mark));
        }
      }

      // Simple keys are allowed after ':' in the block context.
      allowSimpleKey = flowLevel == 0;

      // Reset possible simple key on the current level.
      removePossibleSimpleKey();
    }
    // Add VALUE.
    Mark startMark = reader.getMark();
    reader.forward();
    Mark endMark = reader.getMark();
    Token token = new ValueToken(startMark, endMark);
    addToken(token);
  }

  /**
   * Fetch an alias, which is a reference to an anchor. Aliases take the format:
   *
   * <pre>
   * *(anchor name)
   * </pre>
   *
   * @see <a href="http://www.yaml.org/spec/1.1/#id863390">3.2.2.2. Anchors and Aliases</a>
   */
  private void fetchAlias() {
    // ALIAS could be a simple key.
    savePossibleSimpleKey();

    // No simple keys after ALIAS.
    this.allowSimpleKey = false;

    // Scan and add ALIAS.
    Token tok = scanAnchor(false);
    addToken(tok);
  }

  /**
   * Fetch an anchor. Anchors take the form:
   *
   * <pre>
   * &(anchor name)
   * </pre>
   *
   * @see <a href="http://www.yaml.org/spec/1.1/#id863390">3.2.2.2. Anchors and Aliases</a>
   */
  private void fetchAnchor() {
    // ANCHOR could start a simple key.
    savePossibleSimpleKey();

    // No simple keys after ANCHOR.
    this.allowSimpleKey = false;

    // Scan and add ANCHOR.
    Token tok = scanAnchor(true);
    addToken(tok);
  }

  /**
   * Fetch a tag. Tags take a complex form.
   *
   * @see <a href="http://www.yaml.org/spec/1.1/#id861700">3.2.1.2. Tags</a>
   */
  private void fetchTag() {
    // TAG could start a simple key.
    savePossibleSimpleKey();

    // No simple keys after TAG.
    this.allowSimpleKey = false;

    // Scan and add TAG.
    Token tok = scanTag();
    addToken(tok);
  }

  /**
   * Fetch a literal scalar, denoted with a vertical-bar. This is the type best used for source code
   * and other content, such as binary data, which must be included verbatim.
   *
   * @see <a href="http://www.yaml.org/spec/1.1/#id863975">3.2.3.1. Node Styles</a>
   */
  private void fetchLiteral() {
    fetchBlockScalar('|');
  }

  /**
   * Fetch a folded scalar, denoted with a greater-than sign. This is the type best used for long
   * content, such as the text of a chapter or description.
   *
   * @see <a href="http://www.yaml.org/spec/1.1/#id863975">3.2.3.1. Node Styles</a>
   */
  private void fetchFolded() {
    fetchBlockScalar('>');
  }

  /**
   * Fetch a block scalar (literal or folded).
   *
   * @see <a href="http://www.yaml.org/spec/1.1/#id863975">3.2.3.1. Node Styles</a>
   *
   * @param style
   */
  private void fetchBlockScalar(char style) {
    // A simple key may follow a block scalar.
    this.allowSimpleKey = true;

    // Reset possible simple key on the current level.
    removePossibleSimpleKey();

    // Scan and add SCALAR.
    List<Token> tok = scanBlockScalar(style);
    addAllTokens(tok);
  }

  /**
   * Fetch a single-quoted (') scalar.
   */
  private void fetchSingle() {
    fetchFlowScalar('\'');
  }

  /**
   * Fetch a double-quoted (") scalar.
   */
  private void fetchDouble() {
    fetchFlowScalar('"');
  }

  /**
   * Fetch a flow scalar (single- or double-quoted).
   *
