def execute_predictions(predictions, cache_dict, ofile, case_sensitive, verbose,
                        update_cache):
  """Executes predicted/gold queries and computes performance.

  Writes results to ofile.

  Args:
    predictions: A list of dictionaries defining the predictions made by a
      model.
    cache_dict: A dictionary mapping from gold queries to the resulting tables.
    ofile: A file pointer to be written to.
    case_sensitive: A Boolean indicating whether execution of queries should be
      case sensitive with respect to strings.
    verbose: Whether to print detailed information about evaluation (e.g., for
      debugging).
    update_cache: Whether to execute and cache gold queries.
  """
  # Keeps tracks of metrics throughout all of the evaluation.
  exec_results_same = list()
  string_same = list()

  precision = list()
  recall = list()

  column_f1s = list()
  table_f1s = list()

  conversion_errors = 0

  schema_errors = 0
  syntax_errors = 0
  timeouts = 0

  gold_error = 0

  i = 0

  predictions_iterator = tqdm
  if verbose:
    # Don't use TQDM if verbose: it might mess up the verbose messages
    predictions_iterator = lambda x: x

  for prediction in predictions_iterator(predictions):
    # Attempt to connect to the database for executing.
    try:
      conn = sqlite3.connect(prediction['database_path'])
      conn.text_factory = str
    except sqlite3.OperationalError as e:
      print(prediction['database_path'])
      raise e

    empty_path = prediction['empty_database_path']
    try:
      empty_conn = sqlite3.connect(empty_path)
      empty_conn.text_factory = str
    except sqlite3.OperationalError as e:
      print(empty_path)
      raise e

    empty_cursor = empty_conn.cursor()
    cursor = conn.cursor()

    ofile.write('Example #' + str(i) + '\n')
    printable_utterance = prediction['utterance'].strip()
    ofile.write(printable_utterance + '\n')

    if verbose:
      print('Finding the highest-rated prediction for utterance:\n\t' +
            printable_utterance)

    best_prediction, pred_results, exception_str, execution_time = execute_prediction(
        prediction, empty_cursor, cursor, case_sensitive, verbose)

    ofile.write('Predicted query:\n')
    if best_prediction:
      ofile.write('\t' + best_prediction.strip() + '\n')
    else:
      ofile.write('ERROR: Cannot write prediction %r\n' % best_prediction)

    # If it didn't execute correctly, check why.
    if exception_str:
      ofile.write(exception_str + '\n')

      found_error = False
      for substring in SCHEMA_INCOHERENCE_STRINGS:
        if substring in exception_str.lower():
          schema_errors += 1
          found_error = True
          break

      if not found_error:
        for substring in SYNTAX_INCORRECTNESS_STRINGS:
          if substring in exception_str.lower():
            syntax_errors += 1
            found_error = True
            break

      if not found_error and 'timeout' in exception_str:
        ofile.write('Execution (predicted) took too long.\n')
        found_error = True
        timeouts += 1

      # If the error type hasn't been identified, exit and report it.
      if not found_error:
        print(best_prediction)
        print(exception_str)
        exit(1)

      # Predicted table should be empty for all of these cases.
      pred_results = []

    # Compare to gold and update metrics
    gold_query = prediction['gold']

    ofile.write('Gold query:\n')
    ofile.write('\t' + gold_query.strip() + '\n')

    # Get the gold results
    if cache_dict is None or gold_query not in cache_dict:
      if printable_utterance not in cache_dict:
        if update_cache:
          if verbose:
            print('Trying to execute the gold query:\n\t' + gold_query)
          gold_results, gold_exception_str, execution_time = try_executing_query(
              gold_query, cursor, case_sensitive, verbose)

          if gold_exception_str:
            gold_error += 1
            gold_results = []
          elif cache_dict is not None:
            cache_dict[gold_query] = gold_results
        else:
          print(gold_query)
          print(printable_utterance)
          raise ValueError('Cache miss!')

    gold_results = cache_dict[gold_query]

    if best_prediction:
      string_same.append(string_acc(gold_query, best_prediction))
      col_f1, tab_f1 = col_tab_f1(prediction['schema'], gold_query,
                                  best_prediction)
      column_f1s.append(col_f1)
      table_f1s.append(tab_f1)
      ofile.write('Column F1: %f\n' % col_f1)
      ofile.write('Table F1: %f\n' % tab_f1)

      if 'order by' in gold_query:
        results_equivalent = pred_results == gold_results
      else:
        pred_set = set()
        gold_set = set()
        for pred in pred_results:
          if isinstance(pred, list):
            pred_set.add(' '.join([str(item) for item in pred]))
          else:
            pred_set.add(pred)
        for gold in gold_results:
          if isinstance(gold, list):
            gold_set.add(' '.join([str(item) for item in gold]))
          else:
            gold_set.add(gold)

        results_equivalent = pred_set == gold_set

    else:
      string_same.append(0.)
      ofile.write('Column F1: 0.')
      ofile.write('Table F1: 0.')
      column_f1s.append(0.)
      table_f1s.append(0.)

      conversion_errors += 1

      # Only consider correct if the gold table was empty.
      results_equivalent = gold_results == list()

    exec_results_same.append(int(results_equivalent))
    ofile.write('Execution was correct? ' + str(results_equivalent) + '\n')

    # Add some debugging information about the tables, and compute the
    # precisions.
    if pred_results:
      if not results_equivalent:
        ofile.write('Predicted table:\n')
        ofile.write(result_table_to_string(pred_results))

      precision.append(int(results_equivalent))
    elif best_prediction is None or not results_equivalent:
      ofile.write('Predicted table was EMPTY!\n')

    if gold_results:
      ofile.write('Gold table:\n')
      ofile.write(result_table_to_string(gold_results))

      recall.append(int(results_equivalent))
    else:
      ofile.write('Gold table was EMPTY!\n')

    ofile.write('\n')
    ofile.flush()

    conn.close()
    empty_conn.close()

    i += 1

  # Write the overall metrics to the file.
  num_empty_pred = len(precision)
  num_empty_gold = len(recall)

  precision = np.mean(np.array(precision))
  recall = np.mean(np.array(recall))

  execution_f1 = compute_f1(precision, recall)

  ofile.write('String accuracy: ' +
              '{0:.2f}'.format(100. * np.mean(np.array(string_same))) + '\n')
  ofile.write('Accuracy: ' +
              '{0:.2f}'.format(100. * np.mean(np.array(exec_results_same))) +
              '\n')
  ofile.write('Precision: ' + '{0:.2f}'.format(100. * precision) + ' ; ' +
              str(num_empty_pred) + ' nonempty predicted tables' + '\n')
  ofile.write('Recall: ' + '{0:.2f}'.format(100. * recall) + ' ; ' +
              str(num_empty_gold) + ' nonempty gold tables' + '\n')
  ofile.write('Execution F1: ' + '{0:.2f}'.format(100. * execution_f1) + '\n')
  ofile.write('Timeout: ' +
              '{0:.2f}'.format(timeouts * 100. / len(predictions)) + '\n')
  ofile.write('Gold did not execute: ' +
              '{0:.2f}'.format(gold_error * 100. / len(predictions)) + '\n')
  ofile.write('Average column F1: ' +
              '{0:.2f}'.format(100. * np.mean(np.array(column_f1s))) + '\n')
  ofile.write('Average table F1: ' +
              '{0:.2f}'.format(100. * np.mean(np.array(table_f1s))) + '\n')
  ofile.write('Schema errors: ' +
              '{0:.2f}'.format((schema_errors) * 100. / len(predictions)) +
              '\n')
  ofile.write('Syntax errors:  ' +
              '{0:.2f}'.format((syntax_errors) * 100. / len(predictions)) +
              '\n')
  ofile.write('Conversion errors: ' +
              '{0:.2f}'.format((conversion_errors * 100.) / len(predictions)) +
              '\n')
Executes predicted/gold queries and computes performance.

  Writes results to ofile.

  Args:
    predictions: A list of dictionaries defining the predictions made by a
      model.
    cache_dict: A dictionary mapping from gold queries to the resulting tables.
    ofile: A file pointer to be written to.
    case_sensitive: A Boolean indicating whether execution of queries should be
      case sensitive with respect to strings.
    verbose: Whether to print detailed information about evaluation (e.g., for
      debugging).
    update_cache: Whether to execute and cache gold queries.
  
language/xsp/evaluation/official_evaluation.py