def nonneg_softmax(expr,
                   replace_nonpositives = -10):
  """A softmax operator that is appropriate for NQL outputs.

  NeuralQueryExpressions often evaluate to sparse vectors of small, nonnegative
  values. Softmax for those is dominated by zeros, so this is a fix.  This also
  fixes the problem that minibatches for NQL are one example per column, not one
  example per row.

  Args:
    expr: a Tensorflow expression for some predicted values.
    replace_nonpositives: will replace zeros with this value before computing
      softmax.

  Returns:
    Tensorflow expression for softmax.
  """
  if replace_nonpositives != 0.0:
    ones = tf.ones(tf.shape(input=expr), tf.float32)
    expr = tf.where(expr > 0.0, expr, ones * replace_nonpositives)
  return tf.nn.softmax(expr)
A softmax operator that is appropriate for NQL outputs.

  NeuralQueryExpressions often evaluate to sparse vectors of small, nonnegative
  values. Softmax for those is dominated by zeros, so this is a fix.  This also
  fixes the problem that minibatches for NQL are one example per column, not one
  example per row.

  Args:
    expr: a Tensorflow expression for some predicted values.
    replace_nonpositives: will replace zeros with this value before computing
      softmax.

  Returns:
    Tensorflow expression for softmax.
  
language/nql/nql/__init__.py