mtool/score/smatch.py

import sys;

import score.core;
from smatch.smatch import get_amr_match;

def tuples(graph, prefix, values, faith = True):
  #
  # mimicry of get_triples() in amr.py
  #
  id = 0;
  mapping = dict();
  instances = [];
  relations = [];
  attributes = [];
  n = 0;
  for node in graph.nodes:
    mapping[node.id] = name = prefix + str(id);
    id += 1;
    if "anchors" in values and node.anchors is not None:
      anchor = score.core.anchor(node);
      if graph.input: anchor = score.core.explode(graph.input, anchor);
      attributes.append(("anchor", name, str(anchor)));
    if "labels" in values and node.label is not None:
      instance = node.label;
    else:
      instance = "__()_{}__".format(prefix, n);
      n += 1;
    instances.append(("instance", name, instance));
    if "tops" in values and node.is_top:
      #
      # the native SMATCH code (wrongly, i believe) ties the top property to
      # the node label (see https://github.com/cfmrp/mtool/issues/12).  we get
      # to choose whether to faithfully replicate those scores or not.
      #
      attributes.append(("TOP", name,
                         node.label if node.label and faith else ""));
    if "properties" in values and node.properties and node.values:
      for property, value in zip(node.properties, node.values):
        attributes.append((property, name, value));
  for edge in graph.edges:
    if "edges" in values:
      relations.append((edge.lab, mapping[edge.src], mapping[edge.tgt]));
    if "attributes" in values:
      if edge.attributes and edge.values:
        for attribute, value in zip(edge.attributes, edge.values):
          relations.append((str((attribute, value)),
                            mapping[edge.src], mapping[edge.tgt]));
  return instances, attributes, relations, n;

def smatch(gold, system, limit = 20, values = {}, trace = 0, faith = True):
  gprefix = "g"; sprefix = "s";
  ginstances, gattributes, grelations, gn \
    = tuples(gold, gprefix, values, faith);
  sinstances, sattributes, srelations, sn \
    = tuples(system, sprefix, values, faith);
  if trace > 1:
    print("gold instances [{}]: {}\ngold attributes [{}]: {}\n"
          "gold relations [{}]: {}"
          "".format(len(ginstances), ginstances,
                    len(gattributes), gattributes,
                    len(grelations), grelations),
          file = sys.stderr);
    print("system instances [{}]: {}\nsystem attributes [{}]: {}\n"
          "system relations [{}]: {}"
          "".format(len(sinstances), sinstances,
                    len(sattributes), sattributes,
                    len(srelations), srelations),
          file = sys.stderr);
  correct, gold, system, mapping \
    = get_amr_match(None, None, gold.id, limit = limit,
                    instance1 = ginstances, attributes1 = gattributes,
                    relation1 = grelations, prefix1 = gprefix,
                    instance2 = sinstances, attributes2 = sattributes,
                    relation2 = srelations, prefix2 = sprefix);
  return correct, gold - gn, system - sn, mapping;

def evaluate(golds, systems, format = "json", limit = 20,
             values = {}, trace = 0):
  if limit is None or not limit > 0: limit = 20;
  if trace > 1: print("RRHC limit: {}".format(limit), file = sys.stderr);
  tg = ts = tc = n = 0;
  scores = dict() if trace else None;
  for gold, system in score.core.intersect(golds, systems):
    id = gold.id;
    correct, gold, system, mapping \
      = smatch(gold, system, limit, values, trace);
    tg += gold; ts += system; tc += correct;
    n += 1;
    if trace:
      if id in scores:
        print("smatch.evaluate(): duplicate graph identifier: {}"
              "".format(id), file = sys.stderr);
      scores[id] = {"g": gold, "s": system, "c": correct};
      if trace > 1:
        p, r, f = score.core.fscore(gold, system, correct);
        print("G: {}; S: {}; C: {}; P: {}; R: {}; F: {}"
              "".format(gold, system, correct, p, r, f), file = sys.stderr);
    
  p, r, f = score.core.fscore(tg, ts, tc);
  result = {"n": n, "g": tg, "s": ts, "c": tc, "p": p, "r": r, "f": f};
  if trace: result["scores"] = scores;
  return result;