human-gaze-guided-neural-at.../joint_sentence_compression_.../utils/check_stats.py

44 lines
931 B
Python

import ast
from statistics import mean, pstdev
import sys
import click
from scipy.stats import entropy
from matplotlib import pyplot as plt
def reader(path):
with open(path) as h:
for line in h:
line = line.strip()
try:
s, p, a, f = line.split("\t")
except:
print(f"skipping line: {line}", file=sys.stderr)
else:
try:
yield ast.literal_eval(s), ast.literal_eval(p), ast.literal_eval(a), ast.literal_eval(f)
except:
print(f"malformed line: {s}")
def get_stats(seq):
for s, p, a, f in seq:
print(s)
print(p)
print(len(s), len(p), len(a), len(f))
for x in a:
print(len(x))
print()
@click.command()
@click.argument("path")
def main(path):
get_stats(reader(path))
if __name__ == "__main__":
main()