evaluation.py

# Evaluation Script
from jiwer import wer
import argparse

def ter(decipher_path, plain_text_path):
    with open(decipher_path, 'r', encoding='utf-8') as file1, open(plain_text_path, 'r', encoding='utf-8') as file2:
        lines_decipher = file1.readlines()
        lines_plain_text = file2.readlines()

        total_ter = 0.0

        for line1, line2 in zip(lines_decipher, lines_plain_text):
            # Calculate WER for each pair of lines
            current_ter = wer(line1, line2)
            total_ter += current_ter

        # Calculate the average WER per line
        average_ter = total_ter / len(lines_decipher)

    return average_ter


def main():
    parser = argparse.ArgumentParser(description='evaluate decipherment based on ter')
    parser.add_argument('decipher_path', help='path of decipherment generated by model')
    parser.add_argument('plain_text_path', help='path of plain text ground truth')

    args = parser.parse_args()
    score = ter(args.decipher_path, args.plain_text_path)*100
    print("Average TER score: ", score)
    
if __name__ == "__main__":
    main()