diff --git a/Generation/fluency/CreateLM/sfr2frames.py b/Generation/fluency/CreateLM/sfr2frames.py index 482e68a9f..c02506771 100644 --- a/Generation/fluency/CreateLM/sfr2frames.py +++ b/Generation/fluency/CreateLM/sfr2frames.py @@ -6,7 +6,7 @@ # # adapted by GvN: now undoes MWU, because that is how the model is applied # in the generator -import sys +import io, sys def is_mwu(tag): result=False @@ -27,8 +27,10 @@ def is_mwu(tag): curId = '' frames = [] - sys.stdin.reconfigure(encoding='utf-8') - sys.stdout.reconfigure(encoding='utf-8') + # sys.stdin.reconfigure(encoding='utf-8') + # sys.stdout.reconfigure(encoding='utf-8') + sys.stdin = io.TextIOWrapper(sys.stdin.detach(), encoding='utf-8', newline=None) + sys.stdout = io.TextIOWrapper(sys.stdout.detach(), encoding='utf-8', newline=None, line_buffering=True) for line in sys.stdin: lineParts = line.split('|') diff --git a/Generation/fluency/CreateLM/sfr2words.py b/Generation/fluency/CreateLM/sfr2words.py index 95bd0e4a3..cb0b64ec7 100644 --- a/Generation/fluency/CreateLM/sfr2words.py +++ b/Generation/fluency/CreateLM/sfr2words.py @@ -5,14 +5,16 @@ # # -import sys +import io, sys if __name__ == '__main__': curId = '' frames = [] - sys.stdin.reconfigure(encoding='utf-8') - sys.stdout.reconfigure(encoding='utf-8') + # sys.stdin.reconfigure(encoding='utf-8') + # sys.stdout.reconfigure(encoding='utf-8') + sys.stdin = io.TextIOWrapper(sys.stdin.detach(), encoding='utf-8', newline=None) + sys.stdout = io.TextIOWrapper(sys.stdout.detach(), encoding='utf-8', newline=None, line_buffering=True) for line in sys.stdin: lineParts = line.split('|') diff --git a/Generation/fluency/CreateMaxent/evaluate.py b/Generation/fluency/CreateMaxent/evaluate.py index 65467bd71..be4c3e291 100644 --- a/Generation/fluency/CreateMaxent/evaluate.py +++ b/Generation/fluency/CreateMaxent/evaluate.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import argparse +import io import random import re import sys @@ -165,8 +166,11 @@ def printModelScores(scores, bestScores): # Make results reproducable. random.seed(13) - sys.stdin.reconfigure(encoding='utf-8') - sys.stdout.reconfigure(encoding='utf-8') + # sys.stdin.reconfigure(encoding='utf-8') + # sys.stdout.reconfigure(encoding='utf-8') + sys.stdin = io.TextIOWrapper(sys.stdin.detach(), encoding='utf-8', newline=None) + sys.stdout = io.TextIOWrapper(sys.stdout.detach(), encoding='utf-8', newline=None, line_buffering=True) + with open(options.features, encoding='utf-8') as featureFh: featureWeights = readFeatureWeights(featureFh) diff --git a/Generation/fluency/CreateMaxent/filter_features.py b/Generation/fluency/CreateMaxent/filter_features.py index f9a8e6510..de04bc95d 100644 --- a/Generation/fluency/CreateMaxent/filter_features.py +++ b/Generation/fluency/CreateMaxent/filter_features.py @@ -4,6 +4,7 @@ # import argparse +import io import sys VELLDAL_FEATURES = ['lds', 'ldsb', 'lds_dl', 'lds_skew', 'tngram', 'tngramw'] @@ -38,8 +39,10 @@ tagNgramFeatures = options.tagNgramFeatures velldalFeatures = options.velldalFeatures - sys.stdin.reconfigure(encoding='utf-8') - sys.stdout.reconfigure(encoding='utf-8') + # sys.stdin.reconfigure(encoding='utf-8') + # sys.stdout.reconfigure(encoding='utf-8') + sys.stdin = io.TextIOWrapper(sys.stdin.detach(), encoding='utf-8', newline=None) + sys.stdout = io.TextIOWrapper(sys.stdout.detach(), encoding='utf-8', newline=None, line_buffering=True) for line in sys.stdin: line = line.strip() diff --git a/TreebankTools/bin/dtcanonicalize.py b/TreebankTools/bin/dtcanonicalize.py index 35c384853..5fada3420 100755 --- a/TreebankTools/bin/dtcanonicalize.py +++ b/TreebankTools/bin/dtcanonicalize.py @@ -68,7 +68,7 @@ from xml.dom import minidom from xml.parsers.expat import ExpatError -import sys, os +import io, os, sys import tempfile import html @@ -204,13 +204,11 @@ def dump_content(self): print("comment:", comment) - def write_xml(self, outfile, encoding='UTF-8', version='1.5'): - """write the dependency structure to xml in encoding""" - - outfile.reconfigure(encoding=encoding) + def write_xml(self, outfile, version='1.5'): + """write the dependency structure to xml""" # de xml-header - outfile.write('\n' % encoding) + outfile.write('\n') # de root node if version == "": @@ -766,12 +764,12 @@ def write_xml_data(outfile, data): parser.print_help(sys.stderr) sys.exit(1) - dtparser = DTParser() # de bestanden canoniek maken error_occurred = False if options.stdin: + sys.stdout = io.TextIOWrapper(sys.stdout.detach(), encoding='utf-8', newline=None, line_buffering=True) try: cn = Canonicalizer(dtparser, sys.stdin) cn.canonicalize() @@ -796,7 +794,7 @@ def write_xml_data(outfile, data): os.path.dirname(file)) # - we schrijven de nieuwe file naar die tempfile - tempfileobj = os.fdopen(tempfd, "w") + tempfileobj = os.fdopen(tempfd, "w", encoding="utf-8") cn.write_xml(tempfileobj) tempfileobj.close() diff --git a/webdemo.py b/webdemo.py index ed40219fa..caa76b685 100644 --- a/webdemo.py +++ b/webdemo.py @@ -7,7 +7,7 @@ from urllib import parse #sys.stdin.reconfigure(encoding='utf-8') -sys.stdout = io.TextIOWrapper(sys.stdout.detach(), encoding='utf-8') +sys.stdout = io.TextIOWrapper(sys.stdout.detach(), encoding='utf-8', newline=None, line_buffering=True) form = {} for line in sys.stdin: for key, val in parse.parse_qsl(line,