From c8017572331ac55e6ab74553b291a57b1c60efd6 Mon Sep 17 00:00:00 2001 From: John Collins Date: Fri, 11 Oct 2024 13:11:38 -0400 Subject: [PATCH 1/2] Replace use of winunicode by CodePage's result for system CP --- lib/Biber/CodePage.pm | 10 +++++++--- lib/Biber/Utils.pm | 17 +++++------------ 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/lib/Biber/CodePage.pm b/lib/Biber/CodePage.pm index 993846814..2aa3c84ad 100644 --- a/lib/Biber/CodePage.pm +++ b/lib/Biber/CodePage.pm @@ -4,6 +4,7 @@ use strict; use warnings; use parent qw(Exporter); our @EXPORT = qw( decode_CS_system encode_CS_system get_CS_system + is_Unicode_system set_CP_Win_console set_CS_defaults set_STD_encodings ); our @EXPORT_OK = qw( get_CS_log string_analysis ); @@ -92,6 +93,12 @@ sub get_CS_log() { #-------------------------- +sub is_Unicode_system() { + return (uc($CS_system) eq 'UTF-8') || (uc($CS_system) eq 'CP65001'); +} + +#-------------------------- + sub set_STD_encodings($) { # Set coding scheme for STDOUT, STDERR & STDIN. my $CS = shift; @@ -249,7 +256,4 @@ END { # Don't worry about what to do with STDOUT, etc; it should be unimportant from here. } - - - 1; diff --git a/lib/Biber/Utils.pm b/lib/Biber/Utils.pm index fbb7980b9..fc7201bda 100644 --- a/lib/Biber/Utils.pm +++ b/lib/Biber/Utils.pm @@ -171,9 +171,8 @@ sub slurp_switchr { my ($filename, $encoding) = @_; my $slurp; $encoding //= 'UTF-8'; - if ($^O =~ /Win/ and not Biber::Config->getoption('winunicode')) { + if ($^O =~ /Win/ and not is_Unicode_system() ) { require Win32::Unicode::File; - # JCC ?? Removed NFC my $fh = Win32::Unicode::File->new('<', $filename); $fh->binmode(":encoding($encoding)"); # 100MB block size as the loop over the default 1MB block size seems to fail for @@ -196,9 +195,8 @@ sub slurp_switchr { sub slurp_switchw { my ($filename, $string) = @_; - if ($^O =~ /Win/ and not Biber::Config->getoption('winunicode')) { + if ($^O =~ /Win/ and not is_Unicode_system() ) { require Win32::Unicode::File; - # JCC ?? Removed NFC my $fh = Win32::Unicode::File->new('>', $filename); $fh->binmode(':encoding(UTF-8)'); $fh->write($string); @@ -206,7 +204,6 @@ sub slurp_switchw { $fh->close; } else { - # JCC ?? Removed NFC File::Slurper::write_text($filename, $string); } return; @@ -394,9 +391,8 @@ sub locate_data_file { sub file_exist_check { my $filename = shift; - if ($^O =~ /Win/ and not Biber::Config->getoption('winunicode')) { + if ($^O =~ /Win/ and not is_Unicode_system() ) { require Win32::Unicode::File; - # JCC ?? Add test without NF mapped filename. if (Win32::Unicode::File::statW($filename)) { return $filename; } @@ -408,7 +404,6 @@ sub file_exist_check { } } else { - # JCC ?? Add test without NF mapped filename. if (-e "$filename") { return $filename; } @@ -431,9 +426,8 @@ sub file_exist_check { sub check_empty { my $filename = shift; - if ($^O =~ /Win/ and not Biber::Config->getoption('winunicode')) { + if ($^O =~ /Win/ and not is_Unicode_system() ) { require Win32::Unicode::File; - # JCC ?? Remove NFC return (Win32::Unicode::File::file_size($filename)) ? 1 : 0; } else { @@ -449,9 +443,8 @@ sub check_empty { sub check_exists { my $filename = shift; - if ($^O =~ /Win/ and not Biber::Config->getoption('winunicode')) { + if ($^O =~ /Win/ and not is_Unicode_system() ) { require Win32::Unicode::File; - # JCC ?? Remove NFC return Win32::Unicode::File::statW($filename) ? 1 : 0; } else { From 1db3b6cd6b00ba3ceaa31ca5bbe5bbcbf9db6d97 Mon Sep 17 00:00:00 2001 From: John Collins Date: Fri, 11 Oct 2024 13:13:55 -0400 Subject: [PATCH 2/2] Correct test script for changes in CodePage.pm --- testfiles/Unicode-tests-preserved-NF.zip | Bin 4541 -> 4532 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/testfiles/Unicode-tests-preserved-NF.zip b/testfiles/Unicode-tests-preserved-NF.zip index 48c6a807e1d54d39d4e3dd3da7811ecd7cc236f1..a7f56bac71a2cf8be9b3a1c5bea6b271aa010be7 100644 GIT binary patch delta 1267 zcmV@3IG5A0022@OOXvE2;N%>XZBkOXR$fz2o2R}OIZq$_I+Cf z000@2FbOSxAzKM&D_aR?b$AN^0R#yG000CO00023R?Ti3HxRz-DJE+HyK=R1Y}7>{ z!+`8{gc>j$BeKwofdzNPl^E}m%93kYZG6bB1=?fJITfgnX?x5~*TR04JVJ+D>c^J< z+H?&VBxmNEnQw+etvp*RWWE+LzDD^O3~8QLDutwfaJJdl_L5|YTQHz+!R;Qm{TFTx-o0BA1xMd(K7YRMYEP9F|IV+XX>hiezY}ictE1ri}rC36# zsnCh7KtLpJ)qp6F15u>QUH?>6@5r@O6$;PXOyEobV|xl zn34EZrw#{<7w2^lIedqO6(b2FrUnokt#52@m?$#BxQ?I#__>Gez>i5XxojwU?$*uW zQfz{!V;G72R1-(Q)D6mN)j^Y}GlS%|Q(Mo{cLR)f-3p<>n}x7TIR1V9@XOB&F}MML z$#bFn&mVsKb@BMHy{F@)f4^Tm&Dgn*nM^L1We;`Gyto)UCGh|}Lr)EUz)Wd~C|)(J zZNWvQf`#xF`x$i%zG;hZ$r-&xixjZ$rYZEqR4}A5OIQpFyV8@j2DXI;n7f{FbBF(n zFKs}L7sZJO$dl*1DioGHic+UmFHa7CM1OKQ+vSr>S)__6xi;$%r>4ogkB0^Py(Rp; zr{mwkDr3F+c=gBik5_+9J~_39ft!59m18{26S(^G`u+7!liz1bpPu0Gi<#Ao%iSDX9>ai!xV{9d?N^IW~X@!2}^0C48pW{*8|5& zu>_7EnEyDe@CcW!a9U0M5ad+hg}>X7aYb_yD<=4ML{vP;@X;}nVK2E4)U+X@YQBbr z9L%R{9o`7WJ#fLb21uedLG2=c=Oz}z+@l;}3tEma1rMsCP<5!LqL2TztcEGI!da3n zl!qt{HB)BDTN6jyG@`WcOrtktd|hz;zT5e3CMK$zd5Zc#@;+9D;{$dR*tSl=@#Q-0 zdN+j+P;T7#Z{pZc>Z94PP`@3IG5A005SfL6Hq42)lLsX1jI#X0bWy2n|P%L0P#UYg}Xm z001bHFbOSxb#VMbl?J+l93;VC+Cvc26Tu?ZF->PqZ+tf=TGZxBXJt7%ryf2Df zM2XBZ3U5aa2r`ChA?CrOAx`!t~IElyasIRi+eU(Zn8AHYE_+l(1O<)`P zRQj`jy@Myz0o?ga_$p^S;w%`@w_tZq+U^Ux3h&=9sl4Owww^!VuvMf|*39bRz)3<;#34+Rgzz3>!s*%?D1%dOHSY~pL_6gx;yw0R z1zfF&uoxK?3JO(N89X9zD+fe^7^HdJ0}_va$XI~L!;sQm4Oj$$Vg{IiAn!wzQCfpO zJEt7&p;JJH%7lckIyE?CJin+xn!)$Tb2<_*VsZf9@y6!Xrp_WEjH?VvfS-NT_S}#} zldHO<7j{hxmvT#aJcd!4ohjf5Xx^Y;s|E^0RT)HggBm`I*mcm+RV(<~-z@H4! zzI^=U=Yk7QXY-7g=1`*l%#)ZSD1(!W0}N)vYBVkMKSW!*y^G$$8xrz9MJqv5H- z4_GPn(Th{@t4+8pm5>N$v7hmb_OILGB{`>eU=b-C*l`TKbgCHAkVPy6iCyW4Y8~4` zIhco@VY4Fs<<~Z#@)v}W1BjDjy~-tjQXT}cRjn10!?ZuSn(ea5mB?dBq*$MIh*ec! z-iO12|Gg#udr$X&$5)Z-^{4AUZhpM}Yx3Eu*0sCICtO)N!n}a%KW{$V{51J}cIfjH z0)92J3X$2}zSS1(>4nE@BL_Ir(>P8?I6)gr+%MP{wg461DwwA+&gTWUov!D9?RPqs z&YfN`2^;VYRLWBy&HG1YN#wz_wT#O$>87)#^xt{*TVTB@Gw)`oZ>LJLe#0$ULka0;f5=xfx?SRN|kl-7nHN#$X zpHtn2=vCo0%*9|nU2E_rW!wRO8*F2MASx3)oo8IgDA z-F9qRC+GNbopqht$_F6VVca)is3CRHY?w<|S)srr>bmG@Dq1M?Znxflj%h%e((+8V zijR$na}LKBD2_}z8%=XF@C4Wu-^Qm|;@gFUPieaRul)XdlIMAv5{$J=f$Be33;$@u zfJL(Upikw};63k~7V_u^W2rkLAw#TZ$+rUOa@I;3pp$H@dPAm=JYK6xY*%`$X6SMmVP)h*<9s>#h00000mXbk}-VHJXyLJ4Np${ChFAfv~ l4M&ebS-Br;Tx0|Q04S4C5jzBRaQtSIj}adR*bM*x007VeTN(fW