From 1432a0a14496f28df78970449bba5554d1a0cbaa Mon Sep 17 00:00:00 2001 From: Shougo Matsushita Date: Sun, 17 Dec 2023 12:56:24 +0900 Subject: [PATCH 1/3] Improve skk_dictionary load() --- denops/skkeleton/jisyo.ts | 15 +------------- denops/skkeleton/jisyo/skk_dictionary.ts | 25 ++++++++++++++++++++---- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/denops/skkeleton/jisyo.ts b/denops/skkeleton/jisyo.ts index 5ada960e..6c1d6797 100644 --- a/denops/skkeleton/jisyo.ts +++ b/denops/skkeleton/jisyo.ts @@ -3,7 +3,6 @@ import { JpNum } from "./deps/japanese_numeral.ts"; import { RomanNum } from "./deps/roman.ts"; import { zip } from "./deps/std/collections.ts"; import type { CompletionData, RankData } from "./types.ts"; -import { readFileWithEncoding } from "./util.ts"; import { SkkDictionary } from "./jisyo/skk_dictionary.ts"; import { UserDictionary, UserDictionaryPath } from "./jisyo/user_dictionary.ts"; import { SkkServer } from "./jisyo/skk_server.ts"; @@ -266,19 +265,7 @@ export async function load( globalDictionaryConfig.map(async ([path, encodingName]) => { const dict = new SkkDictionary(); try { - if (path.endsWith(".yaml") || path.endsWith(".yml")) { - const file = await Deno.readTextFile(path); - dict.loadYaml(file); - } else if (path.endsWith(".json")) { - const file = await Deno.readTextFile(path); - dict.loadJson(file); - } else if (path.endsWith(".mpk")) { - const file = await Deno.readFile(path); - dict.loadMsgpack(file); - } else { - const file = await readFileWithEncoding(path, encodingName); - dict.load(file); - } + await dict.load(path, encodingName); } catch (e) { console.error("globalDictionary loading failed"); console.error(`at ${path}`); diff --git a/denops/skkeleton/jisyo/skk_dictionary.ts b/denops/skkeleton/jisyo/skk_dictionary.ts index a788b796..28bb8116 100644 --- a/denops/skkeleton/jisyo/skk_dictionary.ts +++ b/denops/skkeleton/jisyo/skk_dictionary.ts @@ -1,4 +1,5 @@ import { getKanaTable } from "../kana.ts"; +import { readFileWithEncoding } from "../util.ts"; import type { CompletionData } from "../types.ts"; import { Dictionary, @@ -76,7 +77,23 @@ export class SkkDictionary implements Dictionary { return candidates; } - loadJson(data: string) { + async load(path: string, encoding: string) { + if (path.endsWith(".yaml") || path.endsWith(".yml")) { + const file = await Deno.readTextFile(path); + this.loadYaml(file); + } else if (path.endsWith(".json")) { + const file = await Deno.readTextFile(path); + this.loadJson(file); + } else if (path.endsWith(".mpk")) { + const file = await Deno.readFile(path); + this.loadMsgpack(file); + } else { + const file = await readFileWithEncoding(path, encoding); + this.loadString(file); + } + } + + private loadJson(data: string) { const jisyo = JSON.parse(data) as Jisyo; const validator = new jsonschema.Validator(); const result = validator.validate(jisyo, jisyoschema); @@ -89,7 +106,7 @@ export class SkkDictionary implements Dictionary { this.#okuriNasi = new Map(Object.entries(jisyo.okuri_nasi)); } - loadYaml(data: string) { + private loadYaml(data: string) { const jisyo = yaml.parse(data) as Jisyo; const validator = new jsonschema.Validator(); const result = validator.validate(jisyo, jisyoschema); @@ -102,7 +119,7 @@ export class SkkDictionary implements Dictionary { this.#okuriNasi = new Map(Object.entries(jisyo.okuri_nasi)); } - loadMsgpack(data: Uint8Array) { + private loadMsgpack(data: Uint8Array) { const jisyo = msgpack.decode(data) as Jisyo; const validator = new jsonschema.Validator(); const result = validator.validate(jisyo, jisyoschema); @@ -115,7 +132,7 @@ export class SkkDictionary implements Dictionary { this.#okuriNasi = new Map(Object.entries(jisyo.okuri_nasi)); } - load(data: string) { + private loadString(data: string) { let mode = -1; this.#okuriAri = new Map(); this.#okuriNasi = new Map(); From 2834394bb48914f4829f11da5da29006dcb5bd6f Mon Sep 17 00:00:00 2001 From: Shougo Matsushita Date: Sun, 17 Dec 2023 13:01:34 +0900 Subject: [PATCH 2/3] Fix tests --- denops/skkeleton/jisyo/skk_dictionary.ts | 2 ++ denops/skkeleton/jisyo_test.ts | 34 ++++++++++-------------- 2 files changed, 16 insertions(+), 20 deletions(-) diff --git a/denops/skkeleton/jisyo/skk_dictionary.ts b/denops/skkeleton/jisyo/skk_dictionary.ts index 28bb8116..23e497e5 100644 --- a/denops/skkeleton/jisyo/skk_dictionary.ts +++ b/denops/skkeleton/jisyo/skk_dictionary.ts @@ -91,6 +91,8 @@ export class SkkDictionary implements Dictionary { const file = await readFileWithEncoding(path, encoding); this.loadString(file); } + + return this; } private loadJson(data: string) { diff --git a/denops/skkeleton/jisyo_test.ts b/denops/skkeleton/jisyo_test.ts index f9d273e9..a06490e9 100644 --- a/denops/skkeleton/jisyo_test.ts +++ b/denops/skkeleton/jisyo_test.ts @@ -3,7 +3,6 @@ import { assertEquals } from "./deps/std/assert.ts"; import { Library, load as loadJisyo, wrapDictionary } from "./jisyo.ts"; import { SkkDictionary } from "./jisyo/skk_dictionary.ts"; import { UserDictionary } from "./jisyo/user_dictionary.ts"; -import { readFileWithEncoding } from "./util.ts"; const newJisyoJson = join( dirname(fromFileUrl(import.meta.url)), @@ -41,22 +40,11 @@ const numIncludingJisyo = join( "numIncludingJisyo", ); -async function load(path: string, encoding: string): Promise { - const dic = new SkkDictionary(); - if (path.endsWith(".json")) { - dic.loadJson(await readFileWithEncoding(path, encoding)); - } else if (path.endsWith(".yaml") || path.endsWith(".yml")) { - dic.loadYaml(await readFileWithEncoding(path, encoding)); - } else { - dic.load(await readFileWithEncoding(path, encoding)); - } - return dic; -} - Deno.test({ name: "load new JisyoJson", async fn() { - const jisyo = await load(newJisyoJson, "utf-8"); + const dic = new SkkDictionary(); + const jisyo = await dic.load(newJisyoJson, "utf-8"); const manager = new Library([jisyo]); const ari = await manager.getHenkanResult("okuriari", "わるs"); assertEquals(["悪"], ari); @@ -68,7 +56,8 @@ Deno.test({ Deno.test({ name: "load new JisyoYaml", async fn() { - const jisyo = await load(newJisyoYaml, "utf-8"); + const dic = new SkkDictionary(); + const jisyo = await dic.load(newJisyoYaml, "utf-8"); const manager = new Library([jisyo]); const ari = await manager.getHenkanResult("okuriari", "わるs"); assertEquals(["悪"], ari); @@ -80,7 +69,8 @@ Deno.test({ Deno.test({ name: "get candidates", async fn() { - const jisyo = await load(globalJisyo, "euc-jp"); + const dic = new SkkDictionary(); + const jisyo = await dic.load(globalJisyo, "euc-jp"); const manager = new Library([jisyo]); const ari = await manager.getHenkanResult("okuriari", "てすt"); assertEquals(["テスト"], ari); @@ -92,7 +82,8 @@ Deno.test({ Deno.test({ name: "get num candidates", async fn() { - const jisyo = wrapDictionary(await load(numJisyo, "euc-jp")); + const dic = new SkkDictionary(); + const jisyo = wrapDictionary(await dic.load(numJisyo, "euc-jp")); const manager = new Library([jisyo]); const nasi = await manager.getHenkanResult("okurinasi", "101ばん"); assertEquals(nasi, [ @@ -111,7 +102,8 @@ Deno.test({ Deno.test({ name: "get num candidates (Kifu)", async fn() { - const jisyo = wrapDictionary(await load(numJisyo, "euc-jp")); + const dic = new SkkDictionary(); + const jisyo = wrapDictionary(await dic.load(numJisyo, "euc-jp")); const manager = new Library([jisyo]); const nasi1 = await manager.getHenkanResult("okurinasi", "11おうて"); assertEquals(nasi1, ["１一王手"]); @@ -123,7 +115,8 @@ Deno.test({ Deno.test({ name: "get candidates from words that include numbers", async fn() { - const jisyo = wrapDictionary(await load(numIncludingJisyo, "utf-8")); + const dic = new SkkDictionary(); + const jisyo = wrapDictionary(await dic.load(numIncludingJisyo, "utf-8")); const manager = new Library([jisyo]); const nasi1 = await manager.getHenkanResult("okurinasi", "cat2"); assertEquals(nasi1, ["🐈"]); @@ -156,7 +149,8 @@ Deno.test({ Deno.test({ name: "global/local jisyo interop", async fn() { - const jisyo = await load(globalJisyo, "euc-jp"); + const dic = new SkkDictionary(); + const jisyo = await dic.load(globalJisyo, "euc-jp"); const library = new Library([jisyo]); await library.registerHenkanResult("okurinasi", "てすと", "test"); From 38e76654c633e54349c5ae3edc6b28496d0bb2e2 Mon Sep 17 00:00:00 2001 From: Shougo Matsushita Date: Mon, 18 Dec 2023 18:18:49 +0900 Subject: [PATCH 3/3] Improve a bit --- denops/skkeleton/jisyo/skk_dictionary.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/denops/skkeleton/jisyo/skk_dictionary.ts b/denops/skkeleton/jisyo/skk_dictionary.ts index 23e497e5..03f858ca 100644 --- a/denops/skkeleton/jisyo/skk_dictionary.ts +++ b/denops/skkeleton/jisyo/skk_dictionary.ts @@ -135,21 +135,24 @@ export class SkkDictionary implements Dictionary { } private loadString(data: string) { - let mode = -1; this.#okuriAri = new Map(); this.#okuriNasi = new Map(); + + let mode: 0 | 1 | -1 = -1; const a: Map[] = [this.#okuriAri, this.#okuriNasi]; - const lines = data.split("\n"); - for (const line of lines) { + for (const line of data.split("\n")) { if (line === okuriAriMarker) { mode = 0; continue; } + if (line === okuriNasiMarker) { mode = 1; continue; } + if (mode == -1) continue; + const pos = line.indexOf(" "); if (pos !== -1) { a[mode].set(line.substring(0, pos), line.slice(pos + 2, -1).split("/"));