Skip to content

Commit

Permalink
Merge pull request #164 from vim-skk/optimize_skk_dictionary
Browse files Browse the repository at this point in the history 
Improve skk_dictionary
  • Loading branch information
@kuuote
kuuote authored Dec 18, 2023
2 parents df9cee4 + 38e7665 commit b81fe61
Show file tree
Hide file tree
Showing 3 changed files with 44 additions and 41 deletions.
15 changes: 1 addition & 14 deletions denops/skkeleton/jisyo.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ import { JpNum } from "./deps/japanese_numeral.ts";
import { RomanNum } from "./deps/roman.ts";
import { zip } from "./deps/std/collections.ts";
import type { CompletionData, RankData } from "./types.ts";
import { readFileWithEncoding } from "./util.ts";
import { SkkDictionary } from "./jisyo/skk_dictionary.ts";
import { UserDictionary, UserDictionaryPath } from "./jisyo/user_dictionary.ts";
import { SkkServer } from "./jisyo/skk_server.ts";
Expand Down Expand Up @@ -266,19 +265,7 @@ export async function load(
globalDictionaryConfig.map(async ([path, encodingName]) => {
const dict = new SkkDictionary();
try {
if (path.endsWith(".yaml") || path.endsWith(".yml")) {
const file = await Deno.readTextFile(path);
dict.loadYaml(file);
} else if (path.endsWith(".json")) {
const file = await Deno.readTextFile(path);
dict.loadJson(file);
} else if (path.endsWith(".mpk")) {
const file = await Deno.readFile(path);
dict.loadMsgpack(file);
} else {
const file = await readFileWithEncoding(path, encodingName);
dict.load(file);
}
await dict.load(path, encodingName);
} catch (e) {
console.error("globalDictionary loading failed");
console.error(`at ${path}`);
Expand Down
36 changes: 29 additions & 7 deletions denops/skkeleton/jisyo/skk_dictionary.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { getKanaTable } from "../kana.ts";
import { readFileWithEncoding } from "../util.ts";
import type { CompletionData } from "../types.ts";
import {
Dictionary,
Expand Down Expand Up @@ -76,7 +77,25 @@ export class SkkDictionary implements Dictionary {
return candidates;
}

loadJson(data: string) {
async load(path: string, encoding: string) {
if (path.endsWith(".yaml") || path.endsWith(".yml")) {
const file = await Deno.readTextFile(path);
this.loadYaml(file);
} else if (path.endsWith(".json")) {
const file = await Deno.readTextFile(path);
this.loadJson(file);
} else if (path.endsWith(".mpk")) {
const file = await Deno.readFile(path);
this.loadMsgpack(file);
} else {
const file = await readFileWithEncoding(path, encoding);
this.loadString(file);
}

return this;
}

private loadJson(data: string) {
const jisyo = JSON.parse(data) as Jisyo;
const validator = new jsonschema.Validator();
const result = validator.validate(jisyo, jisyoschema);
Expand All @@ -89,7 +108,7 @@ export class SkkDictionary implements Dictionary {
this.#okuriNasi = new Map(Object.entries(jisyo.okuri_nasi));
}

loadYaml(data: string) {
private loadYaml(data: string) {
const jisyo = yaml.parse(data) as Jisyo;
const validator = new jsonschema.Validator();
const result = validator.validate(jisyo, jisyoschema);
Expand All @@ -102,7 +121,7 @@ export class SkkDictionary implements Dictionary {
this.#okuriNasi = new Map(Object.entries(jisyo.okuri_nasi));
}

loadMsgpack(data: Uint8Array) {
private loadMsgpack(data: Uint8Array) {
const jisyo = msgpack.decode(data) as Jisyo;
const validator = new jsonschema.Validator();
const result = validator.validate(jisyo, jisyoschema);
Expand All @@ -115,22 +134,25 @@ export class SkkDictionary implements Dictionary {
this.#okuriNasi = new Map(Object.entries(jisyo.okuri_nasi));
}

load(data: string) {
let mode = -1;
private loadString(data: string) {
this.#okuriAri = new Map();
this.#okuriNasi = new Map();

let mode: 0 | 1 | -1 = -1;
const a: Map<string, string[]>[] = [this.#okuriAri, this.#okuriNasi];
const lines = data.split("\n");
for (const line of lines) {
for (const line of data.split("\n")) {
if (line === okuriAriMarker) {
mode = 0;
continue;
}

if (line === okuriNasiMarker) {
mode = 1;
continue;
}

if (mode == -1) continue;

const pos = line.indexOf(" ");
if (pos !== -1) {
a[mode].set(line.substring(0, pos), line.slice(pos + 2, -1).split("/"));
Expand Down
34 changes: 14 additions & 20 deletions denops/skkeleton/jisyo_test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ import { assertEquals } from "./deps/std/assert.ts";
import { Library, load as loadJisyo, wrapDictionary } from "./jisyo.ts";
import { SkkDictionary } from "./jisyo/skk_dictionary.ts";
import { UserDictionary } from "./jisyo/user_dictionary.ts";
import { readFileWithEncoding } from "./util.ts";

const newJisyoJson = join(
dirname(fromFileUrl(import.meta.url)),
Expand Down Expand Up @@ -41,22 +40,11 @@ const numIncludingJisyo = join(
"numIncludingJisyo",
);

async function load(path: string, encoding: string): Promise<SkkDictionary> {
const dic = new SkkDictionary();
if (path.endsWith(".json")) {
dic.loadJson(await readFileWithEncoding(path, encoding));
} else if (path.endsWith(".yaml") || path.endsWith(".yml")) {
dic.loadYaml(await readFileWithEncoding(path, encoding));
} else {
dic.load(await readFileWithEncoding(path, encoding));
}
return dic;
}

Deno.test({
name: "load new JisyoJson",
async fn() {
const jisyo = await load(newJisyoJson, "utf-8");
const dic = new SkkDictionary();
const jisyo = await dic.load(newJisyoJson, "utf-8");
const manager = new Library([jisyo]);
const ari = await manager.getHenkanResult("okuriari", "わるs");
assertEquals(["悪"], ari);
Expand All @@ -68,7 +56,8 @@ Deno.test({
Deno.test({
name: "load new JisyoYaml",
async fn() {
const jisyo = await load(newJisyoYaml, "utf-8");
const dic = new SkkDictionary();
const jisyo = await dic.load(newJisyoYaml, "utf-8");
const manager = new Library([jisyo]);
const ari = await manager.getHenkanResult("okuriari", "わるs");
assertEquals(["悪"], ari);
Expand All @@ -80,7 +69,8 @@ Deno.test({
Deno.test({
name: "get candidates",
async fn() {
const jisyo = await load(globalJisyo, "euc-jp");
const dic = new SkkDictionary();
const jisyo = await dic.load(globalJisyo, "euc-jp");
const manager = new Library([jisyo]);
const ari = await manager.getHenkanResult("okuriari", "てすt");
assertEquals(["テスト"], ari);
Expand All @@ -92,7 +82,8 @@ Deno.test({
Deno.test({
name: "get num candidates",
async fn() {
const jisyo = wrapDictionary(await load(numJisyo, "euc-jp"));
const dic = new SkkDictionary();
const jisyo = wrapDictionary(await dic.load(numJisyo, "euc-jp"));
const manager = new Library([jisyo]);
const nasi = await manager.getHenkanResult("okurinasi", "101ばん");
assertEquals(nasi, [
Expand All @@ -111,7 +102,8 @@ Deno.test({
Deno.test({
name: "get num candidates (Kifu)",
async fn() {
const jisyo = wrapDictionary(await load(numJisyo, "euc-jp"));
const dic = new SkkDictionary();
const jisyo = wrapDictionary(await dic.load(numJisyo, "euc-jp"));
const manager = new Library([jisyo]);
const nasi1 = await manager.getHenkanResult("okurinasi", "11おうて");
assertEquals(nasi1, ["１一王手"]);
Expand All @@ -123,7 +115,8 @@ Deno.test({
Deno.test({
name: "get candidates from words that include numbers",
async fn() {
const jisyo = wrapDictionary(await load(numIncludingJisyo, "utf-8"));
const dic = new SkkDictionary();
const jisyo = wrapDictionary(await dic.load(numIncludingJisyo, "utf-8"));
const manager = new Library([jisyo]);
const nasi1 = await manager.getHenkanResult("okurinasi", "cat2");
assertEquals(nasi1, ["🐈"]);
Expand Down Expand Up @@ -156,7 +149,8 @@ Deno.test({
Deno.test({
name: "global/local jisyo interop",
async fn() {
const jisyo = await load(globalJisyo, "euc-jp");
const dic = new SkkDictionary();
const jisyo = await dic.load(globalJisyo, "euc-jp");
const library = new Library([jisyo]);
await library.registerHenkanResult("okurinasi", "てすと", "test");

Expand Down

0 comments on commit b81fe61

Please sign in to comment.