Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve skk_dictionary #164

Merged
merged 3 commits into from
Dec 18, 2023
Merged

Improve skk_dictionary #164

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1432a0a
Improve skk_dictionary load()
Shougo Dec 17, 2023
2834394
Fix tests
Shougo Dec 17, 2023
38e7665
Improve a bit
Shougo Dec 18, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 1 addition & 14 deletions denops/skkeleton/jisyo.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ import { JpNum } from "./deps/japanese_numeral.ts";
import { RomanNum } from "./deps/roman.ts";
import { zip } from "./deps/std/collections.ts";
import type { CompletionData, RankData } from "./types.ts";
import { readFileWithEncoding } from "./util.ts";
import { SkkDictionary } from "./jisyo/skk_dictionary.ts";
import { UserDictionary, UserDictionaryPath } from "./jisyo/user_dictionary.ts";
import { SkkServer } from "./jisyo/skk_server.ts";
Expand Down Expand Up @@ -266,19 +265,7 @@ export async function load(
globalDictionaryConfig.map(async ([path, encodingName]) => {
const dict = new SkkDictionary();
try {
if (path.endsWith(".yaml") || path.endsWith(".yml")) {
const file = await Deno.readTextFile(path);
dict.loadYaml(file);
} else if (path.endsWith(".json")) {
const file = await Deno.readTextFile(path);
dict.loadJson(file);
} else if (path.endsWith(".mpk")) {
const file = await Deno.readFile(path);
dict.loadMsgpack(file);
} else {
const file = await readFileWithEncoding(path, encodingName);
dict.load(file);
}
await dict.load(path, encodingName);
} catch (e) {
console.error("globalDictionary loading failed");
console.error(`at ${path}`);
Expand Down
36 changes: 29 additions & 7 deletions denops/skkeleton/jisyo/skk_dictionary.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { getKanaTable } from "../kana.ts";
import { readFileWithEncoding } from "../util.ts";
import type { CompletionData } from "../types.ts";
import {
Dictionary,
Expand Down Expand Up @@ -76,7 +77,25 @@ export class SkkDictionary implements Dictionary {
return candidates;
}

loadJson(data: string) {
async load(path: string, encoding: string) {
if (path.endsWith(".yaml") || path.endsWith(".yml")) {
const file = await Deno.readTextFile(path);
this.loadYaml(file);
} else if (path.endsWith(".json")) {
const file = await Deno.readTextFile(path);
this.loadJson(file);
} else if (path.endsWith(".mpk")) {
const file = await Deno.readFile(path);
this.loadMsgpack(file);
} else {
const file = await readFileWithEncoding(path, encoding);
this.loadString(file);
}

return this;
}

private loadJson(data: string) {
const jisyo = JSON.parse(data) as Jisyo;
const validator = new jsonschema.Validator();
const result = validator.validate(jisyo, jisyoschema);
Expand All @@ -89,7 +108,7 @@ export class SkkDictionary implements Dictionary {
this.#okuriNasi = new Map(Object.entries(jisyo.okuri_nasi));
}

loadYaml(data: string) {
private loadYaml(data: string) {
const jisyo = yaml.parse(data) as Jisyo;
const validator = new jsonschema.Validator();
const result = validator.validate(jisyo, jisyoschema);
Expand All @@ -102,7 +121,7 @@ export class SkkDictionary implements Dictionary {
this.#okuriNasi = new Map(Object.entries(jisyo.okuri_nasi));
}

loadMsgpack(data: Uint8Array) {
private loadMsgpack(data: Uint8Array) {
const jisyo = msgpack.decode(data) as Jisyo;
const validator = new jsonschema.Validator();
const result = validator.validate(jisyo, jisyoschema);
Expand All @@ -115,22 +134,25 @@ export class SkkDictionary implements Dictionary {
this.#okuriNasi = new Map(Object.entries(jisyo.okuri_nasi));
}

load(data: string) {
let mode = -1;
private loadString(data: string) {
this.#okuriAri = new Map();
this.#okuriNasi = new Map();

let mode: 0 | 1 | -1 = -1;
const a: Map<string, string[]>[] = [this.#okuriAri, this.#okuriNasi];
const lines = data.split("\n");
for (const line of lines) {
for (const line of data.split("\n")) {
if (line === okuriAriMarker) {
mode = 0;
continue;
}

if (line === okuriNasiMarker) {
mode = 1;
continue;
}

if (mode == -1) continue;

const pos = line.indexOf(" ");
if (pos !== -1) {
a[mode].set(line.substring(0, pos), line.slice(pos + 2, -1).split("/"));
Expand Down
34 changes: 14 additions & 20 deletions denops/skkeleton/jisyo_test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ import { assertEquals } from "./deps/std/assert.ts";
import { Library, load as loadJisyo, wrapDictionary } from "./jisyo.ts";
import { SkkDictionary } from "./jisyo/skk_dictionary.ts";
import { UserDictionary } from "./jisyo/user_dictionary.ts";
import { readFileWithEncoding } from "./util.ts";

const newJisyoJson = join(
dirname(fromFileUrl(import.meta.url)),
Expand Down Expand Up @@ -41,22 +40,11 @@ const numIncludingJisyo = join(
"numIncludingJisyo",
);

async function load(path: string, encoding: string): Promise<SkkDictionary> {
const dic = new SkkDictionary();
if (path.endsWith(".json")) {
dic.loadJson(await readFileWithEncoding(path, encoding));
} else if (path.endsWith(".yaml") || path.endsWith(".yml")) {
dic.loadYaml(await readFileWithEncoding(path, encoding));
} else {
dic.load(await readFileWithEncoding(path, encoding));
}
return dic;
}

Deno.test({
name: "load new JisyoJson",
async fn() {
const jisyo = await load(newJisyoJson, "utf-8");
const dic = new SkkDictionary();
const jisyo = await dic.load(newJisyoJson, "utf-8");
const manager = new Library([jisyo]);
const ari = await manager.getHenkanResult("okuriari", "わるs");
assertEquals(["悪"], ari);
Expand All @@ -68,7 +56,8 @@ Deno.test({
Deno.test({
name: "load new JisyoYaml",
async fn() {
const jisyo = await load(newJisyoYaml, "utf-8");
const dic = new SkkDictionary();
const jisyo = await dic.load(newJisyoYaml, "utf-8");
const manager = new Library([jisyo]);
const ari = await manager.getHenkanResult("okuriari", "わるs");
assertEquals(["悪"], ari);
Expand All @@ -80,7 +69,8 @@ Deno.test({
Deno.test({
name: "get candidates",
async fn() {
const jisyo = await load(globalJisyo, "euc-jp");
const dic = new SkkDictionary();
const jisyo = await dic.load(globalJisyo, "euc-jp");
const manager = new Library([jisyo]);
const ari = await manager.getHenkanResult("okuriari", "てすt");
assertEquals(["テスト"], ari);
Expand All @@ -92,7 +82,8 @@ Deno.test({
Deno.test({
name: "get num candidates",
async fn() {
const jisyo = wrapDictionary(await load(numJisyo, "euc-jp"));
const dic = new SkkDictionary();
const jisyo = wrapDictionary(await dic.load(numJisyo, "euc-jp"));
const manager = new Library([jisyo]);
const nasi = await manager.getHenkanResult("okurinasi", "101ばん");
assertEquals(nasi, [
Expand All @@ -111,7 +102,8 @@ Deno.test({
Deno.test({
name: "get num candidates (Kifu)",
async fn() {
const jisyo = wrapDictionary(await load(numJisyo, "euc-jp"));
const dic = new SkkDictionary();
const jisyo = wrapDictionary(await dic.load(numJisyo, "euc-jp"));
const manager = new Library([jisyo]);
const nasi1 = await manager.getHenkanResult("okurinasi", "11おうて");
assertEquals(nasi1, ["１一王手"]);
Expand All @@ -123,7 +115,8 @@ Deno.test({
Deno.test({
name: "get candidates from words that include numbers",
async fn() {
const jisyo = wrapDictionary(await load(numIncludingJisyo, "utf-8"));
const dic = new SkkDictionary();
const jisyo = wrapDictionary(await dic.load(numIncludingJisyo, "utf-8"));
const manager = new Library([jisyo]);
const nasi1 = await manager.getHenkanResult("okurinasi", "cat2");
assertEquals(nasi1, ["🐈"]);
Expand Down Expand Up @@ -156,7 +149,8 @@ Deno.test({
Deno.test({
name: "global/local jisyo interop",
async fn() {
const jisyo = await load(globalJisyo, "euc-jp");
const dic = new SkkDictionary();
const jisyo = await dic.load(globalJisyo, "euc-jp");
const library = new Library([jisyo]);
await library.registerHenkanResult("okurinasi", "てすと", "test");

Expand Down
Loading