Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor #11

Merged
merged 26 commits into from
Jul 12, 2024
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Convert to TS
MarvNC committed Jul 11, 2024
commit 11064afc39f9125c6f06e5e8db953a8eda593249
File renamed without changes.
56 changes: 20 additions & 36 deletions src/convertWikipedia.js → src/convertWikipedia.ts
Original file line number Diff line number Diff line change
@@ -3,25 +3,23 @@ import path from 'path';
import readline from 'readline';
import { Dictionary, TermEntry } from 'yomichan-dict-builder';

import { parseLine } from './parseLine.js';
import { languagesAllowed } from './constants.js';
import { getVersion } from './util/getVersion.js';
import { parseLine } from './parse/parseLine';
import { languagesAllowed } from './constants';
import { getVersion } from './util/getVersion';
import type {
StructuredContent,
StructuredContentNode,
} from 'yomichan-dict-builder/dist/types/yomitan/termbank';

const linkCharacter = '⧉';
/**
*
* @param {string} lang
* @param {string} date
* @param {string} version
* @returns
*/
const outputZipName = (lang, date, version) =>

const outputZipName = (lang: string, date: string, version: string) =>
`${lang} Wikipedia [${date}] (v${version}).zip`;
const shortAbstractFile = (lang) =>
const shortAbstractFile = (lang: string) =>
`short-abstracts_lang=${lang.toLowerCase()}.ttl`;

(async () => {
const version = await getVersion();
const version = getVersion();

console.log(`Using version ${version}`);

@@ -74,11 +72,8 @@ div.gloss-sc-div[data-sc-wikipedia=term-specifier] {

/**
* Reads a line and adds that term entry to the dictionary
* @param {string} line
* @param {Dictionary} dict
* @param {string} lang
*/
function processLine(line, dict, lang) {
function processLine(line: string, dict: Dictionary, lang: string) {
const { term, termSlug, termSpecifier, reading, definition } = parseLine(
line,
lang
@@ -87,17 +82,11 @@ function processLine(line, dict, lang) {
const termEntry = new TermEntry(term);
termEntry.setReading(reading);

/**
* @type {import('yomichan-dict-builder/dist/types/yomitan/termbank').StructuredContent[]}
*/
const structuredContentList = [];
const structuredContentList: StructuredContent[] = [];

// Add term specifier heading if exists
if (termSpecifier) {
/**
* @type {import('yomichan-dict-builder/dist/types/yomitan/termbank').StructuredContentNode}
*/
const specifierSCNode = {
const specifierSCNode: StructuredContentNode = {
tag: 'div',
content: `«${termSpecifier}»`,
data: {
@@ -112,10 +101,7 @@ function processLine(line, dict, lang) {
}

const definitionStrings = definition.split('\\n').map((line) => line.trim());
/**
* @type {import('yomichan-dict-builder/dist/types/yomitan/termbank').StructuredContentNode}
*/
const definitionUList = {
const definitionUList: StructuredContentNode = {
tag: 'ul',
content: definitionStrings.map((definitionString) => ({
tag: 'li',
@@ -135,10 +121,7 @@ function processLine(line, dict, lang) {
: lang === languagesAllowed.zh
? '查看更多'
: 'Read more';
/**
* @type {import('yomichan-dict-builder/dist/types/yomitan/termbank').StructuredContentNode}
*/
const linkSC = {
const linkSC: StructuredContentNode = {
tag: 'ul',
content: [
{
@@ -171,17 +154,18 @@ function processLine(line, dict, lang) {

function readArgs() {
// Read arguments: node convertWikipedia.js [language] [date of dump]
const langInput = process.argv[2];
const langInput =
process.argv[2].toLowerCase() as keyof typeof languagesAllowed;
// Assert language is valid
if (!languagesAllowed[langInput.toLowerCase()]) {
if (!languagesAllowed[langInput]) {
throw new Error(
`Language ${langInput} is not allowed. Allowed languages: ${Object.keys(
languagesAllowed
).join(', ')}`
);
}

const lang = languagesAllowed[langInput.toLowerCase()];
const lang = languagesAllowed[langInput];

const dateInput = process.argv[3];
// Assert date is valid in format YYYY-MM-DD