Languages subpackage (#292)

Co-authored-by: coyotte508 <[email protected]>

.github/workflows/agents-publish.yml

@@ -30,7 +30,7 @@ jobs:
           registry-url: "https://registry.npmjs.org"
       - run: pnpm install
       - run: git config --global user.name machineuser
-      - run: git config --global user.email tech@huggingface.co
+      - run: git config --global user.email infra+machineuser@huggingface.co
       - run: |
           PACKAGE_VERSION=$(node -p "require('./package.json').version")
           BUMPED_VERSION=$(node -p "require('semver').inc('$PACKAGE_VERSION', '${{ github.event.inputs.newversion }}')")

.github/workflows/hub-publish.yml

@@ -30,7 +30,7 @@ jobs:
           registry-url: "https://registry.npmjs.org"
       - run: pnpm install
       - run: git config --global user.name machineuser
-      - run: git config --global user.email tech@huggingface.co
+      - run: git config --global user.email infra+machineuser@huggingface.co
       - run: |
           PACKAGE_VERSION=$(node -p "require('./package.json').version")
           BUMPED_VERSION=$(node -p "require('semver').inc('$PACKAGE_VERSION', '${{ github.event.inputs.newversion }}')")

.github/workflows/inference-publish.yml

@@ -30,7 +30,7 @@ jobs:
           registry-url: "https://registry.npmjs.org"
       - run: pnpm install
       - run: git config --global user.name machineuser
-      - run: git config --global user.email tech@huggingface.co
+      - run: git config --global user.email infra+machineuser@huggingface.co
       - run: |
           PACKAGE_VERSION=$(node -p "require('./package.json').version")
           BUMPED_VERSION=$(node -p "require('semver').inc('$PACKAGE_VERSION', '${{ github.event.inputs.newversion }}')")

.github/workflows/languages-publish.yml

@@ -0,0 +1,54 @@
+name: Languages - Version and Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      newversion:
+        description: "Semantic Version Bump Type (major minor patch)"
+        default: patch
+
+defaults:
+  run:
+    working-directory: packages/languages
+
+jobs:
+  version_and_release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          token: ${{ secrets.BOT_ACCESS_TOKEN }}
+      - run: corepack enable
+      - uses: actions/setup-node@v3
+        with:
+          node-version: "18"
+          cache: "pnpm"
+          cache-dependency-path: |
+            packages/languages/pnpm-lock.yaml
+            packages/doc-internal/pnpm-lock.yaml
+          # setting a registry enables the NODE_AUTH_TOKEN env variable where we can set an npm token.  REQUIRED
+          registry-url: "https://registry.npmjs.org"
+      - run: pnpm install
+      - run: git config --global user.name machineuser
+      - run: git config --global user.email [email protected]
+      - run: |
+          PACKAGE_VERSION=$(node -p "require('./package.json').version")
+          BUMPED_VERSION=$(node -p "require('semver').inc('$PACKAGE_VERSION', '${{ github.event.inputs.newversion }}')")
+          # Update package.json with the new version
+          node -e "const fs = require('fs'); const package = JSON.parse(fs.readFileSync('./package.json')); package.version = '$BUMPED_VERSION'; fs.writeFileSync('./package.json', JSON.stringify(package, null, '\t') + '\n');"
+          pnpm --filter doc-internal run fix-cdn-versions
+          git add ../..
+          git commit -m "🔖 @hugginface/languages $BUMPED_VERSION"
+          git tag "languages-v$BUMPED_VERSION"
+      - run: pnpm publish --no-git-checks .
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+      - run: git push --follow-tags
+      # hack - reuse actions/setup-node@v3 just to set a new registry
+      - uses: actions/setup-node@v3
+        with:
+          node-version: "18"
+          registry-url: "https://npm.pkg.github.com"
+      - run: pnpm publish --no-git-checks .
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

CONTRIBUTING.md

@@ -28,4 +28,8 @@ If you want to run only specific tests, you can do `pnpm test -- -t "test name"`
 - Edit [`packages/doc-internal/package.json`](packages/doc-internal/package.json) and add the two commands in the `scripts` section:
   - `prepublish-<package name>`
   - `doc-<package name>`
-- Add the `<package-name>-publish.yml` file in the `.github/workflows` folder, inspired from the other packages
+- Add the `<package-name>-publish.yml` file in the `.github/workflows` folder, inspired from the other packages
+- Add the package in the main README.md
+- Add a `tsup.config.ts` file in the package folder, inspired from the other packages, or just use `tsup src/index.ts --format cjs,esm --clean --dts` as the build command
+- Copy `.prettierignore` from another package
+- Add a `tsconfig.json`

README.md

@@ -40,6 +40,7 @@ This is a collection of JS libraries to interact with the Hugging Face API, with
 - [@huggingface/inference](packages/inference/README.md): Use the Inference API to make calls to 100,000+ Machine Learning models, or your own [inference endpoints](https://hf.co/docs/inference-endpoints/)!
 - [@huggingface/agents](packages/agents/README.md): Interact with HF models through a natural language interface
 - [@huggingface/hub](packages/hub/README.md): Interact with huggingface.co to create or delete repos and commit / download files
+- [@huggingface/languages](packages/languages/README.md): List of all languages supported by the huggingface hub
 
 
 With more to come, like `@huggingface/endpoints` to manage your HF Endpoints!

packages/languages/.prettierignore

@@ -0,0 +1,4 @@
+pnpm-lock.yaml
+# In order to avoid code samples to have tabs, they don't display well on npm
+README.md
+dist

packages/languages/README.md

@@ -0,0 +1,24 @@
+# 🤗 Hugging Face Languages.js
+
+The list of all languages supported by the huggingface hub.
+
+## Install
+
+```console
+pnpm add @huggingface/languages
+
+npm add @huggingface/languages
+
+yarn add @huggingface/languages
+```
+
+### Deno
+
+```ts
+// esm.sh
+import { language, wikiLink, type Language } from "https://esm.sh/@huggingface/languages"
+// or npm:
+import { language, wikiLink, type Language } from "npm:@huggingface/languages"
+```
+
+Check out the [full documentation](https://huggingface.co/docs/huggingface.js/languages/README).

packages/languages/package.json

@@ -0,0 +1,46 @@
+{
+	"name": "@huggingface/languages",
+	"packageManager": "[email protected]",
+	"version": "0.1.0",
+	"description": "List of ISO-639 languages used in the Hub",
+	"repository": "https://github.com/huggingface/huggingface.js.git",
+	"publishConfig": {
+		"access": "public"
+	},
+	"main": "./dist/index.js",
+	"module": "./dist/index.mjs",
+	"types": "./dist/index.d.ts",
+	"exports": {
+		".": {
+			"types": "./dist/index.d.ts",
+			"require": "./dist/index.js",
+			"import": "./dist/index.mjs"
+		}
+	},
+	"source": "index.ts",
+	"scripts": {
+		"lint": "eslint --quiet --fix --ext .cjs,.ts .",
+		"lint:check": "eslint --ext .cjs,.ts .",
+		"format": "prettier --write .",
+		"format:check": "prettier --check .",
+		"prepublishOnly": "pnpm run build",
+		"build": "tsup src/index.ts --format cjs,esm --clean --dts",
+		"test": "vitest run",
+		"type-check": "tsc"
+	},
+	"files": [
+		"dist",
+		"index.ts",
+		"tsconfig.json"
+	],
+	"keywords": [
+		"huggingface",
+		"hub",
+		"languages"
+	],
+	"author": "Hugging Face",
+	"license": "MIT",
+	"devDependencies": {
+		"typescript": "^5.0.4"
+	}
+}

packages/languages/src/index.ts

@@ -0,0 +1,5 @@
+export type { Language } from "./types";
+export { language } from "./language";
+export { wikiLink } from "./wikiLink";
+export { LANGUAGES_ISO_639_1 } from "./languages_iso_639_1";
+export { LANGUAGES_ISO_639_3 } from "./languages_iso_639_3";

packages/languages/src/language.spec.ts

@@ -0,0 +1,28 @@
+import { describe, expect, it } from "vitest";
+import { language } from "./language";
+
+describe("language", () => {
+	it("should return an object for a 2-letter language code", () => {
+		expect(language("en")).toMatchObject({
+			code: "en",
+			name: "English",
+			nativeName: "English",
+		});
+	});
+
+	it("should return an object for a 3-letter language code", () => {
+		expect(language("eng")).toMatchObject({
+			code: "eng",
+			name: "English",
+		});
+
+		expect(language("fra")).toMatchObject({
+			code: "fra",
+			name: "French",
+		});
+	});
+
+	it("should return null for an invalid language code", () => {
+		expect(language("invalid")).toBeNull();
+	});
+});

packages/languages/src/language.ts

@@ -0,0 +1,17 @@
+import { LANGUAGES_ISO_639_1 } from "./languages_iso_639_1";
+import { LANGUAGES_ISO_639_3 } from "./languages_iso_639_3";
+import type { Language } from "./types";
+
+/**
+ * Be careful importing this function in frontend code, as it will import all languages.
+ */
+export function language(code: string): Language | null {
+	if (code in LANGUAGES_ISO_639_1) {
+		return LANGUAGES_ISO_639_1[code as keyof typeof LANGUAGES_ISO_639_1];
+	}
+	if (code in LANGUAGES_ISO_639_3) {
+		return LANGUAGES_ISO_639_3[code as keyof typeof LANGUAGES_ISO_639_3];
+	}
+
+	return null;
+}