✨ Use @huggingface/tasks as source of truth in @huggingface/hub and @…

…huggingface/inference (#380)

packages/hub/package.json

@@ -55,6 +55,7 @@
 	"author": "Hugging Face",
 	"license": "MIT",
 	"devDependencies": {
+		"@huggingface/tasks": "workspace:^",
 		"@types/node": "^18.13.0",
 		"type-fest": "^3.9.0"
 	},

packages/hub/src/index.ts

@@ -5,6 +5,7 @@ export type {
 	AccessTokenRole,
 	AuthType,
 	Credentials,
+	PipelineType,
 	RepoDesignation,
 	RepoFullName,
 	RepoId,
@@ -15,6 +16,5 @@ export type {
 	SpaceRuntime,
 	SpaceSdk,
 	SpaceStage,
-	Task,
 } from "./types/public";
 export { HubApiError, InvalidApiResponseFormatError } from "./error";

packages/hub/src/lib/create-repo.spec.ts

@@ -106,4 +106,4 @@ describe("createRepo", () => {
 			credentials: { accessToken: TEST_ACCESS_TOKEN },
 		});
 	});
-});
+}, 10_000);

packages/hub/src/lib/delete-files.spec.ts

@@ -81,4 +81,4 @@ describe("deleteFiles", () => {
 			});
 		}
 	});
-});
+}, 10_000);

packages/hub/src/lib/list-models.ts

@@ -1,7 +1,7 @@
 import { HUB_URL } from "../consts";
 import { createApiError } from "../error";
 import type { ApiModelInfo } from "../types/api/api-model";
-import type { Credentials, Task } from "../types/public";
+import type { Credentials, PipelineType } from "../types/public";
 import { checkCredentials } from "../utils/checkCredentials";
 import { parseLinkHeader } from "../utils/parseLinkHeader";
 
@@ -12,7 +12,7 @@ export interface ModelEntry {
 	name: string;
 	private: boolean;
 	gated: false | "auto" | "manual";
-	task?: Task;
+	task?: PipelineType;
 	likes: number;
 	downloads: number;
 	updatedAt: Date;
@@ -21,7 +21,7 @@ export interface ModelEntry {
 export async function* listModels(params?: {
 	search?: {
 		owner?: string;
-		task?: Task;
+		task?: PipelineType;
 	};
 	credentials?: Credentials;
 	hubUrl?: string;

packages/hub/src/lib/upload-files.spec.ts

@@ -95,4 +95,4 @@ describe("uploadFiles", () => {
 			});
 		}
 	});
-});
+}, 10_000);

packages/hub/src/types/api/api-model.d.ts

@@ -1,4 +1,4 @@
-import type { Task } from "../public";
+import type { PipelineType } from "../public";
 
 export interface ApiModelInfo {
 	_id: string;
@@ -26,6 +26,6 @@ export interface ApiModelInfo {
 	sha: string;
 	spaces: string[];
 	updatedAt: string; // convert to date
-	pipeline_tag: Task;
+	pipeline_tag: PipelineType;
 	"model-index": unknown;
 }

packages/hub/src/types/public.d.ts

@@ -1,3 +1,5 @@
+import type { PipelineType } from "@huggingface/tasks";
+
 export type RepoType = "space" | "dataset" | "model";
 
 export interface RepoId {
@@ -43,51 +45,7 @@ export type AccessTokenRole = "admin" | "write" | "contributor" | "read";
 
 export type AuthType = "access_token" | "app_token" | "app_token_as_user";
 
-export type Task =
-	| "text-classification"
-	| "token-classification"
-	| "table-question-answering"
-	| "question-answering"
-	| "zero-shot-classification"
-	| "translation"
-	| "summarization"
-	| "conversational"
-	| "feature-extraction"
-	| "text-generation"
-	| "text2text-generation"
-	| "fill-mask"
-	| "sentence-similarity"
-	| "text-to-speech"
-	| "automatic-speech-recognition"
-	| "audio-to-audio"
-	| "audio-classification"
-	| "voice-activity-detection"
-	| "depth-estimation"
-	| "image-classification"
-	| "object-detection"
-	| "image-segmentation"
-	| "text-to-image"
-	| "image-to-text"
-	| "image-to-image"
-	| "image-to-video"
-	| "unconditional-image-generation"
-	| "video-classification"
-	| "reinforcement-learning"
-	| "robotics"
-	| "tabular-classification"
-	| "tabular-regression"
-	| "tabular-to-text"
-	| "table-to-text"
-	| "multiple-choice"
-	| "text-retrieval"
-	| "time-series-forecasting"
-	| "visual-question-answering"
-	| "document-question-answering"
-	| "zero-shot-image-classification"
-	| "graph-ml"
-	| "text-to-3d"
-	| "image-to-3d"
-	| "other";
+export type { PipelineType };
 
 export interface SpaceRuntime {
 	stage: SpaceStage;

packages/inference/package.json

@@ -51,6 +51,7 @@
 		"check": "tsc"
 	},
 	"devDependencies": {
+		"@huggingface/tasks": "workspace:^",
 		"@types/node": "18.13.0"
 	},
 	"resolutions": {}

packages/inference/scripts/generate-dts.ts

@@ -1,8 +1,24 @@
 /** Dirty script to generate pretty .d.ts */
 
 import { readFileSync, writeFileSync, appendFileSync, readdirSync } from "node:fs";
+import { TASKS_DATA } from "@huggingface/tasks";
 
-writeFileSync("./dist/index.d.ts", readFileSync("./src/types.ts", "utf-8") + "\n");
+const tasks = Object.keys(TASKS_DATA)
+	.sort()
+	.filter((task) => task !== "other");
+
+let types = readFileSync("./src/types.ts", "utf-8");
+
+types = types.replace(/import.* "@huggingface\/tasks";\n/g, "");
+types = types.replace(' Exclude<PipelineType, "other">', ["", ...tasks.map((task) => `"${task}"`)].join("\n\t| "));
+
+if (types.includes("PipelineType") || types.includes("@huggingface/tasks")) {
+	console.log(types);
+	console.error("Failed to parse types.ts");
+	process.exit(1);
+}
+
+writeFileSync("./dist/index.d.ts", types + "\n");
 appendFileSync("./dist/index.d.ts", "export class InferenceOutputError extends TypeError {}" + "\n");
 
 const dirs = readdirSync("./src/tasks");

packages/inference/src/types.ts

@@ -1,3 +1,5 @@
+import type { PipelineType } from "@huggingface/tasks";
+
 export interface Options {
 	/**
 	 * (Default: true) Boolean. If a request 503s and wait_for_model is set to false, the request will be retried with the same parameters but with wait_for_model set to true.
@@ -35,39 +37,7 @@ export interface Options {
 	includeCredentials?: string | boolean;
 }
 
-export type InferenceTask =
-	| "audio-classification"
-	| "audio-to-audio"
-	| "automatic-speech-recognition"
-	| "conversational"
-	| "depth-estimation"
-	| "document-question-answering"
-	| "feature-extraction"
-	| "fill-mask"
-	| "image-classification"
-	| "image-segmentation"
-	| "image-to-image"
-	| "image-to-text"
-	| "object-detection"
-	| "video-classification"
-	| "question-answering"
-	| "reinforcement-learning"
-	| "sentence-similarity"
-	| "summarization"
-	| "table-question-answering"
-	| "tabular-classification"
-	| "tabular-regression"
-	| "text-classification"
-	| "text-generation"
-	| "text-to-image"
-	| "text-to-speech"
-	| "text-to-video"
-	| "token-classification"
-	| "translation"
-	| "unconditional-image-generation"
-	| "visual-question-answering"
-	| "zero-shot-classification"
-	| "zero-shot-image-classification";
+export type InferenceTask = Exclude<PipelineType, "other">;
 
 export interface BaseArgs {
 	/**