diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index cff9d79c..bf2c07a4 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -16,7 +16,16 @@ jobs:
     strategy:
       matrix:
         python-version: ["3.9", "3.10", "3.11", "3.12"]
-
+    services:
+      postgres:
+        image: postgres:latest
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: mysecretpassword
+          POSTGRES_DB: postgres
+        ports:
+          - 5432:5432
+        options: --health-cmd "pg_isready -U postgres" --health-interval 10s --health-timeout 5s --health-retries 5
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
@@ -36,3 +45,4 @@ jobs:
         run: tox
         env:
           SANDBOX_ZENODO_TOKEN: ${{ secrets.SANDBOX_ZENODO_TOKEN }}
+          GITHUB_ACTIONS: "true"
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 500c7461..2c4c6669 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,11 @@
 # Hypha Change Log
 
+### 0.20.39
+
+ - Revise artifact manager to use artifact id as the primary key, remove `prefix` based keys.
+ - Support versioning and custom config (e.g. artifact specific s3 credentials) for the artifact manager.
+ - Use SQLModel and support database migration using `alembic`.
+
 ### 0.20.38
 
  - Support event logging in the workspace, use `log_event` to log events in the workspace and use `get_events` to get the events in the workspace. The events will be persists in the SQL database.
diff --git a/alembic.ini b/alembic.ini
new file mode 100644
index 00000000..882d42b0
--- /dev/null
+++ b/alembic.ini
@@ -0,0 +1,116 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts.
+# Use forward slashes (/) also on windows to provide an os agnostic path
+script_location = hypha/migrations
+
+# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
+# Uncomment the line below if you want the files to be prepended with date and time
+# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
+
+# sys.path path, will be prepended to sys.path if present.
+# defaults to the current working directory.
+prepend_sys_path = .
+
+# timezone to use when rendering the date within the migration file
+# as well as the filename.
+# If specified, requires the python>=3.9 or backports.zoneinfo library.
+# Any required deps can installed by adding `alembic[tz]` to the pip requirements
+# string value is passed to ZoneInfo()
+# leave blank for localtime
+# timezone =
+
+# max length of characters to apply to the "slug" field
+# truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; This defaults
+# to migrations/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path.
+# The path separator used here should be the separator specified by "version_path_separator" below.
+# version_locations = %(here)s/bar:%(here)s/bat:migrations/versions
+
+# version path separator; As mentioned above, this is the character used to split
+# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep.
+# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas.
+# Valid values for version_path_separator are:
+#
+# version_path_separator = :
+# version_path_separator = ;
+# version_path_separator = space
+# version_path_separator = newline
+version_path_separator = os  # Use os.pathsep. Default configuration used for new projects.
+
+# set to 'true' to search source files recursively
+# in each "version_locations" directory
+# new in Alembic version 1.10
+# recursive_version_locations = false
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+# Use memory mode for SQLite by default
+sqlalchemy.url = sqlite+aiosqlite:///:memory:
+
+
+[post_write_hooks]
+# post_write_hooks defines scripts or Python functions that are run
+# on newly generated revision scripts.  See the documentation for further
+# detail and examples
+
+# format using "black" - use the console_scripts runner, against the "black" entrypoint
+# hooks = black
+# black.type = console_scripts
+# black.entrypoint = black
+# black.options = -l 79 REVISION_SCRIPT_FILENAME
+
+# lint with attempts to fix using "ruff" - use the exec runner, execute a binary
+# hooks = ruff
+# ruff.type = exec
+# ruff.executable = %(here)s/.venv/bin/ruff
+# ruff.options = --fix REVISION_SCRIPT_FILENAME
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARNING
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARNING
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S
diff --git a/docs/artifact-manager.md b/docs/artifact-manager.md
index 5d968b29..2b89b189 100644
--- a/docs/artifact-manager.md
+++ b/docs/artifact-manager.md
@@ -1,10 +1,9 @@
-# Artifact Manager
 
-The `Artifact Manager` is a built-in Hypha service for indexing, managing, and storing resources such as datasets, AI models, and applications. It provides a structured way to manage datasets and similar resources, enabling efficient listing, uploading, updating, and deleting of files. It also now supports tracking download statistics for each artifact.
+# Artifact Manager
 
-A typical use case for the `Artifact Manager` is as a backend for a single-page web application that displays a gallery of datasets, AI models, applications, or other types of resources. The default metadata of an artifact is designed to render a grid of cards on a webpage. It also supports tracking download statistics.
+The `Artifact Manager` is an essential Hypha service for managing resources such as datasets, AI models, and applications. It allows for structured resource management, providing APIs to create collections, manage datasets, track download statistics, enforce schemas, and control access permissions. The `Artifact Manager` can be used as a backend for web applications, supporting complex operations like indexing, searching, and schema validation.
 
-**Note:** The `Artifact Manager` is only available when your Hypha server has S3 storage enabled.
+**Note:** The `Artifact Manager` is available only when S3 storage is enabled on your Hypha server.
 
 ---
 
@@ -12,62 +11,69 @@ A typical use case for the `Artifact Manager` is as a backend for a single-page
 
 ### Step 1: Connecting to the Artifact Manager Service
 
-To use the `Artifact Manager`, you first need to connect to the Hypha server. This API allows you to create, read, edit, and delete datasets in the artifact registry (stored in an S3 bucket for each workspace).
+To use the `Artifact Manager`, start by connecting to the Hypha server. This connection allows you to interact with the artifact registry, including creating, editing, and deleting datasets, as well as managing permissions and tracking download statistics.
 
 ```python
 from hypha_rpc import connect_to_server
 
 SERVER_URL = "https://hypha.aicell.io"  # Replace with your server URL
 
-# Connect to the Dataset Manager API
+# Connect to the Artifact Manager API
 server = await connect_to_server({"name": "test-client", "server_url": SERVER_URL})
 artifact_manager = await server.get_service("public/artifact-manager")
 ```
 
 ### Step 2: Creating a Dataset Gallery Collection
 
-Once connected, you can create a collection to organize datasets in the gallery.
+Once connected, you can create a collection that organizes datasets, providing metadata and access permissions for each.
 
 ```python
-# Create a collection for the Dataset Gallery
+# Define metadata for the dataset gallery
 gallery_manifest = {
     "name": "Dataset Gallery",
     "description": "A collection for organizing datasets",
-    "type": "collection",
-    "collection": [],
 }
 
-# Create the collection with read permission for everyone and create permission for all authenticated users
-# We set orphan=True to create a collection without a parent
-await artifact_manager.create(prefix="collections/dataset-gallery", manifest=gallery_manifest, permissions={"*": "r", "@": "r+"}, orphan=True)
+# Create the collection with read access for everyone and create access for authenticated users
+collection = await artifact_manager.create(
+    alias="dataset-gallery",
+    type="collection",
+    manifest=gallery_manifest,
+    config={"permissions": {"*": "r", "@": "r+"}}
+)
 print("Dataset Gallery created.")
 ```
 
 ### Step 3: Adding a Dataset to the Gallery
 
-After creating the gallery, you can start adding datasets to it. Each dataset will have its own manifest that describes the dataset and any associated files.
+After creating the gallery, you can add datasets to it, with each dataset having its own metadata and permissions.
 
 ```python
-# Create a new dataset inside the Dataset Gallery
+# Define metadata for the new dataset
 dataset_manifest = {
     "name": "Example Dataset",
-    "description": "A dataset with example data",
-    "type": "dataset",
+    "description": "A dataset containing example data",
 }
 
-await artifact_manager.create(prefix="collections/dataset-gallery/example-dataset", manifest=dataset_manifest, stage=True)
+# Add the dataset to the gallery and stage it for review
+dataset = await artifact_manager.create(
+    parent_id=collection.id,
+    alias="example-dataset",
+    manifest=dataset_manifest,
+    version="stage"
+)
 print("Dataset added to the gallery.")
 ```
 
 ### Step 4: Uploading Files to the Dataset with Download Statistics
 
-Once you have created a dataset, you can upload files to it by generating a pre-signed URL. This URL allows you to upload the actual files to the artifact's S3 bucket.
-
-Additionally, when uploading files to an artifact, you can specify a `download_weight` for each file. This weight determines how the file impacts the artifact's download count when it is accessed. For example, primary files might have a higher `download_weight`, while secondary files might have no impact. The download count is automatically updated whenever users download files from the artifact.
+Each dataset can contain multiple files. Use pre-signed URLs for secure uploads and set `download_weight` to track file downloads.
 
 ```python
-# Get a pre-signed URL to upload a file, with a download_weight assigned
-put_url = await artifact_manager.put_file(prefix="collections/dataset-gallery/example-dataset", file_path="data.csv", options={"download_weight": 0.5})
+# Generate a pre-signed URL to upload a file with a specific download weight
+put_url = await artifact_manager.put_file(
+    dataset.id, file_path="data.csv", download_weight=0.5
+)
 
 # Upload the file using an HTTP PUT request
 with open("path/to/local/data.csv", "rb") as f:
@@ -79,21 +85,21 @@ print("File uploaded to the dataset.")
 
 ### Step 5: Committing the Dataset
 
-After uploading the files, commit the dataset to finalize it. This will check that all files have been uploaded and update the collection.
+After uploading files, commit the dataset to finalize its status in the collection. Committed datasets are accessible based on their permissions.
 
 ```python
-# Finalize and commit the dataset
-await artifact_manager.commit(prefix="collections/dataset-gallery/example-dataset")
+# Commit the dataset to finalize its status
+await artifact_manager.commit(dataset.id)
 print("Dataset committed.")
 ```
 
 ### Step 6: Listing All Datasets in the Gallery
 
-You can retrieve a list of all datasets in the collection to display on a webpage or for further processing.
+Retrieve a list of all datasets in the collection for display or further processing.
 
 ```python
 # List all datasets in the gallery
-datasets = await artifact_manager.list(prefix="collections/dataset-gallery")
+datasets = await artifact_manager.list(collection.id)
 print("Datasets in the gallery:", datasets)
 ```
 
@@ -101,7 +107,7 @@ print("Datasets in the gallery:", datasets)
 
 ## Full Example: Creating and Managing a Dataset Gallery
 
-Here’s a full example that shows how to connect to the service, create a dataset gallery, add a dataset, upload files with download statistics, and commit the dataset.
+Here’s a complete example showing how to connect to the service, create a dataset gallery, add a dataset, upload files, and commit the dataset.
 
 ```python
 import asyncio
@@ -111,7 +117,7 @@ from hypha_rpc import connect_to_server
 SERVER_URL = "https://hypha.aicell.io"
 
 async def main():
-    # Connect to the Dataset Manager API
+    # Connect to the Artifact Manager API
     api = await connect_to_server({"name": "test-client", "server_url": SERVER_URL})
     artifact_manager = await api.get_service("public/artifact-manager")
 
@@ -119,38 +125,41 @@ async def main():
     gallery_manifest = {
         "name": "Dataset Gallery",
         "description": "A collection for organizing datasets",
-        "type": "collection",
-        "collection": [],
     }
-    # Create the collection with read permission for everyone and create permission for all authenticated users
-    # We set orphan=True to create a collection without a parent
-    await artifact_manager.create(prefix="collections/dataset-gallery", manifest=gallery_manifest, permissions={"*": "r+", "@": "r+"}, orphan=True)
+    collection = await artifact_manager.create(
+        type="collection",
+        alias="dataset-gallery",
+        manifest=gallery_manifest,
+        config={"permissions": {"*": "r+", "@": "r+"}}
+    )
     print("Dataset Gallery created.")
 
-    # Create a new dataset inside the Dataset Gallery
+    # Add a dataset to the gallery
     dataset_manifest = {
         "name": "Example Dataset",
-        "description": "A dataset with example data",
-        "type": "dataset",
+        "description": "A dataset containing example data",
     }
-    await artifact_manager.create(prefix="collections/dataset-gallery/example-dataset", manifest=dataset_manifest, stage=True)
+    dataset = await artifact_manager.create(
+        parent_id=collection.id,
+        alias="example-dataset",
+        manifest=dataset_manifest,
+        version="stage"
+    )
     print("Dataset added to the gallery.")
 
-    # Get a pre-signed URL to upload a file, with a download_weight assigned
-    put_url = await artifact_manager.put_file(prefix="collections/dataset-gallery/example-dataset", file_path="data.csv", options={"download_weight": 0.5})
-
-    # Upload the file using an HTTP PUT request
+    # Upload a file to the dataset
+    put_url = await artifact_manager.put_file(dataset.id, file_path="data.csv", download_weight=0.5)
     with open("path/to/local/data.csv", "rb") as f:
         response = requests.put(put_url, data=f)
         assert response.ok, "File upload failed"
     print("File uploaded to the dataset.")
 
-    # Finalize and commit the dataset
-    await artifact_manager.commit(prefix="collections/dataset-gallery/example-dataset")
+    # Commit the dataset
+    await artifact_manager.commit(dataset.id)
     print("Dataset committed.")
 
     # List all datasets in the gallery
-    datasets = await artifact_manager.list(prefix="collections/dataset-gallery")
+    datasets = await artifact_manager.list(collection.id)
     print("Datasets in the gallery:", datasets)
 
 asyncio.run(main())
@@ -160,11 +169,11 @@ asyncio.run(main())
 
 ## Advanced Usage: Enforcing Schema for Dataset Validation
 
-The `Artifact Manager` supports enforcing a schema to ensure that datasets conform to a specific structure. This can be useful in scenarios where you want to maintain consistency across multiple datasets.
+The `Artifact Manager` allows schema enforcement to ensure datasets meet specific requirements. This is helpful when maintaining consistency across multiple datasets.
 
 ### Step 1: Create a Collection with a Schema
 
-You can define a schema for the collection that specifies the required fields for each dataset.
+Define a schema for datasets to specify required fields.
 
 ```python
 # Define a schema for datasets in the gallery
@@ -173,79 +182,97 @@ dataset_schema = {
     "properties": {
         "name": {"type": "string"},
         "description": {"type": "string"},
-        "type": {"type": "string", "enum": ["dataset", "model", "application"]},
+        "record_type": {"type": "string", "enum": ["dataset", "model", "application"]},
     },
-    "required": ["name", "description", "type"]
+    "required": ["name", "description", "record_type"]
 }
 
-# Create a collection with the schema
+# Create a collection with schema validation
 gallery_manifest = {
     "name": "Schema Dataset Gallery",
-    "description": "A gallery of datasets with enforced schema",
-    "type": "collection",
+    "description": "A gallery with schema-enforced datasets",
 }
-# Create the collection with read permission for everyone and create permission for all authenticated users
-await artifact_manager.create(prefix="collections/schema-dataset-gallery", manifest=gallery_manifest, config={"collection_schema": dataset_schema}, permissions={"*": "r+", "@": "r+"}, orphan=True)
+collection = await artifact_manager.create(
+    type="collection",
+    alias="schema-dataset-gallery",
+    manifest=gallery_manifest,
+    config={"collection_schema": dataset_schema, "permissions": {"*": "r+", "@": "r+"}}
+)
 print("Schema-based Dataset Gallery created.")
 ```
 
 ### Step 2: Validate Dataset Against Schema
 
-When you commit a dataset to this collection, it will be validated against the schema.
+Datasets in the collection will be validated against the defined schema during creation and commit.
 
 ```python
-# Create a valid dataset that conforms to the schema
+# Define a valid dataset that conforms to the schema
 valid_dataset_manifest = {
     "name": "Valid Dataset",
-    "description": "A valid dataset conforming to the schema",
-    "type": "dataset",
+    "description": "A valid dataset meeting schema requirements",
+    "record_type": "dataset",
 }
 
-await artifact_manager.create(prefix="collections/schema-dataset-gallery/valid-dataset", manifest=valid_dataset_manifest, stage=True)
+dataset = await artifact_manager.create(
+    parent_id=collection.id,
+    alias="valid-dataset",
+    manifest=valid_dataset_manifest,
+    version="stage"
+)
 print("Valid dataset created.")
 
-# Commit the valid dataset (this should pass schema validation)
-await artifact_manager.commit(prefix="collections/schema-dataset-gallery/valid-dataset")
+# Commit the dataset (will pass schema validation)
+await artifact_manager.commit(dataset.id)
 print("Valid dataset committed.")
 ```
 
+
 ---
 
 ## API References
 
-### `create(prefix: str, manifest: dict, permissions: dict=None, config: dict=None, stage: bool = False, orphan: bool = False, publish_to: str = None) -> None`
+### `create(parent_id: str, alias: str, type: str, manifest: dict, permissions: dict=None, config: dict=None, version: str = None, comment: str = None, publish_to: str = None) -> None`
 
 Creates a new artifact or collection with the specified manifest. The artifact is staged until committed. For collections, the `collection` field should be an empty list.
 
 **Parameters:**
 
-- `prefix`: The path of the artifact, it can be a prefix relative to the current workspace (e.g., `"collections/dataset-gallery/example-dataset"`) or an absolute prefix with the workspace id (e.g., `"/my_workspace_id/collections/dataset-gallery/example-dataset"`). To generate an auto-id, you can use patterns like `"{uuid}"` or `"{timestamp}"`. The following patterns are supported:
+- `parent_id`: The id of the parent collection where the artifact will be created. If the artifact is a top-level collection, leave this field empty or set to None.
+- `alias`: A human readable name for indexing the artifact, it can be a text with lower case letters and numbers. You can set it to absolute alias in the format of `"workspace_id/alias"` or just `"alias"`. In the alias itself, `/` is not allowed, you should use `:` instead in the alias.
+To generate an auto-id, you can use patterns like `"{uuid}"` or `"{timestamp}"`. The following patterns are supported:
   - `{uuid}`: Generates a random UUID.
   - `{timestamp}`: Generates a timestamp in milliseconds.
+  - `{user_id}`: Generate the user id of the creator.
   - `{zenodo_id}`: If `publish_to` is set to `zenodo` or `sandbox_zenodo`, it will use the Zenodo deposit ID (which changes when a new version is created).
   - `{zenodo_conceptrecid}`: If `publish_to` is set to `zenodo` or `sandbox_zenodo`, it will use the Zenodo concept id (which does not change when a new version is created).
   - **Id Parts**: You can also use id parts stored in the parent collection's config['id_parts'] to generate an id. For example, if the parent collection has `{"animals": ["dog", "cat", ...], "colors": ["red", "blue", ...]}`, you can use `"{colors}-{animals}"` to generate an id like `red-dog`.
-- `manifest`: The manifest of the new artifact. Ensure the manifest follows the required schema if applicable (e.g., for collections). 
-
+- `type`: The type of the artifact. Supported values are `collection`, `generic` and any other custom type. By default, it's set to `generic` which contains fields tailored for displaying the artifact as cards on a webpage.
+- `manifest`: The manifest of the new artifact. Ensure the manifest follows the required schema if applicable (e.g., for collections).
 - `config`: Optional. A dictionary containing additional configuration options for the artifact (shared for both staged and committed). For collections, the config can contain the following special fields:
   - `collection_schema`: Optional. A JSON schema that defines the structure of child artifacts in the collection. This schema is used to validate child artifacts when they are created or edited. If a child artifact does not conform to the schema, the creation or edit operation will fail.
-  - `summary_fields`: Optional. A list of fields to include in the summary for each child artifact when calling `list(prefix)`. If not specified, the default summary fields (`id`, `type`, `name`) are used. To include all the fields in the summary, add `"*"` to the list. If you want to include internal fields such as `.created_at`, `.last_modified`, or other download/view statistics such as `.download_count`, you can also specify them individually in the `summary_fields`. If you want to include all fields, you can add `".*"` to the list.
-  - `id_parts`: Optional. A dictionary of id name parts to be used in generating the id for child artifacts. For example: `{"animals": ["dog", "cat", ...], "colors": ["red", "blue", ...]}`. This can be used for creating child artifacts with auto-generated ids based on the id parts. For example, when calling `create`, you can specify the prefix as `collections/my-collection/{colors}-{animals}`, and the id will be generated based on the id parts, e.g., `collections/my-collection/red-dog`.
-  - `archives`: Optional. Configurations for the public archive servers such as Zenodo for the collection. For example, `"archives": {"sandbox_zenodo": {"access_token": "your sandbox zenodo token"}}, "zenodo": {"access_token": "your zenodo token"}}`. This is used for publishing artifacts to Zenodo.
+  - `id_parts`: Optional. A dictionary of id name parts to be used in generating the id for child artifacts. For example: `{"animals": ["dog", "cat", ...], "colors": ["red", "blue", ...]}`. This can be used for creating child artifacts with auto-generated ids based on the id parts. For example, when calling `create`, you can specify the alias as `my-pet-{colors}-{animals}`, and the id will be generated based on the id parts, e.g., `my-pet-red-dog`.
 - `permissions`: Optional. A dictionary containing user permissions. For example `{"*": "r+"}` gives read and create access to everyone, `{"@": "rw+"}` allows all authenticated users to read/write/create, and `{"user_id_1": "r+"}` grants read and create permissions to a specific user. You can also set permissions for specific operations, such as `{"user_id_1": ["read", "create"]}`. See detailed explanation about permissions below.
-- `stage`: Optional. A boolean flag to stage the artifact. Default is `False`. If it's set to `True`, the artifact will be staged and not committed. You will need to call `commit()` to finalize the artifact.
-- `orphan`: Optional. A boolean flag to create the artifact without a parent collection. Default is `False`. If `True`, the artifact will not be associated with any collection. This is mainly used for creating top-level collections, and making sure the artifact is not associated with any parent collection (with inheritance of permissions). To create an orphan artifact, you will need write permissions on the workspace.
-- `publish_to`: Optional. A string specifying the target platform to publish the artifact. Supported values are `zenodo` and `sandbox_zenodo`. If set, the artifact will be published to the specified platform. The artifact must have a valid Zenodo metadata schema to be published.
+- `version`: Optional. The version of the artifact to create. By default, it set to None or `"new"`, it will generate a version `v0`. If you want to create a staged version, you can set it to `"stage"`.
+- `comment`: Optional. A comment to describe the changes made to the artifact.
+- `secrets`: Optional. A dictionary containing secrets to be stored with the artifact. Secrets are encrypted and can only be accessed by the artifact owner or users with appropriate permissions. The following keys can be used:
+  - `ZENODO_ACCESS_TOKEN`: The Zenodo access token to publish the artifact to Zenodo.
+  - `SANDBOX_ZENODO_ACCESS_TOKEN`: The Zenodo access token to publish the artifact to the Zenodo sandbox.
+  - `S3_ENDPOINT_URL`: The endpoint URL of the S3 storage for the artifact.
+  - `S3_ACCESS_KEY_ID`: The access key ID for the S3 storage for the artifact.
+  - `S3_SECRET_ACCESS_KEY`: The secret access key for the S3 storage for the artifact.
+  - `S3_REGION_NAME`: The region name of the S3 storage for the artifact.
+  - `S3_BUCKET`: The bucket name of the S3 storage for the artifact. Default to the hypha workspaces bucket.
+  - `S3_PREFIX`: The prefix of the S3 storage for the artifact. Default: `""`.
 
-**Note 1: If you set `stage=True`, you must call `commit()` to finalize the artifact.**
+- `publish_to`: Optional. A string specifying the target platform to publish the artifact. Supported values are `zenodo` and `sandbox_zenodo`. If set, the artifact will be published to the specified platform. The artifact must have a valid Zenodo metadata schema to be published.
 
-**Note 2: If you set `orphan=True`, the artifact will not be associated with any collection. An non-orphan artifact must have a parent collection.**
+**Note 1: If you set `version="stage"`, you must call `commit()` to finalize the artifact.**
 
 **Example:**
 
 ```python
-# Assuming we have already created a dataset-gallery collection, otherwise create it first or set orphan=True
-await artifact_manager.create(prefix="collections/dataset-gallery/example-dataset", manifest=dataset_manifest, stage=True, orphan=False)
+# Assuming we have already created a dataset-gallery collection, we can add a new dataset to it
+await artifact_manager.create(artifact_id="dataset-gallery", alias="example-dataset", manifest=dataset_manifest, version="stage")
 ```
 
 ### Permissions
@@ -325,7 +352,7 @@ The following list shows how permission expansions work:
 
 ---
 
-### `edit(prefix: str, manifest: dict, permissions: dict = None, config: dict = None, stage: bool = False) -> None`
+### `edit(artifact_id: str, manifest: dict = None, type: str = None,  permissions: dict = None, config: dict = None, secrets: dict = None, version: str = None, comment: str = None) -> None`
 
 Edits an existing artifact's manifest. The new manifest is staged until committed.
 
@@ -333,41 +360,52 @@ Edits an existing artifact's manifest. The new manifest is staged until committe
 
 - `prefix`: The path of the artifact, it can be a prefix relative to the current workspace (e.g., `"collections/dataset-gallery/example-dataset"`) or an absolute prefix with the workspace id (e.g., `"/my_workspace_id/collections/dataset-gallery/example-dataset"`).
 - `manifest`: The updated manifest. Ensure the manifest follows the required schema if applicable (e.g., for collections).
+- `type`: Optional. The type of the artifact. Supported values are `collection`, `generic` and any other custom type. By default, it's set to `generic` which contains fields tailored for displaying the artifact as cards on a webpage.
 - `permissions`: Optional. A dictionary containing user permissions. For example `{"*": "r+"}` gives read and create access to everyone, `{"@": "rw+"}` allows all authenticated users to read/write/create, and `{"user_id_1": "r+"}` grants read and create permissions to a specific user. You can also set permissions for specific operations, such as `{"user_id_1": ["read", "create"]}`. See detailed explanation about permissions below.
+- `secrets`: Optional. A dictionary containing secrets to be stored with the artifact. Secrets are encrypted and can only be accessed by the artifact owner or users with appropriate permissions. See the `create` function for a list of supported secrets.
 - `config`: Optional. A dictionary containing additional configuration options for the artifact.
-- `stage`: Optional. A boolean flag to stage the artifact. Default is `False`. If it's set to `True`, the artifact will be staged and not committed. You will need to call `commit()` to finalize the changes.
+- `version`: Optional. The version of the artifact to edit. By default, it set to None, the version will stay the same. If you want to create a staged version, you can set it to `"stage"`. You can set it to any version in text, e.g. `0.1.0` or `v1`. If you set it to `new`, it will generate a version similar to `v0`, `v1`, etc.
+- `comment`: Optional. A comment to describe the changes made to the artifact.
 
 **Example:**
 
 ```python
-await artifact_manager.edit(prefix="collections/dataset-gallery/example-dataset", manifest=updated_manifest)
+await artifact_manager.edit(artifact_id="example-dataset", manifest=updated_manifest)
 ```
 
 ---
 
-### `commit(prefix: str) -> None`
+### `commit(artifact_id: str, version: str = None, comment: str = None) -> None`
 
 Finalizes and commits an artifact's staged changes. Validates uploaded files and commit the staged manifest. This process also updates view and download statistics.
 
 **Parameters:**
 
-- `prefix`: The path of the artifact, it can be a prefix relative to the current workspace (e.g., `"collections/dataset-gallery/example-dataset"`) or an absolute prefix with the workspace id (e.g., `"/my_workspace_id/collections/dataset-gallery/example-dataset"`).
+- `artifact_id`: The id of the artifact to commit. It can be an uuid generated by `create` or `edit` function, or it can be an alias of the artifact under the current workspace. If you want to refer to an artifact in another workspace, you should use the full alias in the format of `"workspace_id/alias"`.
+- `version`: Optional. The version of the artifact to edit. By default, it set to None, the version will stay the same. If you want to create a staged version, you can set it to `"stage"`. You can set it to any version in text, e.g. `0.1.0` or `v1`. If you set it to `new`, it will generate a version similar to `v0`, `v1`, etc.
+- `comment`: Optional. A comment to describe the changes made to the artifact.
 
 **Example:**
 
 ```python
-await artifact_manager.commit(prefix="collections/dataset-gallery/example-dataset")
+await artifact_manager.commit(artifact_id=artifact.id)
+
+# If "example-dataset" is an alias of the artifact under the current workspace
+await artifact_manager.commit(artifact_id="example-dataset")
+
+# If "example-dataset" is an alias of the artifact under another workspace
+await artifact_manager.commit(artifact_id="other_workspace/example-dataset")
 ```
 
 ---
 
-### `delete(prefix: str, delete_files: bool = False, recursive: bool = False) -> None`
+### `delete(artifact_id: str, delete_files: bool = False, recursive: bool = False) -> None`
 
 Deletes an artifact, its manifest, and all associated files from both the database and S3 storage.
 
 **Parameters:**
 
-- `prefix`: The path of the artifact, it can be a prefix relative to the current workspace (e.g., `"collections/dataset-gallery/example-dataset"`) or an absolute prefix with the workspace id (e.g., `"/my_workspace_id/collections/dataset-gallery/example-dataset"`).
+- `artifact_id`: The id of the artifact to delete. It can be an uuid generated by `create` or `edit` function, or it can be an alias of the artifact under the current workspace. If you want to refer to an artifact in another workspace, you should use the full alias in the format of `"workspace_id/alias"`.
 - `delete_files`: Optional. A boolean flag to delete all files associated with the artifact. Default is `False`.
 - `recursive`: Optional. A boolean flag to delete all child artifacts recursively. Default is `False`.
 
@@ -376,77 +414,104 @@ Deletes an artifact, its manifest, and all associated files from both the databa
 **Example:**
 
 ```python
-await artifact_manager.delete(prefix="collections/dataset-gallery/example-dataset", delete_files=True)
+await artifact_manager.delete(artifact_id=artifact.id, delete_files=True)
+
+# If "example-dataset" is an alias of the artifact under the current workspace
+await artifact_manager.delete(artifact_id="example-dataset", delete_files=True)
+
+# If "example-dataset" is an alias of the artifact under another workspace
+await artifact_manager.delete(artifact_id="other_workspace/example-dataset", delete_files=True)
 ```
 
 ---
 
-### `put_file(prefix: str, file_path: str, options: dict = None) -> str`
+### `put_file(artifact_id: str, file_path: str, download_weight: int = 0) -> str`
 
 Generates a pre-signed URL to upload a file to the artifact in S3. The URL can be used with an HTTP `PUT` request to upload the file. The file is staged until the artifact is committed.
 
 **Parameters:**
 
-- `prefix`: The path of the artifact, it can be a prefix relative to the current workspace (e.g., `"collections/dataset-gallery/example-dataset"`) or an absolute prefix with the workspace id (e.g., `"/my_workspace_id/collections/dataset-gallery/example-dataset"`).
+- `artifact_id`: The id of the artifact to upload the file to. It can be an uuid generated by `create` or `edit` function, or it can be an alias of the artifact under the current workspace. If you want to refer to an artifact in another workspace, you should use the full alias in the format of `"workspace_id/alias"`.
 - `file_path`: The relative path of the file to upload within the artifact (e.g., `"data.csv"`).
-- `options`: Optional. Additional options such as:
-  - `download_weight`: A float value representing the file's impact on download count (0-1). Defaults to `None`.
+- `download_weight`: A float value representing the file's impact on download count (0-1). Defaults to `None`.
 
 **Returns:** A pre-signed URL for uploading the file.
 
 **Example:**
 
 ```python
-put_url = await artifact_manager.put_file(prefix="collections/dataset-gallery/example-dataset", file_path="data.csv", options={"download_weight": 1.0})
+put_url = await artifact_manager.put_file(artifact.id, file_path="data.csv", download_weight=1.0)
+
+# If "example-dataset" is an alias of the artifact under the current workspace
+put_url = await artifact_manager.put_file(artifact_id="example-dataset", file_path="data.csv")
+
+# If "example-dataset" is an alias of the artifact under another workspace
+put_url = await artifact_manager.put_file(artifact_id="other_workspace/example-dataset", file_path="data.csv")
+
+# Upload the file using an HTTP PUT request
+with open("path/to/local/data.csv", "rb") as f:
+    response = requests.put(put_url, data=f)
+    assert response.ok, "File upload failed"
 ```
 
 ---
 
-### `remove_file(prefix: str, file_path: str) -> None`
+### `remove_file(artifact_id: str, file_path: str) -> None`
 
 Removes a file from the artifact and updates the staged manifest. The file is also removed from the S3 storage.
 
 **Parameters:**
 
-- `prefix`: The path of the artifact, it can be a prefix relative to the current workspace (e.g., `"collections/dataset-gallery/example-dataset"`) or an absolute prefix with the workspace id (e.g., `"/my_workspace_id/collections/dataset-gallery/example-dataset"`).
+- `artifact_id`: The id of the artifact to remove the file from. It can be an uuid generated by `create` or `edit` function, or it can be an alias of the artifact under the current workspace. If you want to refer to an artifact in another workspace, you should use the full alias in the format of `"workspace_id/alias"`.
 - `file_path`: The relative path of the file to be removed (e.g., `"data.csv"`).
 
 **Example:**
 
 ```python
-await artifact_manager.remove_file(prefix="collections/dataset-gallery/example-dataset", file_path="data.csv")
+await artifact_manager.remove_file(artifact_id=artifact.id, file_path="data.csv")
+
+# If "example-dataset" is an alias of the artifact under the current workspace
+await artifact_manager.remove_file(artifact_id="example-dataset", file_path="data.csv")
+
+# If "example-dataset" is an alias of the artifact under another workspace
+await artifact_manager.remove_file(artifact_id="other_workspace/example-dataset", file_path="data.csv")
 ```
 
 ---
 
-### `get_file(prefix: str, path: str, options: dict = None) -> str`
+### `get_file(artifact_id: str, path: str, silent: bool = False) -> str`
 
 Generates a pre-signed URL to download a file from the artifact stored in S3.
 
 **Parameters:**
 
-- `prefix`: The path of the artifact, it can be a prefix relative to the current workspace (e.g., `"collections/dataset-gallery/example-dataset"`) or an absolute prefix with the workspace id (e.g., `"/my_workspace_id/collections/dataset-gallery/example-dataset"`).
+- `artifact_id`: The id of the artifact to download the file from. It can be an uuid generated by `create` or `edit` function, or it can be an alias of the artifact under the current workspace. If you want to refer to an artifact in another workspace, you should use the full alias in the format of `"workspace_id/alias"`.
 - `path`: The relative path of the file to download (e.g., `"data.csv"`).
-- `options`: Optional. Controls for the download behavior, such as:
-  - `silent`: A boolean to suppress the download count increment. Default is `False`.
+- `silent`: A boolean to suppress the download count increment. Default is `False`.
 
 **Returns:** A pre-signed URL for downloading the file.
 
 **Example:**
 
 ```python
-get_url = await artifact_manager.get_file(prefix="collections/dataset-gallery/example-dataset", path="data.csv")
+get_url = await artifact_manager.get_file(artifact_id=artifact.id, path="data.csv")
+
+# If "example-dataset" is an alias of the artifact under the current workspace
+get_url = await artifact_manager.get_file(artifact_id="example-dataset", path="data.csv")
+
+# If "example-dataset" is an alias of the artifact under another workspace
+get_url = await artifact_manager.get_file(artifact_id="other_workspace/example-dataset", path="data.csv")
 ```
 
 ---
 
-### `list_files(prefix: str, dir_path: str=None) -> list`
+### `list_files(artifact_id: str, dir_path: str=None) -> list`
 
 Lists all files in the artifact.
 
 **Parameters:**
 
-- `prefix`: The path of the artifact, it can be a prefix relative to the current workspace (e.g., `"collections/dataset-gallery/example-dataset"`) or an absolute prefix with the workspace id (e.g., `"/my_workspace_id/collections/dataset-gallery/example-dataset"`).
+- `artifact_id`: The id of the artifact to list files from. It can be an uuid generated by `create` or `edit` function, or it can be an alias of the artifact under the current workspace. If you want to refer to an artifact in another workspace, you should use the full alias in the format of `"workspace_id/alias"`.
 - `dir_path`: Optional. The directory path within the artifact to list files. Default is `None`.
 
 **Returns:** A list of files in the artifact.
@@ -454,19 +519,25 @@ Lists all files in the artifact.
 **Example:**
 
 ```python
-files = await artifact_manager.list_files(prefix="collections/dataset-gallery/example-dataset")
+files = await artifact_manager.list_files(artifact_id=artifact.id)
+
+# If "example-dataset" is an alias of the artifact under the current workspace
+files = await artifact_manager.list_files(artifact_id="example-dataset")
+
+# If "example-dataset" is an alias of the artifact under another workspace
+files = await artifact_manager.list_files(artifact_id="other_workspace/example-dataset")
 ```
 
 ---
 
-### `read(prefix: str, stage: bool = False, silent: bool = False, include_metadata: bool = False) -> dict`
+### `read(artifact_id: str, stage: bool = False, silent: bool = False, include_metadata: bool = False) -> dict`
 
 Reads and returns the manifest of an artifact or collection. If in staging mode, reads the staged manifest.
 
 **Parameters:**
 
-- `prefix`: The path of the artifact, it can be a prefix relative to the current workspace (e.g., `"collections/dataset-gallery/example-dataset"`) or an absolute prefix with the workspace id (e.g., `"/my_workspace_id/collections/dataset-gallery/example-dataset"`).
-- `stage`: Optional. If `True`, reads the staged manifest. Default is `False`.
+- `artifact_id`: The id of the artifact to read. It can be an uuid generated by `create` or `edit` function, or it can be an alias of the artifact under the current workspace. If you want to refer to an artifact in another workspace, you should use the full alias in the format of `"workspace_id/alias"`.
+- `version`: Optional. The version of the artifact to read. By default, it reads the latest version. If you want to read a staged version, you can set it to `"stage"`.
 - `silent`: Optional. If `True`, suppresses the view count increment. Default is `False`.
 - `include_metadata`: Optional. If `True`, includes metadata such as download statistics in the manifest (fields starting with `"."`). Default is `False`.
 
@@ -475,19 +546,24 @@ Reads and returns the manifest of an artifact or collection. If in staging mode,
 **Example:**
 
 ```python
-manifest = await artifact_manager.read(prefix="collections/dataset-gallery/example-dataset")
+manifest = await artifact_manager.read(artifact_id=artifact.id)
+
+# If "example-dataset" is an alias of the artifact under the current workspace
+manifest = await artifact_manager.read(artifact_id="example-dataset")
+
+# If "example-dataset" is an alias of the artifact under another workspace
+manifest = await artifact_manager.read(artifact_id="other_workspace/example-dataset")
 ```
 
 ---
 
-### `list(prefix: str, keywords: List[str] = None, filters: dict = None, mode: str = "AND", page: int = 0, page_size: int = 100, order_by: str = None, summary_fields: List[str] = None, silent: bool = False) -> list`
+### `list(artifact_id: str, keywords: List[str] = None, filters: dict = None, mode: str = "AND", page: int = 0, page_size: int = 100, order_by: str = None, silent: bool = False) -> list`
 
 Retrieve a list of child artifacts within a specified collection, supporting keyword-based fuzzy search, field-specific filters, and flexible ordering. This function allows detailed control over the search and pagination of artifacts in a collection, including staged artifacts if specified.
 
 **Parameters:**
 
-- `prefix` (str): The path to the artifact, either as a relative prefix within the workspace (e.g., `"collections/dataset-gallery/example-dataset"`) or as an absolute path including the workspace ID (e.g., `"/my_workspace_id/collections/dataset-gallery/example-dataset"`). This defines the collection or artifact directory to search within.
-
+- `artifact_id` (str): The id of the parent artifact or collection to list children from. It can be an uuid generated by `create` or `edit` function, or it can be an alias of the artifact under the current workspace. If you want to refer to an artifact in another workspace, you should use the full alias in the format of `"workspace_id/alias"`.
 - `keywords` (List[str], optional): A list of search terms used for fuzzy searching across all manifest fields. Each term is searched independently, and results matching any term will be included. For example, `["sample", "dataset"]` returns artifacts containing either "sample" or "dataset" in any field of the manifest.
 
 - `filters` (dict, optional): A dictionary where each key is a manifest field name and each value specifies the match for that field. Filters support both exact and range-based matching, depending on the field. You can filter based on the keys inside the manifest, as well as internal fields like permissions and view/download statistics by adding a dot (`.`) before the field name. Supported internal fields include:
@@ -510,12 +586,10 @@ Retrieve a list of child artifacts within a specified collection, supporting key
   - Use a suffix `<` or `>` to specify ascending or descending order, respectively (e.g., `view_count<` for ascending).
   - Default ordering is ascending by prefix if not specified.
 
-- `summary_fields` (List[str], optional): A list of fields to include in the summary for each artifact. If not specified, it will use the `summary_fields` value from the parent collection, otherwise, the default summary fields (`id`, `type`, `name`) are used. You can add `"*"` to the list if you want to include all the manifest fields. If you want to include internal fields such as `.created_at`, `.last_modified`, or other download/view statistics such as `.download_count`, you can also specify them individually in the `summary_fields`. If you want to include all fields, you can add `".*"` to the list.
-
 - `silent` (bool, optional): If `True`, prevents incrementing the view count for the parent artifact when listing children. Default is `False`.
 
 **Returns:**
-A list of artifacts that match the search criteria, each represented by a dictionary containing summary fields. Fields are specified in the `summary_fields` attribute of the parent artifact's manifest; if this attribute is undefined, default summary fields (`id`, `type`, `name`) are used.
+A list of artifacts that match the search criteria, each represented by a dictionary containing all the fields.
 
 **Example Usage:**
 
@@ -523,7 +597,7 @@ A list of artifacts that match the search criteria, each represented by a dictio
 # Retrieve artifacts in the 'dataset-gallery' collection, filtering by creator and keywords,
 # with results ordered by descending view count.
 results = await artifact_manager.list(
-    prefix="collections/dataset-gallery",
+    artifact_id=collection.id, # Or full alias: "my_workspace/dataset-gallery"
     keywords=["example"],
     filters={"created_by": "user123", "stage": False},
     order_by="view_count>",
@@ -535,35 +609,47 @@ results = await artifact_manager.list(
 
 ---
 
-### `reset_stats(prefix: str) -> None`
+### `reset_stats(artifact_id: str) -> None`
 
 Resets the download and view counts for the artifact.
 
 **Parameters:**
 
-- `prefix`: The path of the artifact, it can be a prefix relative to the current workspace (e.g., `"collections/dataset-gallery/example-dataset"`) or an absolute prefix with the workspace id (e.g., `"/my_workspace_id/collections/dataset-gallery/example-dataset"`).
+- `artifact_id`: The id of the artifact to reset statistics for. It can be an uuid generated by `create` or `edit` function, or it can be an alias of the artifact under the current workspace. If you want to refer to an artifact in another workspace, you should use the full alias in the format of `"workspace_id/alias"`.
 
 **Example:**
 
 ```python
-await artifact_manager.reset_stats(prefix="collections/dataset-gallery/example-dataset")
+await artifact_manager.reset_stats(artifact_id=artifact.id)
+
+# If "example-dataset" is an alias of the artifact under the current workspace
+await artifact_manager.reset_stats(artifact_id="example-dataset")
+
+# If "example-dataset" is an alias of the artifact under another workspace
+await artifact_manager.reset_stats(artifact_id="other_workspace/example-dataset")
 ```
 
 ---
 
-### `publish(prefix: str, to: str = None) -> None`
+### `publish(artifact_id: str, to: str = None) -> None`
 
 Publishes the artifact to a specified platform, such as Zenodo. The artifact must have a valid Zenodo metadata schema to be published.
 
 **Parameters:**
 
-- `prefix`: The path of the artifact, it can be a prefix relative to the current workspace (e.g., `"collections/dataset-gallery/example-dataset"`) or an absolute prefix with the workspace id (e.g., `"/my_workspace_id/collections/dataset-gallery/example-dataset"`).
+- `artifact_id`: The id of the artifact to publish. It can be an uuid generated by `create` or `edit` function, or it can be an alias of the artifact under the current workspace. If you want to refer to an artifact in another workspace, you should use the full alias in the format of `"workspace_id/alias"`.
 - `to`: Optional, the target platform to publish the artifact. Supported values are `zenodo` and `sandbox_zenodo`. This parameter should be the same as the `publish_to` parameter in the `create` function or left empty to use the platform specified in the artifact's metadata.
 
 **Example:**
 
 ```python
-await artifact_manager.publish(prefix="collections/dataset-gallery/example-dataset", to="sandbox_zenodo")
+await artifact_manager.publish(artifact_id=artifact.id, to="sandbox_zenodo")
+
+# If "example-dataset" is an alias of the artifact under the current workspace
+await artifact_manager.publish(artifact_id="example-dataset", to="sandbox_zenodo")
+
+# If "example-dataset" is an alias of the artifact under another workspace
+await artifact_manager.publish(artifact_id="other_workspace/example-dataset", to="sandbox_zenodo")
 ```
 
 ---
@@ -581,12 +667,9 @@ artifact_manager = await server.get_service("public/artifact-manager")
 gallery_manifest = {
     "name": "Dataset Gallery",
     "description": "A collection for organizing datasets",
-    "type": "collection",
-    "collection": [],
 }
 # Create the collection with read permission for everyone and create permission for all authenticated users
-# We set orphan=True to create a collection without a parent
-await artifact_manager.create(prefix="collections/dataset-gallery", manifest=gallery_manifest, permissions={"*": "r", "@": "r+"}, orphan=True)
+collection = await artifact_manager.create(type="collection", alias="dataset-gallery", manifest=gallery_manifest, permissions={"*": "r", "@": "r+"})
 
 # Step 3: Add a dataset to the gallery
 dataset_manifest = {
@@ -594,42 +677,33 @@ dataset_manifest = {
     "description": "A dataset with example data",
     "type": "dataset",
 }
-await artifact_manager.create(prefix="collections/dataset-gallery/example-dataset", manifest=dataset_manifest, stage=True)
+dataset = await artifact_manager.create(parent_id=collection.id, alias="example-dataset" manifest=dataset_manifest, version="stage")
 
 # Step 4: Upload a file to the dataset
-put_url = await artifact_manager.put_file(prefix="collections/dataset-gallery/example-dataset", file_path="data.csv")
+put_url = await artifact_manager.put_file(dataset.id, file_path="data.csv")
 with open("path/to/local/data.csv", "rb") as f:
     response = requests.put(put_url, data=f)
     assert response.ok, "File upload failed"
 
 # Step 5: Commit the dataset
-await artifact_manager.commit(prefix="collections/dataset-gallery/example-dataset")
+await artifact_manager.commit(dataset.id)
 
 # Step 6: List all datasets in the gallery
-datasets = await artifact_manager.list(prefix="collections/dataset-gallery")
+datasets = await artifact_manager.list(collection.id)
 print("Datasets in the gallery:", datasets)
 ```
 
 
-### Resetting Download Statistics
-
-You can reset the download statistics of a dataset using the `reset_stats` function.
-
-```python
-await artifact_manager.reset_stats(prefix="collections/dataset-gallery/example-dataset")
-print("Download statistics reset.")
-```
-
 ## HTTP API for Accessing Artifacts and Download Counts
 
 The `Artifact Manager` provides an HTTP endpoint for retrieving artifact manifests, data, and download statistics. This is useful for public-facing web applications that need to access datasets, models, or applications.
 
 ### Endpoints:
 
- - `/{workspace}/artifacts/{prefix:path}` for fetching the artifact manifest.
- - `/{workspace}/artifacts/{prefix:path}/__children__` for listing all artifacts in a collection.
- - `/{workspace}/artifacts/{prefix:path}/__files__` for listing all files in the artifact.
- - `/{workspace}/artifacts/{prefix:path}/__files__/{file_path:path}` for downloading a file from the artifact (will be redirected to a pre-signed URL).
+ - `/{workspace}/artifacts/{artifact_id}` for fetching the artifact manifest.
+ - `/{workspace}/artifacts/{artifact_id}/children` for listing all artifacts in a collection.
+ - `/{workspace}/artifacts/{artifact_id}/files` for listing all files in the artifact.
+ - `/{workspace}/artifacts/{artifact_id}/files/{file_path:path}` for downloading a file from the artifact (will be redirected to a pre-signed URL).
 
 
 ### Request Format:
@@ -643,7 +717,7 @@ The `Artifact Manager` provides an HTTP endpoint for retrieving artifact manifes
 The path parameters are used to specify the artifact or file to access. The following parameters are supported:
 
 - **workspace**: The workspace in which the artifact is stored.
-- **prefix**: The relative prefix to the artifact. For private artifacts, it requires proper authentication by passing the user's token in the request headers.
+- **artifact_id**: The id of the artifact to access. This can be an uuid generated by `create` or `edit` function, or it can be an alias of the artifact under the current workspace. Note that this artifact_id can only be the uuid or the alias without the workspace id.
 - **file_path**: Optional, the relative path to a file within the artifact. This is optional and only required when downloading a file.
 
 ### Query Parameters:
@@ -652,7 +726,6 @@ Qury parameters are passed after the `?` in the URL and are used to control the
 
 - **stage**: A boolean flag to fetch the staged version of the manifest. Default is `False`.
 - **silent**: A boolean flag to suppress the view count increment. Default is `False`.
-- **include_metadata**: A boolean flag to include metadata such as download statistics in the manifest. Default is `False`.
 
 - **keywords**: A list of search terms used for fuzzy searching across all manifest fields, separated by commas.
 - **filters**: A dictionary of filters to apply to the search, in the format of a JSON string.
@@ -660,18 +733,17 @@ Qury parameters are passed after the `?` in the URL and are used to control the
 - **page**: The page number for pagination. Default is `0`.
 - **page_size**: The maximum number of artifacts to return per page. Default is `100`.
 - **order_by**: The field used to order results. Default is ascending by prefix.
-- **summary_fields**: A list of fields to include in the summary for each artifact when listing children. Default to the `summary_fields` value from the parent collection, otherwise, the default summary fields (`id`, `type`, `name`) are used. To include all manifest fields, add `"*"` to the list, and to include all internal fields, add `".*"` to the list.
 - **silent**: A boolean flag to prevent incrementing the view count for the parent artifact when listing children, listing files, or reading the artifact. Default is `False`.
 
 ### Response:
 
-For `/{workspace}/artifacts/{prefix:path}`, the response will be a JSON object representing the artifact manifest. For `/{workspace}/artifacts/{prefix:path}/__files__/{file_path:path}`, the response will be a pre-signed URL to download the file. The artifact manifest will also include any metadata such as download statistics under keys starting with dot (`.`), e.g. `.view_count`, `.download_count`. For private artifacts, make sure if the user has the necessary permissions.
+For `/{workspace}/artifacts/{artifact_id}`, the response will be a JSON object representing the artifact manifest. For `/{workspace}/artifacts/{artifact_id}/__files__/{file_path:path}`, the response will be a pre-signed URL to download the file. The artifact manifest will also include any metadata such as download statistics, e.g. `view_count`, `download_count`. For private artifacts, make sure if the user has the necessary permissions.
 
-For `/{workspace}/artifacts/{prefix:path}/__children__`, the response will be a list of artifacts in the collection.
+For `/{workspace}/artifacts/{artifact_id}/children`, the response will be a list of artifacts in the collection.
 
-For `/{workspace}/artifacts/{prefix:path}/__files__`, the response will be a list of files in the artifact, each file is a dictionary with the `name` and `type` fields.
+For `/{workspace}/artifacts/{artifact_id}/files`, the response will be a list of files in the artifact, each file is a dictionary with the `name` and `type` fields.
 
-For `/{workspace}/artifacts/{prefix:path}/__files__/{file_path:path}`, the response will be a pre-signed URL to download the file.
+For `/{workspace}/artifacts/{artifact_id}/files/{file_path:path}`, the response will be a pre-signed URL to download the file.
 
 ### Example: Fetching a public artifact with download statistics
 
@@ -680,13 +752,11 @@ import requests
 
 SERVER_URL = "https://hypha.aicell.io"
 workspace = "my-workspace"
-response = requests.get(f"{SERVER_URL}/{workspace}/artifacts/collections/dataset-gallery/example-dataset")
+response = requests.get(f"{SERVER_URL}/{workspace}/artifacts/example-dataset")
 if response.ok:
     artifact = response.json()
-    print(artifact["name"])  # Output: Example Dataset
-    print(artifact[".download_count"])  # Output: Download count for the dataset
+    print(artifact["manifest"]["name"])  # Output: Example Dataset
+    print(artifact["download_count"])  # Output: Download count for the dataset
 else:
     print(f"Error: {response.status_code}")
 ```
-
-
diff --git a/docs/getting-started.md b/docs/getting-started.md
index 96bba80e..eb1fdaed 100644
--- a/docs/getting-started.md
+++ b/docs/getting-started.md
@@ -220,7 +220,7 @@ svc = await get_remote_service("http://localhost:9527/ws-user-scintillating-lawy
 Include the following script in your HTML file to load the `hypha-rpc` client:
 
 ```html
-<script src="https://cdn.jsdelivr.net/npm/hypha-rpc@0.20.38/dist/hypha-rpc-websocket.min.js"></script>
+<script src="https://cdn.jsdelivr.net/npm/hypha-rpc@0.20.39/dist/hypha-rpc-websocket.min.js"></script>
 ```
 
 Use the following code in JavaScript to connect to the server and access an existing service:
diff --git a/docs/migration-guide.md b/docs/migration-guide.md
index 979e3be7..7ad14308 100644
--- a/docs/migration-guide.md
+++ b/docs/migration-guide.md
@@ -15,7 +15,7 @@ To connect to the server, instead of installing the `imjoy-rpc` module, you will
 pip install -U hypha-rpc # new install
 ```
 
-We also changed our versioning strategy, we use the same version number for the server and client, so it's easier to match the client and server versions. For example, `hypha-rpc` version `0.20.38` is compatible with Hypha server version `0.20.38`.
+We also changed our versioning strategy, we use the same version number for the server and client, so it's easier to match the client and server versions. For example, `hypha-rpc` version `0.20.39` is compatible with Hypha server version `0.20.39`.
 
 #### 2. Change the imports to use `hypha-rpc`
 
@@ -128,10 +128,10 @@ loop.run_forever()
 To connect to the server, instead of using the `imjoy-rpc` module, you will need to use the `hypha-rpc` module. The `hypha-rpc` module is a standalone module that provides the RPC connection to the Hypha server. You can include it in your HTML using a script tag:
 
 ```html
-<script src="https://cdn.jsdelivr.net/npm/hypha-rpc@0.20.38/dist/hypha-rpc-websocket.min.js"></script>
+<script src="https://cdn.jsdelivr.net/npm/hypha-rpc@0.20.39/dist/hypha-rpc-websocket.min.js"></script>
 ```
 
-We also changed our versioning strategy, we use the same version number for the server and client, so it's easier to match the client and server versions. For example, `hypha-rpc` version `0.20.38` is compatible with Hypha server version `0.20.38`.
+We also changed our versioning strategy, we use the same version number for the server and client, so it's easier to match the client and server versions. For example, `hypha-rpc` version `0.20.39` is compatible with Hypha server version `0.20.39`.
 
 #### 2. Change the connection method and use camelCase for service function names
 
@@ -149,7 +149,7 @@ Here is a suggested list of search and replace operations to update your code:
 Here is an example of how the updated code might look:
 
 ```html
-<script src="https://cdn.jsdelivr.net/npm/hypha-rpc@0.20.38/dist/hypha-rpc-websocket.min.js"></script>
+<script src="https://cdn.jsdelivr.net/npm/hypha-rpc@0.20.39/dist/hypha-rpc-websocket.min.js"></script>
 <script>
 async function main(){
     const server = await hyphaWebsocketClient.connectToServer({"server_url": "https://hypha.amun.ai"});
@@ -197,7 +197,7 @@ We created a tutorial to introduce this new feature: [service type annotation](.
 Here is a quick example in JavaScript:
 
 ```html
-<script src="https://cdn.jsdelivr.net/npm/hypha-rpc@0.20.38/dist/hypha-rpc-websocket.min.js"></script>
+<script src="https://cdn.jsdelivr.net/npm/hypha-rpc@0.20.39/dist/hypha-rpc-websocket.min.js"></script>
 
 <script>
 async function main(){
diff --git a/docs/service-type-annotation.md b/docs/service-type-annotation.md
index d5df4806..a94e1193 100644
--- a/docs/service-type-annotation.md
+++ b/docs/service-type-annotation.md
@@ -134,7 +134,7 @@ if __name__ == "__main__":
 **JavaScript Client: Service Usage**
 
 ```html
-<script src="https://cdn.jsdelivr.net/npm/hypha-rpc@0.20.38/dist/hypha-rpc-websocket.min.js"></script>
+<script src="https://cdn.jsdelivr.net/npm/hypha-rpc@0.20.39/dist/hypha-rpc-websocket.min.js"></script>
 <script>
 async function main() {
     const server = await hyphaWebsocketClient.connectToServer({"server_url": "https://hypha.amun.ai"});
diff --git a/helm-charts/aks-hypha.md b/helm-charts/aks-hypha.md
index 7aebdfac..bed2891e 100644
--- a/helm-charts/aks-hypha.md
+++ b/helm-charts/aks-hypha.md
@@ -146,7 +146,7 @@ replicaCount: 1
 image:
   repository: ghcr.io/amun-ai/hypha
   pullPolicy: IfNotPresent
-  tag: "0.20.38"
+  tag: "0.20.39"
 
 serviceAccount:
   create: true
diff --git a/helm-charts/hypha-server/Chart.yaml b/helm-charts/hypha-server/Chart.yaml
index e11fd706..62b1e20d 100644
--- a/helm-charts/hypha-server/Chart.yaml
+++ b/helm-charts/hypha-server/Chart.yaml
@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.20.38
+version: 0.20.39
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
diff --git a/helm-charts/hypha-server/values.yaml b/helm-charts/hypha-server/values.yaml
index 64658fb8..945e3443 100644
--- a/helm-charts/hypha-server/values.yaml
+++ b/helm-charts/hypha-server/values.yaml
@@ -8,7 +8,7 @@ image:
   repository: ghcr.io/amun-ai/hypha
   pullPolicy: IfNotPresent
   # Overrides the image tag whose default is the chart appVersion.
-  tag: "0.20.38"
+  tag: "0.20.39"
 
 imagePullSecrets: []
 nameOverride: ""
diff --git a/hypha/VERSION b/hypha/VERSION
index 0d9d79ca..7b9c960e 100644
--- a/hypha/VERSION
+++ b/hypha/VERSION
@@ -1,3 +1,3 @@
 {
-    "version": "0.20.38.post19"
+    "version": "0.20.39"
 }
diff --git a/hypha/apps.py b/hypha/apps.py
index 5b00f5fa..746f97e8 100644
--- a/hypha/apps.py
+++ b/hypha/apps.py
@@ -71,28 +71,18 @@ async def setup_applications_collection(self, overwrite=True, context=None):
         # Create an collection in the workspace
         manifest = {
             "id": "applications",
-            "type": "collection",
             "name": "Applications",
             "description": f"A collection of applications for workspace {ws}",
-            "summary_fields": [
-                "id",
-                "name",
-                "description",
-                "app_id",
-                "workspace",
-                "config",
-                "services",
-            ],
         }
-        await self.artifact_manager.create(
-            "applications",
-            manifest,
+        collection = await self.artifact_manager.create(
+            type="collection",
+            alias="applications",
+            manifest=manifest,
             overwrite=overwrite,
-            stage=False,
-            orphan=True,
             context=context,
         )
         logger.info(f"Applications collection created for workspace {ws}")
+        return collection["id"]
 
     async def install(
         self,
@@ -100,9 +90,9 @@ async def install(
         source_hash: str = None,
         config: Optional[Dict[str, Any]] = None,
         workspace: Optional[str] = None,
-        stage: bool = False,
         overwrite: bool = False,
         timeout: float = 60,
+        version: str = None,
         context: Optional[dict] = None,
     ) -> str:
         """Save a server app."""
@@ -196,33 +186,37 @@ async def install(
 
         app_id = f"{mhash}"
 
-        public_url = f"{self.public_base_url}/{workspace_info.id}/applications/{app_id}/{entry_point}"
+        public_url = f"{self.public_base_url}/{workspace_info.id}/artifacts/applications:{app_id}/files/{entry_point}"
         artifact_obj = convert_config_to_artifact(config, app_id, public_url)
         artifact_obj.update(
             {
-                "local_url": f"{self.local_base_url}/{workspace_info.id}/applications/{app_id}/{entry_point}",
+                "local_url": f"{self.local_base_url}/{workspace_info.id}/artifacts/applications:{app_id}/files/{entry_point}",
                 "public_url": public_url,
             }
         )
         ApplicationArtifact.model_validate(artifact_obj)
 
         try:
-            await self.artifact_manager.read("applications", context=context)
+            artifact = await self.artifact_manager.read("applications", context=context)
+            collection_id = artifact["id"]
         except KeyError:
-            await self.setup_applications_collection(overwrite=True, context=context)
+            collection_id = await self.setup_applications_collection(
+                overwrite=True, context=context
+            )
+
         # Create artifact using the artifact controller
-        prefix = f"applications/{app_id}"
-        await self.artifact_manager.create(
-            prefix=prefix,
+        artifact = await self.artifact_manager.create(
+            parent_id=collection_id,
+            alias=f"applications:{app_id}",
             manifest=artifact_obj,
             overwrite=overwrite,
-            stage=True,
+            version="stage",
             context=context,
         )
 
         # Upload the main source file
         put_url = await self.artifact_manager.put_file(
-            prefix=prefix, file_path=config["entry_point"], context=context
+            artifact["id"], file_path=config["entry_point"], context=context
         )
         async with httpx.AsyncClient() as client:
             response = await client.put(put_url, data=source)
@@ -230,11 +224,12 @@ async def install(
                 response.status_code == 200
             ), f"Failed to upload {config['entry_point']}"
 
-        # Commit the artifact if stage is not enabled
-        if not stage:
+        if version != "stage":
+            # Commit the artifact if stage is not enabled
             await self.commit(
                 app_id,
                 timeout=timeout,
+                version=version,
                 context=context,
             )
         return artifact_obj
@@ -247,9 +242,8 @@ async def add_file(
         context: Optional[dict] = None,
     ):
         """Add a file to the installed application."""
-        prefix = f"applications/{app_id}"
         put_url = await self.artifact_manager.put_file(
-            prefix=prefix, file_path=file_path, context=context
+            f"applications:{app_id}", file_path=file_path, context=context
         )
         response = httpx.put(put_url, data=file_content)
         assert response.status_code == 200, f"Failed to upload {file_path} to {app_id}"
@@ -261,41 +255,38 @@ async def remove_file(
         context: Optional[dict] = None,
     ):
         """Remove a file from the installed application."""
-        workspace = context["ws"]
-
-        prefix = f"applications/{app_id}"
         await self.artifact_manager.remove_file(
-            prefix=prefix, file_path=file_path, context=context
+            f"applications:{app_id}", file_path=file_path, context=context
         )
 
     async def list_files(
         self, app_id: str, context: Optional[dict] = None
     ) -> List[dict]:
         """List files of an installed application."""
-        prefix = f"applications/{app_id}"
-        manifest = await self.artifact_manager.read(prefix=prefix, context=context)
-        return manifest["files"]
+        return await self.artifact_manager.list_files(
+            f"applications:{app_id}", context=context
+        )
 
     async def edit(self, app_id: str, context: Optional[dict] = None):
         """Edit an application by re-opening its artifact."""
-        prefix = f"applications/{app_id}"
-        await self.artifact_manager.edit(prefix=prefix, stage=True, context=context)
+        await self.artifact_manager.edit(
+            f"applications:{app_id}", version="stage", context=context
+        )
 
     async def commit(
         self,
         app_id: str,
         timeout: int = 30,
+        version: str = None,
         context: Optional[dict] = None,
     ):
         """Finalize the edits to the application by committing the artifact."""
-
-        prefix = f"applications/{app_id}"
         try:
             info = await self.start(
                 app_id,
                 timeout=timeout,
                 wait_for_service="default",
-                stage=True,
+                version="stage",
                 context=context,
             )
             await self.stop(info["id"], context=context)
@@ -311,12 +302,13 @@ async def commit(
             raise Exception(
                 f"Failed to start the app: {app_id} during installation, error: {exp}"
             )
-        await self.artifact_manager.commit(prefix=prefix, context=context)
+        await self.artifact_manager.commit(
+            f"applications:{app_id}", version=version, context=context
+        )
 
     async def uninstall(self, app_id: str, context: Optional[dict] = None) -> None:
         """Uninstall an application by removing its artifact."""
-        prefix = f"applications/{app_id}"
-        await self.artifact_manager.delete(prefix=prefix, context=context)
+        await self.artifact_manager.delete(f"applications:{app_id}", context=context)
 
     async def launch(
         self,
@@ -347,7 +339,7 @@ async def start(
         app_id,
         client_id=None,
         timeout: float = 60,
-        stage: bool = False,
+        version: str = None,
         wait_for_service: Union[str, bool] = None,
         context: Optional[dict] = None,
     ):
@@ -372,32 +364,31 @@ async def start(
         if client_id is None:
             client_id = random_id(readable=True)
 
-        artifact = await self.artifact_manager.read(
-            prefix=f"applications/{app_id}", stage=stage, context=context
+        artifact_info = await self.artifact_manager.read(
+            f"applications:{app_id}", version=version, context=context
         )
+        artifact = artifact_info.get("manifest", {})
         artifact = ApplicationArtifact.model_validate(artifact)
 
         entry_point = artifact.entry_point
         assert entry_point, f"Entry point not found for app {app_id}."
         server_url = self.local_base_url
         local_url = (
-            f"{self.local_base_url}/{workspace}/applications/{app_id}/{entry_point}?"
+            f"{self.local_base_url}/{workspace}/artifacts/applications:{app_id}/files/{entry_point}?"
             + f"client_id={client_id}&workspace={workspace}"
             + f"&app_id={app_id}"
             + f"&server_url={server_url}"
-            + f"&token={token}"
-            if token
-            else ""
+            + (f"&token={token}" if token else "")
+            + (f"&version={version}" if version else "")
         )
         server_url = self.public_base_url
         public_url = (
-            f"{self.public_base_url}/{workspace}/applications/{app_id}/{entry_point}?"
+            f"{self.public_base_url}/{workspace}/artifacts/applications:{app_id}/files/{entry_point}?"
             + f"client_id={client_id}&workspace={workspace}"
             + f"&app_id={app_id}"
             + f"&server_url={server_url}"
-            + f"&token={token}"
-            if token
-            else ""
+            + (f"&token={token}" if token else "")
+            + (f"&version={version}" if version else "")
         )
 
         runner = random.choice(self._runner)
@@ -450,10 +441,10 @@ def service_added(info: dict):
                 artifact.model_dump(mode="json")
             )
             await self.artifact_manager.edit(
-                prefix=f"applications/{app_id}",
+                f"applications:{app_id}",
+                version=version,
                 manifest=artifact.model_dump(mode="json"),
                 context=context,
-                stage=stage,
             )
 
         except asyncio.TimeoutError:
@@ -518,9 +509,9 @@ async def list_apps(self, context: Optional[dict] = None):
         """List applications in the workspace."""
         try:
             apps = await self.artifact_manager.list_children(
-                prefix="applications", context=context
+                "applications", context=context
             )
-            return apps
+            return [app["manifest"] for app in apps]
         except KeyError:
             return []
         except Exception as exp:
diff --git a/hypha/artifact.py b/hypha/artifact.py
index fb95cf8b..2bff8e85 100644
--- a/hypha/artifact.py
+++ b/hypha/artifact.py
@@ -1,18 +1,13 @@
 import logging
 import time
 import sys
-import json
-import httpx
-import traceback
-import uuid
+import uuid_utils as uuid
 import random
 import re
+import json
 from sqlalchemy import (
     event,
     Column,
-    String,
-    Integer,
-    Float,
     JSON,
     UniqueConstraint,
     select,
@@ -20,119 +15,112 @@
     and_,
     or_,
 )
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.orm.attributes import flag_modified
 from hypha.utils import remove_objects_async, list_objects_async, safe_join
-from hypha.utils.zenodo import ZenodoClient, Deposition
+from hypha.utils.zenodo import ZenodoClient
 from botocore.exceptions import ClientError
+from hypha.s3 import FSFileResponse
+from aiobotocore.session import get_session
 from sqlalchemy import update
-from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.ext.asyncio import (
     async_sessionmaker,
     AsyncSession,
 )
-from sqlalchemy.orm.attributes import flag_modified
+from sqlalchemy.orm import relationship  # For parent-child relationships
 from fastapi import APIRouter, Depends, HTTPException
-from fastapi.responses import RedirectResponse
 from hypha.core import (
     UserInfo,
     UserPermission,
     Artifact,
     CollectionArtifact,
-    ApplicationArtifact,
-    WorkspaceInfo,
 )
 from hypha_rpc.utils import ObjectProxy
 from jsonschema import validate
 from typing import Union, List
+from sqlmodel import SQLModel, Field, Relationship, UniqueConstraint
+from typing import Optional, List
 
+# Logger setup
 logging.basicConfig(stream=sys.stdout)
 logger = logging.getLogger("artifact")
 logger.setLevel(logging.INFO)
 
-Base = declarative_base()
 
-
-# SQLAlchemy model for storing artifacts
-class ArtifactModel(Base):
+# SQLModel model for storing artifacts
+class ArtifactModel(SQLModel, table=True):  # `table=True` makes it a table model
     __tablename__ = "artifacts"
 
-    id = Column(Integer, primary_key=True, autoincrement=True)
-    type = Column(String, nullable=False)
-    workspace = Column(String, nullable=False, index=True)
-    prefix = Column(String, nullable=False)
-    manifest = Column(JSON, nullable=True)  # Store committed manifest
-    stage_manifest = Column(JSON, nullable=True)  # Store staged manifest
-    stage_files = Column(JSON, nullable=True)  # Store staged files during staging
-    download_weights = Column(
-        JSON, nullable=True
-    )  # Store the weights for counting downloads; a dictionary of file paths and their weights 0-1
-    download_count = Column(Float, nullable=False, default=0.0)  # New counter field
-    view_count = Column(Float, nullable=False, default=0.0)  # New counter field
-    created_at = Column(Integer, nullable=False)
-    created_by = Column(String, nullable=True)
-    last_modified = Column(Integer, nullable=False)
-    permissions = Column(
-        JSON, nullable=True
-    )  # New field for storing permissions for the artifact
-    config = Column(JSON, nullable=True)
+    id: str = Field(default_factory=lambda: str(uuid.uuid7()), primary_key=True)
+    type: Optional[str] = Field(default=None)
+    workspace: str = Field(index=True)
+    parent_id: Optional[str] = Field(default=None, foreign_key="artifacts.id")
+    alias: Optional[str] = Field(default=None)
+    manifest: Optional[dict] = Field(
+        default=None, sa_column=Column(JSON, nullable=True)
+    )
+    staging: Optional[dict] = Field(default=None, sa_column=Column(JSON, nullable=True))
+    download_count: float = Field(default=0.0)
+    view_count: float = Field(default=0.0)
+    file_count: int = Field(default=0)
+    created_at: int = Field()
+    created_by: Optional[str] = Field(default=None)
+    last_modified: int = Field()
+    versions: Optional[dict] = Field(
+        default=None, sa_column=Column(JSON, nullable=True)
+    )
+    config: Optional[dict] = Field(default=None, sa_column=Column(JSON, nullable=True))
+    secrets: Optional[dict] = Field(default=None, sa_column=Column(JSON, nullable=True))
+
+    # Relationships
+    parent: Optional["ArtifactModel"] = Relationship(
+        back_populates="children",
+        sa_relationship_kwargs={"remote_side": "ArtifactModel.id"},
+    )
+    children: List["ArtifactModel"] = Relationship(back_populates="parent")
+
+    # Table constraints
     __table_args__ = (
-        UniqueConstraint("workspace", "prefix", name="_workspace_prefix_uc"),
+        UniqueConstraint("workspace", "alias", name="_workspace_alias_uc"),
     )
 
 
-DEFAULT_SUMMARY_FIELDS = ["type", "name"]
-
-
-async def generate_unique_prefix(
-    id_parts,
-    pattern,
-    session,
-    workspace,
-    batch_size=10,
-    max_attempts=10,
-    additional_parts=None,
-):
-    placeholder_pattern = re.compile(r"\{(\w+)\}")
-    placeholders = placeholder_pattern.findall(pattern)
-
-    # Ensure placeholders in pattern exist in id_parts
-    if not all(placeholder in id_parts for placeholder in placeholders):
-        raise ValueError("Some placeholders in the pattern are missing in id_parts")
-
-    for _ in range(max_attempts):
-        # Generate a batch of candidate prefixes
-        candidate_prefixes = set()
-        for _ in range(batch_size):
-            generated_parts = {
-                placeholder: random.choice(id_parts[placeholder])
-                for placeholder in placeholders
-            }
-            if additional_parts:
-                generated_parts.update(additional_parts)
-            candidate_prefix = pattern
-            for placeholder, value in generated_parts.items():
-                candidate_prefix = candidate_prefix.replace(f"{{{placeholder}}}", value)
-            candidate_prefixes.add(candidate_prefix)
-
-        # Check which prefixes already exist in a single query
-        existing_prefixes = await _batch_prefix_exists(
-            session, workspace, candidate_prefixes
-        )
+def update_summary(summary, field, _metadata):
+    if field.startswith("."):
+        field = field[1:]
+    else:
+        field = f"manifest.{field}"
+    if "=" in field:
+        key, value_field = field.split("=", 1)
+        value = get_nested_value(value_field, _metadata)
+        set_nested_value(summary, key, value)
+    else:
+        value = get_nested_value(field, _metadata)
+        set_nested_value(summary, field, value)
 
-        # Find a unique prefix in the batch
-        unique_prefixes = candidate_prefixes - existing_prefixes
-        if unique_prefixes:
-            return unique_prefixes.pop()  # Return any unique prefix found
 
-    raise RuntimeError("Could not generate a unique prefix within the maximum attempts")
+def model_to_dict(model):
+    return {
+        column.name: getattr(model, column.name) for column in model.__table__.columns
+    }
 
 
-async def _batch_prefix_exists(session, workspace, prefixes):
-    """Check which prefixes already exist in the database."""
-    query = select(ArtifactModel.prefix).filter(
-        ArtifactModel.workspace == workspace, ArtifactModel.prefix.in_(prefixes)
-    )
-    result = await session.execute(query)
-    return set(row[0] for row in result.fetchall())
+def get_nested_value(field, _metadata):
+    keys = field.split(".")
+    value = _metadata
+
+    for key in keys:
+        value = value.get(key)
+        if value is None:
+            break
+    return value
+
+
+def set_nested_value(dictionary, field, value):
+    keys = field.split(".")
+    for key in keys[:-1]:
+        dictionary = dictionary.setdefault(key, {})
+    dictionary[keys[-1]] = value
 
 
 class ArtifactController:
@@ -143,6 +131,7 @@ def __init__(
         store,
         s3_controller,
         workspace_bucket="hypha-workspaces",
+        artifacts_dir="artifacts",
     ):
         """Set up controller with SQLAlchemy database and S3 for file storage."""
         self.engine = store.get_sql_engine()
@@ -153,214 +142,306 @@ def __init__(
         self.workspace_bucket = workspace_bucket
         self.store = store
         router = APIRouter()
+        self._artifacts_dir = artifacts_dir
 
-        @router.get("/{workspace}/artifacts/{prefix:path}")
+        # HTTP endpoint for getting an artifact
+        @router.get("/{workspace}/artifacts/{artifact_id}")
         async def get_artifact(
             workspace: str,
-            prefix: str,
-            # The following are used for reading the artifact
-            stage: bool = False,
-            include_metadata: bool = False,
-            # The following are used for listing children
-            keywords: str = None,
-            filters: str = None,
-            mode: str = "AND",
-            page: int = 0,
-            page_size: int = 100,
-            order_by: str = None,
-            summary_fields: str = None,
+            artifact_id: str,
             silent: bool = False,
+            version: str = None,
             user_info: self.store.login_optional = Depends(self.store.login_optional),
         ):
-            """Get artifact manifest or file."""
+            """Get artifact metadata, manifest, and config (excluding secrets)."""
             try:
-                if prefix.endswith("/__files__"):
-                    prefix = prefix.replace("/__files__", "")
-                    return await self.list_files(
-                        prefix,
-                        context={"ws": workspace, "user": user_info.model_dump()},
+                artifact_id = self._validate_artifact_id(artifact_id, {"ws": workspace})
+                session = await self._get_session(read_only=True)
+                async with session.begin():
+                    # Fetch artifact along with parent
+                    artifact, _ = await self._get_artifact_with_permission(
+                        user_info, artifact_id, "read", session
                     )
 
-                if "/__files__/" in prefix:
-                    prefix, path = prefix.split("/__files__/")
-                    try:
-                        url = await self.get_file(
-                            prefix,
-                            path,
-                            options={"silent": silent},
-                            context={"ws": workspace, "user": user_info.model_dump()},
-                        )
-                        # Redirect to the pre-signed URL
-                        return RedirectResponse(url=url)
-                    except FileNotFoundError as e:
-                        return await self.list_files(
-                            prefix,
-                            path,
-                            context={"ws": workspace, "user": user_info.model_dump()},
+                    version_index = self._get_version_index(artifact, version)
+
+                    if version_index == self._get_version_index(artifact, None):
+                        # Prepare artifact representation
+                        artifact_data = self._generate_artifact_data(artifact)
+                    else:
+                        s3_config = self._get_s3_config(artifact)
+                        artifact_data = await self._load_version_from_s3(
+                            artifact, version_index, s3_config
                         )
 
-                if prefix.endswith("/__children__"):
-                    assert not stage, "Cannot list children of a staged artifact."
-                    prefix = prefix.replace("/__children__", "")
-                    if keywords:
-                        keywords = keywords.split(",")
-                    if filters:
-                        filters = json.loads(filters)
-                    if summary_fields:
-                        summary_fields = summary_fields.split(",")
-                    return await self.list_children(
-                        prefix,
-                        page=page,
-                        page_size=page_size,
-                        filters=filters,
-                        mode=mode,
-                        order_by=order_by,
-                        keywords=keywords,
-                        summary_fields=summary_fields,
-                        silent=silent,
-                        context={"ws": workspace, "user": user_info.model_dump()},
-                    )
+                    # Increment view count unless silent
+                    if not silent:
+                        await self._increment_stat(session, artifact.id, "view_count")
+                        await session.commit()
 
-                artifact = await self._get_artifact_with_permission(
-                    workspace, user_info, prefix, "read"
+                    return artifact_data
+
+            except KeyError:
+                raise HTTPException(status_code=404, detail="Artifact not found")
+            except PermissionError:
+                raise HTTPException(status_code=403, detail="Permission denied")
+            except SQLAlchemyError as e:
+                logger.error(f"Database error: {str(e)}")
+                raise HTTPException(status_code=500, detail="Internal server error.")
+            except ClientError as e:
+                logger.error(f"S3 client error: {str(e)}")
+                raise HTTPException(status_code=500, detail="File storage error.")
+            except Exception as e:
+                logger.error(f"Unhandled exception: {str(e)}")
+                raise HTTPException(
+                    status_code=500, detail="An unexpected error occurred."
                 )
-                manifest = await self._read_manifest(
-                    artifact,
-                    stage=stage,
-                    increment_view_count=not silent,
-                    include_metadata=include_metadata,
+
+            finally:
+                await session.close()
+
+        # HTTP endpoint for listing child artifacts
+        @router.get("/{workspace}/artifacts/{artifact_id}/children")
+        async def list_children(
+            workspace: str,
+            artifact_id: str,
+            page: int = 0,
+            page_size: int = 100,
+            order_by: str = None,
+            user_info: self.store.login_optional = Depends(self.store.login_optional),
+        ):
+            """List child artifacts of a specified artifact."""
+            try:
+                artifact_id = self._validate_artifact_id(artifact_id, {"ws": workspace})
+                session = await self._get_session(read_only=True)
+                async with session.begin():
+                    parent_artifact, _ = await self._get_artifact_with_permission(
+                        user_info, artifact_id, "list", session
+                    )
+                    # Query child artifacts
+                    query = (
+                        select(ArtifactModel)
+                        .where(
+                            ArtifactModel.workspace == workspace,
+                            ArtifactModel.parent_id == parent_artifact.id,
+                        )
+                        .limit(page_size)
+                        .offset(page * page_size)
+                    )
+
+                    if order_by:
+                        if order_by.endswith("<"):
+                            query = query.order_by(
+                                getattr(ArtifactModel, order_by[:-1]).asc()
+                            )
+                        else:
+                            query = query.order_by(
+                                getattr(ArtifactModel, order_by).desc()
+                            )
+
+                    result = await session.execute(query)
+                    children = result.scalars().all()
+
+                    # Return children data, excluding secrets
+                    return [self._generate_artifact_data(child) for child in children]
+
+            except KeyError:
+                raise HTTPException(status_code=404, detail="Parent artifact not found")
+            except PermissionError:
+                raise HTTPException(status_code=403, detail="Permission denied")
+            except SQLAlchemyError as e:
+                logger.error(f"Database error: {str(e)}")
+                raise HTTPException(status_code=500, detail="Internal server error.")
+            except ClientError as e:
+                logger.error(f"S3 client error: {str(e)}")
+                raise HTTPException(status_code=500, detail="File storage error.")
+            except Exception as e:
+                logger.error(f"Unhandled exception: {str(e)}")
+                raise HTTPException(
+                    status_code=500, detail="An unexpected error occurred."
                 )
 
-                return manifest
-            except KeyError as e:
-                raise HTTPException(status_code=404, detail=str(e))
-            except PermissionError as e:
-                raise HTTPException(status_code=403, detail=str(e))
+            finally:
+                await session.close()
+
+        # HTTP endpoint for retrieving files within an artifact
+        @router.get("/{workspace}/artifacts/{artifact_id}/files/{path:path}")
+        async def get_file(
+            workspace: str,
+            artifact_id: str,
+            path: str = "",
+            silent: bool = False,
+            version: str = None,
+            token: str = None,
+            user_info: self.store.login_optional = Depends(self.store.login_optional),
+        ):
+            """Retrieve a file within an artifact or list files in a directory."""
+            try:
+                artifact_id = self._validate_artifact_id(artifact_id, {"ws": workspace})
+                session = await self._get_session(read_only=True)
+                if token:
+                    user_info = await self.store.parse_user_token(token)
+                async with session.begin():
+                    # Fetch artifact and check permissions
+                    (
+                        artifact,
+                        parent_artifact,
+                    ) = await self._get_artifact_with_permission(
+                        user_info, artifact_id, "get_file", session
+                    )
+                    version_index = self._get_version_index(artifact, version)
+                    s3_config = self._get_s3_config(artifact, parent_artifact)
+                    file_key = safe_join(
+                        s3_config["prefix"],
+                        workspace,
+                        f"{self._artifacts_dir}/{artifact.id}/v{version_index}",
+                        path,
+                    )
+                    if path.endswith("/") or path == "":
+                        async with self._create_client_async(
+                            s3_config,
+                        ) as s3_client:
+                            items = await list_objects_async(
+                                s3_client,
+                                s3_config["bucket"],
+                                file_key.rstrip("/") + "/",
+                                max_length=1000,
+                            )
+                            if not items:
+                                raise HTTPException(
+                                    status_code=404,
+                                    detail="File or directory not found",
+                                )
+                            return items
+
+                    s3_client = self._create_client_async(s3_config)
+                    # Increment download count unless silent
+                    if not silent:
+                        if artifact.config and "download_weights" in artifact.config:
+                            download_weights = artifact.config.get(
+                                "download_weights", {}
+                            )
+                        else:
+                            download_weights = {}
+                        download_weight = download_weights.get(path) or 0
+                        if download_weight > 0:
+                            await self._increment_stat(
+                                session,
+                                artifact.id,
+                                "download_count",
+                                increment=download_weight,
+                            )
+                        await session.commit()
+                    return FSFileResponse(s3_client, s3_config["bucket"], file_key)
+
+            except KeyError:
+                raise HTTPException(status_code=404, detail="Artifact not found")
+            except PermissionError:
+                raise HTTPException(status_code=403, detail="Permission denied")
+            except HTTPException as e:
+                raise e  # Re-raise HTTPExceptions to be handled by FastAPI
             except Exception as e:
-                raise HTTPException(status_code=500, detail=str(traceback.format_exc()))
+                logger.error(f"Unhandled exception in get_file: {str(e)}")
+                raise HTTPException(status_code=500, detail="Internal server error")
+            finally:
+                await session.close()
 
-        self.store.set_artifact_manager(self)
+        # Register public services
         artifact_service = self.get_artifact_service()
         self.store.register_public_service(artifact_service)
+        self.store.set_artifact_manager(self)
         self.store.register_router(router)
 
     async def init_db(self):
         """Initialize the database and create tables."""
         async with self.engine.begin() as conn:
-            await conn.run_sync(Base.metadata.create_all)
+            await conn.run_sync(ArtifactModel.metadata.create_all)
             logger.info("Database tables created successfully.")
 
     async def _get_session(self, read_only=False):
         """Return an SQLAlchemy async session. If read_only=True, ensure no modifications are allowed."""
         session = self.SessionLocal()
-
         if read_only:
-            sync_session = session.sync_session  # Access the synchronous session object
+            sync_session = session.sync_session
 
             @event.listens_for(sync_session, "before_flush")
             def prevent_flush(session, flush_context, instances):
-                """Prevent any flush operations to keep the session read-only."""
                 raise RuntimeError("This session is read-only.")
 
             @event.listens_for(sync_session, "after_transaction_end")
             def end_transaction(session, transaction):
-                """Ensure rollback after a transaction in a read-only session."""
                 if not transaction._parent:
                     session.rollback()
 
         return session
 
-    async def _get_artifact(self, session, workspace, prefix):
-        query = select(ArtifactModel).filter(
-            ArtifactModel.workspace == workspace,
-            ArtifactModel.prefix == prefix,
-        )
+    async def _get_artifact(self, session, artifact_id):
+        """
+        Retrieve an artifact by its unique ID or by parent ID and alias.
+        """
+        if "/" in artifact_id:
+            assert len(artifact_id.split("/")) == 2, "Invalid artifact ID format."
+            ws, alias = artifact_id.split("/")
+            query = select(ArtifactModel).where(
+                ArtifactModel.workspace == ws,
+                ArtifactModel.alias == alias,
+            )
+        else:
+            query = select(ArtifactModel).where(
+                ArtifactModel.id == artifact_id,
+            )
         result = await session.execute(query)
-        return result.scalar_one_or_none()
-
-    async def _get_session(self, read_only=False):
-        """Return an SQLAlchemy async session. If read_only=True, ensure no modifications are allowed."""
-        session = self.SessionLocal()
-
-        if read_only:
-            sync_session = session.sync_session  # Access the synchronous session object
-
-            @event.listens_for(sync_session, "before_flush")
-            def prevent_flush(session, flush_context, instances):
-                """Prevent any flush operations to keep the session read-only."""
-                raise RuntimeError("This session is read-only.")
-
-            @event.listens_for(sync_session, "after_transaction_end")
-            def end_transaction(session, transaction):
-                """Ensure rollback after a transaction in a read-only session."""
-                if not transaction._parent:
-                    session.rollback()
+        artifact = result.scalar_one_or_none()
+        if not artifact:
+            raise KeyError(f"Artifact with ID '{artifact_id}' does not exist.")
+        return artifact
 
-        return session
+    async def _get_artifact_with_parent(self, session, artifact_id):
+        """
+        Retrieve an artifact and its parent artifact in a single SQL query.
+        """
+        if "/" in artifact_id:
+            assert (
+                len(artifact_id.split("/")) == 2
+            ), "Invalid artifact ID format, it should be `workspace/alias`."
+            ws, alias = artifact_id.split("/")
+            query = select(ArtifactModel).where(
+                ArtifactModel.workspace == ws,
+                ArtifactModel.alias == alias,
+            )
+        else:
+            query = select(ArtifactModel).where(
+                ArtifactModel.id == artifact_id,
+            )
 
-    def _generate_metadata(self, artifact):
-        metadata = {
-            "type": artifact.type,
-            "workspace": artifact.workspace,
-            "prefix": artifact.prefix,
-            "created_by": artifact.created_by,
-            "created_at": int(artifact.created_at or 0),
-            "modified_at": int(artifact.last_modified or 0),
-            "download_count": int(artifact.download_count or 0),
-            "view_count": int(artifact.view_count or 0),
-            "permissions": artifact.permissions or {},
-            "download_weights": artifact.download_weights or {},
-            "stage": artifact.stage_manifest is not None,
-            "stage_files": artifact.stage_files or [],
-            "config": artifact.config or {},
-        }
-        return {f".{k}": v for k, v in metadata.items()}
+        result = await session.execute(query)
+        artifact = result.scalar_one_or_none()
+        if not artifact:
+            raise KeyError(f"Artifact with ID '{artifact_id}' does not exist.")
+        parent_artifact = None
+        if artifact and artifact.parent_id:
+            parent_query = select(ArtifactModel).where(
+                ArtifactModel.id == artifact.parent_id,
+            )
+            parent_result = await session.execute(parent_query)
+            parent_artifact = parent_result.scalar_one_or_none()
+        return artifact, parent_artifact
 
-    async def _read_manifest(
-        self, artifact, stage=False, increment_view_count=False, include_metadata=False
-    ):
-        manifest = artifact.stage_manifest if stage else artifact.manifest
-        prefix = artifact.prefix
-        if not manifest:
-            raise KeyError(f"No manifest found for artifact '{prefix}'.")
-        try:
-            session = await self._get_session()
-            async with session.begin():
-                if stage:
-                    manifest["_stage_files"] = artifact.stage_files
-                elif increment_view_count:
-                    # increase view count
-                    stmt = (
-                        update(ArtifactModel)
-                        .where(ArtifactModel.id == artifact.id)
-                        # atomically increment the view count
-                        .values(view_count=ArtifactModel.view_count + 1)
-                        .execution_options(synchronize_session="fetch")
-                    )
-                    await session.execute(stmt)
-                    await session.commit()
-                    artifact.view_count += 1
-                if include_metadata:
-                    manifest.update(self._generate_metadata(artifact))
-                return manifest
-        except Exception as e:
-            raise e
-        finally:
-            await session.close()
+    def _generate_artifact_data(self, artifact):
+        artifact_data = model_to_dict(artifact)
+        # Exclude 'secrets' from artifact_data to prevent exposure
+        artifact_data.pop("secrets", None)
+        return artifact_data
 
     def _expand_permission(self, permission):
-        """Get the alias for the operation."""
+        """Get the list of operations allowed by the permission code."""
         if isinstance(permission, str):
-            if permission == "n":
-                return []
-            elif permission == "l":
-                return ["list"]
-            elif permission == "l+":
-                return ["list", "create", "commit"]
-            elif permission == "r":
-                return ["read", "get_file", "list_files", "list"]
-            elif permission == "r+":
-                return [
+            permission_map = {
+                "n": [],
+                "l": ["list"],
+                "l+": ["list", "create", "commit"],
+                "r": ["read", "get_file", "list_files", "list"],
+                "r+": [
                     "read",
                     "get_file",
                     "put_file",
@@ -368,9 +449,8 @@ def _expand_permission(self, permission):
                     "list",
                     "create",
                     "commit",
-                ]
-            elif permission == "rw":
-                return [
+                ],
+                "rw": [
                     "read",
                     "get_file",
                     "list_files",
@@ -379,9 +459,8 @@ def _expand_permission(self, permission):
                     "commit",
                     "put_file",
                     "remove_file",
-                ]
-            elif permission == "rw+":
-                return [
+                ],
+                "rw+": [
                     "read",
                     "get_file",
                     "list_files",
@@ -391,9 +470,8 @@ def _expand_permission(self, permission):
                     "put_file",
                     "remove_file",
                     "create",
-                ]
-            elif permission == "*":
-                return [
+                ],
+                "*": [
                     "read",
                     "get_file",
                     "list_files",
@@ -405,495 +483,767 @@ def _expand_permission(self, permission):
                     "create",
                     "reset_stats",
                     "publish",
-                ]
-            else:
-                raise ValueError(f"Permission '{permission}' is not supported.")
+                ],
+            }
+            return permission_map.get(permission, [])
         else:
-            return permission
+            return permission  # Assume it's already a list
+
+    async def _check_permissions(self, artifact, user_info, operation):
+        """Check whether a user has permission to perform an operation on an artifact."""
+        # Check specific artifact permissions
+        permissions = artifact.config.get("permissions", {}) if artifact.config else {}
+
+        # Specific user permission
+        if user_info.id in permissions:
+            if operation in self._expand_permission(permissions[user_info.id]):
+                return True
+
+        # Check based on user's authentication status
+        if not user_info.is_anonymous and "@" in permissions:
+            if operation in self._expand_permission(permissions["@"]):
+                return True
+
+        # Global permission for all users
+        if "*" in permissions:
+            if operation in self._expand_permission(permissions["*"]):
+                return True
+        return False
 
     async def _get_artifact_with_permission(
         self,
-        workspace: str,
         user_info: UserInfo,
-        prefix: str,
+        artifact_id: str,
         operation: Union[str, List[str]],
+        session: AsyncSession,
     ):
         """
         Check whether a user has the required permission to perform an operation on an artifact.
-        :param workspace: The workspace where the artifact is located.
-        :param user_info: The user info object, including user permissions.
-        :param prefix: The prefix of the artifact being accessed.
-        :param operation: The operation being checked ('read', 'write', etc.).
-        :return: True if the user has permission, False otherwise.
+        Fetches artifact and parent artifacts, checking permissions hierarchically.
         """
-
-        # Map the operation to the corresponding UserPermission enum
-        if operation in ["list", "read", "get_file", "list_files"]:  # list
-            perm = UserPermission.read
-        elif operation in ["create", "edit", "commit", "put_file", "remove_file"]:
-            perm = UserPermission.read_write
-        elif operation in ["delete", "reset_stats", "publish"]:
-            perm = UserPermission.admin
-        else:
+        # Map operation to permission level
+        operation_map = {
+            "list": UserPermission.read,
+            "read": UserPermission.read,
+            "get_file": UserPermission.read,
+            "list_files": UserPermission.read,
+            "create": UserPermission.read_write,
+            "edit": UserPermission.read_write,
+            "commit": UserPermission.read_write,
+            "put_file": UserPermission.read_write,
+            "remove_file": UserPermission.read_write,
+            "delete": UserPermission.admin,
+            "reset_stats": UserPermission.admin,
+            "publish": UserPermission.admin,
+        }
+        required_perm = operation_map.get(operation)
+        if required_perm is None:
             raise ValueError(f"Operation '{operation}' is not supported.")
 
-        session = await self._get_session(read_only=True)
-        try:
-            async with session.begin():
-                artifact = await self._get_artifact(session, workspace, prefix)
-                if not artifact:
-                    raise KeyError(f"Artifact with prefix '{prefix}' does not exist.")
-                else:
-                    # Step 1: Check artifact-specific permissions
-                    # Check if there are specific permissions for this artifact
-                    if artifact.permissions:
-                        # Check if this specific user is in the permissions dictionary
-                        if user_info.id in artifact.permissions and (
-                            operation
-                            in self._expand_permission(
-                                artifact.permissions[user_info.id]
-                            )
-                        ):
-                            return artifact
-
-                        # check if the user is logged in and has permission
-                        if (
-                            not user_info.is_anonymous
-                            and "@" in artifact.permissions
-                            and (
-                                operation
-                                in self._expand_permission(artifact.permissions["@"])
-                            )
-                        ):
-                            return artifact
-
-                        # If all users are allowed for this operation, return True
-                        if "*" in artifact.permissions and (
-                            operation
-                            in self._expand_permission(artifact.permissions["*"])
-                        ):
-                            return artifact
-
-                # Step 2: Check workspace-level permission
-                if user_info.check_permission(workspace, perm):
-                    return artifact
-
-                # Step 3: Check parent artifact's permissions if no explicit permission defined
-                parent_prefix = "/".join(prefix.split("/")[:-1])
-                depth = 5
-                while parent_prefix:
-                    parent_artifact = await self._get_artifact(
-                        session, workspace, parent_prefix
-                    )
-                    if parent_artifact:
-                        # Check parent artifact permissions if they exist
-                        if parent_artifact.permissions:
-                            # Check if this specific user is in the parent artifact permissions dictionary
-                            if user_info.id in parent_artifact.permissions and (
-                                operation
-                                in self._expand_permission(
-                                    parent_artifact.permissions[user_info.id]
-                                )
-                            ):
-                                return artifact
-
-                            # check if the user is logged in and has permission
-                            if (
-                                not user_info.is_anonymous
-                                and "@" in parent_artifact.permissions
-                                and (
-                                    operation
-                                    in self._expand_permission(
-                                        parent_artifact.permissions["@"]
-                                    )
-                                )
-                            ):
-                                return artifact
-
-                            # If all users are allowed for this operation, return True
-                            if "*" in parent_artifact.permissions and (
-                                operation
-                                in self._expand_permission(
-                                    parent_artifact.permissions["*"]
-                                )
-                            ):
-                                return artifact
-                    else:
-                        break
-                    # Move up the hierarchy to check the next parent
-                    parent_prefix = "/".join(parent_prefix.split("/")[:-1])
-                    depth -= 1
-                    if depth <= 0:
-                        break
+        # Fetch the primary artifact
+        artifact, parent_artifact = await self._get_artifact_with_parent(
+            session, artifact_id
+        )
+        if not artifact:
+            raise KeyError(f"Artifact with ID '{artifact_id}' does not exist.")
 
-        except Exception as e:
-            raise e
-        finally:
-            await session.close()
+        # Check permissions on the artifact itself
+        if await self._check_permissions(artifact, user_info, operation):
+            return artifact, parent_artifact
+
+        # Finally, check workspace-level permission
+        if user_info.check_permission(artifact.workspace, required_perm):
+            return artifact, parent_artifact
 
         raise PermissionError(
             f"User does not have permission to perform the operation '{operation}' on the artifact."
         )
 
-    def _get_zenodo_client(self, parent_artifact, publish_to):
-        archives_config = parent_artifact.config.get("archives")
+    async def _increment_stat(self, session, artifact_id, field, increment=1.0):
+        """
+        Increment the specified statistic field for an artifact.
+        :param session: Database session.
+        :param artifact_id: ID of the artifact.
+        :param field: 'view_count' or 'download_count'.
+        :param increment: Amount to increment.
+        """
+        if field not in ["view_count", "download_count"]:
+            raise ValueError("Field must be 'view_count' or 'download_count'.")
+        stmt = (
+            update(ArtifactModel)
+            .where(ArtifactModel.id == artifact_id)
+            .values({field: getattr(ArtifactModel, field) + increment})
+            .execution_options(synchronize_session="fetch")
+        )
+        await session.execute(stmt)
+
+    def _get_zenodo_client(self, artifact, parent_artifact=None, publish_to=None):
+        """
+        Get a Zenodo client using credentials from the artifact's or parent artifact's secrets.
+        """
+        # merge secrets from parent and artifact
+        if parent_artifact and parent_artifact.secrets:
+            secrets = parent_artifact.secrets.copy()
+        else:
+            secrets = {}
+        secrets.update(artifact.secrets or {})
+
         if publish_to == "zenodo":
-            if (
-                "zenodo" not in archives_config
-                or "access_token" not in archives_config["zenodo"]
-            ):
-                raise ValueError("Zenodo access token is not configured.")
-            zenodo_token = archives_config["zenodo"]["access_token"]
+            zenodo_token = secrets.get("ZENODO_ACCESS_TOKEN")
+            if not zenodo_token:
+                raise ValueError("Zenodo access token is not configured in secrets.")
             zenodo_client = ZenodoClient(zenodo_token, "https://zenodo.org")
-            assert zenodo_client, "Zenodo access token is not configured."
         elif publish_to == "sandbox_zenodo":
-            if (
-                "sandbox_zenodo" not in archives_config
-                or "access_token" not in archives_config["sandbox_zenodo"]
-            ):
-                raise ValueError("Sandbox Zenodo access token is not configured.")
-            sandbox_zenodo_token = archives_config["sandbox_zenodo"]["access_token"]
-            zenodo_client = ZenodoClient(
-                sandbox_zenodo_token, "https://sandbox.zenodo.org"
-            )
+            zenodo_token = secrets.get("SANDBOX_ZENODO_ACCESS_TOKEN")
+            if not zenodo_token:
+                raise ValueError(
+                    "Sandbox Zenodo access token is not configured in secrets."
+                )
+            zenodo_client = ZenodoClient(zenodo_token, "https://sandbox.zenodo.org")
         else:
             raise ValueError(f"Publishing to '{publish_to}' is not supported.")
+
         return zenodo_client
 
+    def _get_s3_config(self, artifact, parent_artifact=None):
+        """
+        Get S3 configuration using credentials from the artifact's or parent artifact's secrets.
+        """
+        # merge secrets from parent and artifact
+        if parent_artifact and parent_artifact.secrets:
+            secrets = parent_artifact.secrets.copy()
+        else:
+            secrets = {}
+        secrets.update(artifact.secrets or {})
+
+        endpoint_url = secrets.get("S3_ENDPOINT_URL")
+        access_key_id = secrets.get("S3_ACCESS_KEY_ID")
+        secret_access_key = secrets.get("S3_SECRET_ACCESS_KEY")
+        region_name = secrets.get("S3_REGION_NAME")
+        bucket = secrets.get("S3_BUCKET")
+        prefix = secrets.get("S3_PREFIX")
+
+        if not access_key_id or not secret_access_key:
+            return {
+                "endpoint_url": self.s3_controller.endpoint_url,
+                "access_key_id": self.s3_controller.access_key_id,
+                "secret_access_key": self.s3_controller.secret_access_key,
+                "region_name": self.s3_controller.region_name,
+                "bucket": self.workspace_bucket,
+                "prefix": "",
+            }
+        else:
+            return {
+                "endpoint_url": endpoint_url,
+                "access_key_id": access_key_id,
+                "secret_access_key": secret_access_key,
+                "region_name": region_name,
+                "bucket": bucket or self.workspace_bucket,
+                "prefix": prefix or "",
+            }
+
+    def _create_client_async(self, s3_config):
+        """
+        Create an S3 client using credentials from secrets.
+        """
+        endpoint_url = s3_config.get("endpoint_url")
+        access_key_id = s3_config.get("access_key_id")
+        secret_access_key = s3_config.get("secret_access_key")
+        region_name = s3_config.get("region_name")
+
+        assert (
+            endpoint_url and access_key_id and secret_access_key and region_name
+        ), "S3 credentials are not configured."
+
+        return get_session().create_client(
+            "s3",
+            endpoint_url=endpoint_url,
+            aws_access_key_id=access_key_id,
+            aws_secret_access_key=secret_access_key,
+            region_name=region_name,
+        )
+
+    async def _generate_candidate_aliases(
+        self,
+        alias_pattern: str,
+        id_parts: dict,
+        additional_parts: dict = None,
+        max_candidates: int = 10,
+    ):
+        """
+        Generate a list of candidate aliases based on the alias pattern and id_parts.
+        """
+        placeholder_pattern = re.compile(r"\{(\w+)\}")
+        placeholders = placeholder_pattern.findall(alias_pattern)
+        if not placeholders:
+            return [alias_pattern]
+
+        if additional_parts:
+            id_parts = {**id_parts, **additional_parts}
+
+        candidates = set()
+        attempts = 0
+        while len(candidates) < max_candidates and attempts < max_candidates * 10:
+            attempts += 1
+            alias = alias_pattern
+            for placeholder in placeholders:
+                if placeholder in id_parts and isinstance(id_parts[placeholder], list):
+                    value = random.choice(id_parts[placeholder])
+                elif placeholder in id_parts:
+                    value = id_parts[placeholder]
+                else:
+                    value = str(uuid.uuid4())
+                alias = alias.replace(f"{{{placeholder}}}", value)
+            candidates.add(alias)
+        return list(candidates)
+
+    async def _batch_alias_exists(self, session, workspace, aliases):
+        """
+        Check which aliases already exist under the given parent artifact.
+        """
+        query = select(ArtifactModel.alias).where(
+            ArtifactModel.workspace == workspace,
+            ArtifactModel.alias.in_(aliases),
+        )
+        result = await session.execute(query)
+        existing_aliases = set(row[0] for row in result.fetchall())
+        return existing_aliases
+
+    async def _generate_unique_alias(
+        self,
+        session,
+        workspace,
+        alias_pattern,
+        id_parts,
+        additional_parts=None,
+        max_attempts=10,
+    ):
+        """
+        Generate a unique alias under the given parent artifact.
+        """
+        for _ in range(max_attempts):
+            candidates = await self._generate_candidate_aliases(
+                alias_pattern, id_parts, additional_parts
+            )
+            existing_aliases = await self._batch_alias_exists(
+                session, workspace, candidates
+            )
+            unique_aliases = set(candidates) - existing_aliases
+            if unique_aliases:
+                return unique_aliases.pop()
+        raise RuntimeError(
+            "Could not generate a unique alias within the maximum attempts."
+        )
+
+    async def _handle_existing_artifact(
+        self, session, workspace, parent_id, alias, overwrite
+    ):
+        """Remove existing artifact if overwrite is enabled."""
+        try:
+            existing_artifact = await self._get_artifact(
+                session, f"{workspace}/{alias}"
+            )
+            if parent_id != existing_artifact.parent_id:
+                raise FileExistsError(
+                    f"Artifact with alias '{alias}' already exists in a different parent artifact."
+                )
+            if not overwrite:
+                raise FileExistsError(
+                    f"Artifact with alias '{alias}' already exists. Use overwrite=True to overwrite."
+                )
+            logger.info(f"Overwriting artifact with alias: {alias}")
+            await session.delete(existing_artifact)
+            await session.flush()
+            return existing_artifact.id
+        except KeyError:
+            return None
+
+    async def _save_version_to_s3(self, version_index: int, artifact, s3_config):
+        """
+        Save the staged version to S3 and update the artifact's manifest.
+        """
+        assert version_index >= 0, "Version index must be non-negative."
+        async with self._create_client_async(s3_config) as s3_client:
+            version_key = safe_join(
+                s3_config["prefix"],
+                artifact.workspace,
+                f"{self._artifacts_dir}",
+                artifact.id,
+                f"v{version_index}.json",
+            )
+            await s3_client.put_object(
+                Bucket=s3_config["bucket"],
+                Key=version_key,
+                Body=json.dumps(model_to_dict(artifact)),
+            )
+
+    async def _load_version_from_s3(
+        self, artifact, version_index: int, s3_config, include_secrets=False
+    ):
+        """
+        Load the version from S3.
+        """
+        assert version_index >= 0, "Version index must be non-negative."
+        s3_config = self._get_s3_config(artifact)
+        version_key = safe_join(
+            s3_config["prefix"],
+            artifact.workspace,
+            f"{self._artifacts_dir}",
+            artifact.id,
+            f"v{version_index}.json",
+        )
+        async with self._create_client_async(s3_config) as s3_client:
+            response = await s3_client.get_object(
+                Bucket=s3_config["bucket"], Key=version_key
+            )
+            body = await response["Body"].read()
+            data = json.loads(body)
+            if not include_secrets:
+                data.pop("secrets", None)
+            return data
+
+    def _get_version_index(self, artifact, version):
+        """Get the version index based on the version string."""
+        if version is None:
+            version_index = max(0, len(artifact.versions) - 1)
+        elif version == "latest":
+            version_index = len(artifact.versions) - 1
+            return version_index
+        elif version == "stage":
+            version_index = len(artifact.versions)
+        elif isinstance(version, str):
+            # find the version index
+            version_index = -1
+            for i, v in enumerate(artifact.versions):
+                if v["version"] == version:
+                    version_index = i
+                    break
+            if version_index < 0:
+                if version.isdigit():
+                    version_index = int(version)
+                raise ValueError(f"Artifact version '{version}' does not exist.")
+        elif isinstance(version, int):
+            assert version >= 0, "Version must be non-negative."
+            version_index = version
+        else:
+            raise ValueError("Version must be a string or an integer.")
+
+        return version_index
+
+    async def _delete_version_files_from_s3(
+        self, version_index: int, artifact, s3_config, delete_files=False
+    ):
+        """
+        Delete the staged version from S3.
+        """
+        assert version_index >= 0, "Version index must be non-negative."
+        async with self._create_client_async(s3_config) as s3_client:
+            version_key = safe_join(
+                s3_config["prefix"],
+                artifact.workspace,
+                f"{self._artifacts_dir}",
+                artifact.id,
+                f"v{version_index}",
+            )
+            await s3_client.delete_object(
+                Bucket=s3_config["bucket"], Key=version_key + ".json"
+            )
+            if delete_files:
+                await remove_objects_async(
+                    s3_client,
+                    s3_config["bucket"],
+                    version_key + "/",
+                )
+
     async def create(
         self,
-        prefix,
-        manifest: dict,
+        alias: str = None,
+        workspace: str = None,
+        parent_id: str = None,
+        manifest: dict = None,
         overwrite=False,
-        stage=False,
-        orphan=False,
-        permissions: dict = None,
+        type="generic",
         config: dict = None,
+        secrets: dict = None,
         publish_to=None,
+        version: str = None,
+        comment: str = None,
         context: dict = None,
     ):
         """Create a new artifact and store its manifest in the database."""
         if context is None or "ws" not in context:
             raise ValueError("Context must include 'ws' (workspace).")
 
-        assert "__files__" not in prefix.split(
-            "/"
-        ), "Artifact prefix cannot contain '__files__'."
-
-        assert "__children__" not in prefix.split(
-            "/"
-        ), "Artifact prefix cannot contain '__children__'."
-
-        if prefix.startswith("/"):
-            ws = prefix.split("/")[1]
-            prefix = "/".join(prefix.split("/")[2:])
-        else:
-            ws = context["ws"]
         user_info = UserInfo.model_validate(context["user"])
 
-        # Validate based on the type of manifest
-        if manifest["type"] == "collection":
+        # Validate the manifest
+        if type == "collection":
             CollectionArtifact.model_validate(manifest)
-        elif manifest["type"] == "application":
-            ApplicationArtifact.model_validate(manifest)
-        else:
+        elif type == "generic":
             Artifact.model_validate(manifest)
 
-        # Convert ObjectProxy to dict if necessary
         if isinstance(manifest, ObjectProxy):
             manifest = ObjectProxy.toDict(manifest)
+
+        if alias:
+            if re.match(
+                r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$",
+                alias,
+            ):
+                raise ValueError(
+                    "Alias should be a human readable string, it cannot be a UUID."
+                )
+
         session = await self._get_session()
         try:
             async with session.begin():
-                # if permissions is not set, try to use the parent artifact's permissions setting
-                parent_prefix = "/".join(prefix.split("/")[:-1])
-                assert (
-                    "{" not in parent_prefix and "}" not in parent_prefix
-                ), "Curly braces are not allowed in artifact prefix."
-                if parent_prefix:
-                    try:
-                        parent_artifact = await self._get_artifact_with_permission(
-                            ws, user_info, parent_prefix, "create"
+                parent_artifact = None
+                if parent_id:
+                    parent_artifact, _ = await self._get_artifact_with_permission(
+                        user_info, parent_id, "create", session
+                    )
+                    if workspace:
+                        assert (
+                            workspace == parent_artifact.workspace
+                        ), "Workspace must match parent artifact's workspace."
+                    workspace = parent_artifact.workspace
+                    if not parent_artifact.manifest:
+                        raise ValueError(
+                            f"Parent artifact with ID '{parent_id}' must be committed before creating a child artifact."
                         )
-                        if not parent_artifact.manifest:
-                            raise ValueError(
-                                f"Parent artifact under prefix '{parent_prefix}' must be committed before creating a child artifact."
-                            )
-
-                        if permissions is None:
-                            permissions = parent_artifact.permissions
-                        else:
-                            # Merge the parent artifact's permissions with the new permissions
-                            for key, value in parent_artifact.permissions.items():
-                                if key not in permissions:
-                                    permissions[key] = value
-                    except KeyError:
-                        parent_artifact = None
                 else:
-                    parent_artifact = None
-
-                if not orphan and not parent_artifact:
-                    raise ValueError(
-                        f"Parent artifact not found (prefix: {parent_prefix}) for non-orphan artifact, please create the parent artifact first or set orphan=True."
-                    )
-
-                if parent_artifact and orphan:
-                    raise ValueError(
-                        f"Parent artifact found (prefix: {parent_prefix}) for orphan artifact, please set orphan=False."
-                    )
-
-                if orphan:
-                    # Make sure the user has permission to the workspace
-                    if not user_info.check_permission(ws, UserPermission.read_write):
+                    workspace = workspace or context["ws"]
+                    if not user_info.check_permission(
+                        workspace, UserPermission.read_write
+                    ):
                         raise PermissionError(
-                            f"User does not have permission to create an orphan artifact in the workspace '{ws}'."
+                            f"User does not have permission to create an orphan artifact in the workspace '{workspace}'."
                         )
 
-                if publish_to:
-                    assert parent_artifact, "A parent artifact is required"
-                    zenodo_client = self._get_zenodo_client(parent_artifact, publish_to)
-                    deposition_info = await zenodo_client.create_deposition()
-                    if config:
-                        assert (
-                            "zenodo" not in config
-                        ), "Zenodo config already exists in the artifact."
-                        config["zenodo"] = deposition_info
-                    else:
-                        config = {"zenodo": deposition_info}
-                else:
-                    deposition_info = None
+                winfo = await self.store.get_workspace_info(workspace, load=True)
+                if not winfo.persistent:
+                    raise PermissionError(
+                        f"Cannot create artifact in a non-persistent workspace `{workspace}`, you may need to login to a persistent workspace."
+                    )
 
-                if "{" in prefix and "}" in prefix:
-                    default_parts = {
-                        "uuid": str(uuid.uuid4()),
-                        "timestamp": str(time.time()),
+                config = config or {}
+                if alias and "{" in alias and "}" in alias:
+                    id_parts = {}
+                    additional_parts = {
+                        "uuid": str(uuid.uuid7()),
+                        "timestamp": str(int(time.time())),
+                        "user_id": user_info.id,
                     }
-                    if deposition_info:
-                        default_parts["zenodo_id"] = deposition_info["id"]
-                        default_parts["zenodo_conceptrecid"] = deposition_info[
-                            "conceptrecid"
-                        ]
-                    # check if parent artifact is a collection
-                    if (
-                        parent_artifact
-                        and parent_artifact.type == "collection"
-                        and parent_artifact.config
-                    ):
-                        id_parts = parent_artifact.config.get("id_parts")
-                        if id_parts:
-                            # id parts example: {"animals": ["dog", "cat", ...], "colors": ["red", "blue", ...]}
-                            # say if the prefix is "collections/{animals}-{colors}"
-                            # then the prefix will be generate as "collections/dog-red", "collections/dog-blue", ...
-                            try:
-                                prefix = await generate_unique_prefix(
-                                    id_parts,
-                                    prefix,
-                                    session,
-                                    ws,
-                                    additional_parts=default_parts,
-                                )
-                            except Exception as e:
-                                raise e
-                        else:
-                            prefix = prefix.format(**default_parts)
-                    else:
-                        prefix = prefix.format(**default_parts)
-                    existing_artifact = await self._get_artifact(session, ws, prefix)
-                    if existing_artifact:
-                        raise RuntimeError(
-                            f"Failed to generate unique prefix for artifact '{prefix}'."
+                    if publish_to:
+                        zenodo_client = self._get_zenodo_client(
+                            parent_artifact, publish_to=publish_to
                         )
-                else:
-                    existing_artifact = await self._get_artifact(session, ws, prefix)
-
-                if existing_artifact:
-                    if not overwrite:
-                        raise FileExistsError(
-                            f"Artifact under prefix '{prefix}' already exists. Use overwrite=True to overwrite."
+                        deposition_info = await zenodo_client.create_deposition()
+                        additional_parts["zenodo_id"] = str(deposition_info["id"])
+                        additional_parts["zenodo_conceptrecid"] = str(
+                            deposition_info["conceptrecid"]
                         )
-                    else:
-                        logger.info(f"Overwriting artifact under prefix: {prefix}")
-                        await session.delete(existing_artifact)
-                        await session.commit()
+                        config["zenodo"] = deposition_info
 
-            async with session.begin():
-                permissions = permissions or {}
-                permissions[user_info.id] = "*"
+                    if parent_artifact and parent_artifact.config:
+                        id_parts.update(parent_artifact.config.get("id_parts", {}))
 
-                new_artifact = ArtifactModel(
-                    workspace=ws,
-                    prefix=prefix,
-                    manifest=None if stage else manifest,
-                    stage_manifest=manifest if stage else None,
-                    stage_files=[] if stage else None,
-                    download_weights=None,
-                    created_by=user_info.id,
-                    created_at=int(time.time()),
-                    last_modified=int(time.time()),
-                    permissions=permissions,
-                    config=config,
-                    type=manifest["type"],
+                    alias = await self._generate_unique_alias(
+                        session,
+                        workspace,
+                        alias,
+                        id_parts,
+                        additional_parts=additional_parts,
+                    )
+                id = str(uuid.uuid7())
+                # Modified main section:
+                if alias:
+                    existing_id = await self._handle_existing_artifact(
+                        session, workspace, parent_id, alias, overwrite
+                    )
+                    if existing_id:
+                        id = existing_id
+
+                parent_permissions = (
+                    parent_artifact.config["permissions"] if parent_artifact else {}
+                )
+                permissions = config.get("permissions", {}) if config else {}
+                permissions[user_info.id] = "*"
+                permissions.update(parent_permissions)
+                config["permissions"] = permissions
+
+                versions = []
+                if version != "stage" and version is not None:
+                    if version == "new":
+                        version = f"v{len(versions)}"
+                    comment = comment or f"Initial version"
+                    versions.append(
+                        {
+                            "version": version,
+                            "comment": comment,
+                            "created_at": int(time.time()),
+                        }
+                    )
+                assert manifest, "Manifest must be provided."
+                new_artifact = ArtifactModel(
+                    id=id,
+                    workspace=workspace,
+                    parent_id=parent_id,
+                    alias=alias,
+                    staging=[] if version == "stage" else None,
+                    manifest=manifest,
+                    created_by=user_info.id,
+                    created_at=int(time.time()),
+                    last_modified=int(time.time()),
+                    config=config,
+                    secrets=secrets,
+                    versions=versions,
+                    type=type,
                 )
+                version_index = self._get_version_index(new_artifact, version)
                 session.add(new_artifact)
                 await session.commit()
-            logger.info(f"Created artifact under prefix: {prefix}")
+                await self._save_version_to_s3(
+                    version_index,
+                    new_artifact,
+                    self._get_s3_config(new_artifact, parent_artifact),
+                )
+                if version != "stage":
+                    logger.info(
+                        f"Created artifact with ID: {id} (committed), alias: {alias}, parent: {parent_id}"
+                    )
+                else:
+                    logger.info(
+                        f"Created artifact with ID: {id} (staged), alias: {alias}, parent: {parent_id}"
+                    )
+                return self._generate_artifact_data(new_artifact)
         except Exception as e:
             raise e
         finally:
             await session.close()
-        return self._generate_metadata(new_artifact)
 
-    async def reset_stats(self, prefix, context: dict):
-        """Reset the artifact's manifest's download count and view count."""
-        if context is None or "ws" not in context:
-            raise ValueError("Context must include 'ws' (workspace).")
-        if prefix.startswith("/"):
-            ws = prefix.split("/")[1]
-            prefix = "/".join(prefix.split("/")[2:])
+    def _validate_artifact_id(self, artifact_id, context: dict):
+        """Check if the artifact ID is valid and add workspace if missing."""
+        assert isinstance(artifact_id, str), "Artifact ID must be a string."
+        # Check if the artifact ID is a valid UUID, e.g.: 0192f4be-11f7-7610-9da2-ffc39feeb009
+        if re.match(
+            r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$",
+            artifact_id,
+        ):
+            return artifact_id
+        elif "/" in artifact_id:
+            assert (
+                len(artifact_id.split("/")) == 2
+            ), "Invalid artifact ID format, it should be `workspace/alias`."
+            return artifact_id
         else:
-            ws = context["ws"]
+            workspace = context["ws"]
+            return f"{workspace}/{artifact_id}"
 
+    async def reset_stats(self, artifact_id, context: dict):
+        """Reset the artifact's download count and view count."""
+        if context is None or "ws" not in context:
+            raise ValueError("Context must include 'ws' (workspace).")
         user_info = UserInfo.model_validate(context["user"])
-        artifact = await self._get_artifact_with_permission(
-            ws, user_info, prefix, "reset_stats"
-        )
+        artifact_id = self._validate_artifact_id(artifact_id, context)
+
         session = await self._get_session()
         try:
             async with session.begin():
-                stmt = (
-                    update(ArtifactModel)
-                    .where(ArtifactModel.id == artifact.id)
-                    .values(download_count=0, view_count=0)
-                    .execution_options(synchronize_session="fetch")
+                artifact, _ = await self._get_artifact_with_permission(
+                    user_info, artifact_id, "reset_stats", session
                 )
-                await session.execute(stmt)
+                artifact.download_count = 0
+                artifact.view_count = 0
+                artifact.last_modified = int(time.time())
+                session.add(artifact)
                 await session.commit()
-            logger.info(f"Reset artifact under prefix: {prefix}")
         except Exception as e:
             raise e
         finally:
             await session.close()
 
-    async def read(
-        self,
-        prefix,
-        stage=False,
-        silent=False,
-        include_metadata=False,
-        context: dict = None,
-    ):
-        """Read the artifact's manifest from the database and populate collections dynamically."""
-        if context is None or "ws" not in context:
-            raise ValueError("Context must include 'ws' (workspace).")
-        if prefix.startswith("/"):
-            ws = prefix.split("/")[1]
-            prefix = "/".join(prefix.split("/")[2:])
-        else:
-            ws = context["ws"]
-
-        user_info = UserInfo.model_validate(context["user"])
-        artifact = await self._get_artifact_with_permission(
-            ws, user_info, prefix, "read"
-        )
-        manifest = await self._read_manifest(
-            artifact,
-            stage=stage,
-            increment_view_count=not silent,
-            include_metadata=include_metadata,
-        )
-        return manifest
-
     async def edit(
         self,
-        prefix,
+        artifact_id,
         manifest=None,
-        permissions: dict = None,
+        type=None,
         config: dict = None,
-        stage: bool = False,
+        secrets: dict = None,
+        version: str = None,
+        comment: str = None,
         context: dict = None,
     ):
         """Edit the artifact's manifest and save it in the database."""
         if context is None or "ws" not in context:
             raise ValueError("Context must include 'ws' (workspace).")
-        if prefix.startswith("/"):
-            ws = prefix.split("/")[1]
-            prefix = "/".join(prefix.split("/")[2:])
-        else:
-            ws = context["ws"]
-
         user_info = UserInfo.model_validate(context["user"])
-        artifact = await self._get_artifact_with_permission(
-            ws, user_info, prefix, "edit"
-        )
-
-        if manifest:
-            # Validate the manifest
-            if manifest["type"] == "collection":
-                CollectionArtifact.model_validate(manifest)
-            elif manifest["type"] == "application":
-                ApplicationArtifact.model_validate(manifest)
-            elif manifest["type"] == "workspace":
-                WorkspaceInfo.model_validate(manifest)
-
-            # Convert ObjectProxy to dict if necessary
-            if isinstance(manifest, ObjectProxy):
-                manifest = ObjectProxy.toDict(manifest)
-
+        artifact_id = self._validate_artifact_id(artifact_id, context)
         session = await self._get_session()
         try:
             async with session.begin():
-                if stage:
-                    artifact.stage_manifest = manifest
-                    flag_modified(
-                        artifact, "stage_manifest"
-                    )  # Mark JSON field as modified
-                elif manifest:
+                artifact, parent_artifact = await self._get_artifact_with_permission(
+                    user_info, artifact_id, "edit", session
+                )
+                artifact.type = type or artifact.type
+                if manifest:
+                    if artifact.type == "collection":
+                        CollectionArtifact.model_validate(manifest)
+                    elif artifact.type == "generic":
+                        Artifact.model_validate(manifest)
+                    if isinstance(manifest, ObjectProxy):
+                        manifest = ObjectProxy.toDict(manifest)
+
+                if manifest:
                     artifact.manifest = manifest
                     flag_modified(artifact, "manifest")
-                if config:
+
+                if version is not None:
+                    if version != "stage":
+                        # Increase the version number
+                        versions = artifact.versions or []
+                        if version == "new":
+                            version = f"v{len(versions)}"
+                        versions.append(
+                            {
+                                "version": version,
+                                "comment": comment,
+                                "created_at": int(time.time()),
+                            }
+                        )
+                        artifact.versions = versions
+                        flag_modified(artifact, "versions")
+                        version_index = self._get_version_index(artifact, "latest")
+                    else:
+                        version_index = self._get_version_index(artifact, "stage")
+                else:
+                    version_index = self._get_version_index(artifact, None)
+
+                if config is not None:
+                    if parent_artifact:
+                        parent_permissions = (
+                            parent_artifact.config["permissions"]
+                            if parent_artifact
+                            else {}
+                        )
+                        permissions = config.get("permissions", {}) if config else {}
+                        permissions[user_info.id] = "*"
+                        permissions.update(parent_permissions)
+                        config["permissions"] = permissions
                     artifact.config = config
                     flag_modified(artifact, "config")
+                if secrets is not None:
+                    artifact.secrets = secrets
+                    flag_modified(artifact, "secrets")
                 artifact.last_modified = int(time.time())
-                if permissions is not None:
-                    artifact.permissions = permissions
-                    flag_modified(artifact, "permissions")
-                session.add(artifact)
-                await session.commit()
-            logger.info(f"Edited artifact under prefix: {prefix}")
+                if version != "stage":
+                    artifact.staging = None
+                    flag_modified(artifact, "staging")
+                    session.add(artifact)
+                    await session.commit()
+                    logger.info(
+                        f"Edited artifact with ID: {artifact_id} (committed), alias: {artifact.alias}, version: {version}"
+                    )
+                else:
+                    artifact.staging = artifact.staging or []
+                    logger.info(
+                        f"Edited artifact with ID: {artifact_id} (staged), alias: {artifact.alias}"
+                    )
+                await self._save_version_to_s3(
+                    version_index,
+                    artifact,
+                    self._get_s3_config(artifact, parent_artifact),
+                )
+                return self._generate_artifact_data(artifact)
         except Exception as e:
             raise e
         finally:
             await session.close()
 
-    async def commit(self, prefix, context: dict):
-        """Commit the artifact by finalizing the staged manifest and files."""
+    async def read(
+        self,
+        artifact_id,
+        silent=False,
+        version: str = None,
+        context: dict = None,
+    ):
+        """Read the artifact's data including manifest and config."""
         if context is None or "ws" not in context:
             raise ValueError("Context must include 'ws' (workspace).")
-        if prefix.startswith("/"):
-            ws = prefix.split("/")[1]
-            prefix = "/".join(prefix.split("/")[2:])
-        else:
-            ws = context["ws"]
-
+        artifact_id = self._validate_artifact_id(artifact_id, context)
         user_info = UserInfo.model_validate(context["user"])
-        artifact = await self._get_artifact_with_permission(
-            ws, user_info, prefix, "commit"
-        )
+        session = await self._get_session()
+        try:
+            async with session.begin():
+                artifact, _ = await self._get_artifact_with_permission(
+                    user_info, artifact_id, "read", session
+                )
+                if not silent:
+                    await self._increment_stat(session, artifact_id, "view_count")
+
+                version_index = self._get_version_index(artifact, version)
+                if version_index == self._get_version_index(artifact, None):
+                    artifact_data = self._generate_artifact_data(artifact)
+                else:
+                    s3_config = self._get_s3_config(artifact)
+                    artifact_data = await self._load_version_from_s3(
+                        artifact, version_index, s3_config
+                    )
 
+                if not silent:
+                    await session.commit()
+
+                return artifact_data
+        except Exception as e:
+            raise e
+        finally:
+            await session.close()
+
+    async def commit(
+        self,
+        artifact_id,
+        version: str = None,
+        comment: str = None,
+        context: dict = None,
+    ):
+        """Commit the artifact by finalizing the staged manifest and files."""
+        if context is None or "ws" not in context:
+            raise ValueError("Context must include 'ws' (workspace).")
+        artifact_id = self._validate_artifact_id(artifact_id, context)
+        user_info = UserInfo.model_validate(context["user"])
         session = await self._get_session()
+        assert version != "stage", "Version cannot be 'stage' when committing."
         try:
             async with session.begin():
-                manifest = artifact.stage_manifest
-
-                download_weights = {}
-                # Validate files exist in S3 if the staged files list is present
-                if artifact.stage_files:
-                    async with self.s3_controller.create_client_async() as s3_client:
-                        for file_info in artifact.stage_files:
-                            file_key = safe_join(ws, f"{prefix}/{file_info['path']}")
+                artifact, parent_artifact = await self._get_artifact_with_permission(
+                    user_info, artifact_id, "commit", session
+                )
+                # the new version index
+                version_index = self._get_version_index(artifact, "stage")
+                # Load the staged version
+                s3_config = self._get_s3_config(artifact)
+                artifact_data = await self._load_version_from_s3(
+                    artifact, version_index, s3_config
+                )
+                artifact = ArtifactModel(**artifact_data)
+
+                versions = artifact.versions or []
+                artifact.config = artifact.config or {}
+                if artifact.staging:
+                    s3_config = self._get_s3_config(artifact, parent_artifact)
+                    async with self._create_client_async(
+                        s3_config,
+                    ) as s3_client:
+                        download_weights = {}
+                        for file_info in artifact.staging:
+                            file_key = safe_join(
+                                s3_config["prefix"],
+                                artifact.workspace,
+                                f"{self._artifacts_dir}/{artifact.id}/v{version_index}/{file_info['path']}",
+                            )
                             try:
                                 await s3_client.head_object(
-                                    Bucket=self.workspace_bucket, Key=file_key
+                                    Bucket=s3_config["bucket"], Key=file_key
                                 )
                             except ClientError:
                                 raise FileNotFoundError(
@@ -903,84 +1253,282 @@ async def commit(self, prefix, context: dict):
                                 download_weights[file_info["path"]] = file_info[
                                     "download_weight"
                                 ]
-                # Validate the schema if the artifact belongs to a collection
-                parent_prefix = "/".join(prefix.split("/")[:-1])
-                if parent_prefix:
-                    parent_artifact = await self._get_artifact(
-                        session, ws, parent_prefix
+                        if download_weights:
+                            artifact.config["download_weights"] = download_weights
+                            flag_modified(artifact, "config")
+                        artifact.file_count = len(artifact.staging)
+
+                parent_artifact_config = (
+                    parent_artifact.config if parent_artifact else {}
+                )
+                if (
+                    parent_artifact_config
+                    and "collection_schema" in parent_artifact_config
+                ):
+                    collection_schema = parent_artifact_config["collection_schema"]
+                    try:
+                        validate(instance=artifact.manifest, schema=collection_schema)
+                    except Exception as e:
+                        raise ValueError(f"ValidationError: {str(e)}")
+                assert artifact.manifest, "Artifact must be in staging mode to commit."
+
+                if version is not None:
+                    if version == "new":
+                        version = f"v{len(versions)}"
+                    versions.append(
+                        {
+                            "version": version,
+                            "comment": comment,
+                            "created_at": int(time.time()),
+                        }
                     )
-                    if (
-                        parent_artifact
-                        and parent_artifact.config
-                        and "collection_schema" in parent_artifact.config
-                    ):
-                        collection_schema = parent_artifact.config["collection_schema"]
-                        try:
-                            validate(instance=manifest, schema=collection_schema)
-                        except Exception as e:
-                            raise ValueError(f"ValidationError: {str(e)}")
-
-                # Finalize the manifest
-                artifact.manifest = manifest
-                artifact.stage_manifest = None
-                artifact.stage_files = None
-                artifact.download_weights = download_weights
+                artifact.versions = versions
+                flag_modified(artifact, "versions")
+                artifact.staging = None
                 artifact.last_modified = int(time.time())
                 flag_modified(artifact, "manifest")
-                flag_modified(artifact, "stage_files")
-                flag_modified(artifact, "download_weights")
-                session.add(artifact)
+                flag_modified(artifact, "staging")
+                await session.merge(artifact)
                 await session.commit()
-            logger.info(f"Committed artifact under prefix: {prefix}")
+                await self._save_version_to_s3(
+                    self._get_version_index(artifact, None),
+                    artifact,
+                    self._get_s3_config(artifact, parent_artifact),
+                )
+                logger.info(
+                    f"Committed artifact with ID: {artifact_id}, alias: {artifact.alias}, version: {version}"
+                )
+                return self._generate_artifact_data(artifact)
         except Exception as e:
             raise e
         finally:
             await session.close()
 
     async def delete(
-        self, prefix, delete_files=False, recursive=False, context: dict = None
+        self,
+        artifact_id,
+        delete_files=False,
+        recursive=False,
+        version=None,
+        context: dict = None,
     ):
         """Delete an artifact from the database and S3."""
         if context is None or "ws" not in context:
             raise ValueError("Context must include 'ws' (workspace).")
-        if prefix.startswith("/"):
-            ws = prefix.split("/")[1]
-            prefix = "/".join(prefix.split("/")[2:])
-        else:
-            ws = context["ws"]
-
-        if delete_files and not recursive:
-            raise ValueError("Delete files requires recursive=True.")
-
+        artifact_id = self._validate_artifact_id(artifact_id, context)
         user_info = UserInfo.model_validate(context["user"])
-        artifact = await self._get_artifact_with_permission(
-            ws, user_info, prefix, "delete"
-        )
 
-        if recursive:
-            # Remove all child artifacts
-            children = await self.list_children(
-                prefix, summary_fields=[".prefix"], context=context
+        try:
+            session = await self._get_session()
+            # Get artifact first
+            artifact, parent_artifact = await self._get_artifact_with_permission(
+                user_info, artifact_id, "delete", session
             )
-            for child in children:
-                await self.delete(
-                    child[".prefix"], delete_files=delete_files, context=context
+
+            s3_config = self._get_s3_config(artifact, parent_artifact)
+            if version is None:
+                # Handle recursive deletion first
+                if recursive:
+                    children = await self.list_children(artifact_id, context=context)
+                    for child in children:
+                        await self.delete(
+                            child["id"], delete_files=delete_files, context=context
+                        )
+
+                # Delete files if requested
+                if delete_files and artifact.file_count > 0:
+                    artifact_path = safe_join(
+                        s3_config["prefix"],
+                        artifact.workspace,
+                        f"{self._artifacts_dir}/{artifact.id}",
+                    )
+                    async with self._create_client_async(
+                        s3_config,
+                    ) as s3_client:
+                        await remove_objects_async(
+                            s3_client, s3_config["bucket"], artifact_path + "/"
+                        )
+
+                # Delete the artifact
+                await session.refresh(artifact)
+                artifact.parent_id = None
+                await session.flush()
+                await session.delete(artifact)
+            else:
+                version_index = self._get_version_index(artifact, version)
+                artifact.versions.pop(version_index)
+                flag_modified(artifact, "versions")
+                session.add(artifact)
+                # Delete the version from S3
+                self._delete_version_files_from_s3(
+                    version_index, artifact, s3_config, delete_files=delete_files
                 )
+            await session.commit()
+            logger.info(
+                f"Deleted artifact with ID: {artifact_id}, alias: {artifact.alias}, version: {version}"
+            )
+        except Exception as e:
+            if session:
+                await session.rollback()
+            raise e
+        finally:
+            if session:
+                await session.close()
 
-        if delete_files:
-            # Remove all files in the artifact's S3 prefix
-            artifact_path = safe_join(ws, f"{prefix}") + "/"
-            async with self.s3_controller.create_client_async() as s3_client:
-                await remove_objects_async(
-                    s3_client, self.workspace_bucket, artifact_path
+    async def put_file(
+        self, artifact_id, file_path, download_weight: float = 0, context: dict = None
+    ):
+        """Generate a pre-signed URL to upload a file to an artifact in S3."""
+        if context is None or "ws" not in context:
+            raise ValueError("Context must include 'ws' (workspace).")
+        artifact_id = self._validate_artifact_id(artifact_id, context)
+        user_info = UserInfo.model_validate(context["user"])
+        session = await self._get_session()
+        try:
+            async with session.begin():
+                artifact, parent_artifact = await self._get_artifact_with_permission(
+                    user_info, artifact_id, "put_file", session
                 )
+                assert artifact.staging is not None, "Artifact must be in staging mode."
+                # The new version is not committed yet, so we use a new version index
+                version_index = self._get_version_index(artifact, "stage")
+
+                s3_config = self._get_s3_config(artifact, parent_artifact)
+                async with self._create_client_async(
+                    s3_config,
+                ) as s3_client:
+                    file_key = safe_join(
+                        s3_config["prefix"],
+                        artifact.workspace,
+                        f"{self._artifacts_dir}/{artifact.id}/v{version_index}/{file_path}",
+                    )
+                    presigned_url = await s3_client.generate_presigned_url(
+                        "put_object",
+                        Params={"Bucket": s3_config["bucket"], "Key": file_key},
+                        ExpiresIn=3600,
+                    )
+
+                    artifact.staging = artifact.staging or []
+                    if not any(f["path"] == file_path for f in artifact.staging):
+                        artifact.staging.append(
+                            {
+                                "path": file_path,
+                                "download_weight": download_weight,
+                            }
+                        )
+                        # flag_modified(artifact, "staging")
+                    # save the artifact to s3
+                    await self._save_version_to_s3(version_index, artifact, s3_config)
+                    session.add(artifact)
+                    await session.commit()
+                    logger.info(
+                        f"Put file '{file_path}' to artifact with ID: {artifact_id}"
+                    )
+            return presigned_url
+        except Exception as e:
+            raise e
+        finally:
+            await session.close()
 
+    async def remove_file(self, artifact_id, file_path, context: dict):
+        """Remove a file from the artifact and update the staged manifest."""
+        if context is None or "ws" not in context:
+            raise ValueError("Context must include 'ws' (workspace).")
+        artifact_id = self._validate_artifact_id(artifact_id, context)
+        user_info = UserInfo.model_validate(context["user"])
         session = await self._get_session()
         try:
             async with session.begin():
-                await session.delete(artifact)
+                artifact, parent_artifact = await self._get_artifact_with_permission(
+                    user_info, artifact_id, "remove_file", session
+                )
+
+                assert artifact.staging is not None, "Artifact must be in staging mode."
+                # The new version is not committed yet, so we use a new version index
+                version_index = self._get_version_index(artifact, "stage")
+
+                s3_config = self._get_s3_config(artifact, parent_artifact)
+                async with self._create_client_async(
+                    s3_config,
+                ) as s3_client:
+                    file_key = safe_join(
+                        s3_config["prefix"],
+                        artifact.workspace,
+                        f"{self._artifacts_dir}/{artifact.id}/v{version_index}/{file_path}",
+                    )
+                    await s3_client.delete_object(
+                        Bucket=s3_config["bucket"], Key=file_key
+                    )
+
+                if artifact.staging:
+                    artifact.staging = [
+                        f for f in artifact.staging if f["path"] != file_path
+                    ]
+                    flag_modified(artifact, "staging")
+
+                session.add(artifact)
                 await session.commit()
-            logger.info(f"Deleted artifact under prefix: {prefix}")
+                logger.info(
+                    f"Removed file '{file_path}' from artifact with ID: {artifact_id}"
+                )
+        except Exception as e:
+            raise e
+        finally:
+            await session.close()
+
+    async def get_file(
+        self, artifact_id, path, silent=False, version=None, context: dict = None
+    ):
+        """Generate a pre-signed URL to download a file from an artifact in S3."""
+        if context is None or "ws" not in context:
+            raise ValueError("Context must include 'ws' (workspace).")
+        artifact_id = self._validate_artifact_id(artifact_id, context)
+        user_info = UserInfo.model_validate(context["user"])
+        session = await self._get_session()
+        try:
+            async with session.begin():
+                artifact, parent_artifact = await self._get_artifact_with_permission(
+                    user_info, artifact_id, "get_file", session
+                )
+                version_index = self._get_version_index(artifact, version)
+                s3_config = self._get_s3_config(artifact, parent_artifact)
+                async with self._create_client_async(
+                    s3_config,
+                ) as s3_client:
+                    file_key = safe_join(
+                        s3_config["prefix"],
+                        artifact.workspace,
+                        f"{self._artifacts_dir}/{artifact.id}/v{version_index}/{path}",
+                    )
+                    try:
+                        await s3_client.head_object(
+                            Bucket=s3_config["bucket"], Key=file_key
+                        )
+                    except ClientError:
+                        raise FileNotFoundError(
+                            f"File '{path}' does not exist in the artifact."
+                        )
+                    presigned_url = await s3_client.generate_presigned_url(
+                        "get_object",
+                        Params={"Bucket": s3_config["bucket"], "Key": file_key},
+                        ExpiresIn=3600,
+                    )
+                # Increment download count unless silent
+                if not silent:
+                    if artifact.config and "download_weights" in artifact.config:
+                        download_weights = artifact.config.get("download_weights", {})
+                    else:
+                        download_weights = {}
+                    download_weight = download_weights.get(path) or 0
+                    if download_weight > 0:
+                        await self._increment_stat(
+                            session,
+                            artifact.id,
+                            "download_count",
+                            increment=download_weight,
+                        )
+                    await session.commit()
+            return presigned_url
         except Exception as e:
             raise e
         finally:
@@ -988,76 +1536,119 @@ async def delete(
 
     async def list_files(
         self,
-        prefix: str,
+        artifact_id: str,
         dir_path: str = None,
         max_length: int = 1000,
+        version: str = None,
         context: dict = None,
     ):
-        """List files in the specified S3 prefix."""
+        """List files in the specified artifact's S3 path."""
         if context is None or "ws" not in context:
             raise ValueError("Context must include 'ws' (workspace).")
-        if prefix.startswith("/"):
-            ws = prefix.split("/")[1]
-            prefix = "/".join(prefix.split("/")[2:])
-        else:
-            ws = context["ws"]
+        artifact_id = self._validate_artifact_id(artifact_id, context)
         user_info = UserInfo.model_validate(context["user"])
-        await self._get_artifact_with_permission(ws, user_info, prefix, "list_files")
-        async with self.s3_controller.create_client_async() as s3_client:
-            if dir_path:
-                full_path = safe_join(ws, prefix, dir_path) + "/"
-            else:
-                full_path = safe_join(ws, prefix) + "/"
-            items = await list_objects_async(
-                s3_client, self.workspace_bucket, full_path, max_length=max_length
-            )
-            # TODO: If stage should we return only the staged files?
+        session = await self._get_session()
+        try:
+            async with session.begin():
+                artifact, parent_artifact = await self._get_artifact_with_permission(
+                    user_info, artifact_id, "list_files", session
+                )
+                version_index = self._get_version_index(artifact, version)
+                s3_config = self._get_s3_config(artifact, parent_artifact)
+                async with self._create_client_async(
+                    s3_config,
+                ) as s3_client:
+                    if dir_path:
+                        full_path = (
+                            safe_join(
+                                s3_config["prefix"],
+                                artifact.workspace,
+                                f"{self._artifacts_dir}/{artifact.id}/v{version_index}/{dir_path}",
+                            )
+                            + "/"
+                        )
+                    else:
+                        full_path = (
+                            safe_join(
+                                s3_config["prefix"],
+                                artifact.workspace,
+                                f"{self._artifacts_dir}/{artifact.id}/v{version_index}",
+                            )
+                            + "/"
+                        )
+                    items = await list_objects_async(
+                        s3_client,
+                        s3_config["bucket"],
+                        full_path,
+                        max_length=max_length,
+                    )
             return items
+        except Exception as e:
+            raise e
+        finally:
+            await session.close()
 
     async def list_children(
         self,
-        prefix,
+        parent_id=None,
         keywords=None,
         filters=None,
         mode="AND",
         page: int = 0,
         page_size: int = 100,
         order_by=None,
-        summary_fields=None,
         silent=False,
         context: dict = None,
     ):
         """
-        List artifacts within a collection under a specific prefix.
+        List artifacts within a collection under a specific artifact_id.
         """
         if context is None or "ws" not in context:
             raise ValueError("Context must include 'ws' (workspace).")
-        if prefix.startswith("/"):
-            ws = prefix.split("/")[1]
-            prefix = "/".join(prefix.split("/")[2:])
-        else:
-            ws = context["ws"]
-
+        if parent_id:
+            parent_id = self._validate_artifact_id(parent_id, context)
         user_info = UserInfo.model_validate(context["user"])
-        parent_artifact = await self._get_artifact_with_permission(
-            ws, user_info, prefix, "list"
-        )
+
         session = await self._get_session(read_only=True)
         stage = False
+
         try:
             async with session.begin():
-                # Get the database backend from the SQLAlchemy engine
+                if parent_id:
+                    parent_artifact, _ = await self._get_artifact_with_permission(
+                        user_info, parent_id, "list", session
+                    )
+                else:
+                    parent_artifact = None
                 backend = (
                     self.engine.dialect.name
-                )  # This will return 'postgresql', 'sqlite', etc.
-
-                base_query = select(ArtifactModel).filter(
-                    ArtifactModel.workspace == ws,
-                    ArtifactModel.prefix.like(f"{prefix}/%"),
-                )
-
-                # Handle keyword-based search (fuzzy search across all manifest fields)
+                )  # Database type (e.g., 'postgresql', 'sqlite')
+
+                # Prepare base query for children artifacts
+                if parent_artifact:
+                    config = parent_artifact.config or {}
+                    list_fields = config.get("list_fields")
+                    if list_fields:
+                        assert isinstance(
+                            list_fields, list
+                        ), "Invalid list_fields, it should be a list."
+                        assert (
+                            "secrets" not in list_fields
+                        ), "Secrets cannot be included in list_fields."
+                        # If list_fields is specified and is a list, select only those fields
+                        query = select(
+                            *[getattr(ArtifactModel, field) for field in list_fields]
+                        ).where(ArtifactModel.parent_id == parent_artifact.id)
+                    else:
+                        # If list_fields is empty or not specified, select all columns
+                        query = select(ArtifactModel).where(
+                            ArtifactModel.parent_id == parent_artifact.id
+                        )
+                else:
+                    query = select(ArtifactModel).where(ArtifactModel.parent_id == None)
                 conditions = []
+
+                # Handle keyword-based search across manifest fields
                 if keywords:
                     for keyword in keywords:
                         if backend == "postgresql":
@@ -1068,229 +1659,161 @@ async def list_children(
                             )
                         conditions.append(condition)
 
-                # Handle filter-based search (specific key-value matching)
+                # Handle filter-based search with specific key-value matching
                 if filters:
                     for key, value in filters.items():
-                        if key == ".stage":
-                            if value:
-                                condition = ArtifactModel.stage_manifest != None
-                                stage = True
-                            else:
-                                condition = ArtifactModel.manifest != None
-                            conditions.append(condition)
-                            continue
-
-                        elif key == ".type":
-                            # check ArtifactModel.type
-                            condition = ArtifactModel.type == value
-                            conditions.append(condition)
+                        if key == "stage":
+                            stage = bool(value)
                             continue
 
-                        elif key == ".created_by":
-                            # check ArtifactModel.created_by
-                            condition = ArtifactModel.created_by == value
-                            conditions.append(condition)
+                        if key == "manifest" and isinstance(value, dict):
+                            # Nested search within manifest fields
+                            for manifest_key, manifest_value in value.items():
+                                if "*" in manifest_value:  # Fuzzy matching in manifest
+                                    if backend == "postgresql":
+                                        condition = text(
+                                            f"manifest->>'{manifest_key}' ILIKE '{manifest_value.replace('*', '%')}'"
+                                        )
+                                    else:
+                                        condition = text(
+                                            f"json_extract(manifest, '$.{manifest_key}') LIKE '{manifest_value.replace('*', '%')}'"
+                                        )
+                                else:  # Exact matching in manifest
+                                    if backend == "postgresql":
+                                        condition = text(
+                                            f"manifest->>'{manifest_key}' = '{manifest_value}'"
+                                        )
+                                    else:
+                                        condition = text(
+                                            f"json_extract(manifest, '$.{manifest_key}') = '{manifest_value}'"
+                                        )
+                                conditions.append(condition)
                             continue
 
-                        elif key == ".created_at":
-                            # if the value is a list, it should be a range
-                            # otherwise, it should be a single value which is the lower bound
-                            if isinstance(value, list):
-                                assert (
-                                    len(value) == 2
-                                ), "created_at range should have 2 values"
-                                if value[0] is None:
-                                    condition = ArtifactModel.created_at <= value[1]
-                                elif value[1] is None:
-                                    condition = ArtifactModel.created_at >= value[0]
-                                else:
-                                    condition = and_(
-                                        ArtifactModel.created_at >= value[0],
-                                        ArtifactModel.created_at <= value[1],
-                                    )
-                            else:
-                                condition = ArtifactModel.created_at >= value
-
-                            conditions.append(condition)
+                        elif key == "config" and isinstance(value, dict):
+                            # Nested search within config fields, specifically permissions
+                            for config_key, config_value in value.items():
+                                if config_key == "permissions" and isinstance(
+                                    config_value, dict
+                                ):
+                                    for user_id, permission in config_value.items():
+                                        if backend == "postgresql":
+                                            condition = text(
+                                                f"permissions->>'{user_id}' = '{permission}'"
+                                            )
+                                        else:
+                                            condition = text(
+                                                f"json_extract(permissions, '$.{user_id}') = '{permission}'"
+                                            )
+                                        conditions.append(condition)
                             continue
 
-                        elif key == ".last_modified":
-                            if isinstance(value, list):
-                                assert (
-                                    len(value) == 2
-                                ), "last_modified range should have 2 values"
-                                if value[0] is None:
-                                    condition = ArtifactModel.last_modified <= value[1]
-                                elif value[1] is None:
-                                    condition = ArtifactModel.last_modified >= value[0]
-                                else:
-                                    condition = and_(
-                                        ArtifactModel.last_modified >= value[0],
-                                        ArtifactModel.last_modified <= value[1],
-                                    )
-                            else:
-                                condition = ArtifactModel.last_modified >= value
-
-                            conditions.append(condition)
-                            continue
-
-                        elif key == ".download_count":
-                            if isinstance(value, list):
-                                assert (
-                                    len(value) == 2
-                                ), "download_count range should have 2 values"
-                                if value[0] is None:
-                                    condition = ArtifactModel.download_count <= value[1]
-                                elif value[1] is None:
-                                    condition = ArtifactModel.download_count >= value[0]
-                                else:
+                        else:
+                            fixed_fields = {
+                                "type": ArtifactModel.type,
+                                "alias": ArtifactModel.alias,
+                                "workspace": ArtifactModel.workspace,
+                                "parent_id": ArtifactModel.parent_id,
+                                "created_by": ArtifactModel.created_by,
+                            }
+                            range_fields = {
+                                "created_at": ArtifactModel.created_at,
+                                "last_modified": ArtifactModel.last_modified,
+                                "download_count": ArtifactModel.download_count,
+                                "view_count": ArtifactModel.view_count,
+                            }
+                            if key in fixed_fields:
+                                model_field = fixed_fields[key]
+                                condition = model_field == value
+                                conditions.append(condition)
+                                continue
+                            elif key in range_fields:
+                                model_field = range_fields[key]
+                                if isinstance(value, list):
                                     condition = and_(
-                                        ArtifactModel.download_count >= value[0],
-                                        ArtifactModel.download_count <= value[1],
+                                        model_field >= value[0]
+                                        if value[0] is not None
+                                        else True,
+                                        model_field <= value[1]
+                                        if value[1] is not None
+                                        else True,
                                     )
-                            else:
-                                condition = ArtifactModel.download_count >= value
-
-                            conditions.append(condition)
-                            continue
-
-                        elif key == ".view_count":
-                            if isinstance(value, list):
-                                assert (
-                                    len(value) == 2
-                                ), "view_count range should have 2 values"
-                                if value[0] is None:
-                                    condition = ArtifactModel.view_count <= value[1]
-                                elif value[1] is None:
-                                    condition = ArtifactModel.view_count >= value[0]
                                 else:
-                                    condition = and_(
-                                        ArtifactModel.view_count >= value[0],
-                                        ArtifactModel.view_count <= value[1],
-                                    )
+                                    condition = model_field >= value
+                                conditions.append(condition)
+                                continue
                             else:
-                                condition = ArtifactModel.view_count >= value
+                                raise ValueError(f"Invalid filter key: {key}")
 
-                            conditions.append(condition)
-                            continue
-                        elif key.startswith(".permissions/"):
-                            # format: permissions/user.id=permission
-                            user_id = key.split("/")[1]
-                            if backend == "postgresql":
-                                condition = text(
-                                    f"permissions->>'{user_id}' = '{value}'"
-                                )
-                            else:
-                                condition = text(
-                                    f"json_extract(permissions, '$.{user_id}') = '{value}'"
-                                )
-                            conditions.append(condition)
-                            continue
-                        elif key.startswith("."):
-                            raise ValueError(f"Invalid filter key: {key}")
+                if backend == "sqlite":
+                    if stage:
+                        stage_condition = and_(
+                            ArtifactModel.staging.isnot(None), text("staging != 'null'")
+                        )
+                    else:
+                        stage_condition = or_(
+                            ArtifactModel.staging.is_(None), text("staging = 'null'")
+                        )
+                else:
+                    if stage:
+                        stage_condition = and_(
+                            ArtifactModel.staging.isnot(None),
+                            text("staging::text != 'null'"),
+                        )
+                    else:
+                        stage_condition = or_(
+                            ArtifactModel.staging.is_(None),
+                            text("staging::text = 'null'"),
+                        )
 
-                        if "*" in value:  # Fuzzy matching
-                            if backend == "postgresql":
-                                condition = text(
-                                    f"manifest->>'{key}' ILIKE '{value.replace('*', '%')}'"
-                                )
-                            else:
-                                condition = text(
-                                    f"json_extract(manifest, '$.{key}') LIKE '{value.replace('*', '%')}'"
-                                )
-                        else:  # Exact matching
-                            if backend == "postgresql":
-                                condition = text(f"manifest->>'{key}' = '{value}'")
-                            else:
-                                condition = text(
-                                    f"json_extract(manifest, '$.{key}') = '{value}'"
-                                )
-                        conditions.append(condition)
+                query = query.where(stage_condition)
 
-                # Combine conditions using AND/OR mode
+                # Combine conditions based on mode (AND/OR)
                 if conditions:
-                    if mode == "OR":
-                        query = base_query.where(or_(*conditions))
-                    else:  # Default is AND
-                        query = base_query.where(and_(*conditions))
-                else:
-                    query = base_query
+                    query = (
+                        query.where(or_(*conditions))
+                        if mode == "OR"
+                        else query.where(and_(*conditions))
+                    )
 
+                # Pagination and ordering
                 offset = page * page_size
-                if order_by is None:
-                    query = query.order_by(ArtifactModel.prefix.asc())
-                elif order_by.startswith("view_count"):
-                    if order_by.endswith("<"):
-                        query = query.order_by(ArtifactModel.view_count.asc())
-                    else:
-                        query = query.order_by(ArtifactModel.view_count.desc())
-                elif order_by.startswith("download_count"):
-                    if order_by.endswith("<"):
-                        query = query.order_by(ArtifactModel.download_count.asc())
-                    else:
-                        query = query.order_by(ArtifactModel.download_count.desc())
-                elif order_by.startswith("last_modified"):
-                    if order_by.endswith("<"):
-                        query = query.order_by(ArtifactModel.last_modified.asc())
-                    else:
-                        query = query.order_by(ArtifactModel.last_modified.desc())
-                elif order_by.startswith("created_at"):
-                    if order_by.endswith("<"):
-                        query = query.order_by(ArtifactModel.created_at.asc())
-                    else:
-                        query = query.order_by(ArtifactModel.created_at.desc())
-                elif order_by.startswith("prefix"):
-                    if order_by.endswith("<"):
-                        query = query.order_by(ArtifactModel.prefix.asc())
-                    else:
-                        query = query.order_by(ArtifactModel.prefix.desc())
-                else:
-                    raise ValueError(f"Invalid order_by field: {order_by}")
+                order_field_map = {
+                    "id": ArtifactModel.id,
+                    "view_count": ArtifactModel.view_count,
+                    "download_count": ArtifactModel.download_count,
+                    "last_modified": ArtifactModel.last_modified,
+                    "created_at": ArtifactModel.created_at,
+                }
+                order_by = order_by or "id"
+                order_field = order_field_map.get(
+                    order_by.split("<")[0] if order_by else "id"
+                )
+                ascending = "<" in order_by if order_by else True
+                query = (
+                    query.order_by(
+                        order_field.asc() if ascending else order_field.desc()
+                    )
+                    .limit(page_size)
+                    .offset(offset)
+                )
 
-                query = query.limit(page_size).offset(offset)
                 # Execute the query
                 result = await session.execute(query)
                 artifacts = result.scalars().all()
 
-                if summary_fields is None:
-                    if parent_artifact.manifest.get("type") == "collection":
-                        # Generate the results with summary_fields
-                        summary_fields = (
-                            parent_artifact.config
-                            and parent_artifact.config.get("summary_fields")
-                            or DEFAULT_SUMMARY_FIELDS
-                        )
-                    else:
-                        summary_fields = DEFAULT_SUMMARY_FIELDS
+                # Compile summary results for each artifact
                 results = []
                 for artifact in artifacts:
-                    manifest = artifact.stage_manifest if stage else artifact.manifest
-                    if not manifest:
-                        continue
-                    summary = {}
-                    _metadata = self._generate_metadata(artifact)
-                    for field in summary_fields:
-                        if field == ".*":
-                            summary.update(_metadata)
-                            continue
-                        if field == "*":
-                            summary.update(manifest)
-                            continue
-                        if field.startswith("."):
-                            summary[field] = _metadata.get(field)
-                        else:
-                            summary[field] = manifest.get(field)
+                    _artifact_data = self._generate_artifact_data(artifact)
+                    results.append(_artifact_data)
 
-                    results.append(summary)
-
-                if not stage:
-                    # Increment the view count for the parent artifact
-                    await self._read_manifest(
-                        parent_artifact,
-                        stage=False,
-                        increment_view_count=not silent,
-                        include_metadata=False,
+                # Increment view count for parent artifact if not in stage mode
+                if not silent and parent_artifact:
+                    await self._increment_stat(
+                        session, parent_artifact.id, "view_count"
                     )
+                    await session.commit()
 
                 return results
         except Exception as e:
@@ -1300,309 +1823,77 @@ async def list_children(
         finally:
             await session.close()
 
-    async def put_file(
-        self, prefix, file_path, options: dict = None, context: dict = None
-    ):
-        """Generate a pre-signed URL to upload a file to an artifact in S3 and update the manifest."""
-        if prefix.startswith("/"):
-            ws = prefix.split("/")[1]
-            prefix = "/".join(prefix.split("/")[2:])
-        else:
-            ws = context["ws"]
-        user_info = UserInfo.model_validate(context["user"])
-        artifact = await self._get_artifact_with_permission(
-            ws, user_info, prefix, "put_file"
-        )
-        assert artifact.stage_manifest, "Artifact must be in staging mode to put files."
-
-        options = options or {}
-
-        async with self.s3_controller.create_client_async() as s3_client:
-            file_key = safe_join(ws, f"{prefix}/{file_path}")
-            presigned_url = await s3_client.generate_presigned_url(
-                "put_object",
-                Params={"Bucket": self.workspace_bucket, "Key": file_key},
-                ExpiresIn=3600,
-            )
-            if self.s3_controller.enable_s3_proxy:
-                presigned_url = f"{self.store.public_base_url}/s3/{self.workspace_bucket}/{file_key}?{presigned_url.split('?')[1]}"
-
-        session = await self._get_session()
-        try:
-            async with session.begin():
-                artifact.stage_files = artifact.stage_files or []
-                artifact.last_modified = int(time.time())
-
-                if not any(f["path"] == file_path for f in artifact.stage_files):
-                    artifact.stage_files.append(
-                        {
-                            "path": file_path,
-                            "download_weight": options.get("download_weight"),
-                        }
-                    )
-                    flag_modified(artifact, "stage_files")
-
-                session.add(artifact)
-                await session.commit()
-            logger.info(f"Generated pre-signed URL for file upload: {file_path}")
-        except Exception as e:
-            raise e
-        finally:
-            await session.close()
-
-        return presigned_url
-
-    async def get_file(self, prefix, path, options: dict = None, context: dict = None):
-        """Generate a pre-signed URL to download a file from an artifact in S3."""
-        if prefix.startswith("/"):
-            ws = prefix.split("/")[1]
-            prefix = "/".join(prefix.split("/")[2:])
-        else:
-            ws = context["ws"]
-
-        user_info = UserInfo.model_validate(context["user"])
-        artifact = await self._get_artifact_with_permission(
-            ws, user_info, prefix, "get_file"
-        )
-        async with self.s3_controller.create_client_async() as s3_client:
-            file_key = safe_join(ws, f"{prefix}/{path}")
-            # check if the file exists
-            try:
-                await s3_client.head_object(Bucket=self.workspace_bucket, Key=file_key)
-            except ClientError:
-                raise FileNotFoundError(
-                    f"File '{path}' does not exist in the artifact."
-                )
-            presigned_url = await s3_client.generate_presigned_url(
-                "get_object",
-                Params={"Bucket": self.workspace_bucket, "Key": file_key},
-                ExpiresIn=3600,
-            )
-            if self.s3_controller.enable_s3_proxy:
-                presigned_url = f"{self.store.public_base_url}/s3/{self.workspace_bucket}/{file_key}?{presigned_url.split('?')[1]}"
-
-        logger.info(f"Generated pre-signed URL for file download: {path}")
-
-        if options is None or not options.get("silent"):
-            session = await self._get_session()
-            try:
-                async with session.begin():
-                    artifact = await self._get_artifact(session, ws, prefix)
-                    if artifact.download_weights and path in artifact.download_weights:
-                        # if it has download_weights, increment the download count by the weight
-                        stmt = (
-                            update(ArtifactModel)
-                            .where(ArtifactModel.id == artifact.id)
-                            # atomically increment the download count by the weight
-                            .values(
-                                download_count=ArtifactModel.download_count
-                                + artifact.download_weights[path]
-                            )
-                            .execution_options(synchronize_session="fetch")
-                        )
-                        await session.execute(stmt)
-                        await session.commit()
-            except Exception as e:
-                raise e
-            finally:
-                await session.close()
-        return presigned_url
-
-    async def remove_file(self, prefix, file_path, context: dict):
-        """Remove a file from the artifact and update the staged manifest."""
-        if prefix.startswith("/"):
-            ws = prefix.split("/")[1]
-            prefix = "/".join(prefix.split("/")[2:])
-        else:
-            ws = context["ws"]
-
-        user_info = UserInfo.model_validate(context["user"])
-        artifact = await self._get_artifact_with_permission(
-            ws, user_info, prefix, "remove_file"
-        )
-        assert (
-            artifact.stage_manifest
-        ), "Artifact must be in staging mode to remove files."
-
-        session = await self._get_session()
-        try:
-            async with session.begin():
-                artifact = await self._get_artifact(session, ws, prefix)
-                artifact.last_modified = int(time.time())
-                if not artifact or not artifact.stage_manifest:
-                    raise KeyError(
-                        f"Artifact under prefix '{prefix}' is not in staging mode."
-                    )
-                if artifact.stage_files:
-                    # remove the file from the staged files list
-                    artifact.stage_files = [
-                        f for f in artifact.stage_files if f["path"] != file_path
-                    ]
-                    flag_modified(artifact, "stage_files")
-                if artifact.download_weights:
-                    # remove the file from download_weights if it's there
-                    artifact.download_weights = {
-                        k: v
-                        for k, v in artifact.download_weights.items()
-                        if k != file_path
-                    }
-                    flag_modified(artifact, "download_weights")
-                session.add(artifact)
-                await session.commit()
-        except Exception as e:
-            raise e
-        finally:
-            await session.close()
-
-        async with self.s3_controller.create_client_async() as s3_client:
-            file_key = safe_join(ws, f"{prefix}/{file_path}")
-            await s3_client.delete_object(Bucket=self.workspace_bucket, Key=file_key)
-
-        logger.info(f"Removed file from artifact: {file_key}")
-
     async def publish(
-        self, prefix: str, to: str = None, metadata=None, context: dict = None
+        self, artifact_id: str, to: str = None, metadata=None, context: dict = None
     ):
-        """Publish the artifact to public archive."""
-        ws = context["ws"]
+        """Publish the artifact to a public archive like Zenodo."""
+        if context is None or "ws" not in context:
+            raise ValueError("Context must include 'ws' (workspace).")
+        artifact_id = self._validate_artifact_id(artifact_id, context)
         user_info = UserInfo.model_validate(context["user"])
-        # read the artifact
-        artifact = await self._get_artifact_with_permission(
-            ws, user_info, prefix, "publish"
-        )
-        manifest = await self._read_manifest(
-            artifact,
-            stage=False,
-            increment_view_count=False,
-            include_metadata=True,
-        )
-
-        parent_prefix = "/".join(prefix.split("/")[:-1])
-        assert parent_prefix, "Parent artifact not found."
         session = await self._get_session()
         try:
             async with session.begin():
-                parent_artifact = await self._get_artifact(session, ws, parent_prefix)
-                zenodo_client = self._get_zenodo_client(parent_artifact, to)
-                if not parent_artifact:
-                    raise ValueError(
-                        f"Parent artifact under prefix '{parent_prefix}' not found."
-                    )
-        except Exception as e:
-            raise e
-        finally:
-            await session.close()
-
-        config = artifact.config or {}
-        if config and "zenodo" in config:
-            # Infer the target Zenodo instance from the existing config
-            deposition_info = config["zenodo"]
-            if deposition_info["links"]["self"].startswith(
-                "https://sandbox.zenodo.org"
-            ):
-                assert (
-                    to is None or to == "sandbox_zenodo"
-                ), "Cannot publish to Zenodo from Sandbox Zenodo."
-                to = "sandbox_zenodo"
-            else:
-                assert (
-                    to is None or to == "zenodo"
-                ), "Cannot publish to Sandbox Zenodo from Zenodo."
-                to = "zenodo"
-
-        if "zenodo" in config:
-            deposition_id = config["zenodo"]["id"]
-            deposition_info = await zenodo_client.load_deposition(deposition_id)
-        else:
-            deposition_info = await zenodo_client.create_deposition()
-        deposition = Deposition(zenodo_client, deposition_info)
-
-        assert manifest, "Manifest is empty or not committed."
-        assert "name" in manifest, "Manifest must have a name."
-        assert "type" in manifest, "Manifest must have a type."
-        assert "description" in manifest, "Manifest must have a description."
-
-        # Map authors to creators format required by Zenodo
-        creators = []
-        if "authors" in manifest:
-            creators = [
-                {
-                    "name": author.get("name"),
-                    "affiliation": author.get("affiliation", ""),
+                artifact, parent_artifact = await self._get_artifact_with_permission(
+                    user_info, artifact_id, "publish", session
+                )
+                manifest = artifact.manifest
+                assert manifest, "Manifest is empty or not committed."
+                assert "name" in manifest, "Manifest must have a name."
+                assert "description" in manifest, "Manifest must have a description."
+
+                config = artifact.config or {}
+                zenodo_client = self._get_zenodo_client(
+                    artifact, parent_artifact, publish_to=to
+                )
+                deposition_info = (
+                    config.get("zenodo") or await zenodo_client.create_deposition()
+                )
+                _metadata = {
+                    "title": manifest.get("name", "Untitled"),
+                    "upload_type": "dataset" if artifact.type == "dataset" else "other",
+                    "description": manifest.get(
+                        "description", "No description provided."
+                    ),
+                    "creators": [
+                        {"name": author.get("name")}
+                        for author in manifest.get("authors", [{"name": user_info.id}])
+                    ],
+                    "access_right": "open",
+                    "license": manifest.get("license", "cc-by"),
+                    "keywords": manifest.get("tags", []),
+                    "notes": f"Published automatically from Hypha ({self.store.public_base_url}).",
                 }
-                for author in manifest["authors"]
-            ]
-        else:
-            creators = [{"name": user_info.id}]
-        # Update the metadata
-        _metadata = {
-            "title": manifest.get("name", "Untitled"),
-            "upload_type": "dataset" if manifest.get("type") == "dataset" else "other",
-            "description": manifest.get("description", "No description provided."),
-            "creators": creators,
-            "access_right": "open",
-            "license": manifest.get("license", "cc-by"),
-            "keywords": manifest.get("tags", []),
-            "notes": f"Published automatically from Hypha ({self.store.public_base_url}).",
-        }
-        if metadata:
-            _metadata.update(metadata)
-        await deposition.update_metadata(_metadata)
-
-        # Add a file to the deposition
-        # list the files in the artifact then upload one by one
-        async def upload_files(dir_path=""):
-            files = await self.list_files(
-                prefix, dir_path=dir_path, max_length=100, context=context
-            )
-            for file_info in files:
-                name = file_info["name"]
-                if file_info["type"] == "directory":
-                    await upload_files(safe_join(dir_path, name))
-                else:
-                    url = await self.get_file(
-                        prefix, safe_join(dir_path, name), context=context
+                if metadata:
+                    _metadata.update(metadata)
+                await zenodo_client.update_metadata(deposition_info, _metadata)
+
+                async def upload_files(dir_path=""):
+                    files = await self.list_files(
+                        artifact.id, dir_path=dir_path, context=context
                     )
-                    async with httpx.AsyncClient() as client:
-                        async with client.stream("GET", url) as response:
-
-                            async def s3_response_chunk_reader(
-                                response, chunk_size: int = 2048
-                            ):
-                                async for chunk in response.aiter_bytes(chunk_size):
-                                    yield chunk
-
-                            put_response = await zenodo_client.client.put(
-                                f"{deposition.bucket_url}/{name}",
-                                params=zenodo_client.params,
-                                headers={"Content-Type": "application/octet-stream"},
-                                data=s3_response_chunk_reader(response),
+                    for file_info in files:
+                        name = file_info["name"]
+                        if file_info["type"] == "directory":
+                            await upload_files(safe_join(dir_path, name))
+                        else:
+                            url = await self.get_file(
+                                artifact.id, safe_join(dir_path, name), context=context
                             )
-                            put_response.raise_for_status()
-                            logger.info(f"Uploaded file: {name} to Zenodo.")
-
-        await upload_files()
-        # Publish the deposition
-        record = await deposition.publish()
-        logger.info(
-            f"Published artifact under prefix: {prefix} to Zenodo, DOI: {record.get('doi')}"
-        )
+                            await zenodo_client.import_file(deposition_info, name, url)
+
+                await upload_files()
+                record = await zenodo_client.publish(deposition_info)
 
-        # Save the record to the artifact's config for future reference
-        session = await self._get_session()
-        try:
-            async with session.begin():
-                artifact = await self._get_artifact(session, ws, prefix)
-                artifact.config = artifact.config or {}
                 artifact.config["zenodo"] = record
                 flag_modified(artifact, "config")
                 session.add(artifact)
                 await session.commit()
+                return record
         except Exception as e:
             raise e
         finally:
             await session.close()
-        return record
 
     def get_artifact_service(self):
         """Return the artifact service definition."""
diff --git a/hypha/core/__init__.py b/hypha/core/__init__.py
index e5b79a55..2860130e 100644
--- a/hypha/core/__init__.py
+++ b/hypha/core/__init__.py
@@ -228,7 +228,7 @@ class Artifact(BaseModel):
 
     model_config = ConfigDict(extra="allow")
 
-    type: str
+    type: Optional[str] = "generic"
     format_version: str = "0.2.1"
     name: Optional[str] = None
     id: Optional[str] = None
@@ -262,9 +262,8 @@ def model_validate(cls, data):
 class CollectionArtifact(Artifact):
     """Represent collection artifact."""
 
-    type: str = "collection"
+    type: Optional[str] = "collection"
     collection: Optional[List[str]] = []
-    summary_fields: Optional[List[str]] = None
     collection_schema: Optional[Dict[str, Any]] = None
 
     @classmethod
@@ -284,7 +283,7 @@ def model_validate(cls, data):
 class ApplicationArtifact(Artifact):
     """Represent application artifact."""
 
-    type: str = "application"
+    type: Optional[str] = "application"
     entry_point: Optional[str] = None  # entry point for the application
     services: Optional[List[SerializeAsAny[ServiceInfo]]] = None  # for application
 
diff --git a/hypha/core/auth.py b/hypha/core/auth.py
index b240d580..2d12bfd5 100644
--- a/hypha/core/auth.py
+++ b/hypha/core/auth.py
@@ -6,7 +6,6 @@
 import sys
 from calendar import timegm
 import datetime
-import traceback
 from os import environ as env
 from typing import List, Union, Dict
 from urllib.request import urlopen
diff --git a/hypha/core/store.py b/hypha/core/store.py
index 7ef09c44..a5cb907a 100644
--- a/hypha/core/store.py
+++ b/hypha/core/store.py
@@ -7,7 +7,7 @@
 import datetime
 from typing import List, Union
 from pydantic import BaseModel
-from fastapi import Header, Cookie, Query
+from fastapi import Header, Cookie
 
 from hypha_rpc import RPC
 from hypha_rpc.utils.schema import schema_method
@@ -255,180 +255,6 @@ async def upgrade(self):
             }
         )
 
-        # SQL database upgrade to replace 'public' column with 'permissions' column
-        def _upgrade_schema_if_needed(conn):
-            """Run schema upgrade by replacing the 'public' column with 'permissions' column."""
-            inspector = inspect(conn)
-            columns = inspector.get_columns("artifacts")
-
-            # Check if 'created_at' column exists
-            # If not, insert 'created_at' column (integer type) and 'last_modified' column (integer type) with the current timestamp = int(time.time())
-            if "created_at" not in [col["name"] for col in columns]:
-                logger.info(
-                    "Adding 'created_at' and 'last_modified' columns to 'artifacts' table."
-                )
-
-                current_time = int(time.time())
-                # Step 1: Add 'created_at' and 'last_modified' columns as nullable
-                conn.execute(
-                    text("ALTER TABLE artifacts ADD COLUMN created_at INTEGER")
-                )
-                conn.execute(
-                    text("ALTER TABLE artifacts ADD COLUMN last_modified INTEGER")
-                )
-                logger.info(
-                    "Successfully added 'created_at' and 'last_modified' columns to 'artifacts' table as nullable."
-                )
-
-                # Step 2: Populate existing rows with the current timestamp
-                conn.execute(
-                    text(
-                        "UPDATE artifacts SET created_at = :time, last_modified = :time WHERE created_at IS NULL OR last_modified IS NULL"
-                    ),
-                    {"time": current_time},
-                )
-                logger.info(
-                    "Successfully set default timestamp values for 'created_at' and 'last_modified'."
-                )
-
-                # Step 3: Alter columns to be non-nullable
-                conn.execute(
-                    text("ALTER TABLE artifacts ALTER COLUMN created_at SET NOT NULL")
-                )
-                conn.execute(
-                    text(
-                        "ALTER TABLE artifacts ALTER COLUMN last_modified SET NOT NULL"
-                    )
-                )
-                logger.info("Set 'created_at' and 'last_modified' columns to NOT NULL.")
-                database_change_log.append(
-                    {
-                        "time": datetime.datetime.now().isoformat(),
-                        "version": __version__,
-                        "change": "Added 'created_at' and 'last_modified' columns to 'artifacts' table",
-                    }
-                )
-            else:
-                logger.info(
-                    "'created_at' and 'last_modified' columns already exist in 'artifacts' table."
-                )
-
-            # Check if 'created_by' column exists
-            if "created_by" not in [col["name"] for col in columns]:
-                logger.info("Adding 'created_by' column to 'artifacts' table.")
-
-                # Step 1: Add 'created_by' column (string type)
-                conn.execute(text("ALTER TABLE artifacts ADD COLUMN created_by TEXT"))
-                logger.info(
-                    "Successfully added 'created_by' column to 'artifacts' table."
-                )
-
-                database_change_log.append(
-                    {
-                        "time": datetime.datetime.now().isoformat(),
-                        "version": __version__,
-                        "change": "Added 'created_by' column to 'artifacts' table",
-                    }
-                )
-            else:
-                logger.info("'created_by' column already exists in 'artifacts' table")
-
-            # Check if 'config' column exists (JSON type, nullable)
-            if "config" not in [col["name"] for col in columns]:
-                logger.info("Adding 'config' column to 'artifacts' table.")
-
-                # Step 1: Add 'config' column (JSON type)
-                conn.execute(text("ALTER TABLE artifacts ADD COLUMN config JSON"))
-                logger.info("Successfully added 'config' column to 'artifacts' table.")
-
-                database_change_log.append(
-                    {
-                        "time": datetime.datetime.now().isoformat(),
-                        "version": __version__,
-                        "change": "Added 'config' column to 'artifacts' table",
-                    }
-                )
-            # Check if 'permissions' column exists
-            if "permissions" not in [col["name"] for col in columns]:
-                logger.info("Adding 'permissions' column to 'artifacts' table.")
-
-                # Step 1: Add 'permissions' column (JSON type)
-                conn.execute(text("ALTER TABLE artifacts ADD COLUMN permissions JSON"))
-                logger.info(
-                    "Successfully added 'permissions' column to 'artifacts' table."
-                )
-
-                database_change_log.append(
-                    {
-                        "time": datetime.datetime.now().isoformat(),
-                        "version": __version__,
-                        "change": "Added 'permissions' column to 'artifacts' table",
-                    }
-                )
-            else:
-                logger.info("'permissions' column already exists in 'artifacts' table.")
-
-            # Check if 'public' column exists
-            if "public" in [col["name"] for col in columns]:
-                logger.info(
-                    "Replacing 'public' column with 'permissions' column in 'artifacts' table."
-                )
-
-                # Step 2: Update existing records: if 'public' is True, set the permissions to {'*': "r+"}
-                conn.execute(
-                    text(
-                        """
-                    UPDATE artifacts 
-                    SET permissions = '{"*": "r+"}' 
-                    WHERE public IS TRUE
-                    """
-                    )
-                )
-                logger.info(
-                    "Updated 'permissions' column for records where 'public' is TRUE."
-                )
-
-                # Step 3: Drop the 'public' column
-                conn.execute(text("ALTER TABLE artifacts DROP COLUMN public"))
-                logger.info(
-                    "Successfully removed 'public' column from 'artifacts' table."
-                )
-
-                # Log the schema change
-                database_change_log.append(
-                    {
-                        "time": datetime.datetime.now().isoformat(),
-                        "version": __version__,
-                        "change": "Replaced 'public' column with 'permissions' column in 'artifacts' table",
-                    }
-                )
-            else:
-                logger.info(
-                    "'public' column does not exist in 'artifacts' table, no need for schema upgrade."
-                )
-
-        async with self._sql_engine.begin() as conn:
-            try:
-                # Use run_sync to run synchronous code on the connection
-                await conn.run_sync(_upgrade_schema_if_needed)
-            except NoSuchTableError:
-                pass
-            except NoInspectionAvailable as e:
-                logger.error(f"Error inspecting the database: {str(e)}")
-
-        # For <=0.20.38, upgrade workspaces so it contains `type` key but not containing `applications` key
-        workspaces = await self._redis.hgetall("workspaces")
-        for k, v in workspaces.items():
-            workspace_info = json.loads(v.decode())
-            workspace_info["type"] = "workspace"
-            if "applications" in workspace_info:
-                del workspace_info["applications"]
-            if workspace_info.get("persistent"):
-                await self._redis.hset("workspaces", k, json.dumps(workspace_info))
-            else:
-                # remove non-persistent workspaces
-                await self._redis.hdel("workspaces", k)
-
         old_keys = await self._redis.keys(
             f"services:*|*:{self._root_user.get_workspace()}/workspace-client-*:*@*"
         )
diff --git a/hypha/core/workspace.py b/hypha/core/workspace.py
index 0a980c17..cfe3a3c3 100644
--- a/hypha/core/workspace.py
+++ b/hypha/core/workspace.py
@@ -15,14 +15,11 @@
 from pydantic import BaseModel, Field
 from sqlalchemy import (
     Column,
-    String,
-    Integer,
     JSON,
-    DateTime,
     select,
     func,
 )
-from sqlalchemy.ext.declarative import declarative_base
+from sqlmodel import SQLModel, Field
 from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
 
 from hypha.core import (
@@ -42,8 +39,6 @@
 logger = logging.getLogger("workspace")
 logger.setLevel(logging.INFO)
 
-Base = declarative_base()
-
 
 SERVICE_SUMMARY_FIELD = ["id", "name", "type", "description", "config"]
 
@@ -51,27 +46,32 @@
 _allowed_characters = re.compile(r"^[a-zA-Z0-9-_/|*]*$")
 
 
-# SQLAlchemy model for storing events
-class EventLog(Base):
+# Function to return a timezone-naive datetime
+def naive_utc_now():
+    return datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
+
+
+# SQLModel model for storing events
+class EventLog(SQLModel, table=True):
     __tablename__ = "event_logs"
 
-    id = Column(Integer, primary_key=True, autoincrement=True)
-    event_type = Column(String, nullable=False)
-    workspace = Column(String, nullable=False)
-    user_id = Column(String, nullable=False)
-    timestamp = Column(
-        DateTime, default=datetime.datetime.now(datetime.timezone.utc), index=True
+    id: Optional[int] = Field(default=None, primary_key=True)
+    event_type: str = Field(nullable=False)
+    workspace: str = Field(nullable=False)
+    user_id: str = Field(nullable=False)
+    timestamp: datetime.datetime = Field(default_factory=naive_utc_now, index=True)
+    data: Optional[Dict[str, Any]] = Field(
+        default=None, sa_column=Column(JSON, nullable=True)
     )
-    data = Column(JSON, nullable=True)  # Store any additional event metadata
 
-    def to_dict(self):
-        """Convert the SQLAlchemy model instance to a dictionary."""
+    def to_dict(self) -> dict:
+        """Convert the SQLModel model instance to a dictionary."""
         return {
             "id": self.id,
             "event_type": self.event_type,
             "workspace": self.workspace,
             "user_id": self.user_id,
-            "timestamp": self.timestamp.isoformat(),  # Convert datetime to ISO string
+            "timestamp": self.timestamp.isoformat() if self.timestamp else None,
             "data": self.data,
         }
 
@@ -156,7 +156,7 @@ async def setup(
         )
         if self._sql_engine:
             async with self._sql_engine.begin() as conn:
-                await conn.run_sync(Base.metadata.create_all)
+                await conn.run_sync(EventLog.metadata.create_all)
                 logger.info("Database tables created successfully.")
         self._initialized = True
         return rpc
@@ -1252,7 +1252,7 @@ async def _launch_application_for_service(
             "applications", context={"ws": workspace, "user": user_info.model_dump()}
         )
         applications = {
-            item["id"]: ApplicationArtifact.model_validate(item)
+            item["manifest"]["id"]: ApplicationArtifact.model_validate(item["manifest"])
             for item in applications
         }
         if app_id not in applications:
diff --git a/hypha/http.py b/hypha/http.py
index 227d0f95..e8fbcaf9 100644
--- a/hypha/http.py
+++ b/hypha/http.py
@@ -183,19 +183,22 @@ def _get_authorization_header(self, scope):
 
     def _get_access_token_from_cookies(self, scope):
         """Extract the access_token cookie from the scope."""
-        cookies = None
-        for key, value in scope.get("headers", []):
-            if key.decode("utf-8").lower() == "cookie":
-                cookies = value.decode("utf-8")
-                break
-        if not cookies:
+        try:
+            cookies = None
+            for key, value in scope.get("headers", []):
+                if key.decode("utf-8").lower() == "cookie":
+                    cookies = value.decode("utf-8")
+                    break
+            if not cookies:
+                return None
+            # Split cookies and find the access_token
+            cookie_dict = {
+                k.strip(): v.strip()
+                for k, v in (cookie.split("=") for cookie in cookies.split(";"))
+            }
+            return cookie_dict.get("access_token")
+        except Exception as exp:
             return None
-        # Split cookies and find the access_token
-        cookie_dict = {
-            k.strip(): v.strip()
-            for k, v in (cookie.split("=") for cookie in cookies.split(";"))
-        }
-        return cookie_dict.get("access_token")
 
     async def __call__(self, scope, receive, send):
         if scope["type"] == "http":
@@ -280,7 +283,7 @@ async def __call__(self, scope, receive, send):
                             )
                         return
                 except Exception as exp:
-                    logger.error(f"Error in ASGI service: {exp}")
+                    logger.exception(f"Error in ASGI service: {exp}")
                     await send(
                         {
                             "type": "http.response.start",
@@ -582,7 +585,7 @@ async def get_workspace_apps(
             """Route for get apps under a workspace."""
             try:
                 async with self.store.get_workspace_interface(
-                    user_info, user_info.scope.current_workspace
+                    user_info, workspace or user_info.scope.current_workspace
                 ) as manager:
                     try:
                         controller = await manager.get_service("public/server-apps")
@@ -683,56 +686,6 @@ async def get_app_info(
                 url=f"{base_path.rstrip('/')}/{workspace}/apps/{service_id}/"
             )
 
-        @app.get(norm_url("/{workspace}/applications/{app_id}/{path:path}"))
-        async def get_browser_app_file(
-            workspace: str, app_id: str, path: str, token: str = None
-        ) -> Response:
-            """Route for getting browser app files."""
-            if token is None:
-                return JSONResponse(
-                    status_code=403,
-                    content={
-                        "success": False,
-                        "detail": (f"Token not provided for {workspace}/{path}"),
-                    },
-                )
-            try:
-                user_info = await store.parse_user_token(token)
-            except jose.exceptions.JWTError:
-                return JSONResponse(
-                    status_code=403,
-                    content={
-                        "success": False,
-                        "detail": (
-                            f"Invalid token not provided for {workspace}/{path}"
-                        ),
-                    },
-                )
-            if not user_info.check_permission(workspace, UserPermission.read):
-                return JSONResponse(
-                    status_code=403,
-                    content={
-                        "success": False,
-                        "detail": (
-                            f"{user_info['username']} has no"
-                            f" permission to access {workspace}"
-                        ),
-                    },
-                )
-            key = safe_join(workspace, "applications", app_id, path)
-            assert self.s3_enabled, "S3 is not enabled."
-            from hypha.s3 import FSFileResponse
-            from aiobotocore.session import get_session
-
-            s3_client = get_session().create_client(
-                "s3",
-                endpoint_url=self.endpoint_url,
-                aws_access_key_id=self.access_key_id,
-                aws_secret_access_key=self.secret_access_key,
-                region_name=self.region_name,
-            )
-            return FSFileResponse(s3_client, self.workspace_bucket, key)
-
         @app.get(norm_url("/{workspace}/services/{service_id}/{function_key}"))
         @app.post(norm_url("/{workspace}/services/{service_id}/{function_key}"))
         async def call_service_function(
diff --git a/hypha/migrations/env.py b/hypha/migrations/env.py
new file mode 100644
index 00000000..e3ec155b
--- /dev/null
+++ b/hypha/migrations/env.py
@@ -0,0 +1,99 @@
+import asyncio
+import os
+from logging.config import fileConfig
+
+from sqlalchemy import pool
+from sqlalchemy.engine import Connection
+from sqlalchemy.ext.asyncio import async_engine_from_config
+from sqlmodel import SQLModel
+
+from alembic import context
+
+from hypha.core.workspace import EventLog
+from hypha.artifact import ArtifactModel
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+if config.config_file_name is not None:
+    fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+target_metadata = SQLModel.metadata
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_offline() -> None:
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def do_run_migrations(connection: Connection) -> None:
+    context.configure(connection=connection, target_metadata=target_metadata)
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+async def run_async_migrations() -> None:
+    """In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+
+    connectable = async_engine_from_config(
+        config.get_section(config.config_ini_section, {}),
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
+
+    async with connectable.connect() as connection:
+        await connection.run_sync(do_run_migrations)
+
+    await connectable.dispose()
+
+
+def run_migrations_online() -> None:
+    """Run migrations in 'online' mode."""
+    # check if the loop is already running
+    if asyncio.get_event_loop().is_running():
+        # create a task to run the async migrations
+        asyncio.create_task(run_async_migrations())
+    else:
+        loop = asyncio.get_event_loop()
+        loop.run_until_complete(run_async_migrations())
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()
diff --git a/hypha/migrations/script.py.mako b/hypha/migrations/script.py.mako
new file mode 100644
index 00000000..6ce33510
--- /dev/null
+++ b/hypha/migrations/script.py.mako
@@ -0,0 +1,27 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+import sqlmodel
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision: str = ${repr(up_revision)}
+down_revision: Union[str, None] = ${repr(down_revision)}
+branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
+depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    ${downgrades if downgrades else "pass"}
diff --git a/hypha/migrations/versions/9096f050eb04_init.py b/hypha/migrations/versions/9096f050eb04_init.py
new file mode 100644
index 00000000..ff7c44d5
--- /dev/null
+++ b/hypha/migrations/versions/9096f050eb04_init.py
@@ -0,0 +1,74 @@
+"""init
+
+Revision ID: 9096f050eb04
+Revises: 
+Create Date: 2024-11-15 05:58:12.181737
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+import sqlmodel
+
+
+# revision identifiers, used by Alembic.
+revision: str = "9096f050eb04"
+down_revision: Union[str, None] = None
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "artifacts",
+        sa.Column("id", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+        sa.Column("type", sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+        sa.Column("workspace", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+        sa.Column("parent_id", sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+        sa.Column("alias", sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+        sa.Column("manifest", sa.JSON(), nullable=True),
+        sa.Column("staging", sa.JSON(), nullable=True),
+        sa.Column("download_count", sa.Float(), nullable=False),
+        sa.Column("view_count", sa.Float(), nullable=False),
+        sa.Column("file_count", sa.Integer(), nullable=False),
+        sa.Column("created_at", sa.Integer(), nullable=False),
+        sa.Column("created_by", sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+        sa.Column("last_modified", sa.Integer(), nullable=False),
+        sa.Column("versions", sa.JSON(), nullable=True),
+        sa.Column("config", sa.JSON(), nullable=True),
+        sa.Column("secrets", sa.JSON(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["parent_id"],
+            ["artifacts.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("workspace", "alias", name="_workspace_alias_uc"),
+    )
+    op.create_index(
+        op.f("ix_artifacts_workspace"), "artifacts", ["workspace"], unique=False
+    )
+    op.create_table(
+        "event_logs",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("event_type", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+        sa.Column("workspace", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+        sa.Column("user_id", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+        sa.Column("timestamp", sa.DateTime(), nullable=False),
+        sa.Column("data", sa.JSON(), nullable=True),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        op.f("ix_event_logs_timestamp"), "event_logs", ["timestamp"], unique=False
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f("ix_event_logs_timestamp"), table_name="event_logs")
+    op.drop_table("event_logs")
+    op.drop_index(op.f("ix_artifacts_workspace"), table_name="artifacts")
+    op.drop_table("artifacts")
+    # ### end Alembic commands ###
diff --git a/hypha/server.py b/hypha/server.py
index 0ff892e0..e20d3559 100644
--- a/hypha/server.py
+++ b/hypha/server.py
@@ -379,6 +379,12 @@ def get_argparser(add_help=True):
         default=None,
         help="set database URI for the artifact manager",
     )
+    parser.add_argument(
+        "--migrate-database",
+        type=str,
+        default=None,
+        help="migrate the database using alembic to the specified revision, e.g. head",
+    )
     parser.add_argument(
         "--workspace-bucket",
         type=str,
@@ -423,6 +429,17 @@ def get_argparser(add_help=True):
 
     arg_parser = get_argparser()
     opt = arg_parser.parse_args()
-    app = create_application(opt)
 
+    # Apply database migrations
+    if opt.migrate_database is not None:
+        from alembic.config import Config
+        from alembic import command
+
+        alembic_cfg = Config()
+        migration_dir = Path(__file__).parent / "migrations"
+        alembic_cfg.set_main_option("script_location", str(migration_dir))
+        alembic_cfg.set_main_option("sqlalchemy.url", opt.database_uri)
+        command.upgrade(alembic_cfg, "head")
+
+    app = create_application(opt)
     uvicorn.run(app, host=opt.host, port=int(opt.port))
diff --git a/hypha/templates/hypha-core-app/hypha-app-webpython.js b/hypha/templates/hypha-core-app/hypha-app-webpython.js
index 27daf5d5..25489962 100644
--- a/hypha/templates/hypha-core-app/hypha-app-webpython.js
+++ b/hypha/templates/hypha-core-app/hypha-app-webpython.js
@@ -27,7 +27,7 @@ loadPyodide().then(async (pyodide) => {
     pyodide.setStderr({ batched: (msg) => console.error(msg) });
     await pyodide.loadPackage("micropip");
     const micropip = pyodide.pyimport("micropip");
-    await micropip.install('hypha-rpc==0.20.38');
+    await micropip.install('hypha-rpc==0.20.39');
     const isWindow = typeof window !== "undefined";
     
     setTimeout(() => {
diff --git a/hypha/templates/ws/index.html b/hypha/templates/ws/index.html
index 65edebed..c327bb4e 100644
--- a/hypha/templates/ws/index.html
+++ b/hypha/templates/ws/index.html
@@ -160,7 +160,7 @@
 <body class="bg-black text-white font-poppins">
     <div id="app"></div>
     <script type="module">
-        import { HyphaCore } from "https://cdn.jsdelivr.net/npm/hypha-core@0.20.38/dist/hypha-core.mjs";
+        import { HyphaCore } from "https://cdn.jsdelivr.net/npm/hypha-core@0.20.39/dist/hypha-core.mjs";
 
         const defaultService = {
             getServerConfig(context){
diff --git a/hypha/utils/zenodo.py b/hypha/utils/zenodo.py
index 9b8d0137..a556391a 100644
--- a/hypha/utils/zenodo.py
+++ b/hypha/utils/zenodo.py
@@ -15,25 +15,21 @@ def __init__(
         self.params = {"access_token": self.access_token}
         self.client = httpx.AsyncClient()
 
-    async def create_deposition(self):
-        """Creates a new empty deposition."""
+    async def create_deposition(self) -> Dict[str, Any]:
+        """Creates a new empty deposition and returns its info."""
         url = f"{self.zenodo_server}/api/deposit/depositions"
         response = await self.client.post(
             url, params=self.params, json={}, headers=self.headers
         )
         response.raise_for_status()
+        return response.json()
 
-        deposition_info = response.json()
-        return deposition_info
-
-    async def load_deposition(self, deposition_id: str) -> "Deposition":
-        """Loads an existing deposition by ID and returns a Deposition object."""
+    async def load_deposition(self, deposition_id: str) -> Dict[str, Any]:
+        """Loads an existing deposition by ID and returns its information."""
         url = f"{self.zenodo_server}/api/deposit/depositions/{deposition_id}"
         response = await self.client.get(url, params=self.params)
         response.raise_for_status()
-
-        deposition_info = response.json()
-        return deposition_info
+        return response.json()
 
     async def load_published_record(self, record_id: str) -> Optional[str]:
         """Loads an existing published record to retrieve the concept_id."""
@@ -45,61 +41,44 @@ async def load_published_record(self, record_id: str) -> Optional[str]:
         concept_id = record_info.get("conceptrecid")
         if not concept_id:
             raise ValueError(f"Record {record_id} does not have a conceptrecid.")
-
         return concept_id
 
-    async def create_new_version(self, record_id: str) -> "Deposition":
+    async def create_new_version(self, record_id: str) -> Dict[str, Any]:
         """Creates a new version of an existing published record."""
-        # Use the concept_id to initiate a new version
         url = f"{self.zenodo_server}/api/deposit/depositions/{record_id}/actions/newversion"
         response = await self.client.post(url, params=self.params)
         response.raise_for_status()
 
-        # Fetch the latest draft version from the latest_draft URL
         draft_url = response.json()["links"]["latest_draft"]
         draft_response = await self.client.get(draft_url, params=self.params)
         draft_response.raise_for_status()
+        return draft_response.json()
 
-        draft_info = draft_response.json()
-        return draft_info
-
-    async def delete_deposition(self, deposition_id: str) -> None:
-        """Deletes a deposition. Only unpublished depositions can be deleted."""
+    async def update_metadata(
+        self, deposition_info: Dict[str, Any], metadata: Dict[str, Any]
+    ) -> None:
+        """Updates metadata for a specified deposition."""
+        deposition_id = deposition_info["id"]
         url = f"{self.zenodo_server}/api/deposit/depositions/{deposition_id}"
-        response = await self.client.delete(url, params=self.params)
-        response.raise_for_status()
-
-
-class Deposition:
-    def __init__(self, zenodo_client: ZenodoClient, deposition_info: Dict[str, Any]):
-        self.zenodo_client = zenodo_client
-        self.deposition_id = str(deposition_info["id"])
-        self.bucket_url = deposition_info["links"]["bucket"]
-        self.doi = deposition_info["metadata"].get("prereserve_doi", {}).get("doi")
-        self.concept_doi = deposition_info.get("conceptrecid")
-
-    async def update_metadata(self, metadata: Dict[str, Any]) -> None:
-        """Updates metadata for this deposition."""
-        url = f"{self.zenodo_client.zenodo_server}/api/deposit/depositions/{self.deposition_id}"
-        response = await self.zenodo_client.client.put(
+        response = await self.client.put(
             url,
-            params=self.zenodo_client.params,
+            params=self.params,
             json={"metadata": metadata},
-            headers=self.zenodo_client.headers,
+            headers=self.headers,
         )
         response.raise_for_status()
 
-    async def list_files(self) -> List[Dict[str, Any]]:
-        """Lists all files in this deposition."""
-        url = f"{self.zenodo_client.zenodo_server}/api/deposit/depositions/{self.deposition_id}/files"
-        response = await self.zenodo_client.client.get(
-            url, params=self.zenodo_client.params
-        )
+    async def list_files(self, deposition_info: Dict[str, Any]) -> List[Dict[str, Any]]:
+        """Lists all files in a specified deposition."""
+        deposition_id = deposition_info["id"]
+        url = f"{self.zenodo_server}/api/deposit/depositions/{deposition_id}/files"
+        response = await self.client.get(url, params=self.params)
         response.raise_for_status()
         return response.json()
 
-    async def add_file(self, file_path: str) -> None:
-        """Uploads a file to this deposition using the new bucket API."""
+    async def add_file(self, deposition_info: Dict[str, Any], file_path: str) -> None:
+        """Uploads a file to the deposition using the bucket API."""
+        bucket_url = deposition_info["links"]["bucket"]
         filename = PurePosixPath(file_path).name
 
         async def file_chunk_reader(file_path: str, chunk_size: int = 1024):
@@ -110,29 +89,43 @@ async def file_chunk_reader(file_path: str, chunk_size: int = 1024):
                         break
                     yield chunk
 
-        await self.zenodo_client.client.put(
-            f"{self.bucket_url}/{filename}",
-            params=self.zenodo_client.params,
+        await self.client.put(
+            f"{bucket_url}/{filename}",
+            params=self.params,
             data=file_chunk_reader(file_path),
         )
 
-    async def delete_file(self, file_id: str) -> None:
-        """Deletes a file from this deposition."""
-        url = f"{self.zenodo_client.zenodo_server}/api/deposit/depositions/{self.deposition_id}/files/{file_id}"
-        response = await self.zenodo_client.client.delete(
-            url, params=self.zenodo_client.params
-        )
+    async def import_file(self, deposition_info, name, target_url):
+        bucket_url = deposition_info["links"]["bucket"]
+        async with httpx.AsyncClient() as client:
+            async with client.stream("GET", target_url) as response:
+
+                async def s3_response_chunk_reader(response, chunk_size: int = 2048):
+                    async for chunk in response.aiter_bytes(chunk_size):
+                        yield chunk
+
+                put_response = await self.client.put(
+                    f"{bucket_url}/{name}",
+                    params=self.params,
+                    headers={"Content-Type": "application/octet-stream"},
+                    data=s3_response_chunk_reader(response),
+                )
+                put_response.raise_for_status()
+
+    async def delete_deposition(self, deposition_id: str) -> None:
+        """Deletes a deposition. Only unpublished depositions can be deleted."""
+        url = f"{self.zenodo_server}/api/deposit/depositions/{deposition_id}"
+        response = await self.client.delete(url, params=self.params)
         response.raise_for_status()
 
-    async def publish(self) -> None:
-        """Publishes this deposition."""
-        url = f"{self.zenodo_client.zenodo_server}/api/deposit/depositions/{self.deposition_id}/actions/publish"
-        response = await self.zenodo_client.client.post(
-            url, params=self.zenodo_client.params
-        )
+    async def publish(self, deposition_info: Dict[str, Any]) -> Dict[str, Any]:
+        """Publishes the deposition."""
+        deposition_id = deposition_info["id"]
+        url = f"{self.zenodo_server}/api/deposit/depositions/{deposition_id}/actions/publish"
+        response = await self.client.post(url, params=self.params)
         if response.status_code == 400:
             raise RuntimeError(
-                f"Failed to publish deposition: {response.json()}, you might forgot to update the metadata."
+                f"Failed to publish deposition: {response.json()}, you might have forgotten to update the metadata."
             )
         response.raise_for_status()
         return response.json()
@@ -150,8 +143,7 @@ async def main():
 
     # Create a new deposition
     deposition_info = await zenodo_client.create_deposition()
-    deposition = Deposition(zenodo_client, deposition_info)
-    print(f"Created new deposition with ID: {deposition.deposition_id}")
+    print(f"Created new deposition with ID: {deposition_info['id']}")
 
     # Update metadata
     metadata = {
@@ -163,28 +155,29 @@ async def main():
         "license": "cc-by",
         "keywords": ["example", "dataset"],
     }
-    await deposition.update_metadata(metadata)
+    await zenodo_client.update_metadata(deposition_info, metadata)
 
     # Add a file to the deposition
-    await deposition.add_file("setup.py")
+    await zenodo_client.add_file(deposition_info, "setup.py")
 
     # Publish the deposition
-    record = await deposition.publish()
-    print(f"Published deposition with DOI: {deposition.doi}")
+    record = await zenodo_client.publish(deposition_info)
+    print(f"Published deposition with DOI: {record['doi']}")
 
+    # Create a new version of the published record
     record_id = record["id"]
     deposition_info = await zenodo_client.create_new_version(record_id)
-    new_version = Deposition(zenodo_client, deposition_info)
-    print(f"Created new version with ID: {new_version.deposition_id}")
+    print(f"Created new version with ID: {deposition_info['id']}")
 
-    await new_version.update_metadata(metadata)
+    # Update metadata for the new version
+    await zenodo_client.update_metadata(deposition_info, metadata)
 
     # Add another file to the new version
-    await new_version.add_file("requirements.txt")
+    await zenodo_client.add_file(deposition_info, "requirements.txt")
 
     # Publish the new version
-    await new_version.publish()
-    print(f"Published new version with DOI: {new_version.doi}")
+    record = await zenodo_client.publish(deposition_info)
+    print(f"Published new version with DOI: {record['doi']}")
 
 
 if __name__ == "__main__":
diff --git a/requirements.txt b/requirements.txt
index 0d92ff2c..1eb46631 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,8 +1,9 @@
 aioboto3==13.1.1
 aiofiles==23.2.1
+websockets==13.1
 base58==2.1.1
 fastapi==0.106.0
-hypha-rpc==0.20.38
+hypha-rpc==0.20.39
 jinja2==3.1.4
 lxml==4.9.3
 msgpack==1.0.8
@@ -27,3 +28,8 @@ jsonschema==3.2.0
 sqlalchemy==2.0.35
 aiosqlite==0.20.0
 prometheus-client==0.21.0
+uuid-utils==0.9.0
+psycopg2==2.9.9
+asyncpg==0.30.0
+sqlmodel==0.0.22
+alembic==1.14.0
\ No newline at end of file
diff --git a/setup.py b/setup.py
index 099d60ca..e8af8837 100644
--- a/setup.py
+++ b/setup.py
@@ -12,7 +12,7 @@
 REQUIREMENTS = [
     "aiofiles",
     "fastapi>=0.70.0,<=0.106.0",
-    "hypha-rpc>=0.20.38",
+    "hypha-rpc>=0.20.39",
     "msgpack>=1.0.2",
     "numpy",
     "pydantic[email]>=2.6.1",
@@ -39,6 +39,9 @@
     "sqlalchemy>=2.0.35",
     "aiosqlite>=0.20.0",
     "prometheus-client>=0.21.0",
+    "uuid-utils>=0.9.0",
+    "sqlmodel>=0.0.22",
+    "alembic>=1.14.0",
 ]
 
 ROOT_DIR = Path(__file__).parent.resolve()
@@ -75,6 +78,7 @@
             "base58>=2.1.0",
             "pymultihash>=0.8.2",
         ],
+        "postgres": ["psycopg2-binary>=2.9.10", "asyncpg>=0.30.0"],
     },
     zip_safe=False,
     entry_points={"console_scripts": ["hypha = hypha.__main__:main"]},
diff --git a/tests/__init__.py b/tests/__init__.py
index c2f993f5..fe7eff3b 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -5,11 +5,13 @@
 SIO_PORT2 = 38223
 SIO_PORT_REDIS_1 = SIO_PORT + 8
 SIO_PORT_REDIS_2 = SIO_PORT + 9
+SIO_PORT_SQLITE = SIO_PORT + 20
 TRITON_PORT = 38284
 WS_SERVER_URL = f"ws://127.0.0.1:{SIO_PORT}/ws"
 SERVER_URL = f"http://127.0.0.1:{SIO_PORT}"
 SERVER_URL_REDIS_1 = f"http://127.0.0.1:{SIO_PORT_REDIS_1}"
 SERVER_URL_REDIS_2 = f"http://127.0.0.1:{SIO_PORT_REDIS_2}"
+SERVER_URL_SQLITE = f"http://127.0.0.1:{SIO_PORT_SQLITE}"
 
 MINIO_PORT = 38483
 MINIO_SERVER_URL = f"http://127.0.0.1:{MINIO_PORT}"
@@ -18,6 +20,13 @@
 MINIO_ROOT_PASSWORD = str(uuid.uuid4())
 REDIS_PORT = 6333
 
+POSTGRES_PORT = 5432
+POSTGRES_USER = "postgres"
+POSTGRES_PASSWORD = "mysecretpassword"
+POSTGRES_DB = "postgres"
+
+POSTGRES_URI = f"postgresql+asyncpg://{POSTGRES_USER}:{POSTGRES_PASSWORD}@localhost:{POSTGRES_PORT}/{POSTGRES_DB}"
+
 
 def find_item(items, key, value):
     """Find an item with key or attributes in an object list."""
diff --git a/tests/conftest.py b/tests/conftest.py
index 2edf78e4..396e6d5f 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -12,6 +12,8 @@
 import pytest_asyncio
 import requests
 from requests import RequestException
+import pytest_asyncio
+from sqlalchemy import create_engine
 
 from hypha.core import UserInfo, auth, UserPermission
 from hypha.core.auth import generate_presigned_token, create_scope
@@ -29,7 +31,13 @@
     SIO_PORT2,
     SIO_PORT_REDIS_1,
     SIO_PORT_REDIS_2,
+    SIO_PORT_SQLITE,
     TRITON_PORT,
+    POSTGRES_PORT,
+    POSTGRES_USER,
+    POSTGRES_PASSWORD,
+    POSTGRES_DB,
+    POSTGRES_URI,
 )
 
 JWT_SECRET = str(uuid.uuid4())
@@ -122,6 +130,89 @@ def triton_server():
     yield
 
 
+@pytest_asyncio.fixture(name="postgres_server", scope="session")
+def postgres_server():
+    """Start a fresh PostgreSQL server as a test fixture and tear down after the test."""
+    # Check if the container is already running
+    existing_container = subprocess.run(
+        ["docker", "ps", "-q", "-f", "name=hypha-postgres"],
+        capture_output=True,
+        text=True,
+    ).stdout.strip()
+
+    # Stop and remove the existing container if it is running
+    if existing_container:
+        if os.environ.get("GITHUB_ACTIONS") != "true":
+            print(
+                "Stopping and removing existing PostgreSQL container:",
+                existing_container,
+            )
+            subprocess.run(["docker", "stop", "hypha-postgres"], check=True)
+            # Attempt to remove the container, but ignore errors if it does not exist
+            subprocess.run(["docker", "rm", "hypha-postgres"], check=False)
+
+            # Start a new PostgreSQL container
+            print("Starting a new PostgreSQL container")
+            subprocess.Popen(
+                [
+                    "docker",
+                    "run",
+                    "--name",
+                    "hypha-postgres",
+                    "--rm",
+                    "-e",
+                    f"POSTGRES_PASSWORD={POSTGRES_PASSWORD}",
+                    "-p",
+                    f"{POSTGRES_PORT}:5432",
+                    "-d",
+                    "postgres",
+                ]
+            )
+            time.sleep(2)  # Give the container time to initialize
+        else:
+            print("Using existing PostgreSQL container:", existing_container)
+    else:
+        # Start a new PostgreSQL container
+        print("Starting a new PostgreSQL container")
+        subprocess.Popen(
+            [
+                "docker",
+                "run",
+                "--name",
+                "hypha-postgres",
+                "--rm",
+                "-e",
+                f"POSTGRES_PASSWORD={POSTGRES_PASSWORD}",
+                "-p",
+                f"{POSTGRES_PORT}:5432",
+                "-d",
+                "postgres",
+            ]
+        )
+        time.sleep(2)
+
+    # Wait for the PostgreSQL server to become available
+    timeout = 10
+    while timeout > 0:
+        try:
+            engine = create_engine(
+                f"postgresql://{POSTGRES_USER}:{POSTGRES_PASSWORD}@localhost:{POSTGRES_PORT}/{POSTGRES_DB}"
+            )
+            with engine.connect() as connection:
+                connection.execute("SELECT 1")
+            break
+        except Exception:
+            timeout -= 1
+            time.sleep(1)
+
+    yield  # Test code executes here
+
+    if os.environ.get("GITHUB_ACTIONS") != "true":
+        # Stop the PostgreSQL container
+        print("Stopping PostgreSQL container")
+        subprocess.run(["docker", "stop", "hypha-postgres"], check=True)
+
+
 @pytest_asyncio.fixture(name="redis_server", scope="session")
 def redis_server():
     """Start a redis server as test fixture and tear down after test."""
@@ -151,12 +242,8 @@ def redis_server():
 
 
 @pytest_asyncio.fixture(name="fastapi_server", scope="session")
-def fastapi_server_fixture(minio_server):
+def fastapi_server_fixture(minio_server, postgres_server):
     """Start server as test fixture and tear down after test."""
-    # create a temporary directory for the artifacts database
-    dirpath = tempfile.mkdtemp()
-    print(f"Artifacts database directory: {dirpath}")
-    db_path = f"sqlite+aiosqlite:///{dirpath}/artifacts.db"
     with subprocess.Popen(
         [
             sys.executable,
@@ -165,7 +252,8 @@ def fastapi_server_fixture(minio_server):
             f"--port={SIO_PORT}",
             "--enable-server-apps",
             "--enable-s3",
-            f"--database-uri={db_path}",
+            f"--database-uri={POSTGRES_URI}",
+            "--migrate-database=head",
             "--reset-redis",
             f"--endpoint-url={MINIO_SERVER_URL}",
             f"--access-key-id={MINIO_ROOT_USER}",
@@ -201,6 +289,54 @@ def fastapi_server_fixture(minio_server):
         proc.terminate()
 
 
+@pytest_asyncio.fixture(name="fastapi_server_sqlite", scope="session")
+def fastapi_server_sqlite_fixture(minio_server):
+    """Start server as test fixture and tear down after test."""
+    # create a temporary directory for the artifacts database
+    dirpath = tempfile.mkdtemp()
+    print(f"Artifacts database directory: {dirpath}")
+    db_path = f"sqlite+aiosqlite:///{dirpath}/artifacts.db"
+    with subprocess.Popen(
+        [
+            sys.executable,
+            "-m",
+            "hypha.server",
+            f"--port={SIO_PORT_SQLITE}",
+            "--enable-server-apps",
+            "--enable-s3",
+            f"--database-uri={db_path}",
+            "--migrate-database=head",
+            "--reset-redis",
+            f"--endpoint-url={MINIO_SERVER_URL}",
+            f"--access-key-id={MINIO_ROOT_USER}",
+            f"--secret-access-key={MINIO_ROOT_PASSWORD}",
+            f"--endpoint-url-public={MINIO_SERVER_URL_PUBLIC}",
+            f"--workspace-bucket=my-workspaces",
+            "--s3-admin-type=generic",
+        ],
+        env=test_env,
+    ) as proc:
+        timeout = 20
+        while timeout > 0:
+            try:
+                response = requests.get(
+                    f"http://127.0.0.1:{SIO_PORT_SQLITE}/health/readiness"
+                )
+                if response.ok:
+                    break
+            except RequestException:
+                pass
+            timeout -= 0.1
+            time.sleep(0.1)
+        if timeout <= 0:
+            raise TimeoutError("Server (fastapi_server) did not start in time")
+        response = requests.get(f"http://127.0.0.1:{SIO_PORT_SQLITE}/health/liveness")
+        assert response.ok
+        yield
+        proc.kill()
+        proc.terminate()
+
+
 @pytest_asyncio.fixture(name="fastapi_server_redis_1", scope="session")
 def fastapi_server_redis_1(redis_server, minio_server):
     """Start server as test fixture and tear down after test."""
diff --git a/tests/test_artifact.py b/tests/test_artifact.py
index 2351dfdf..7823df4b 100644
--- a/tests/test_artifact.py
+++ b/tests/test_artifact.py
@@ -4,152 +4,324 @@
 import os
 from hypha_rpc import connect_to_server
 
-from . import SERVER_URL, find_item
+from . import SERVER_URL, SERVER_URL_SQLITE, find_item
 
 # All test coroutines will be treated as marked.
 pytestmark = pytest.mark.asyncio
 
-import pytest
-import requests
-from hypha_rpc import connect_to_server
-from jsonschema import validate, ValidationError
 
-from . import SERVER_URL, find_item
+async def test_sqlite_create_and_search_artifacts(
+    minio_server, fastapi_server_sqlite, test_user_token
+):
+    """Test creating a collection, adding datasets with files, and performing a search on manifest fields."""
 
-# All test coroutines will be treated as marked.
-pytestmark = pytest.mark.asyncio
+    # Set up connection and artifact manager
+    api = await connect_to_server(
+        {
+            "name": "sqlite test client",
+            "server_url": SERVER_URL_SQLITE,
+            "token": test_user_token,
+        }
+    )
+    artifact_manager = await api.get_service("public/artifact-manager")
 
+    # Create a collection for the SQLite test
+    collection_manifest = {
+        "name": "SQLite Test Collection",
+        "description": "A collection for testing SQLite backend",
+    }
+    collection = await artifact_manager.create(
+        type="collection",
+        manifest=collection_manifest,
+        config={"permissions": {"*": "r", "@": "r+"}},
+    )
+
+    # Add multiple datasets to the collection
+    dataset_manifests = [
+        {"name": "Dataset 1", "description": "First dataset for SQLite testing"},
+        {"name": "Dataset 2", "description": "Second dataset for SQLite testing"},
+        {"name": "Dataset 3", "description": "Third dataset for SQLite testing"},
+    ]
+    datasets = []
+    for dataset_manifest in dataset_manifests:
+        dataset = await artifact_manager.create(
+            type="dataset",
+            parent_id=collection.id,
+            manifest=dataset_manifest,
+            version="stage",
+        )
+        datasets.append(dataset)
+
+    # create an application artifact
+    application = await artifact_manager.create(
+        type="application",
+        parent_id=collection.id,
+        manifest={
+            "name": "Application 1",
+            "description": "First application for SQLite testing",
+        },
+        version="stage",
+    )
 
-"""Test Artifact services."""
-import pytest
-import requests
-from hypha_rpc import connect_to_server
+    # Test read operation on the datasets
+    for dataset in datasets:
+        dataset_data = await artifact_manager.read(
+            artifact_id=dataset.id, version="stage"
+        )
+        assert dataset_data["manifest"]["name"] in [
+            dataset_manifest["name"] for dataset_manifest in dataset_manifests
+        ]
+
+    application_data = await artifact_manager.read(
+        artifact_id=application.id, version="stage"
+    )
+    assert application_data["manifest"]["name"] == "Application 1"
+
+    # Add files to each dataset
+    file_contents = [
+        "Contents of file 1 for Dataset 1",
+        "Contents of file 2 for Dataset 2",
+        "Contents of file 3 for Dataset 3",
+    ]
+    for idx, dataset in enumerate(datasets):
+        put_url = await artifact_manager.put_file(
+            artifact_id=dataset.id, file_path=f"file_{idx+1}.txt"
+        )
+        response = requests.put(put_url, data=file_contents[idx])
+        assert response.ok
 
-from . import SERVER_URL, find_item
+    search_results = await artifact_manager.list(
+        parent_id=collection.id,
+        filters={"stage": True, "manifest": {"description": "*dataset*"}},
+    )
 
-# All test coroutines will be treated as marked.
-pytestmark = pytest.mark.asyncio
+    assert len(search_results) == len(datasets)
+
+    # list application only
+    search_results = await artifact_manager.list(
+        parent_id=collection.id, filters={"stage": True, "type": "application"}
+    )
+    assert len(search_results) == 1
+
+    # Commit the datasets
+    for dataset in datasets:
+        await artifact_manager.commit(artifact_id=dataset.id)
+
+    await artifact_manager.commit(artifact_id=application.id)
 
+    # Search for artifacts based on manifest fields
+    search_results = await artifact_manager.list(
+        parent_id=collection.id,
+        keywords=["SQLite"],
+        filters={"manifest": {"description": "*dataset*"}},
+        mode="AND",
+    )
 
-async def test_serve_artifact_endpoint(minio_server, fastapi_server):
+    # Validate the search results contain the added datasets
+    assert len(search_results) == len(datasets)
+    assert set([result["manifest"]["name"] for result in search_results]) == set(
+        [dataset_manifest["name"] for dataset_manifest in dataset_manifests]
+    )
+
+    # Search for application
+    search_results = await artifact_manager.list(
+        parent_id=collection.id,
+        keywords=["SQLite"],
+        filters={"type": "application"},
+        mode="AND",
+    )
+    assert len(search_results) == 1
+
+    # Clean up by deleting datasets and the collection
+    for dataset in datasets:
+        await artifact_manager.delete(artifact_id=dataset.id)
+    await artifact_manager.delete(artifact_id=collection.id)
+
+
+async def test_serve_artifact_endpoint(minio_server, fastapi_server, test_user_token):
     """Test the artifact serving endpoint."""
-    api = await connect_to_server({"name": "test-client", "server_url": SERVER_URL})
+    api = await connect_to_server(
+        {"name": "test-client", "server_url": SERVER_URL, "token": test_user_token}
+    )
     artifact_manager = await api.get_service("public/artifact-manager")
 
-    # Create a public collection
+    # Create a public collection and retrieve the UUID
     collection_manifest = {
         "name": "Public Dataset Gallery",
         "description": "A public collection for organizing datasets",
-        "type": "collection",
     }
-    await artifact_manager.create(
-        prefix="public/collections/dataset-gallery",
+    collection_info = await artifact_manager.create(
+        type="collection",
+        alias="public-dataset-gallery",
         manifest=collection_manifest,
-        # allow public read access and create access for authenticated users
-        permissions={"*": "r", "@": "r+"},
-        orphan=True,
+        config={"permissions": {"*": "r", "@": "r+"}},
     )
 
+    # list artifact in current workspace
+    artifacts = await artifact_manager.list()
+    assert collection_info.id in [artifact["id"] for artifact in artifacts]
     # Create an artifact inside the public collection
     dataset_manifest = {
         "name": "Public Example Dataset",
         "description": "A public dataset with example data",
-        "type": "dataset",
     }
-    await artifact_manager.create(
-        prefix="public/collections/dataset-gallery/public-example-dataset",
+    dataset = await artifact_manager.create(
+        type="dataset",
+        parent_id=collection_info.id,
         manifest=dataset_manifest,
-        stage=True,
-        # permissions={"*": "r+"}, # This is not necessary since the collection is already public
+        version="stage",
     )
 
     # Commit the artifact
-    await artifact_manager.commit(
-        prefix="public/collections/dataset-gallery/public-example-dataset"
-    )
+    await artifact_manager.commit(artifact_id=dataset.id)
 
     # Ensure the public artifact is available via HTTP
     response = requests.get(
-        f"{SERVER_URL}/{api.config.workspace}/artifacts/public/collections/dataset-gallery/public-example-dataset"
+        f"{SERVER_URL}/{api.config.workspace}/artifacts/{dataset.id}"
     )
     assert response.status_code == 200
-    assert "Public Example Dataset" in response.json()["name"]
+    assert "Public Example Dataset" in response.json()["manifest"]["name"]
 
     # Now create a non-public collection
     private_collection_manifest = {
         "name": "Private Dataset Gallery",
         "description": "A private collection for organizing datasets",
-        "type": "collection",
     }
-    await artifact_manager.create(
-        prefix="collections/private-dataset-gallery",
+    private_collection = await artifact_manager.create(
+        type="collection",
         manifest=private_collection_manifest,
-        permissions={},
-        orphan=True,
     )
 
     # Create an artifact inside the private collection
     private_dataset_manifest = {
         "name": "Private Example Dataset",
         "description": "A private dataset with example data",
-        "type": "dataset",
         "custom_key": 192,
     }
-    await artifact_manager.create(
-        prefix="collections/private-dataset-gallery/private-example-dataset",
+    private_dataset = await artifact_manager.create(
+        type="dataset",
+        parent_id=private_collection.id,
         manifest=private_dataset_manifest,
-        stage=True,
+        version="stage",
     )
 
     # Commit the private artifact
-    await artifact_manager.commit(
-        prefix="collections/private-dataset-gallery/private-example-dataset"
-    )
+    await artifact_manager.commit(artifact_id=private_dataset.id)
 
     token = await api.generate_token()
     # Ensure the private artifact is available via HTTP (requires authentication or special permissions)
     response = requests.get(
-        f"{SERVER_URL}/{api.config.workspace}/artifacts/collections/private-dataset-gallery/private-example-dataset",
+        f"{SERVER_URL}/{api.config.workspace}/artifacts/{private_dataset.id}",
         headers={"Authorization": f"Bearer {token}"},
     )
-
     assert response.status_code == 200
-    assert "Private Example Dataset" in response.json()["name"]
-    assert response.json()["custom_key"] == 192
+    assert "Private Example Dataset" in response.json()["manifest"]["name"]
+    assert response.json()["manifest"]["custom_key"] == 192
 
-    # If no authentication is provided, the server should return a 401 Unauthorized status code
+    # If no authentication is provided, the server should return a 403 Forbidden status code
     response = requests.get(
-        f"{SERVER_URL}/{api.config.workspace}/artifacts/collections/private-dataset-gallery/private-example-dataset"
+        f"{SERVER_URL}/{api.config.workspace}/artifacts/{private_dataset.id}"
     )
     assert response.status_code == 403
 
 
+async def test_http_file_and_directory_endpoint(
+    minio_server, fastapi_server, test_user_token
+):
+    """Test the HTTP file serving and directory listing endpoint."""
+
+    # Connect and get the artifact manager service
+    api = await connect_to_server(
+        {"name": "test-client", "server_url": SERVER_URL, "token": test_user_token}
+    )
+    artifact_manager = await api.get_service("public/artifact-manager")
+
+    # Create a collection and retrieve the UUID
+    collection_manifest = {
+        "name": "test-collection",
+        "description": "A test collection",
+    }
+    collection = await artifact_manager.create(
+        type="collection",
+        manifest=collection_manifest,
+        config={"permissions": {"*": "r", "@": "r+"}},
+    )
+
+    # Create a dataset within the collection
+    dataset_manifest = {
+        "name": "test-dataset",
+        "description": "A test dataset",
+    }
+    dataset = await artifact_manager.create(
+        type="dataset",
+        parent_id=collection.id,
+        manifest=dataset_manifest,
+        version="stage",
+    )
+
+    # Add a file to the dataset artifact
+    file_contents = "file contents of example.txt"
+    put_url = await artifact_manager.put_file(
+        artifact_id=dataset.id,
+        file_path="example.txt",
+        download_weight=1,
+    )
+    response = requests.put(put_url, data=file_contents)
+    assert response.ok
+
+    # Commit the dataset artifact
+    await artifact_manager.commit(artifact_id=dataset.id)
+
+    files = await artifact_manager.list_files(artifact_id=dataset.id)
+    assert len(files) == 1
+
+    # Retrieve the file via HTTP
+    response = requests.get(
+        f"{SERVER_URL}/{api.config.workspace}/artifacts/{dataset.id}/files/example.txt"
+    )
+    assert response.status_code == 200
+    assert response.text == file_contents
+
+    # Check download count increment
+    artifact = await artifact_manager.read(
+        artifact_id=dataset.id,
+    )
+    assert artifact["download_count"] == 1
+
+    # Attempt to list directory contents (should be successful after attempting file)
+    response = requests.get(
+        f"{SERVER_URL}/{api.config.workspace}/artifacts/{dataset.id}/files/"
+    )
+    assert response.status_code == 200
+    assert "example.txt" in [file["name"] for file in response.json()]
+
+
 async def test_artifact_permissions(
     minio_server, fastapi_server, test_user_token, test_user_token_2
 ):
     """Test artifact permissions."""
     api = await connect_to_server(
-        {"name": "test-client", "server_url": SERVER_URL, "token": test_user_token}
+        {
+            "name": "test-client",
+            "server_url": SERVER_URL,
+            "token": test_user_token,
+            "token": test_user_token,
+        }
     )
     artifact_manager = await api.get_service("public/artifact-manager")
 
-    # Create a public collection
+    # Create a public collection and get its UUID
     collection_manifest = {
         "name": "Public Dataset Gallery",
         "description": "A public collection for organizing datasets",
-        "type": "collection",
     }
-    await artifact_manager.create(
-        prefix="public/collections/dataset-gallery",
+    collection = await artifact_manager.create(
+        type="collection",
         manifest=collection_manifest,
-        # allow public read access and create access for authenticated users
-        permissions={"*": "r", "@": "r+"},
-        orphan=True,
-    )
-    await artifact_manager.reset_stats(
-        prefix=f"/{api.config.workspace}/public/collections/dataset-gallery"
+        config={"permissions": {"*": "r", "@": "r+"}},
     )
 
+    # Verify that anonymous access to the collection works
     api_anonymous = await connect_to_server(
         {"name": "test-client", "server_url": SERVER_URL}
     )
@@ -157,136 +329,240 @@ async def test_artifact_permissions(
         "public/artifact-manager"
     )
 
-    collection = await artifact_manager_anonymous.read(
-        prefix=f"/{api.config.workspace}/public/collections/dataset-gallery"
-    )
-    assert collection["name"] == "Public Dataset Gallery"
+    collection = await artifact_manager_anonymous.read(artifact_id=collection.id)
+    assert collection["manifest"]["name"] == "Public Dataset Gallery"
 
-    # Create an artifact inside the public collection
+    # Attempt to create an artifact inside the collection as an anonymous user
     dataset_manifest = {
         "name": "Public Example Dataset",
         "description": "A public dataset with example data",
-        "type": "dataset",
     }
     with pytest.raises(
-        Exception, match=r".*PermissionError: User does not have permission.*"
+        Exception, match=r".*User does not have permission to perform the operation.*"
     ):
         await artifact_manager_anonymous.create(
-            prefix=f"/{api.config.workspace}/public/collections/dataset-gallery/public-example-dataset",
+            type="dataset",
+            parent_id=collection.id,
             manifest=dataset_manifest,
         )
 
+    # Verify that anonymous user cannot reset stats on the public collection
     with pytest.raises(
         Exception,
         match=r".*PermissionError: User does not have permission to perform the operation.*",
     ):
-        await artifact_manager_anonymous.reset_stats(
-            prefix=f"/{api.config.workspace}/public/collections/dataset-gallery"
-        )
+        await artifact_manager_anonymous.reset_stats(artifact_id=collection.id)
 
+    # Authenticated user 2 can create a child artifact within the collection
     api_2 = await connect_to_server(
-        {"name": "test-client", "server_url": SERVER_URL, "token": test_user_token_2}
+        {
+            "name": "test-client",
+            "server_url": SERVER_URL,
+            "token": test_user_token,
+            "token": test_user_token_2,
+        }
     )
     artifact_manager_2 = await api_2.get_service("public/artifact-manager")
 
-    # authenticaed user can create artifact
-    await artifact_manager_2.create(
-        prefix=f"/{api.config.workspace}/public/collections/dataset-gallery/public-example-dataset",
+    # Create a dataset as a child of the public collection
+    dataset = await artifact_manager_2.create(
+        parent_id=collection.id,
         manifest=dataset_manifest,
     )
-    # but can't reset stats
+
+    await artifact_manager_anonymous.read(artifact_id=dataset.id)
+
+    # User 2 cannot reset stats on the newly created dataset
     with pytest.raises(
         Exception,
         match=r".*PermissionError: User does not have permission to perform the operation.*",
     ):
-        await artifact_manager_anonymous.reset_stats(
-            prefix=f"/{api.config.workspace}/public/collections/dataset-gallery/public-example-dataset"
-        )
+        await artifact_manager_anonymous.reset_stats(artifact_id=dataset.id)
+
+
+async def test_versioning_artifacts(
+    minio_server, fastapi_server_sqlite, test_user_token
+):
+    """Test versioning features for artifacts."""
+
+    # Set up connection and artifact manager
+    api = await connect_to_server(
+        {
+            "name": "sqlite test client",
+            "server_url": SERVER_URL_SQLITE,
+            "token": test_user_token,
+        }
+    )
+    artifact_manager = await api.get_service("public/artifact-manager")
+
+    # Create an artifact with an initial version
+    initial_manifest = {
+        "name": "Versioned Artifact",
+        "description": "Artifact to test versioning",
+    }
+    artifact = await artifact_manager.create(
+        type="dataset",
+        manifest=initial_manifest,
+        version="v1",
+    )
+
+    # Verify the initial version
+    artifact_data = await artifact_manager.read(artifact_id=artifact.id, version="v1")
+    assert artifact_data["versions"][-1]["version"] == "v1"
+
+    # Stage a new version
+    updated_manifest = {
+        "name": "Versioned Artifact",
+        "description": "Updated version description",
+    }
+    await artifact_manager.edit(
+        artifact_id=artifact.id,
+        manifest=updated_manifest,
+        version="stage",
+    )
+
+    # Read the staged version to confirm the update
+    staged_data = await artifact_manager.read(artifact_id=artifact.id, version="stage")
+    assert staged_data["manifest"]["description"] == "Updated version description"
+
+    # Commit the staged version
+    await artifact_manager.commit(artifact_id=artifact.id, version="v2")
+
+    # Verify the staged version is accessible
+    staged_data = await artifact_manager.read(artifact_id=artifact.id, version="v2")
+    assert staged_data["manifest"]["description"] == "Updated version description"
+
+    # Test reading the latest version
+    latest_data = await artifact_manager.read(artifact_id=artifact.id, version="latest")
+    assert latest_data["manifest"]["description"] == "Updated version description"
+
+    # Attempt to read a non-existent version
+    with pytest.raises(Exception, match=r".*Artifact version 'v999' does not exist.*"):
+        await artifact_manager.read(artifact_id=artifact.id, version="v999")
+
+    # Clean up by deleting the artifact
+    await artifact_manager.delete(artifact_id=artifact.id)
 
 
-async def test_artifact_prefix_pattern(minio_server, fastapi_server):
-    """Test artifact prefix pattern."""
-    api = await connect_to_server({"name": "test-client", "server_url": SERVER_URL})
+async def test_artifact_alias_pattern(minio_server, fastapi_server, test_user_token):
+    """Test artifact alias pattern functionality."""
+    api = await connect_to_server(
+        {"name": "test-client", "server_url": SERVER_URL, "token": test_user_token}
+    )
     artifact_manager = await api.get_service("public/artifact-manager")
-    # Create a collection for testing prefix pattern
+
+    # Create a collection with an alias pattern configuration
     collection_manifest = {
-        "name": "Prefix Pattern Collection",
-        "description": "A collection to test prefix pattern",
-        "type": "collection",
+        "name": "Alias Pattern Collection",
+        "description": "A collection to test alias pattern functionality",
     }
-    await artifact_manager.create(
-        prefix="collections/prefix-pattern-collection",
+    collection_config = {
+        "id_parts": {
+            "animals": ["dog", "cat", "bird"],
+            "colors": ["red", "blue", "green"],
+        }
+    }
+    collection = await artifact_manager.create(
+        type="collection",
         manifest=collection_manifest,
-        config={
-            "id_parts": {
-                "animals": ["dog", "cat", "bird"],
-                "colors": ["red", "blue", "green"],
-            }
-        },
-        stage=False,
-        orphan=True,
+        config=collection_config,
     )
-    metadata = await artifact_manager.create(
-        prefix="collections/prefix-pattern-collection/{colors}-{animals}",
-        manifest={"name": "My test data", "type": "dataset"},
-        stage=True,
+
+    # Create an artifact with alias pattern under the collection
+    dataset_manifest = {"name": "My test data"}
+    alias_pattern = "{colors}-{animals}"
+    dataset = await artifact_manager.create(
+        type="dataset",
+        parent_id=collection.id,
+        manifest=dataset_manifest,
+        alias=alias_pattern,
+        version="stage",
+    )
+
+    # Fetch metadata to confirm the generated alias
+    dataset_metadata = await artifact_manager.read(
+        artifact_id=dataset.id, version="stage"
     )
-    prefix = metadata[".prefix"]
-    color, animal = prefix.split("/")[-1].split("-")
+    generated_alias = dataset_metadata["alias"].split("/")[-1]
+
+    # Extract and validate parts of the generated alias
+    color, animal = generated_alias.split("-")
     assert color in ["red", "blue", "green"]
     assert animal in ["dog", "cat", "bird"]
 
+    # read the artifact using the generated alias
+    dataset = await artifact_manager.read(dataset_metadata["alias"], version="stage")
+    # Verify the alias pattern is correctly applied
+    assert dataset["manifest"]["name"] == "My test data"
+
 
-async def test_publish_artifact(minio_server, fastapi_server):
+async def test_publish_artifact(minio_server, fastapi_server, test_user_token):
     """Test publishing an artifact."""
-    api = await connect_to_server({"name": "test-client", "server_url": SERVER_URL})
+    api = await connect_to_server(
+        {"name": "test-client", "server_url": SERVER_URL, "token": test_user_token}
+    )
     artifact_manager = await api.get_service("public/artifact-manager")
+
     # Create a collection for testing publishing
     collection_manifest = {
         "name": "Publish Test Collection",
         "description": "A collection to test publishing",
-        "type": "collection",
     }
 
-    access_token = os.environ.get("SANDBOX_ZENODO_TOKEN")
-    assert access_token, "Please set SANDBOX_ZENODO_TOKEN environment variable"
-    collection_config = {"archives": {"sandbox_zenodo": {"access_token": access_token}}}
-    await artifact_manager.create(
-        prefix="test-collection",
-        manifest=collection_manifest,
-        config=collection_config,
-        stage=False,
-        orphan=True,
+    # Define secrets, including the Zenodo sandbox access token
+    secrets = {"SANDBOX_ZENODO_ACCESS_TOKEN": os.environ.get("SANDBOX_ZENODO_TOKEN")}
+    assert secrets[
+        "SANDBOX_ZENODO_ACCESS_TOKEN"
+    ], "Please set SANDBOX_ZENODO_TOKEN environment variable"
+
+    # Create the collection artifact with the necessary secrets
+    collection = await artifact_manager.create(
+        type="collection", manifest=collection_manifest, secrets=secrets
     )
 
-    # Create an artifact (orphan)
+    # Create an artifact within the collection with a publish target to sandbox Zenodo
     dataset_manifest = {
         "name": "Test Dataset",
         "description": "A test dataset to publish",
-        "type": "dataset",
     }
 
-    metadata = await artifact_manager.create(
-        prefix="test-collection/{zenodo_id}",
+    dataset = await artifact_manager.create(
+        type="dataset",
+        parent_id=collection.id,
         manifest=dataset_manifest,
-        stage=True,
+        version="stage",
         publish_to="sandbox_zenodo",
     )
-    # add files
+
+    # Add a file to the artifact
     put_url = await artifact_manager.put_file(
-        prefix=metadata[".prefix"],
+        artifact_id=dataset.id,
         file_path="example.txt",
     )
     source = "file contents of example.txt"
     response = requests.put(put_url, data=source)
     assert response.ok
-    await artifact_manager.commit(prefix=metadata[".prefix"])
-    await artifact_manager.publish(prefix=metadata[".prefix"], to="sandbox_zenodo")
 
+    # Commit the artifact after adding the file
+    await artifact_manager.commit(artifact_id=dataset.id)
+
+    # Publish the artifact to Zenodo Sandbox
+    await artifact_manager.publish(artifact_id=dataset.id, to="sandbox_zenodo")
+
+    # Retrieve and validate the updated metadata, confirming it includes publication info
+    dataset_metadata = await artifact_manager.read(artifact_id=dataset.id)
+    assert (
+        "zenodo" in dataset_metadata["config"]
+    ), "Zenodo publication information missing in config"
 
-async def test_artifact_filtering(minio_server, fastapi_server):
+
+async def test_artifact_filtering(
+    minio_server, fastapi_server, test_user_token, test_user_token_2
+):
     """Test artifact filtering with various fields, including type, created_by, view_count, and stage."""
-    api = await connect_to_server({"name": "test-client", "server_url": SERVER_URL})
+    api = await connect_to_server(
+        {"name": "test-client", "server_url": SERVER_URL, "token": test_user_token}
+    )
     artifact_manager = await api.get_service("public/artifact-manager")
     user_id1 = api.config.user["id"]
 
@@ -294,15 +570,13 @@ async def test_artifact_filtering(minio_server, fastapi_server):
     collection_manifest = {
         "name": "Filter Test Collection",
         "description": "A collection to test filter functionality",
-        "type": "collection",
     }
-    await artifact_manager.create(
-        prefix="collections/filter-test-collection",
+    collection = await artifact_manager.create(
+        type="collection",
         manifest=collection_manifest,
-        config={"summary_fields": ["name", "description", ".*", "type"]},
-        stage=False,
-        orphan=True,
-        permissions={"*": "r+"},
+        config={
+            "permissions": {"*": "r+"},
+        },
     )
 
     # Create artifacts with varied attributes inside the collection (created by user_id1)
@@ -311,282 +585,296 @@ async def test_artifact_filtering(minio_server, fastapi_server):
         artifact_manifest = {
             "name": f"Artifact {i}",
             "description": f"Description for artifact {i}",
-            "type": "dataset" if i % 2 == 0 else "application",
         }
-        await artifact_manager.create(
-            prefix=f"collections/filter-test-collection/artifact-{i}",
+        artifact = await artifact_manager.create(
+            type="dataset" if i % 2 == 0 else "application",
+            parent_id=collection.id,
             manifest=artifact_manifest,
-            stage=(i % 2 == 0),
+            version="stage" if (i % 2 == 0) else None,
         )
-        await artifact_manager.commit(
-            prefix=f"collections/filter-test-collection/artifact-{i}"
+        await artifact_manager.read(
+            artifact_id=artifact.id, version="stage" if (i % 2 == 0) else None
         )
-
-        if i % 2 == 0:
-            await artifact_manager.read(
-                prefix=f"collections/filter-test-collection/artifact-{i}"
-            )
-        created_artifacts.append(artifact_manifest)
+        created_artifacts.append(artifact)
 
     # Create a second client (user_id2) to create additional artifacts
-    api2 = await connect_to_server({"name": "test-client-2", "server_url": SERVER_URL})
+    api2 = await connect_to_server(
+        {"name": "test-client-2", "server_url": SERVER_URL, "token": test_user_token_2}
+    )
     artifact_manager2 = await api2.get_service("public/artifact-manager")
-    # user_id2 = api2.config.user["id"]
 
     for i in range(5, 10):
         artifact_manifest = {
             "name": f"Artifact {i}",
             "description": f"Description for artifact {i}",
-            "type": "dataset" if i % 2 == 0 else "application",
         }
-        await artifact_manager2.create(
-            prefix=f"/{api.config.workspace}/collections/filter-test-collection/artifact-{i}",
+        artifact = await artifact_manager2.create(
+            type="dataset" if i % 2 == 0 else "application",
+            parent_id=collection.id,
             manifest=artifact_manifest,
-            stage=(i % 2 == 0),
+            version="stage" if (i % 2 == 0) else None,
         )
-        created_artifacts.append(artifact_manifest)
+        if i % 2 == 0:
+            await artifact_manager2.commit(artifact_id=artifact.id)
+        created_artifacts.append(artifact)
 
     # Filter by `type`: Only datasets should be returned
     results = await artifact_manager.list(
-        prefix="collections/filter-test-collection",
-        filters={".type": "dataset"},
-        mode="AND",
+        parent_id=collection.id, filters={"type": "dataset"}, mode="AND"
     )
-    assert len(results) == 3
+    assert len(results) == 2
     for result in results:
-        assert result[".type"] == "dataset"
+        assert result["type"] == "dataset"
 
     # Filter by `created_by`: Only artifacts created by user_id1 should be returned
     results = await artifact_manager.list(
-        prefix="collections/filter-test-collection",
-        filters={".created_by": user_id1, ".stage": False},
+        parent_id=collection.id,
+        filters={"created_by": user_id1, "stage": False},
         mode="AND",
     )
-    assert len(results) == 3
+    assert len(results) == 2
     for result in results:
-        assert result[".created_by"] == user_id1
+        assert result["created_by"] == user_id1
 
-    # Filter by `view_count`: Return artifacts that have been viewed at least once (incremented for every even-indexed artifact)
+    # Filter by `view_count`: Return artifacts that have been viewed at least once
     results = await artifact_manager.list(
-        prefix="collections/filter-test-collection",
-        filters={".view_count": [1, None]},  # Filter for any view count >= 1
+        parent_id=collection.id,
+        filters={"view_count": [1, None]},  # Filter for any view count >= 1
         mode="AND",
     )
-    assert len(results) == 3
+    assert len(results) == 2
     for result in results:
-        assert result[".view_count"] >= 1
+        assert result["view_count"] >= 1
 
     # Filter by `stage`: Only staged artifacts should be returned
     results = await artifact_manager.list(
-        prefix="collections/filter-test-collection",
-        filters={".stage": True},
-        mode="AND",
+        parent_id=collection.id, filters={"stage": True}, mode="AND"
     )
-    assert len(results) == 2
+    assert len(results) == 3
 
     # Clean up by deleting the artifacts and the collection
     for i in range(10):
-        await artifact_manager.delete(
-            prefix=f"collections/filter-test-collection/artifact-{i}"
-        )
-    await artifact_manager.delete(prefix="collections/filter-test-collection")
+        artifact = created_artifacts[i]
+        await artifact_manager.delete(artifact_id=artifact.id)
+    await artifact_manager.delete(artifact_id=collection.id)
 
 
-async def test_http_artifact_endpoint(minio_server, fastapi_server):
-    """Test the HTTP artifact serving endpoint."""
+async def test_http_artifact_endpoint(minio_server, fastapi_server, test_user_token):
+    """Test the HTTP artifact serving endpoint for retrieving files and listing directories."""
 
-    # create a artifact with a file
+    # Set up connection and artifact manager
     api = await connect_to_server(
-        {"name": "test deploy client", "server_url": SERVER_URL}
+        {
+            "name": "test deploy client",
+            "server_url": SERVER_URL,
+            "token": test_user_token,
+        }
     )
     artifact_manager = await api.get_service("public/artifact-manager")
 
+    # Create a collection
     collection_manifest = {
-        "name": "test-collection",
-        "description": "A test collection",
-        "type": "collection",
+        "name": "Test Collection",
+        "description": "A test collection for HTTP endpoint",
     }
-    await artifact_manager.create(
-        prefix="collections/test-collection",
+    collection = await artifact_manager.create(
+        type="collection",
         manifest=collection_manifest,
-        stage=False,
-        permissions={"*": "r", "@": "r+"},
-        orphan=True,
+        config={"permissions": {"*": "r", "@": "r+"}},
     )
 
+    # Create an artifact within the collection
     dataset_manifest = {
-        "name": "test-dataset",
-        "description": "A test dataset to edit",
-        "type": "dataset",
+        "name": "Test Dataset",
+        "description": "A test dataset for HTTP endpoint",
     }
-    await artifact_manager.create(
-        prefix="collections/test-collection/test-dataset",
+    dataset = await artifact_manager.create(
+        type="dataset",
+        parent_id=collection.id,
         manifest=dataset_manifest,
-        stage=True,
-        permissions={"*": "r", "@": "r+"},
+        version="stage",
+        config={"permissions": {"*": "r", "@": "r+"}},
     )
+
     # Add a file to the artifact
-    source = "file contents of example.txt"
+    file_content = "File contents of example.txt"
     put_url = await artifact_manager.put_file(
-        prefix="collections/test-collection/test-dataset",
-        file_path="example.txt",
-        options={"download_weight": 1},
+        artifact_id=dataset.id, file_path="example.txt", download_weight=1
     )
-    response = requests.put(put_url, data=source)
+    response = requests.put(put_url, data=file_content)
     assert response.ok
-    await artifact_manager.commit(prefix="collections/test-collection/test-dataset")
 
-    # get the dataset manifest via http
+    # Commit the artifact
+    await artifact_manager.commit(artifact_id=dataset.id)
+
+    # Retrieve the dataset manifest via HTTP
     response = requests.get(
-        f"{SERVER_URL}/{api.config.workspace}/artifacts/collections/test-collection/test-dataset"
+        f"{SERVER_URL}/{api.config.workspace}/artifacts/{dataset.id}"
     )
     assert response.status_code == 200
-    assert "test-dataset" in response.json()["name"]
+    assert "Test Dataset" in response.json()["manifest"]["name"]
 
-    # get file list
+    # List the files in the dataset directory via HTTP
     response = requests.get(
-        f"{SERVER_URL}/{api.config.workspace}/artifacts/collections/test-collection/test-dataset/__files__"
+        f"{SERVER_URL}/{api.config.workspace}/artifacts/{dataset.id}/files/"
     )
     assert response.status_code == 200
-    assert "example.txt" in response.json()[0]["name"]
+    assert "example.txt" in [file["name"] for file in response.json()]
 
-    # get the file via http
+    # Retrieve the file content via HTTP
     response = requests.get(
-        f"{SERVER_URL}/{api.config.workspace}/artifacts/collections/test-collection/test-dataset/__files__/example.txt"
+        f"{SERVER_URL}/{api.config.workspace}/artifacts/{dataset.id}/files/example.txt"
     )
     assert response.status_code == 200
-    assert response.text == source
+    assert response.text == file_content
 
-    # check download count
-    artifact = await artifact_manager.read(
-        prefix="collections/test-collection/test-dataset",
-        include_metadata=True,
+    # Verify that download count has incremented
+    artifact_data = await artifact_manager.read(
+        artifact_id=dataset.id,
     )
-    assert artifact[".download_count"] == 1
+    assert artifact_data["download_count"] == 1
 
-    # get childre of the collection
+    # Retrieve the collection's children to verify the dataset presence
     response = requests.get(
-        f"{SERVER_URL}/{api.config.workspace}/artifacts/collections/test-collection/__children__"
+        f"{SERVER_URL}/{api.config.workspace}/artifacts/{collection.id}/children"
     )
     assert response.status_code == 200
-    assert "test-dataset" in response.json()[0]["name"]
+    item = find_item(response.json(), "id", dataset.id)
+    manifest = item["manifest"]
+    assert "Test Dataset" == manifest["name"]
 
 
-async def test_edit_existing_artifact(minio_server, fastapi_server):
+async def test_edit_existing_artifact(minio_server, fastapi_server, test_user_token):
     """Test editing an existing artifact."""
+
+    # Set up connection and artifact manager
     api = await connect_to_server(
-        {"name": "test deploy client", "server_url": SERVER_URL}
+        {
+            "name": "test deploy client",
+            "server_url": SERVER_URL,
+            "token": test_user_token,
+        }
     )
     artifact_manager = await api.get_service("public/artifact-manager")
 
-    # Create a collection first
+    # Create a collection for testing
     collection_manifest = {
-        "name": "edit-test-collection",
-        "description": "A test collection for editing artifacts",
-        "type": "collection",
+        "name": "Edit Test Collection",
+        "description": "A collection for testing artifact edits",
     }
-    await artifact_manager.create(
-        prefix="collections/edit-test-collection",
+    collection = await artifact_manager.create(
+        type="collection",
         manifest=collection_manifest,
-        stage=False,
-        orphan=True,
+        config={"permissions": {"*": "r", "@": "r+"}},
     )
 
-    # Create an artifact (a dataset in this case) within the collection
+    # Create an artifact (dataset) within the collection
     dataset_manifest = {
-        "name": "edit-test-dataset",
-        "description": "A test dataset to edit",
-        "type": "dataset",
+        "name": "Edit Test Dataset",
+        "description": "A dataset to test editing",
     }
-
-    await artifact_manager.create(
-        prefix="collections/edit-test-collection/edit-test-dataset",
+    dataset = await artifact_manager.create(
+        type="dataset",
+        parent_id=collection.id,
         manifest=dataset_manifest,
-        stage=True,
+        version="stage",
     )
 
-    # Commit the dataset artifact (finalize it)
-    await artifact_manager.commit(
-        prefix="collections/edit-test-collection/edit-test-dataset"
-    )
-
-    # Ensure that the dataset appears in the collection's index
-    items = await artifact_manager.list(
-        prefix="collections/edit-test-collection",
-        order_by="last_modified",
-        summary_fields=[".prefix"],
-    )
-    assert find_item(
-        items,
-        ".prefix",
-        f"collections/edit-test-collection/edit-test-dataset",
-    )
+    # Commit the artifact
+    dataset = await artifact_manager.commit(artifact_id=dataset.id)
+    versions = dataset["versions"]
 
-    collection = await artifact_manager.read(
-        prefix="collections/edit-test-collection", include_metadata=True
-    )
-    initial_view_count = collection[".view_count"]
-    assert initial_view_count > 0
-    collection = await artifact_manager.read(
-        prefix="collections/edit-test-collection", include_metadata=True
-    )
-    assert collection[".view_count"] == initial_view_count + 1
+    # Verify the artifact is listed under the collection
+    items = await artifact_manager.list(parent_id=collection.id)
+    item = find_item(items, "id", dataset.id)
+    assert item["manifest"]["name"] == "Edit Test Dataset"
 
     # Edit the artifact's manifest
     edited_manifest = {
-        "name": "edit-test-dataset",
-        "description": "Edited description of the test dataset",
-        "type": "dataset",
+        "name": "Edit Test Dataset",
+        "description": "Updated description of the test dataset",
         "custom_key": 19222,
     }
 
-    # Call edit on the artifact
+    # Edit the artifact using the new manifest
     await artifact_manager.edit(
-        prefix="collections/edit-test-collection/edit-test-dataset",
+        type="dataset",
+        artifact_id=dataset.id,
         manifest=edited_manifest,
-        stage=True,
+        version="stage",
     )
 
-    # Ensure that the changes are reflected in the manifest (read the artifact in stage mode)
-    updated_manifest = await artifact_manager.read(
-        prefix="collections/edit-test-collection/edit-test-dataset", stage=True
+    # Verify the manifest updates are staged (not committed yet)
+    staged_artifact = await artifact_manager.read(
+        artifact_id=dataset.id, version="stage"
     )
-    assert updated_manifest["description"] == "Edited description of the test dataset"
-    assert updated_manifest["custom_key"] == 19222
+    staged_manifest = staged_artifact["manifest"]
+    assert staged_manifest["description"] == "Updated description of the test dataset"
+    assert staged_manifest["custom_key"] == 19222
 
     # Add a file to the artifact
-    source = "file contents of example.txt"
+    file_content = "File contents of example.txt"
     put_url = await artifact_manager.put_file(
-        prefix="collections/edit-test-collection/edit-test-dataset",
-        file_path="example.txt",
+        artifact_id=dataset.id, file_path="example.txt"
     )
-
-    # Use the pre-signed URL to upload the file
-    response = requests.put(put_url, data=source)
+    response = requests.put(put_url, data=file_content)
     assert response.ok
 
-    # Commit the artifact again after editing
-    await artifact_manager.commit(
-        prefix="collections/edit-test-collection/edit-test-dataset"
+    # Commit the artifact after editing, no version specified, so it will overwrite the latest version
+    dataset = await artifact_manager.commit(artifact_id=dataset.id)
+    assert len(versions) == len(dataset["versions"])
+
+    # Verify the committed manifest reflects the edited data
+    committed_artifact = await artifact_manager.read(artifact_id=dataset.id)
+    committed_manifest = committed_artifact["manifest"]
+    assert (
+        committed_manifest["description"] == "Updated description of the test dataset"
     )
+    assert committed_manifest["custom_key"] == 19222
 
-    # Ensure that the file is included in the manifest
-    manifest_data = await artifact_manager.read(
-        prefix="collections/edit-test-collection/edit-test-dataset"
+    # Retrieve and verify the uploaded file
+    get_url = await artifact_manager.get_file(
+        artifact_id=dataset.id, path="example.txt"
     )
-    assert manifest_data["custom_key"] == 19222
+    response = requests.get(get_url)
+    assert response.status_code == 200
+    assert response.text == file_content
 
-    # Clean up by deleting the dataset and collection
-    await artifact_manager.delete(
-        prefix="collections/edit-test-collection/edit-test-dataset"
+    # Edit it directly
+    await artifact_manager.edit(
+        type="dataset",
+        artifact_id=dataset.id,
+        manifest={"name": "Edit Test Dataset"},
+        version="new",  # auto will create a new version
     )
-    await artifact_manager.delete(prefix="collections/edit-test-collection")
+
+    # Verify the manifest updates are committed
+    committed_artifact = await artifact_manager.read(artifact_id=dataset.id)
+    committed_manifest = committed_artifact["manifest"]
+    assert committed_manifest["name"] == "Edit Test Dataset"
+    assert len(versions) + 1 == len(committed_artifact["versions"])
+
+    # Check list of artifacts
+    items = await artifact_manager.list(parent_id=collection.id)
+    item = find_item(items, "id", committed_artifact.id)
+
+    # Clean up by deleting the dataset and collection
+    await artifact_manager.delete(artifact_id=dataset.id)
+    await artifact_manager.delete(artifact_id=collection.id)
 
 
-async def test_artifact_schema_validation(minio_server, fastapi_server):
+async def test_artifact_schema_validation(
+    minio_server, fastapi_server, test_user_token
+):
     """Test schema validation when committing artifacts to a collection."""
+
+    # Set up connection and artifact manager
     api = await connect_to_server(
-        {"name": "test deploy client", "server_url": SERVER_URL}
+        {
+            "name": "test deploy client",
+            "server_url": SERVER_URL,
+            "token": test_user_token,
+        }
     )
     artifact_manager = await api.get_service("public/artifact-manager")
 
@@ -596,101 +884,86 @@ async def test_artifact_schema_validation(minio_server, fastapi_server):
         "properties": {
             "name": {"type": "string"},
             "description": {"type": "string"},
-            "type": {"type": "string", "enum": ["dataset", "model", "application"]},
+            "record_type": {
+                "type": "string",
+                "enum": ["dataset", "model", "application"],
+            },
         },
-        "required": ["name", "description", "type"],
+        "required": ["name", "description", "record_type"],
     }
 
     # Create a collection with the schema
     collection_manifest = {
         "name": "Schema Test Collection",
-        "description": "A test collection with schema validation",
-        "type": "collection",
+        "description": "A collection with schema validation",
     }
-
-    await artifact_manager.create(
-        prefix="collections/schema-test-collection",
+    collection = await artifact_manager.create(
+        type="collection",
         manifest=collection_manifest,
-        config={"collection_schema": collection_schema},
-        stage=False,
-        orphan=True,
+        config={
+            "collection_schema": collection_schema,
+            "permissions": {"*": "r", "@": "r+"},
+        },
     )
 
     # Create a valid dataset artifact that conforms to the schema
     valid_dataset_manifest = {
+        "record_type": "dataset",
         "name": "Valid Dataset",
-        "description": "A dataset that conforms to the collection schema",
-        "type": "dataset",
+        "description": "A dataset that conforms to the schema",
     }
-
-    await artifact_manager.create(
-        prefix="collections/schema-test-collection/valid-dataset",
-        manifest=valid_dataset_manifest,
-        stage=True,
+    valid_dataset = await artifact_manager.create(
+        parent_id=collection.id, manifest=valid_dataset_manifest, version="stage"
     )
 
     # Add a file to the valid dataset artifact
-    source = "file contents of valid_file.txt"
+    file_content = "Contents of valid_file.txt"
     put_url = await artifact_manager.put_file(
-        prefix="collections/schema-test-collection/valid-dataset",
-        file_path="valid_file.txt",
+        artifact_id=valid_dataset.id, file_path="valid_file.txt"
     )
-
-    # Use the pre-signed URL to upload the file
-    response = requests.put(put_url, data=source)
+    response = requests.put(put_url, data=file_content)
     assert response.ok
 
-    # Commit the valid dataset artifact (finalize it)
-    await artifact_manager.commit(
-        prefix="collections/schema-test-collection/valid-dataset"
-    )
+    # Commit the valid dataset artifact (should succeed as it conforms to the schema)
+    await artifact_manager.commit(artifact_id=valid_dataset.id)
 
-    # Ensure that the dataset appears in the collection's index
-    items = await artifact_manager.list(
-        prefix="collections/schema-test-collection", summary_fields=[".prefix"]
-    )
-    assert find_item(
-        items,
-        ".prefix",
-        f"collections/schema-test-collection/valid-dataset",
-    )
+    # Confirm the dataset is listed in the collection
+    items = await artifact_manager.list(parent_id=collection.id)
+    item = find_item(items, "id", valid_dataset.id)
+    assert item["manifest"]["name"] == "Valid Dataset"
 
-    # Now, create an invalid dataset artifact that does not conform to the schema (missing required fields)
+    # Attempt to create an invalid dataset artifact (missing required 'description' field)
     invalid_dataset_manifest = {
         "name": "Invalid Dataset",
-        # "description" field is missing, which are required by the schema
-        "type": "dataset",
+        "record_type": "dataset",
     }
-
-    await artifact_manager.create(
-        prefix="collections/schema-test-collection/invalid-dataset",
-        manifest=invalid_dataset_manifest,
-        stage=True,
+    invalid_dataset = await artifact_manager.create(
+        parent_id=collection.id, manifest=invalid_dataset_manifest, version="stage"
     )
 
-    # Commit should raise a ValidationError due to the schema violation
+    # Commit should raise a validation error due to schema violation
     with pytest.raises(
-        Exception,
-        match=r".*ValidationError: 'description' is a required property.*",
+        Exception, match=r".*ValidationError: 'description' is a required property.*"
     ):
-        await artifact_manager.commit(
-            prefix="collections/schema-test-collection/invalid-dataset"
-        )
+        await artifact_manager.commit(artifact_id=invalid_dataset.id)
 
-    # Clean up by deleting the artifacts and the collection
-    await artifact_manager.delete(
-        prefix="collections/schema-test-collection/valid-dataset"
-    )
-    await artifact_manager.delete(
-        prefix="collections/schema-test-collection/invalid-dataset"
-    )
-    await artifact_manager.delete(prefix="collections/schema-test-collection")
+    # Clean up by deleting the valid dataset and the collection
+    await artifact_manager.delete(artifact_id=valid_dataset.id)
+    await artifact_manager.delete(artifact_id=collection.id)
 
 
-async def test_artifact_manager_with_collection(minio_server, fastapi_server):
-    """Test artifact controller with collections."""
+async def test_artifact_manager_with_collection(
+    minio_server, fastapi_server, test_user_token
+):
+    """Test artifact management within collections."""
+
+    # Connect to the server and set up the artifact manager
     api = await connect_to_server(
-        {"name": "test deploy client", "server_url": SERVER_URL}
+        {
+            "name": "test deploy client",
+            "server_url": SERVER_URL,
+            "token": test_user_token,
+        }
     )
     artifact_manager = await api.get_service("public/artifact-manager")
 
@@ -698,498 +971,468 @@ async def test_artifact_manager_with_collection(minio_server, fastapi_server):
     collection_manifest = {
         "name": "test-collection",
         "description": "A test collection",
-        "type": "collection",
     }
-
-    await artifact_manager.create(
-        prefix="collections/test-collection",
+    collection = await artifact_manager.create(
+        type="collection",
         manifest=collection_manifest,
-        config={"summary_fields": ["name", "description", ".*"]},
-        stage=False,
-        permissions={"*": "r", "@": "r+"},
-        orphan=True,
+        config={
+            "permissions": {"*": "r", "@": "r+"},
+        },
     )
 
-    # get the collection via http
+    # Retrieve the collection via HTTP to confirm creation
     response = requests.get(
-        f"{SERVER_URL}/{api.config.workspace}/artifacts/collections/test-collection"
+        f"{SERVER_URL}/{api.config.workspace}/artifacts/{collection.id}"
     )
     assert response.status_code == 200
-    assert "test-collection" in response.json()["name"]
+    assert "test-collection" in response.json()["manifest"]["name"]
 
-    # Create an artifact (a dataset in this case) within the collection
+    # Create a dataset artifact within the collection
     dataset_manifest = {
         "name": "test-dataset",
-        "description": "A test dataset in the collection",
-        "type": "dataset",
+        "description": "A test dataset within the collection",
     }
-
-    await artifact_manager.create(
-        prefix="collections/test-collection/test-dataset",
+    dataset = await artifact_manager.create(
+        type="dataset",
+        alias="test-dataset",
+        parent_id=collection.id,
         manifest=dataset_manifest,
-        stage=True,
+        version="stage",
     )
 
-    # Add a file to the artifact (test file)
-    source = "file contents of test.txt"
+    # Add a file to the dataset artifact
+    file_content = "file contents of test.txt"
     put_url = await artifact_manager.put_file(
-        prefix="collections/test-collection/test-dataset", file_path="test.txt"
+        artifact_id=dataset.id, file_path="test.txt"
     )
-
-    # Use the pre-signed URL to upload the file
-    response = requests.put(put_url, data=source)
+    response = requests.put(put_url, data=file_content)
     assert response.ok
 
-    # Ensure that the file is included in the manifest
-    manifest_data = await artifact_manager.read(
-        prefix="collections/test-collection/test-dataset", stage=True
-    )
-    assert manifest_data["name"] == "test-dataset"
-
-    # Test using absolute prefix
-    manifest_data = await artifact_manager.read(
-        prefix=f"/{api.config.workspace}/collections/test-collection/test-dataset",
-        stage=True,
+    # Confirm the dataset's inclusion in the collection
+    collection_items = await artifact_manager.list(
+        parent_id=collection.id, filters={"stage": True}
     )
-    assert manifest_data["name"] == "test-dataset"
+    item = find_item(collection_items, "id", dataset.id)
+    assert item["manifest"]["name"] == "test-dataset"
 
-    files = await artifact_manager.list_files(
-        prefix="collections/test-collection/test-dataset"
-    )
-    assert find_item(files, "name", "test.txt")
+    # Commit the dataset artifact to finalize it
+    await artifact_manager.commit(artifact_id=dataset.id)
 
-    items = await artifact_manager.list(
-        prefix="collections/test-collection",
-        filters={".stage": True},
-        summary_fields=[".prefix"],
-    )
-    assert find_item(
-        items,
-        ".prefix",
-        f"collections/test-collection/test-dataset",
-    )
+    # Verify the dataset is listed in the collection after commit
+    collection_items = await artifact_manager.list(parent_id=collection.id)
+    item = find_item(collection_items, "id", dataset.id)
+    dataset_item = item["manifest"]
+    assert dataset_item["description"] == "A test dataset within the collection"
+    assert item["view_count"] == 0
 
-    # Commit the artifact (finalize it)
-    await artifact_manager.commit(prefix="collections/test-collection/test-dataset")
+    # Increase the view count by reading the dataset
+    dataset_summary = await artifact_manager.read(artifact_id=dataset.id)
+    assert dataset_summary["view_count"] == 1
 
-    # Ensure that the dataset appears in the collection's index
-    items = await artifact_manager.list(prefix="collections/test-collection")
-    assert find_item(
-        items,
-        ".prefix",
-        f"collections/test-collection/test-dataset",
-    )
-    # all the summary fields should be in the collection
-    dataset_summary = find_item(items, "name", "test-dataset")
-    assert dataset_summary["name"] == "test-dataset"
-    assert dataset_summary["description"] == "A test dataset in the collection"
-    assert dataset_summary[".view_count"] == 0
+    # Verify that view count is reflected in the collection list
+    collection_items = await artifact_manager.list(parent_id=collection.id)
+    dataset_item = next(item for item in collection_items if item["id"] == dataset.id)
+    assert dataset_item["view_count"] == 1
 
-    # view the dataset
-    dataset_summary = await artifact_manager.read(
-        prefix="collections/test-collection/test-dataset", include_metadata=True
-    )
-    assert dataset_summary[".view_count"] == 1
-
-    # now we should have 1 view count
-    items = await artifact_manager.list(prefix="collections/test-collection")
-    dataset_summary = find_item(items, "name", "test-dataset")
-    assert dataset_summary[".view_count"] == 1
-
-    # Ensure that the manifest.yaml is finalized and the artifact is validated
-    artifacts = await artifact_manager.list(prefix="collections/test-collection")
-    assert find_item(artifacts, "name", "test-dataset")
-
-    # Retrieve the file via the get_file method
-    get_url = await artifact_manager.get_file(
-        prefix="collections/test-collection/test-dataset", path="test.txt"
-    )
+    # Retrieve the file in the dataset via HTTP to verify contents
+    get_url = await artifact_manager.get_file(artifact_id=dataset.id, path="test.txt")
     response = requests.get(get_url)
     assert response.ok
-    assert response.text == source
+    assert response.text == file_content
 
-    # Test overwriting the existing artifact within the collection
-    new_dataset_manifest = {
+    # Test overwriting the existing dataset artifact within the collection
+    updated_dataset_manifest = {
         "name": "test-dataset",
-        "description": "Overwritten test dataset in collection",
-        "type": "dataset",
+        "description": "Overwritten description for dataset",
     }
-
     await artifact_manager.create(
-        prefix="collections/test-collection/test-dataset",
-        manifest=new_dataset_manifest,
+        type="dataset",
+        parent_id=collection.id,
+        alias="test-dataset",
+        manifest=updated_dataset_manifest,
         overwrite=True,
-        stage=False,
     )
 
-    # Ensure that the old manifest has been overwritten
-    manifest_data = await artifact_manager.read(
-        prefix="collections/test-collection/test-dataset"
+    # Confirm that the description update is reflected
+    updated_manifest = await artifact_manager.read(
+        artifact_id=f"{api.config.workspace}/test-dataset"
     )
-    assert manifest_data["description"] == "Overwritten test dataset in collection"
+    updated_manifest = updated_manifest["manifest"]
+    assert updated_manifest["description"] == "Overwritten description for dataset"
 
-    # Remove the dataset artifact
-    await artifact_manager.delete(prefix="collections/test-collection/test-dataset")
+    # Remove the dataset artifact from the collection
+    await artifact_manager.delete(artifact_id=dataset.id)
 
-    # Ensure that the dataset artifact is removed
-    artifacts = await artifact_manager.list(prefix="collections/test-collection")
-    assert not find_item(artifacts, "name", "test-dataset")
-
-    # Ensure the collection is updated after removing the dataset
-    items = await artifact_manager.list(
-        prefix="collections/test-collection", summary_fields=[".prefix"]
-    )
-    assert not find_item(
-        items,
-        ".prefix",
-        "collections/test-collection/test-dataset",
-    )
+    # Confirm deletion by checking the collection’s children
+    collection_items = await artifact_manager.list(parent_id=collection.id)
+    item = find_item(collection_items, "id", dataset.id)
+    assert item is None
 
     # Clean up by deleting the collection
-    await artifact_manager.delete(prefix="collections/test-collection")
+    await artifact_manager.delete(artifact_id=collection.id)
+
 
+async def test_artifact_edge_cases_with_collection(
+    minio_server, fastapi_server, test_user_token
+):
+    """Test edge cases with artifact collections."""
 
-async def test_artifact_edge_cases_with_collection(minio_server, fastapi_server):
-    """Test artifact edge cases with collections."""
+    # Connect to the server and set up the artifact manager
     api = await connect_to_server(
-        {"name": "test deploy client", "server_url": SERVER_URL}
+        {
+            "name": "test deploy client",
+            "server_url": SERVER_URL,
+            "token": test_user_token,
+        }
     )
     artifact_manager = await api.get_service("public/artifact-manager")
 
-    # Create a collection first
+    # Attempt to create a collection with orphan=False, which should fail
     collection_manifest = {
         "name": "edge-case-collection",
         "description": "A test collection for edge cases",
-        "type": "collection",
     }
 
     with pytest.raises(
-        Exception,
-        match=r".*Parent artifact not found.*",
+        Exception, match=r".*Artifact with ID 'non-existent-parent' does not exist.*"
     ):
         await artifact_manager.create(
-            prefix="collections/edge-case-collection",
+            type="collection",
+            parent_id="non-existent-parent",
             manifest=collection_manifest,
-            stage=False,
-            orphan=False,
+            config={"permissions": {"*": "r", "@": "r+"}},
         )
 
-    await artifact_manager.create(
-        prefix="collections/edge-case-collection",
-        manifest=collection_manifest,
-        stage=False,
-        orphan=True,
+    # Successfully create the collection with orphan set by default
+    collection = await artifact_manager.create(
+        manifest=collection_manifest, config={"permissions": {"*": "r", "@": "r+"}}
     )
 
-    # Try to create an artifact that already exists within the collection without overwriting
-    manifest = {
+    # Create a dataset artifact within the collection
+    dataset_manifest = {
         "name": "edge-case-dataset",
-        "description": "Edge case test dataset",
-        "type": "dataset",
+        "description": "A dataset for testing edge cases",
     }
-
-    # Create the artifact first
-    await artifact_manager.create(
-        prefix="collections/edge-case-collection/edge-case-dataset",
-        manifest=manifest,
-        stage=True,
+    dataset = await artifact_manager.create(
+        type="dataset",
+        parent_id=collection.id,
+        alias="edge-case-dataset",
+        manifest=dataset_manifest,
+        version="stage",
     )
 
-    # Attempt to create the same artifact without overwrite
+    # Try to create the same artifact again without overwrite, expecting it to fail
     with pytest.raises(
-        Exception,
-        match=r".*FileExistsError: Artifact under prefix .*",
+        Exception, match=r".*already exists. Use overwrite=True to overwrite.*"
     ):
         await artifact_manager.create(
-            prefix="collections/edge-case-collection/edge-case-dataset",
-            manifest=manifest,
+            type="dataset",
+            parent_id=collection.id,
+            alias="edge-case-dataset",
+            manifest=dataset_manifest,
             overwrite=False,
-            stage=True,
+            version="stage",
         )
 
-    # Add a file to the artifact
+    # Add a file to the dataset artifact
     put_url = await artifact_manager.put_file(
-        prefix="collections/edge-case-collection/edge-case-dataset",
-        file_path="example.txt",
+        artifact_id=dataset.id, file_path="example.txt"
     )
-    source = "some content"
-    response = requests.put(put_url, data=source)
+    file_content = "sample file content"
+    response = requests.put(put_url, data=file_content)
     assert response.ok
 
-    # Commit the artifact
-    await artifact_manager.commit(
-        prefix="collections/edge-case-collection/edge-case-dataset"
-    )
+    # Commit the dataset artifact to finalize it
+    await artifact_manager.commit(artifact_id=dataset.id)
 
-    # Ensure that the collection index is updated
-    items = await artifact_manager.list(
-        prefix="collections/edge-case-collection", summary_fields=[".prefix"]
-    )
-    assert find_item(
-        items,
-        ".prefix",
-        f"collections/edge-case-collection/edge-case-dataset",
-    )
+    # Ensure the dataset is listed under the collection
+    collection_items = await artifact_manager.list(parent_id=collection.id)
+    item = find_item(collection_items, "id", dataset.id)
+    assert item["manifest"]["name"] == "edge-case-dataset"
 
-    # Test validation without uploading a file
+    # Test committing a dataset artifact with missing files, expecting a failure
     incomplete_manifest = {
         "name": "incomplete-dataset",
-        "description": "This dataset is incomplete",
-        "type": "dataset",
+        "description": "Dataset missing a required file",
     }
-
-    await artifact_manager.create(
-        prefix="collections/edge-case-collection/incomplete-dataset",
+    incomplete_dataset = await artifact_manager.create(
+        type="dataset",
+        parent_id=collection.id,
         manifest=incomplete_manifest,
-        stage=True,
+        version="stage",
     )
 
     await artifact_manager.put_file(
-        prefix="collections/edge-case-collection/incomplete-dataset",
-        file_path="missing_file.txt",
+        artifact_id=incomplete_dataset.id, file_path="missing_file.txt"
     )
 
-    # Commit should raise an error due to the missing file
-    with pytest.raises(
-        Exception,
-        match=r".*FileNotFoundError: .*",
-    ):
-        await artifact_manager.commit(
-            prefix="collections/edge-case-collection/incomplete-dataset"
-        )
+    # Commit should fail due to missing file
+    with pytest.raises(Exception, match=r".*FileNotFoundError: .*"):
+        await artifact_manager.commit(artifact_id=incomplete_dataset.id)
 
-    # Clean up the artifacts
-    await artifact_manager.delete(
-        prefix="collections/edge-case-collection/edge-case-dataset"
-    )
-    await artifact_manager.delete(
-        prefix="collections/edge-case-collection/incomplete-dataset"
-    )
-    await artifact_manager.delete(prefix="collections/edge-case-collection")
+    # Clean up by deleting the artifacts and the collection
+    await artifact_manager.delete(artifact_id=dataset.id)
+    await artifact_manager.delete(artifact_id=incomplete_dataset.id)
+    await artifact_manager.delete(artifact_id=collection.id)
 
 
-async def test_artifact_search_in_manifest(minio_server, fastapi_server):
-    """Test search functionality within the 'manifest' field of artifacts with multiple keywords and both AND and OR modes."""
-    api = await connect_to_server({"name": "test-client", "server_url": SERVER_URL})
+async def test_artifact_search_in_manifest(
+    minio_server, fastapi_server, test_user_token
+):
+    """Test search functionality within the 'manifest' field of artifacts using keywords and filters in both AND and OR modes."""
+
+    # Connect to the server and set up the artifact manager
+    api = await connect_to_server(
+        {"name": "test-client", "server_url": SERVER_URL, "token": test_user_token}
+    )
     artifact_manager = await api.get_service("public/artifact-manager")
 
     # Create a collection for testing search
     collection_manifest = {
         "name": "Search Test Collection",
-        "description": "A collection to test search functionality",
-        "type": "collection",
+        "description": "A collection for testing search functionality",
     }
-    await artifact_manager.create(
-        prefix="collections/search-test-collection",
+    collection = await artifact_manager.create(
+        type="collection",
         manifest=collection_manifest,
-        stage=False,
-        orphan=True,
+        config={"permissions": {"*": "r", "@": "r+"}},
     )
 
-    # Create multiple artifacts inside the collection
+    # Create multiple artifacts within the collection for search tests
     for i in range(5):
         dataset_manifest = {
             "name": f"Test Dataset {i}",
             "description": f"A test dataset {i}",
-            "type": "dataset",
         }
-        await artifact_manager.create(
-            prefix=f"collections/search-test-collection/test-dataset-{i}",
+        artifact = await artifact_manager.create(
+            type="dataset",
+            alias=f"test-dataset-{i}",
+            parent_id=collection.id,
             manifest=dataset_manifest,
-            stage=True,
-        )
-        await artifact_manager.commit(
-            prefix=f"collections/search-test-collection/test-dataset-{i}"
+            version="stage",
         )
+        await artifact_manager.commit(artifact_id=artifact.id)
 
-    # Use the search function to find datasets with 'Dataset 3' in the 'name' field using AND mode
+    # Test search by exact match keyword using AND mode
     search_results = await artifact_manager.list(
-        prefix="collections/search-test-collection", keywords=["Dataset 3"], mode="AND"
+        parent_id=collection.id, keywords=["Dataset 3"], mode="AND"
     )
-
-    # Assert that the search results contain only the relevant dataset
     assert len(search_results) == 1
-    assert search_results[0]["name"] == "Test Dataset 3"
+    assert search_results[0]["manifest"]["name"] == "Test Dataset 3"
 
-    # Test search for multiple results by 'description' field using OR mode
+    # Test search for multiple results by description using OR mode
     search_results = await artifact_manager.list(
-        prefix="collections/search-test-collection",
-        keywords=["test", "dataset"],
-        mode="OR",
+        parent_id=collection.id, keywords=["test", "dataset"], mode="OR"
     )
+    assert (
+        len(search_results) == 5
+    )  # All datasets have "test" and "dataset" in their description
 
-    # Assert that all datasets are returned because both keywords appear in the description
-    assert len(search_results) == 5
+    # Test search with multiple keywords using AND mode (returns fewer results)
+    search_results = await artifact_manager.list(
+        parent_id=collection.id, keywords=["Test Dataset", "3"], mode="AND"
+    )
+    assert len(search_results) == 1
+    assert search_results[0]["manifest"]["name"] == "Test Dataset 3"
 
-    # Test search with multiple keywords using AND mode (this should return fewer results)
+    # Test search using filters to match manifest field directly
     search_results = await artifact_manager.list(
-        prefix="collections/search-test-collection",
-        keywords=["Test Dataset", "3"],
+        parent_id=collection.id,
+        filters={"manifest": {"name": "Test Dataset 3"}},
         mode="AND",
     )
+    assert len(search_results) == 1
+    assert search_results[0]["manifest"]["name"] == "Test Dataset 3"
+
+    # Test fuzzy search using filters within the manifest
+    search_results = await artifact_manager.list(
+        parent_id=collection.id, filters={"manifest": {"name": "Test*"}}, mode="OR"
+    )
+    assert len(search_results) == 5  # All datasets match "Test*" in the name
 
-    # Assert that only the dataset with 'Test Dataset 3' is returned
+    # Test search with multiple filters in AND mode for exact match on both name and description
+    search_results = await artifact_manager.list(
+        parent_id=collection.id,
+        filters={
+            "manifest": {"name": "Test Dataset 3", "description": "A test dataset 3"}
+        },
+        mode="AND",
+    )
     assert len(search_results) == 1
-    assert search_results[0]["name"] == "Test Dataset 3"
+    assert search_results[0]["manifest"]["name"] == "Test Dataset 3"
 
-    # Clean up by deleting the datasets and the collection
+    # Test search with multiple filters in OR mode for matching any field
+    search_results = await artifact_manager.list(
+        parent_id=collection.id,
+        filters={
+            "manifest": {"name": "Test Dataset 3", "description": "A test dataset 1"}
+        },
+        mode="OR",
+    )
+    assert (
+        len(search_results) == 2
+    )  # Matches for either dataset 1 or 3 based on OR filter
+
+    # Clean up by deleting all artifacts and the collection
     for i in range(5):
-        await artifact_manager.delete(
-            prefix=f"collections/search-test-collection/test-dataset-{i}"
-        )
-    await artifact_manager.delete(prefix="collections/search-test-collection")
+        await artifact_manager.delete(artifact_id=f"test-dataset-{i}")
+    await artifact_manager.delete(artifact_id=collection.id)
 
 
-async def test_artifact_search_with_filters(minio_server, fastapi_server):
-    """Test search functionality with specific key-value filters in the manifest with multiple filters and AND/OR modes."""
-    api = await connect_to_server({"name": "test-client", "server_url": SERVER_URL})
+async def test_artifact_search_with_filters(
+    minio_server, fastapi_server, test_user_token
+):
+    """Test search functionality with specific key-value filters in the manifest, supporting multiple filters and AND/OR modes."""
+
+    # Connect to the server and initialize the artifact manager
+    api = await connect_to_server(
+        {"name": "test-client", "server_url": SERVER_URL, "token": test_user_token}
+    )
     artifact_manager = await api.get_service("public/artifact-manager")
 
-    # Create a collection for testing search
+    # Create a collection for testing filter-based search
     collection_manifest = {
-        "name": "Search Test Collection",
-        "description": "A collection to test search functionality",
-        "type": "collection",
+        "name": "Filter Test Collection",
+        "description": "A collection to test filter functionality",
     }
-    await artifact_manager.create(
-        prefix="collections/search-test-collection",
+    collection = await artifact_manager.create(
+        type="collection",
         manifest=collection_manifest,
-        stage=False,
-        orphan=True,
+        config={"permissions": {"*": "r", "@": "r+"}},
     )
 
-    # Create multiple artifacts inside the collection
+    # Create multiple artifacts with varied attributes inside the collection
     for i in range(5):
         dataset_manifest = {
-            "name": f"Test Dataset {i}",
-            "description": f"A test dataset {i}",
-            "type": "dataset",
+            "name": f"Dataset {i}",
+            "description": f"Description for dataset {i}",
         }
         await artifact_manager.create(
-            prefix=f"collections/search-test-collection/test-dataset-{i}",
+            type="dataset" if i % 2 == 0 else "application",
+            alias=f"dataset-{i}",
+            parent_id=collection.id,
             manifest=dataset_manifest,
-            stage=True,
-        )
-        await artifact_manager.commit(
-            prefix=f"collections/search-test-collection/test-dataset-{i}"
         )
 
-    # Use the search function to find datasets with an exact name match using AND mode
+    # Filter by type to return only datasets
     search_results = await artifact_manager.list(
-        prefix="collections/search-test-collection",
-        filters={"name": "Test Dataset 3"},
-        mode="AND",
+        parent_id=collection.id, filters={"type": "dataset"}, mode="AND"
     )
+    assert len(search_results) == 3
+    for result in search_results:
+        assert result["type"] == "dataset"
 
-    # Assert that the search results contain only the relevant dataset
+    # Filter by a field in the manifest: exact name match
+    search_results = await artifact_manager.list(
+        parent_id=collection.id, filters={"manifest": {"name": "Dataset 3"}}, mode="AND"
+    )
     assert len(search_results) == 1
-    assert search_results[0]["name"] == "Test Dataset 3"
+    assert search_results[0]["manifest"]["name"] == "Dataset 3"
 
-    # Test search with fuzzy match on name using OR mode
+    # Filter by multiple fields in the manifest using OR mode
     search_results = await artifact_manager.list(
-        prefix="collections/search-test-collection",
-        filters={"name": "Test*"},
+        parent_id=collection.id,
+        filters={
+            "manifest": {
+                "name": "Dataset 3",
+                "description": "Description for dataset 1",
+            }
+        },
         mode="OR",
     )
+    assert (
+        len(search_results) == 2
+    )  # Matches for either dataset 1 or 3 based on OR filter
 
-    # Assert that all datasets are returned since the fuzzy match applies to all
-    assert len(search_results) == 5
-
-    # Test search with multiple filters in AND mode (exact match on name and description)
+    # Filter by multiple fields in the manifest using AND mode
     search_results = await artifact_manager.list(
-        prefix="collections/search-test-collection",
-        filters={"name": "Test Dataset 3", "description": "A test dataset 3"},
+        parent_id=collection.id,
+        filters={
+            "manifest": {
+                "name": "Dataset 3",
+                "description": "Description for dataset 3",
+            }
+        },
         mode="AND",
     )
-
-    # Assert that only one dataset is returned
     assert len(search_results) == 1
-    assert search_results[0]["name"] == "Test Dataset 3"
+    assert search_results[0]["manifest"]["name"] == "Dataset 3"
 
-    # Test search with multiple filters in OR mode (match any of the fields)
+    # Filter with a fuzzy match in the description
     search_results = await artifact_manager.list(
-        prefix="collections/search-test-collection",
-        filters={"name": "Test Dataset 3", "description": "A test dataset 1"},
-        mode="OR",
+        parent_id=collection.id,
+        filters={"manifest": {"description": "Description for dataset*"}},
+        mode="AND",
     )
-
-    # Assert that two datasets are returned (matching either name or description)
-    assert len(search_results) == 2
+    assert (
+        len(search_results) == 5
+    )  # All items match since the description pattern is common
 
     # Clean up by deleting the datasets and the collection
     for i in range(5):
-        await artifact_manager.delete(
-            prefix=f"collections/search-test-collection/test-dataset-{i}"
-        )
-    await artifact_manager.delete(prefix="collections/search-test-collection")
+        await artifact_manager.delete(artifact_id=f"dataset-{i}")
+    await artifact_manager.delete(artifact_id=collection.id)
 
 
-async def test_download_count(minio_server, fastapi_server):
+async def test_download_count(minio_server, fastapi_server, test_user_token):
     """Test the download count functionality for artifacts."""
-    api = await connect_to_server({"name": "test-client", "server_url": SERVER_URL})
+    api = await connect_to_server(
+        {"name": "test-client", "server_url": SERVER_URL, "token": test_user_token}
+    )
     artifact_manager = await api.get_service("public/artifact-manager")
 
     # Create a collection for testing download count
     collection_manifest = {
         "name": "Download Test Collection",
         "description": "A collection to test download count functionality",
-        "type": "collection",
     }
-    await artifact_manager.create(
-        prefix="collections/download-test-collection",
+    collection = await artifact_manager.create(
+        type="collection",
         manifest=collection_manifest,
-        stage=False,
-        orphan=True,
+        config={"permissions": {"*": "r", "@": "r+"}},
     )
 
     # Create an artifact inside the collection
     dataset_manifest = {
         "name": "Download Test Dataset",
         "description": "A test dataset for download count",
-        "type": "dataset",
     }
-    await artifact_manager.create(
-        prefix="collections/download-test-collection/download-test-dataset",
+    artifact = await artifact_manager.create(
+        type="dataset",
+        parent_id=collection.id,
         manifest=dataset_manifest,
-        stage=True,
+        version="stage",
     )
 
-    # Put a file in the artifact
+    # Put a primary file in the artifact
     put_url = await artifact_manager.put_file(
-        prefix="collections/download-test-collection/download-test-dataset",
+        artifact_id=artifact.id,
         file_path="example.txt",
-        options={
-            "download_weight": 0.5
-        },  # Set the file as primary so downloading it will be count as a download
+        download_weight=0.5,  # Set as primary for download count
     )
     source = "file contents of example.txt"
     response = requests.put(put_url, data=source)
     assert response.ok
 
-    # put another file in the artifact but not setting weights
+    # Put another non-primary file in the artifact
     put_url = await artifact_manager.put_file(
-        prefix="collections/download-test-collection/download-test-dataset",
-        file_path="example2.txt",
+        artifact_id=artifact.id, file_path="example2.txt"
     )
-    source = "file contents of example2.txt"
-    response = requests.put(put_url, data=source)
+    response = requests.put(put_url, data="file contents of example2.txt")
     assert response.ok
 
     # Commit the artifact
-    await artifact_manager.commit(
-        prefix="collections/download-test-collection/download-test-dataset"
-    )
+    await artifact_manager.commit(artifact_id=artifact.id)
 
     # Ensure that the download count is initially zero
     artifact = await artifact_manager.read(
-        prefix="collections/download-test-collection/download-test-dataset",
-        include_metadata=True,
+        artifact_id=artifact.id,
     )
-    assert artifact[".download_count"] == 0
+    assert artifact["download_count"] == 0
 
     # Increment the download count of the artifact by download the primary file
     get_url = await artifact_manager.get_file(
-        prefix="collections/download-test-collection/download-test-dataset",
+        artifact_id=artifact.id,
         path="example.txt",
     )
     response = requests.get(get_url)
@@ -1197,15 +1440,14 @@ async def test_download_count(minio_server, fastapi_server):
 
     # Ensure that the download count is incremented
     artifact = await artifact_manager.read(
-        prefix="collections/download-test-collection/download-test-dataset",
+        artifact_id=artifact.id,
         silent=True,
-        include_metadata=True,
     )
-    assert artifact[".download_count"] == 0
+    assert artifact["download_count"] == 0.5
 
     # download twice will increment the download count by 1
     get_url = await artifact_manager.get_file(
-        prefix="collections/download-test-collection/download-test-dataset",
+        artifact_id=artifact.id,
         path="example.txt",
     )
     response = requests.get(get_url)
@@ -1213,38 +1455,36 @@ async def test_download_count(minio_server, fastapi_server):
 
     # Ensure that the download count is incremented
     artifact = await artifact_manager.read(
-        prefix="collections/download-test-collection/download-test-dataset",
+        artifact_id=artifact.id,
         silent=True,
-        include_metadata=True,
     )
-    assert artifact[".download_count"] == 1
+    assert artifact["download_count"] == 1
 
     # If we get the example file in silent mode, the download count won't increment
     get_url = await artifact_manager.get_file(
-        prefix="collections/download-test-collection/download-test-dataset",
+        artifact_id=artifact.id,
         path="example.txt",
-        options={"silent": True},
+        silent=True,
     )
     response = requests.get(get_url)
     assert response.ok
     get_url = await artifact_manager.get_file(
-        prefix="collections/download-test-collection/download-test-dataset",
+        artifact_id=artifact.id,
         path="example.txt",
-        options={"silent": True},
+        silent=True,
     )
     response = requests.get(get_url)
     assert response.ok
 
     # Ensure that the download count is not incremented
     artifact = await artifact_manager.read(
-        prefix="collections/download-test-collection/download-test-dataset",
-        include_metadata=True,
+        artifact_id=artifact.id,
     )
-    assert artifact[".download_count"] == 1
+    assert artifact["download_count"] == 1
 
     # download example 2 won't increment the download count
     get_url = await artifact_manager.get_file(
-        prefix="collections/download-test-collection/download-test-dataset",
+        artifact_id=artifact.id,
         path="example2.txt",
     )
     response = requests.get(get_url)
@@ -1252,27 +1492,25 @@ async def test_download_count(minio_server, fastapi_server):
 
     # Ensure that the download count is incremented
     artifact = await artifact_manager.read(
-        prefix="collections/download-test-collection/download-test-dataset",
-        include_metadata=True,
+        artifact_id=artifact.id,
     )
-    assert artifact[".download_count"] == 1
+    assert artifact["download_count"] == 1
 
     # Let's call reset_stats to reset the download count
     await artifact_manager.reset_stats(
-        prefix="collections/download-test-collection/download-test-dataset"
+        artifact_id=artifact.id,
     )
 
     # Ensure that the download count is reset
     artifact = await artifact_manager.read(
-        prefix="collections/download-test-collection/download-test-dataset",
-        include_metadata=True,
+        artifact_id=artifact.id,
     )
-    assert artifact[".download_count"] == 0
+    assert artifact["download_count"] == 0
 
     # Clean up by deleting the dataset and the collection
     await artifact_manager.delete(
-        prefix="collections/download-test-collection/download-test-dataset",
+        artifact_id=artifact.id,
         delete_files=True,
         recursive=True,
     )
-    await artifact_manager.delete(prefix="collections/download-test-collection")
+    await artifact_manager.delete(artifact_id=collection.id)
diff --git a/tests/test_server_apps.py b/tests/test_server_apps.py
index 97ff47a4..a219fdb2 100644
--- a/tests/test_server_apps.py
+++ b/tests/test_server_apps.py
@@ -37,14 +37,21 @@
 """
 
 
-async def test_server_apps_unauthorized(fastapi_server, root_user_token):
+async def test_server_apps_unauthorized(
+    fastapi_server, test_user_token, root_user_token
+):
     """Test the server apps."""
     api = await connect_to_server(
-        {"name": "test client", "server_url": WS_SERVER_URL, "method_timeout": 30}
+        {
+            "name": "test client",
+            "server_url": WS_SERVER_URL,
+            "method_timeout": 30,
+            "token": test_user_token,
+        }
     )
     controller = await api.get_service("public/server-apps")
 
-    await controller.launch(
+    app_info = await controller.launch(
         source=TEST_APP_CODE,
         config={"type": "window"},
         overwrite=True,
@@ -60,6 +67,7 @@ async def test_server_apps_unauthorized(fastapi_server, root_user_token):
         workspace_info = find_item(workspaces, "name", api.config["workspace"])
         assert workspace_info
 
+    await controller.stop(app_info.id)
     # Now disconnect it
     await api.disconnect()
     await asyncio.sleep(0.1)
@@ -84,7 +92,6 @@ async def test_server_apps(fastapi_server, test_user_token):
             "token": test_user_token,
         }
     )
-    workspace = api.config["workspace"]
 
     # objects = await api.list_remote_objects()
     # assert len(objects) == 1
@@ -99,6 +106,7 @@ async def test_server_apps(fastapi_server, test_user_token):
         source=TEST_APP_CODE,
         config={"type": "window"},
         wait_for_service="default",
+        overwrite=True,
     )
     assert "id" in config.keys()
     app = await api.get_app(config.id)
@@ -144,7 +152,6 @@ async def test_web_python_apps(fastapi_server, test_user_token):
     api = await connect_to_server(
         {"name": "test client", "server_url": WS_SERVER_URL, "token": test_user_token}
     )
-    workspace = api.config["workspace"]
 
     controller = await api.get_service("public/server-apps")
     source = (
@@ -201,22 +208,22 @@ async def test_non_persistent_workspace(fastapi_server, root_user_token):
     )
     workspace = api.config["workspace"]
 
-    # Test app with custom template
-    controller = await api.get_service("public/server-apps")
+    # # Test app with custom template
+    # controller = await api.get_service("public/server-apps")
 
-    source = (
-        (Path(__file__).parent / "testWebWorkerPlugin.imjoy.html")
-        .open(encoding="utf-8")
-        .read()
-    )
+    # source = (
+    #     (Path(__file__).parent / "testWebWorkerPlugin.imjoy.html")
+    #     .open(encoding="utf-8")
+    #     .read()
+    # )
 
-    config = await controller.launch(
-        source=source,
-        wait_for_service="default",
-    )
+    # config = await controller.launch(
+    #     source=source,
+    #     wait_for_service="default",
+    # )
 
-    app = await api.get_app(config.id)
-    assert app is not None
+    # app = await api.get_app(config.id)
+    # assert app is not None
 
     # It should exist in the stats
     async with connect_to_server(
diff --git a/tox.ini b/tox.ini
index bb4892ce..732262e6 100644
--- a/tox.ini
+++ b/tox.ini
@@ -10,6 +10,9 @@ python =
   3.12: py312
 
 [testenv]
+passenv = 
+  SANDBOX_ZENODO_TOKEN
+  GITHUB_ACTIONS
 commands =
   playwright install
   pytest -v --timeout=0 --cov=hypha --cov-report=xml --asyncio-mode=strict {posargs}
@@ -18,6 +21,9 @@ deps =
   -rrequirements_test.txt
 
 [testenv:lint]
+passenv =
+  SANDBOX_ZENODO_TOKEN
+  GITHUB_ACTIONS
 basepython = python3
 ignore_errors = True
 commands =