diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 0000000..331f6de
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,166 @@
+_skbuild/
+
+.envrc
+
+models/
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+# Usually these files are written by a python script from a template
+# before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+# For a library or package, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+# However, in case of collaboration, if having platform-specific dependencies or dependencies
+# having no cross-platform support, pipenv may install dependencies that don't work, or not
+# install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+# This is especially recommended for binary packages to ensure reproducibility, and is more
+# commonly ignored for libraries.
+# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+# in version control.
+# https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+# and can be added to the global gitignore or merged into this file. For a more nuclear
+# option (not recommended) you can uncomment the following to ignore the entire idea folder.
+.idea/
\ No newline at end of file
diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml
new file mode 100644
index 0000000..3cc0b5b
--- /dev/null
+++ b/.github/workflows/cicd.yml
@@ -0,0 +1,94 @@
+name: CI/CD
+run-name: ${{ github.actor }} is triggering CI/CD Action
+on:
+ push:
+ branches:
+ - main
+ - 'releases/**'
+ pull_request:
+
+env:
+ UV_SYSTEM_PYTHON: 1
+ REGISTRY: ghcr.io
+ IMAGE_NAME: ${{ github.repository }}
+ UV_CACHE_DIR: /tmp/.uv-cache
+
+jobs:
+ code_quality:
+ name: Check code quality
+ runs-on: ubuntu-latest
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Set up uv
+ run: curl -LsSf https://astral.sh/uv/install.sh | sh
+
+ - name: Install dependencies
+ run: uv tool install ruff==0.6.1
+
+ - name: Check ruff formatting
+ run: uv run ruff format --config pyproject.toml --line-length 88 --exclude src/notebook .
+
+ - name: Check ruff linting
+ run: uv run ruff check --fix --respect-gitignore --config pyproject.toml --exclude src/notebook .
+
+ api_test:
+ name: Test API
+ runs-on: ubuntu-latest
+ needs: [ code_quality ]
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Set up uv
+ run: curl -LsSf https://astral.sh/uv/install.sh | sh
+
+ - name: Install dependencies
+ run: uv sync --extra test
+
+ - name: Run API tests
+ run: uv run pytest .
+
+ build-and-push-image:
+ name: Build and push Docker image
+ runs-on: ubuntu-latest
+ needs: [ api_test, code_quality ]
+ permissions:
+ contents: read
+ packages: write
+ attestations: write
+ id-token: write
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+
+ - name: Log in to the Container registry
+ uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567
+ with:
+ registry: ${{ env.REGISTRY }}
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Extract metadata (tags, labels) for Docker
+ id: meta
+ uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81
+ with:
+ images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+ - name: Build and push Docker image
+ id: push
+ uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85
+ with:
+ context: .
+ push: true
+ tags: ${{ steps.meta.outputs.tags }}
+ labels: ${{ steps.meta.outputs.labels }}
+
+ - name: Generate artifact attestation
+ uses: actions/attest-build-provenance@v1.4.1
+ with:
+ subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}
+ subject-digest: ${{ steps.push.outputs.digest }}
+ push-to-registry: true
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..ae30662
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,146 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# ruff
+.ruff_cache/
+
+# PyInstaller
+# Usually these files are written by a python script from a template
+# before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+Pipfile.lock
+
+# poetry
+poetry.lock
+
+# pdm
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# uv
+uv.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
\ No newline at end of file
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 0000000..21c2eae
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,28 @@
+FROM python:3.12-slim AS build
+
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv
+
+RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends \
+ python3 \
+ python3-pip \
+ ninja-build \
+ libopenblas-dev \
+ build-essential \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/* /tmp/*
+
+ENV VIRTUAL_ENV=/opt/venv \
+ PATH="/opt/venv/bin:$PATH"
+
+COPY ./pyproject.toml .
+
+RUN uv venv opt/venv && uv pip install -r pyproject.toml
+
+FROM python:3.12-slim-bullseye
+COPY --from=build /opt/venv /opt/venv
+
+COPY ./src .
+
+ENV PATH="/opt/venv/bin:$PATH"
+
+ENTRYPOINT ["gunicorn", "-w", "2", "web.app:app"]
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..962c411
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,24 @@
+.PHONY: run setup clean
+
+run:
+ echo "Running the application"
+
+dev:
+ flask --app src.web.app run --debug
+
+test:
+ pytest --verbose
+
+# Install the dependencies to the virtual environment
+# Remove unnecessary dependencies
+setup: pyproject.toml
+ uv sync --all-extras
+
+quality:
+ ruff format --config pyproject.toml --line-length 88 --exclude src/notebook .
+ ruff check --fix --respect-gitignore --exclude src/notebook --config pyproject.toml .
+
+clean:
+ rm -rf __pycache__
+ rm -rf .venv
+
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..23b9197
--- /dev/null
+++ b/README.md
@@ -0,0 +1,94 @@
+# Gemma-2 2B fine-tuned for Structured Data Extraction
+
+This project is a collection of notebook and a simple flask web server to serve
+**Gemma-2** using **llama-cpp**.
+
+The goal of this project is to fine-tune a model to get a better result on the task of
+to the task of extracting data into a structured format (JSON).
+
+You will need to provide the **output schema** in openapi format and the **text** (context).
+
+## ⛩️ Project Architecture
+
+The project is divided between notebook for the fine-tuning, quantization and evaluation and python files.
+
+| **Source** | **Description** |
+|----------------------------------|-----------------------------------------------------------------------------------------------------------------------|
+| [➡️ Gemma-2 Finetuning](https://github.com/bastienpo/llm-finetune/blob/main/src/notebook/gemma_2_finetuning.ipynb) | A notebook that shows how tofine-tune and quantize gemma2-2b-it using the unsloth and hugging-face libraries. |
+| [➡️ Server](https://github.com/bastienpo/llm-finetune/blob/main/src/web/app.py) | A simple flask REST server using llama-cpp with a 4 bit quantized model. |
+| [➡️ CI/CD](https://github.com/bastienpo/llm-finetune/blob/main/.github/workflows/cicd.yml) | A github action consisting of a formatting/linting step with ruff, testing with pytest and building the docker image. |
+| [➡️ Dockerfile](https://github.com/bastienpo/llm-finetune/blob/main/Dockerfile) | A mutlistage dockerfile to build the server with gunicorn. |
+
+
+## 📊 Details about the Dataset
+
+The different finetuned models can be found in safetensors and GGUF format (4bit, 8bit) on the hugging-face hub at [bastienp/Gemma-2-2B-it-JSON-data-extration](https://huggingface.co/bastienp/Gemma-2-2B-it-JSON-data-extration).
+
+**Note**: It also gives more details on how to use it with **llama-cpp** or **unsloth**.
+
+## 💻 Installation
+
+### Dev setup
+
+Recommended: Use the fast Python package installer and resolver **uv** from astral.
+
+Alternatively, you can replace this command with *pip*. You can find the documentation
+for installing uv [here](https://github.com/astral-sh/uv?tab=readme-ov-file#getting-started).
+
+1. Sync the dependencies with uv
+
+```bash
+uv venv .venv
+```
+
+```bash
+source .venv/bin/activate
+```
+
+```bash
+uv sync --all-extras --dev # in addition it adds pytest and ruff
+```
+
+2. Launch a flask dev server
+
+```bash
+flask --app src.web.app run --debug
+```
+
+To reproduce the fine-tuning, the easiest way is to use Google Collab (the free version is sufficient).
+
+3. Run the tests (API testing)
+
+```bash
+pytest
+```
+**Note**: An example of how to call the API and the prompt format can be found in `examplesexample_api_call.py`.
+
+## 👥 Deployment setup
+
+In order to deploy the model the easiest way to go is to use the provided docker image.
+
+1. Pull the image from github (buit from the CI):
+
+```bash
+docker pull ghcr.io/bastienpo/unsloth_finetuning:main
+```
+
+
+**Note**: Otherwise you can build the image yourself
+```bash
+docker build -tag unsloth_finetuning:0.0.1 .
+```
+
+
+2. Run the docker image
+
+```bash
+docker run -p 8000:8000 -d unsloth_finetuning:main # or 0.0.1
+```
+
+3. Make a post request
+
+```bash
+curl -i -H "Content-Type: application/json" -X POST -d '{"query": "How are you ?"}' http://localhost:8000/api/v1/chat/completions
+```
\ No newline at end of file
diff --git a/examples/example_api_call.py b/examples/example_api_call.py
new file mode 100644
index 0000000..9aa6a19
--- /dev/null
+++ b/examples/example_api_call.py
@@ -0,0 +1,60 @@
+"""Example of making an API call to the server."""
+
+import json
+from typing import Any
+
+import requests
+from pydantic import BaseModel
+
+
+class Requirement(BaseModel):
+ description: str
+ tags: list[str]
+
+
+JSON_PROMPT = """Below is a text paired with input that provides further context. Write JSON output that matches the schema to extract information.
+
+### Input:
+{}
+
+### Schema:
+{}
+
+### Response:
+"""
+
+
+def make_api_call(
+ url: str, query: str, hostname: str = "localhost", port: int = "5000"
+) -> dict[str, Any]:
+ """Make an API call to the specified endpoint.
+
+ Args:
+ url (str): The endpoint to make the API call to.
+ query (str): The query to send to the endpoint.
+ hostname (str): The hostname of the server.
+ port (int): The port of the server.
+
+ Returns:
+ dict[str, Any]: The JSON response from the server.
+ """
+ response = requests.post(
+ f"http://{hostname}:{port}/{url}",
+ json={
+ "query": query,
+ },
+ )
+
+ return response.json()
+
+
+if __name__ == "__main__":
+ prompt = JSON_PROMPT.format(
+ "A car is a vehicle that has four wheels and an engine.",
+ str(Requirement.model_json_schema()),
+ )
+
+ res = make_api_call(url="api/v1/chat/completions", query=prompt)
+
+ # parse the json response
+ print(json.dumps(res["response"]))
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 0000000..fcaaaea
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,35 @@
+[project]
+name = "unsloth_finetuning"
+version = "0.1.0"
+description = "Finetuning of Gemma-2 2B for structured output"
+readme = "README.md"
+requires-python = ">=3.11"
+license = { text = "MIT License" }
+dependencies = [
+ "Flask>=3.0.3",
+ "gunicorn>=23.0.0",
+ "llama-cpp-python>=0.2.88",
+ "huggingface-hub>=0.24.6",
+ "httpx>=0.27.0",
+ "pydantic>=2.8.2",
+]
+
+[project.optional-dependencies]
+test = ["pytest>=8.3.2"]
+
+[project.urls]
+respository = "https://github.com/bastienpo/unsloth_finetuning"
+
+[tool.ruff]
+# Same as black
+line-length = 88
+
+[tool.ruff.format]
+docstring-code-format = true
+docstring-code-line-length = "dynamic"
+
+[tool.ruff.lint.pydocstyle]
+convention = "google"
+
+[tool.uv]
+dev-dependencies = ["ruff>=0.6.1"]
diff --git a/src/notebook/__init__.py b/src/notebook/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/src/notebook/gemma_2_finetuning.ipynb b/src/notebook/gemma_2_finetuning.ipynb
new file mode 100644
index 0000000..f762c84
--- /dev/null
+++ b/src/notebook/gemma_2_finetuning.ipynb
@@ -0,0 +1,15512 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "iAbrzmBxhKTT"
+ },
+ "source": [
+ "# Fine-tuning gemma2-2b for structured data extraction"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "eDCNXuLRvHQc"
+ },
+ "source": [
+ "**Use case**:\n",
+ "\n",
+ "The goal is to extract structured output data from a text in a structured way (JSON format is the most common).\n",
+ "\n",
+ "\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "XkVf6OURuTQw"
+ },
+ "source": [
+ "I decided to use gemma2-2b because it is a distilled model with great performance on the various benchmarks.\n",
+ "\n",
+ "As we are on limited hardware, we can use the 2B parameter version of the model, which allows for speed of fine tuning."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {
+ "id": "2eSvM9zX_2d3"
+ },
+ "outputs": [],
+ "source": [
+ "%%capture\n",
+ "# Installs Unsloth, Xformers (Flash Attention) and all other packages!\n",
+ "!pip install \"unsloth[colab-new] @ git+https://github.com/unslothai/unsloth.git\"\n",
+ "!pip install --no-deps \"xformers<0.0.27\" \"trl<0.9.0\" peft accelerate bitsandbytes\n",
+ "\n",
+ "import torch\n",
+ "\n",
+ "if torch.cuda.get_device_capability()[0] >= 8:\n",
+ " !pip install --no-deps packaging ninja einops \"flash-attn>=2.6.3\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {
+ "id": "krTDWxUxMECs"
+ },
+ "outputs": [],
+ "source": [
+ "# Known issue with Gemma-2 and cache issue\n",
+ "torch._dynamo.config.accumulated_cache_size_limit = 2048"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {
+ "id": "dnzizcdW5yiP"
+ },
+ "outputs": [],
+ "source": [
+ "from google.colab import userdata\n",
+ "\n",
+ "HF_TOKEN_READ = userdata.get(\"HF_TOKEN_READ\")\n",
+ "HF_TOKEN_WRITE = userdata.get(\"HF_TOKEN_WRITE\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "PFbVNO8c5hE-"
+ },
+ "source": [
+ "# Gemma-2-2b fine-tuning with unsloth\n",
+ "\n",
+ "Unsloth is a framework used to simplify the process of fine-tuning a model by reducing the amount of boilerplate."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 351,
+ "referenced_widgets": [
+ "dc9055bec8ea477aba7ab70034c18df9",
+ "08fd9c21a3ae415092a8b41ffa6ef29d",
+ "1c8e1ca467aa46488f5d32e0a682dc3f",
+ "0021a7be57c04c8b8ff4e5de32c00497",
+ "b77abea6eb244bfb868de91c5e82f781",
+ "c0aaa5b5d4a34211b72e1b521b207715",
+ "1abb1facad464fc6b5510fdced3efb49",
+ "90eb98a240624715af2e90603b5c6906",
+ "fa4b8433880a406fa4c13787a248c4f4",
+ "918572f119124c9f9634c9709332a74a",
+ "7c8b90c1581047f7a2e21bd7b8b8e4b4",
+ "7c8579a5dc2b4b79b24481c2e1fa23d4",
+ "2392c612ef7044f2989f5883cc8b736e",
+ "7c0817cb6fe0442db7d851bd850d89a9",
+ "366bc6a885824b3fa5f8843bb9b7c61b",
+ "a4ffc744a4c548e9b2d44e3433b80eae",
+ "0886edc52de14957a9914288e7492b16",
+ "4eeb0b014bf8407a8bde13417830df08",
+ "44b0833a4b204ec8b560845ed1082614",
+ "e0ab6b3a02614974a092e45fb053e9d4",
+ "8be842db91c7448faf59d5e66530efef",
+ "d592881ea24f4da29f99e2a98f925320",
+ "c78567588a3c4413a226dadb7ae9aa22",
+ "2756c1445733413aa71fdb9c11f503f3",
+ "9f3622b3cc18445ca9756f394aa887aa",
+ "8b86863ce86f403290e7fea742fea27b",
+ "6fdba7ae94d74eb9a9f848804e1be10e",
+ "e8d28dd151154ec486608553f4d3c891",
+ "a52bab18d6ea44fbb367f9e8a73101d6",
+ "20df323a5a5b4c449ba0d6e315b5dcbc",
+ "4f4dede403ac4a328ad1fa99540995c1",
+ "72b64777f8ed447697293138f07fe160",
+ "bf984de52aca4250b030cfd4e4075ed3",
+ "17da28a6e453436faaa2546599d88603",
+ "0a64e28fcfbc4514a116ce37eeaa2a3c",
+ "ad023bc35c8043c7a8c52a8c331143c5",
+ "863f3be87d0a4001b8b621aad120e53a",
+ "10d70171d5a24913b4fb3c143cd25277",
+ "99638ffedf4a46deadc124e7e12a4e31",
+ "a7c3eabac93e472685c1ffae00b826ac",
+ "f4d08ad70914489abc952b502ab8f647",
+ "b15f3ac2cdd14a84a7dd411dacb814cd",
+ "ed033dab226a47c9b6a910d2af724eb8",
+ "3d6ff58c3ce54930b9bab7060dfcfbfe",
+ "fc80466ebadd418ebd6998dce68846fc",
+ "864aa80df0794cd5ac06ae8ca2c8795e",
+ "b66a6321ab204361a4e6e148ebecff22",
+ "a52f92eabf534de1861b620b91d95b81",
+ "c1a6c63e05714daabc94dd21bf6c2ed6",
+ "8a5053f3a8d94f08a2a085bf256b03b2",
+ "a9ca65f50e96436f962411f9879f943c",
+ "2d501e35e65c4472b7a20f26723fdf31",
+ "770fd2f3f88c4c3492b3c6dbed30ac39",
+ "3a88a3baa76a4f2fa6ee9df3fb0772b4",
+ "3c3c45703f3b4b21aaa5acd33ebf8481",
+ "e4cec12ac63943dcaddf1ee59a33efd0",
+ "cdc5eeb2e8264c00ac12b2f67974a1ed",
+ "3951007a15bf41f1a9efbee7a982121b",
+ "dc5eaf08f5084a2e8a109df53969b2e0",
+ "749faa012cb44294bb4cf67b6b006ef5",
+ "0fd44346d27f44afbd20bbc27fe88a94",
+ "f5a72fe815184355a05dd47ded386b1c",
+ "0e6ed9d556cb46f7859f98104a0569fb",
+ "6e1c96a8931c45a3bb173e23e94ff707",
+ "5a9d00df1f30402d995bea0e9c755d88",
+ "b3c8033fed004d4a99aa6af7a6589e45"
+ ]
+ },
+ "id": "QmUBVEnvCDJv",
+ "outputId": "7e5f8b19-1444-446d-e606-d39104e5166e"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n",
+ "==((====))== Unsloth 2024.8: Fast Gemma2 patching. Transformers = 4.44.0.\n",
+ " \\\\ /| GPU: Tesla T4. Max memory: 14.748 GB. Platform = Linux.\n",
+ "O^O/ \\_/ \\ Pytorch: 2.3.1+cu121. CUDA = 7.5. CUDA Toolkit = 12.1.\n",
+ "\\ / Bfloat16 = FALSE. FA [Xformers = 0.0.26.post1. FA2 = False]\n",
+ " \"-____-\" Free Apache license: http://github.com/unslothai/unsloth\n",
+ "Unsloth: Fast downloading is enabled - ignore downloading bars which are red colored!\n"
+ ]
+ },
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "dc9055bec8ea477aba7ab70034c18df9",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ "model.safetensors: 0%| | 0.00/5.23G [00:00\n",
+ " \n",
+ " \n",
+ " [60/60 18:35, Epoch 2/3]\n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " | Step | \n",
+ " Training Loss | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 1 | \n",
+ " 0.769400 | \n",
+ "
\n",
+ " \n",
+ " | 2 | \n",
+ " 0.730800 | \n",
+ "
\n",
+ " \n",
+ " | 3 | \n",
+ " 0.806400 | \n",
+ "
\n",
+ " \n",
+ " | 4 | \n",
+ " 0.525800 | \n",
+ "
\n",
+ " \n",
+ " | 5 | \n",
+ " 0.665000 | \n",
+ "
\n",
+ " \n",
+ " | 6 | \n",
+ " 0.582900 | \n",
+ "
\n",
+ " \n",
+ " | 7 | \n",
+ " 0.572300 | \n",
+ "
\n",
+ " \n",
+ " | 8 | \n",
+ " 0.666000 | \n",
+ "
\n",
+ " \n",
+ " | 9 | \n",
+ " 0.536900 | \n",
+ "
\n",
+ " \n",
+ " | 10 | \n",
+ " 0.557000 | \n",
+ "
\n",
+ " \n",
+ " | 11 | \n",
+ " 0.558300 | \n",
+ "
\n",
+ " \n",
+ " | 12 | \n",
+ " 0.516000 | \n",
+ "
\n",
+ " \n",
+ " | 13 | \n",
+ " 0.536700 | \n",
+ "
\n",
+ " \n",
+ " | 14 | \n",
+ " 0.535600 | \n",
+ "
\n",
+ " \n",
+ " | 15 | \n",
+ " 0.547000 | \n",
+ "
\n",
+ " \n",
+ " | 16 | \n",
+ " 0.429500 | \n",
+ "
\n",
+ " \n",
+ " | 17 | \n",
+ " 0.439600 | \n",
+ "
\n",
+ " \n",
+ " | 18 | \n",
+ " 0.485100 | \n",
+ "
\n",
+ " \n",
+ " | 19 | \n",
+ " 0.524400 | \n",
+ "
\n",
+ " \n",
+ " | 20 | \n",
+ " 0.483800 | \n",
+ "
\n",
+ " \n",
+ " | 21 | \n",
+ " 0.489800 | \n",
+ "
\n",
+ " \n",
+ " | 22 | \n",
+ " 0.477000 | \n",
+ "
\n",
+ " \n",
+ " | 23 | \n",
+ " 0.442600 | \n",
+ "
\n",
+ " \n",
+ " | 24 | \n",
+ " 0.422000 | \n",
+ "
\n",
+ " \n",
+ " | 25 | \n",
+ " 0.443400 | \n",
+ "
\n",
+ " \n",
+ " | 26 | \n",
+ " 0.462900 | \n",
+ "
\n",
+ " \n",
+ " | 27 | \n",
+ " 0.417800 | \n",
+ "
\n",
+ " \n",
+ " | 28 | \n",
+ " 0.442900 | \n",
+ "
\n",
+ " \n",
+ " | 29 | \n",
+ " 0.465600 | \n",
+ "
\n",
+ " \n",
+ " | 30 | \n",
+ " 0.375100 | \n",
+ "
\n",
+ " \n",
+ " | 31 | \n",
+ " 0.419700 | \n",
+ "
\n",
+ " \n",
+ " | 32 | \n",
+ " 0.490900 | \n",
+ "
\n",
+ " \n",
+ " | 33 | \n",
+ " 0.384300 | \n",
+ "
\n",
+ " \n",
+ " | 34 | \n",
+ " 0.391300 | \n",
+ "
\n",
+ " \n",
+ " | 35 | \n",
+ " 0.390200 | \n",
+ "
\n",
+ " \n",
+ " | 36 | \n",
+ " 0.386200 | \n",
+ "
\n",
+ " \n",
+ " | 37 | \n",
+ " 0.350800 | \n",
+ "
\n",
+ " \n",
+ " | 38 | \n",
+ " 0.432100 | \n",
+ "
\n",
+ " \n",
+ " | 39 | \n",
+ " 0.443100 | \n",
+ "
\n",
+ " \n",
+ " | 40 | \n",
+ " 0.389300 | \n",
+ "
\n",
+ " \n",
+ " | 41 | \n",
+ " 0.387000 | \n",
+ "
\n",
+ " \n",
+ " | 42 | \n",
+ " 0.420000 | \n",
+ "
\n",
+ " \n",
+ " | 43 | \n",
+ " 0.358700 | \n",
+ "
\n",
+ " \n",
+ " | 44 | \n",
+ " 0.450000 | \n",
+ "
\n",
+ " \n",
+ " | 45 | \n",
+ " 0.427300 | \n",
+ "
\n",
+ " \n",
+ " | 46 | \n",
+ " 0.420900 | \n",
+ "
\n",
+ " \n",
+ " | 47 | \n",
+ " 0.444100 | \n",
+ "
\n",
+ " \n",
+ " | 48 | \n",
+ " 0.348900 | \n",
+ "
\n",
+ " \n",
+ " | 49 | \n",
+ " 0.330900 | \n",
+ "
\n",
+ " \n",
+ " | 50 | \n",
+ " 0.399800 | \n",
+ "
\n",
+ " \n",
+ " | 51 | \n",
+ " 0.395200 | \n",
+ "
\n",
+ " \n",
+ " | 52 | \n",
+ " 0.371800 | \n",
+ "
\n",
+ " \n",
+ " | 53 | \n",
+ " 0.352200 | \n",
+ "
\n",
+ " \n",
+ " | 54 | \n",
+ " 0.406200 | \n",
+ "
\n",
+ " \n",
+ " | 55 | \n",
+ " 0.372200 | \n",
+ "
\n",
+ " \n",
+ " | 56 | \n",
+ " 0.397000 | \n",
+ "
\n",
+ " \n",
+ " | 57 | \n",
+ " 0.446500 | \n",
+ "
\n",
+ " \n",
+ " | 58 | \n",
+ " 0.351100 | \n",
+ "
\n",
+ " \n",
+ " | 59 | \n",
+ " 0.350900 | \n",
+ "
\n",
+ " \n",
+ " | 60 | \n",
+ " 0.352600 | \n",
+ "
\n",
+ " \n",
+ "
"
+ ],
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "trainer_stats = trainer.train()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "pCqnaKmlO1U9",
+ "outputId": "73335853-d855-4659-9cd6-5ec66d4c7450"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1489.5908 seconds used for training.\n",
+ "24.83 minutes used for training.\n",
+ "Peak reserved memory = 12.311 GB.\n",
+ "Peak reserved memory for training = 6.717 GB.\n",
+ "Peak reserved memory % of max memory = 83.476 %.\n",
+ "Peak reserved memory for training % of max memory = 45.545 %.\n"
+ ]
+ }
+ ],
+ "source": [
+ "used_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3)\n",
+ "used_memory_for_lora = round(used_memory - start_gpu_memory, 3)\n",
+ "used_percentage = round(used_memory / max_memory * 100, 3)\n",
+ "lora_percentage = round(used_memory_for_lora / max_memory * 100, 3)\n",
+ "print(f\"{trainer_stats.metrics['train_runtime']} seconds used for training.\")\n",
+ "print(\n",
+ " f\"{round(trainer_stats.metrics['train_runtime']/60, 2)} minutes used for training.\"\n",
+ ")\n",
+ "print(f\"Peak reserved memory = {used_memory} GB.\")\n",
+ "print(f\"Peak reserved memory for training = {used_memory_for_lora} GB.\")\n",
+ "print(f\"Peak reserved memory % of max memory = {used_percentage} %.\")\n",
+ "print(f\"Peak reserved memory for training % of max memory = {lora_percentage} %.\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "ekOmTR1hSNcr"
+ },
+ "source": [
+ "### Inference\n",
+ "\n",
+ "Now let's run the model! On an example of the training data.\n",
+ "\n",
+ "Using an openapi json format schema."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {
+ "id": "L7rLzg2on6gd"
+ },
+ "outputs": [],
+ "source": [
+ "schema = \"\"\"\n",
+ "{\n",
+ " '$schema': 'http://json-schema.org/draft-07/schema#',\n",
+ " '$defs': {\n",
+ " 'Patient': {\n",
+ " 'type': 'object',\n",
+ " 'properties': {\n",
+ " 'patientId': {\n",
+ " 'type': 'string',\n",
+ " 'minLength': 1,\n",
+ " 'maxLength': 255\n",
+ " },\n",
+ " 'firstName': {\n",
+ " 'type': 'string',\n",
+ " 'minLength': 1,\n",
+ " 'maxLength': 255\n",
+ " },\n",
+ " 'lastName': {\n",
+ " 'type': 'string',\n",
+ " 'minLength': 1,\n",
+ " 'maxLength': 255\n",
+ " },\n",
+ " 'dateOfBirth': {\n",
+ " 'type': 'string',\n",
+ " 'format': 'date'\n",
+ " },\n",
+ " 'gender': {\n",
+ " 'type': 'string',\n",
+ " 'enum': [\n",
+ " 'male',\n",
+ " 'female',\n",
+ " 'other'\n",
+ " ]\n",
+ " },\n",
+ " 'medicalConditions': {\n",
+ " 'type': 'array',\n",
+ " 'items': {\n",
+ " 'type': 'string',\n",
+ " 'minLength': 1,\n",
+ " 'maxLength': 255\n",
+ " },\n",
+ " 'minItems': 0,\n",
+ " 'uniqueItems': true\n",
+ " }\n",
+ " },\n",
+ " 'required': [\n",
+ " 'patientId',\n",
+ " 'firstName',\n",
+ " 'lastName',\n",
+ " 'dateOfBirth',\n",
+ " 'gender'\n",
+ " ]\n",
+ " },\n",
+ " 'Medication': {\n",
+ " 'type': 'object',\n",
+ " 'properties': {\n",
+ " 'medicationId': {\n",
+ " 'type': 'string',\n",
+ " 'minLength': 1,\n",
+ " 'maxLength': 255\n",
+ " },\n",
+ " 'name': {\n",
+ " 'type': 'string',\n",
+ " 'minLength': 1,\n",
+ " 'maxLength': 255\n",
+ " },\n",
+ " 'dosage': {\n",
+ " 'type': 'number',\n",
+ " 'minimum': 0\n",
+ " },\n",
+ " 'units': {\n",
+ " 'type': 'string',\n",
+ " 'minLength': 1,\n",
+ " 'maxLength': 255\n",
+ " },\n",
+ " 'instructions': {\n",
+ " 'type': 'string',\n",
+ " 'minLength': 1,\n",
+ " 'maxLength': 255\n",
+ " },\n",
+ " 'refills': {\n",
+ " 'type': 'number',\n",
+ " 'minimum': 0\n",
+ " }\n",
+ " },\n",
+ " 'required': [\n",
+ " 'medicationId',\n",
+ " 'name',\n",
+ " 'dosage',\n",
+ " 'units',\n",
+ " 'instructions',\n",
+ " 'refills'\n",
+ " ]\n",
+ " },\n",
+ " 'Pharmacy': {\n",
+ " 'type': 'object',\n",
+ " 'properties': {\n",
+ " 'pharmacyId': {\n",
+ " 'type': 'string',\n",
+ " 'minLength': 1,\n",
+ " 'maxLength': 255\n",
+ " },\n",
+ " 'name': {\n",
+ " 'type': 'string',\n",
+ " 'minLength': 1,\n",
+ " 'maxLength': 255\n",
+ " },\n",
+ " 'address': {\n",
+ " 'type': 'string',\n",
+ " 'minLength': 1,\n",
+ " 'maxLength': 255\n",
+ " },\n",
+ " 'phone': {\n",
+ " 'type': 'string',\n",
+ " 'minLength': 1,\n",
+ " 'maxLength': 255\n",
+ " }\n",
+ " },\n",
+ " 'required': [\n",
+ " 'pharmacyId',\n",
+ " 'name',\n",
+ " 'address'\n",
+ " ]\n",
+ " },\n",
+ " 'Prescription': {\n",
+ " 'type': 'object',\n",
+ " 'properties': {\n",
+ " 'prescriptionId': {\n",
+ " 'type': 'string',\n",
+ " 'minLength': 1,\n",
+ " 'maxLength': 255\n",
+ " },\n",
+ " 'patient': {\n",
+ " '$ref': '#/$defs/Patient'\n",
+ " },\n",
+ " 'medication': {\n",
+ " '$ref': '#/$defs/Medication'\n",
+ " },\n",
+ " 'pharmacy': {\n",
+ " '$ref': '#/$defs/Pharmacy'\n",
+ " },\n",
+ " 'datePrescribed': {\n",
+ " 'type': 'string',\n",
+ " 'format': 'date'\n",
+ " },\n",
+ " 'dateExpires': {\n",
+ " 'type': 'string',\n",
+ " 'format': 'date'\n",
+ " },\n",
+ " 'status': {\n",
+ " 'type': 'string',\n",
+ " 'enum': [\n",
+ " 'active',\n",
+ " 'inactive',\n",
+ " 'expired'\n",
+ " ]\n",
+ " }\n",
+ " },\n",
+ " 'required': [\n",
+ " 'prescriptionId',\n",
+ " 'patient',\n",
+ " 'medication',\n",
+ " 'pharmacy',\n",
+ " 'datePrescribed'\n",
+ " ]\n",
+ " }\n",
+ " },\n",
+ " 'type': 'object',\n",
+ " 'properties': {\n",
+ " 'prescription': {\n",
+ " '$ref': '#/$defs/Prescription'\n",
+ " }\n",
+ " },\n",
+ " 'required': [\n",
+ " 'prescription'\n",
+ " ]\n",
+ "}\n",
+ "\"\"\""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "3jOtfu4IBiTE"
+ },
+ "source": [
+ "Let's use the json_prompt used for supervised fine-tuning."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {
+ "id": "-BYrEbC_qBDv"
+ },
+ "outputs": [],
+ "source": [
+ "prompt = json_prompt.format(\n",
+ " \"**Medical Document** **Patient Information** * Patient ID: PT123456 * Name: Jane Doe * Date of Birth: 1980-01-01 * Gender: Female * Medical Conditions: * Asthma * Hypertension **Prescription Information** * Prescription ID: RX123456 * Date Prescribed: 2023-03-08 * Date Expires: 2023-09-07 * Status: Active **Medication Information** * Medication ID: MD123456 * Name: Albuterol * Dosage: 200 mcg * Units: mcg * Instructions: Inhale 2 puffs every 4-6 hours as needed for shortness of breath. * Refills: 3 **Pharmacy Information** * Pharmacy ID: PH123456 * Name: CVS Pharmacy * Address: 123 Main Street, Anytown, CA 12345 * Phone: (123) 456-7890 **Additional Information** * The patient has been using Albuterol for the past 5 years to manage her asthma. * The patient has been advised to use a spacer device with the Albuterol inhaler to improve the delivery of the medication to the lungs. * The patient should avoid using Albuterol more than 4 times per day. * The patient should contact her doctor if her asthma symptoms worsen or if she experiences any side effects from the medication. **Instructions for the Patient** * Take Albuterol exactly as prescribed by your doctor. * Do not take more than the prescribed dosage. * Use a spacer device with the Albuterol inhaler. * Avoid using Albuterol more than 4 times per day. * Contact your doctor if your asthma symptoms worsen or if you experience any side effects from the medication. **Signature** [Doctor's Name] [Date]\",\n",
+ " schema,\n",
+ " \"\",\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "MMx38S-_B5M_"
+ },
+ "source": [
+ "The FastLanguageModel interface is similar to the Hugging Face Transformer library interface, but enables FlashAttention by default and improves inference on the model supported by unsloth."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 18,
+ "metadata": {
+ "id": "kR3gIAX-SM2q"
+ },
+ "outputs": [],
+ "source": [
+ "FastLanguageModel.for_inference(model)\n",
+ "\n",
+ "inputs = tokenizer([prompt], return_tensors=\"pt\").to(\"cuda\")\n",
+ "\n",
+ "outputs = model.generate(**inputs, max_new_tokens=1024, use_cache=True)\n",
+ "result = tokenizer.batch_decode(outputs)[0][len(prompt) + 5 : -5]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "1aZEG0KAMT4I"
+ },
+ "source": [
+ "Let's parse the json output."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 19,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "BvvqhcRVwaZP",
+ "outputId": "742a14dc-bf0d-4726-e779-d256960ebef4"
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "{'prescription': {'prescriptionId': 'RX123456',\n",
+ " 'patient': {'patientId': 'PT123456',\n",
+ " 'firstName': 'Jane',\n",
+ " 'lastName': 'Doe',\n",
+ " 'dateOfBirth': '1980-01-01',\n",
+ " 'gender': 'female',\n",
+ " 'medicalConditions': ['Asthma', 'Hypertension']},\n",
+ " 'medication': {'medicationId': 'MD123456',\n",
+ " 'name': 'Albuterol',\n",
+ " 'dosage': 200,\n",
+ " 'units': 'mcg',\n",
+ " 'instructions': 'Inhale 2 puffs every 4-6 hours as needed for shortness of breath.',\n",
+ " 'refills': 3},\n",
+ " 'pharmacy': {'pharmacyId': 'PH123456',\n",
+ " 'name': 'CVS Pharmacy',\n",
+ " 'address': '123 Main Street, Anytown, CA 12345',\n",
+ " 'phone': '(123) 456-7890'},\n",
+ " 'datePrescribed': '2023-03-08',\n",
+ " 'dateExpires': '2023-09-07',\n",
+ " 'status': 'active'}}"
+ ]
+ },
+ "execution_count": 19,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "import json\n",
+ "\n",
+ "json.loads(result)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "uMuVrWbjAzhc"
+ },
+ "source": [
+ "### Saving, loading finetuned models"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {
+ "id": "0Q0xKVH5Mtlu"
+ },
+ "outputs": [],
+ "source": [
+ "USERNAME = \"bastienp\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 21,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 246,
+ "referenced_widgets": [
+ "bb80eaaaaa9148178e2ef6ede2f9d7d6",
+ "ab2a8bc1ffec4e15b5f862b4f1652f64",
+ "97a7ca0e058f464c89f90f6e3bb58018",
+ "3a0caa99aae14bcc9f33df988eb02cf7",
+ "85fb6a9046d34c39a0766dde8f3af0d2",
+ "680d92acf35f47e9a652f8343b5bda25",
+ "4770804493964f67a3f9494fea85fbb9",
+ "37f2dc522f2848438bd9feeba3e6f80e",
+ "ee7afdf438fb420d9e9fa201177e0c97",
+ "9e19a27268724a74b806554fff57a45c",
+ "a509b5dd339f45bf9197b23f18b34062",
+ "ca995c4089594d4f84caf42f0ce065c0",
+ "86adc5622c614544aaa7f37a4ad3c4fc",
+ "997ab76dac984b76a3aaf0264fb360aa",
+ "9d0016260eee452dba935419222d4be8",
+ "f9cb8f5ea1494c18b3c281cd86b0f2ed",
+ "0736d67a663543cc872654d4351681ca",
+ "24433e3e6f164eef9ea6fcc770e3ebb3",
+ "57404ed7bb5e498a82c3fee8d5fab99a",
+ "4c1e2a1fe9e84d74afe23772be397d28",
+ "c7c1a79894a7441998802fcf5f5a38f8",
+ "4de7cf85e1b948d2be0cd645c4618ad2",
+ "d083bba277b64d81bb7f38f67ac1d6fb",
+ "2728693354a94ca1925dab18bce68485",
+ "c61ff9e60d0041a9b1bedd5eb79d56ed",
+ "c5f6249284d341af9e7de4079af7faf5",
+ "2c9a19c039ac482da52f4c332394386a",
+ "4177fe06c1094d4fb91094c88e4c3f59",
+ "1e79044dd61d493eb3abdd67f127c922",
+ "3a0fc9f2f93c46b5bf082696e99dc81b",
+ "fdeba8bdaf6e4b1ba435e9d5d7e1cccf",
+ "c97279a6e27a417dba83143f13d75d60",
+ "99b765f5150d451c8410039d8538c713",
+ "ba65b1d208d749d6a480560e1927e309",
+ "d7e8cbce9188440c9c57ecbf61182470",
+ "7c6fd3233970472393cbdc9006b3a7bd",
+ "0ad7d36b87cd4c87aa6805f58ca46e2d",
+ "b780658eb4074d919eac12c5ee604181",
+ "d7aa327d766d42bc97e7fbbae120e3e6",
+ "2feba1fa82274a9288c88b2437d0cc87",
+ "5a81e69081c04ea79506faed49df58ff",
+ "d88966d8da27434086c1e2f7afbd126e",
+ "a6f6723a7fc3420fbfb51947cb40c767",
+ "57c64f5cfac742df8ac867e66df2fe35",
+ "a6803f3f34cb4b8a9196010c1a5bb433",
+ "3b27b885c3284ad094021932a32d1aeb",
+ "0085c4c5c46e461d88fb54e30f4c3564",
+ "bda1050ba65e4e538fe788a335e38ba1",
+ "5d032fc1945f48d8a2e7d00fe252e3f3",
+ "68111f3a2d8e4ebc93549cc965a13edd",
+ "2b8428a7a26540159588e31d8ec585d9",
+ "472b50a1c14647a1bf712ba5ac8be8b4",
+ "9c1ec8b7689148768c955ff1ca9c890c",
+ "3b1b96d89c664599b0fec0494c80452c",
+ "78540c110c654fcfa6d7f941198b00ca",
+ "1948d3a81ec94c058c31998e1ab20582",
+ "7b3c622f99694a0eba22e21399c9ce9c",
+ "eb1401b37dfc49cea6bce8f85bf0295f",
+ "75859490fb3145c9a8466ded1e0e7b6e",
+ "1246605857e5436c9849e30178aec428",
+ "9dc014f15e58496c8af9193035c870fb",
+ "61de9234bbda480c82b75946c50e4ec9",
+ "460e85d82ac0425195aaaf55799ba4d1",
+ "99875aa951d340e69c6bb8fbe92cadb8",
+ "f41aee3b9a9d4a40aa40e26aa823d619",
+ "381936bfb2cf4dbc8c70bf166e35c9b2"
+ ]
+ },
+ "id": "upcOlWe7A1vc",
+ "outputId": "9fa32490-e18d-42c1-a40a-cbaada9e14a8"
+ },
+ "outputs": [
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "bb80eaaaaa9148178e2ef6ede2f9d7d6",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ "README.md: 0%| | 0.00/565 [00:00 F16, shape = {2304, 256000}\n",
+ "INFO:hf-to-gguf:blk.0.attn_q.weight, torch.float16 --> F16, shape = {2304, 2048}\n",
+ "INFO:hf-to-gguf:blk.0.attn_k.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.0.attn_v.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.0.attn_output.weight, torch.float16 --> F16, shape = {2048, 2304}\n",
+ "INFO:hf-to-gguf:blk.0.ffn_gate.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.0.ffn_up.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.0.ffn_down.weight, torch.float16 --> F16, shape = {9216, 2304}\n",
+ "INFO:hf-to-gguf:blk.0.attn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.0.post_attention_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.0.ffn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.0.post_ffw_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.1.attn_q.weight, torch.float16 --> F16, shape = {2304, 2048}\n",
+ "INFO:hf-to-gguf:blk.1.attn_k.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.1.attn_v.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.1.attn_output.weight, torch.float16 --> F16, shape = {2048, 2304}\n",
+ "INFO:hf-to-gguf:blk.1.ffn_gate.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.1.ffn_up.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.1.ffn_down.weight, torch.float16 --> F16, shape = {9216, 2304}\n",
+ "INFO:hf-to-gguf:blk.1.attn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.1.post_attention_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.1.ffn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.1.post_ffw_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.2.attn_q.weight, torch.float16 --> F16, shape = {2304, 2048}\n",
+ "INFO:hf-to-gguf:blk.2.attn_k.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.2.attn_v.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.2.attn_output.weight, torch.float16 --> F16, shape = {2048, 2304}\n",
+ "INFO:hf-to-gguf:blk.2.ffn_gate.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.2.ffn_up.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.2.ffn_down.weight, torch.float16 --> F16, shape = {9216, 2304}\n",
+ "INFO:hf-to-gguf:blk.2.attn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.2.post_attention_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.2.ffn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.2.post_ffw_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.3.attn_q.weight, torch.float16 --> F16, shape = {2304, 2048}\n",
+ "INFO:hf-to-gguf:blk.3.attn_k.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.3.attn_v.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.3.attn_output.weight, torch.float16 --> F16, shape = {2048, 2304}\n",
+ "INFO:hf-to-gguf:blk.3.ffn_gate.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.3.ffn_up.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.3.ffn_down.weight, torch.float16 --> F16, shape = {9216, 2304}\n",
+ "INFO:hf-to-gguf:blk.3.attn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.3.post_attention_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.3.ffn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.3.post_ffw_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.4.attn_q.weight, torch.float16 --> F16, shape = {2304, 2048}\n",
+ "INFO:hf-to-gguf:blk.4.attn_k.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.4.attn_v.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.4.attn_output.weight, torch.float16 --> F16, shape = {2048, 2304}\n",
+ "INFO:hf-to-gguf:blk.4.ffn_gate.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.4.ffn_up.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.4.ffn_down.weight, torch.float16 --> F16, shape = {9216, 2304}\n",
+ "INFO:hf-to-gguf:blk.4.attn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.4.post_attention_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.4.ffn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.4.post_ffw_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.5.attn_q.weight, torch.float16 --> F16, shape = {2304, 2048}\n",
+ "INFO:hf-to-gguf:blk.5.attn_k.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.5.attn_v.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.5.attn_output.weight, torch.float16 --> F16, shape = {2048, 2304}\n",
+ "INFO:hf-to-gguf:blk.5.ffn_gate.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.5.ffn_up.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.5.ffn_down.weight, torch.float16 --> F16, shape = {9216, 2304}\n",
+ "INFO:hf-to-gguf:blk.5.attn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.5.post_attention_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.5.ffn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.5.post_ffw_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.6.attn_q.weight, torch.float16 --> F16, shape = {2304, 2048}\n",
+ "INFO:hf-to-gguf:blk.6.attn_k.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.6.attn_v.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.6.attn_output.weight, torch.float16 --> F16, shape = {2048, 2304}\n",
+ "INFO:hf-to-gguf:blk.6.ffn_gate.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.6.ffn_up.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.6.ffn_down.weight, torch.float16 --> F16, shape = {9216, 2304}\n",
+ "INFO:hf-to-gguf:blk.6.attn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.6.post_attention_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.6.ffn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.6.post_ffw_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.7.attn_q.weight, torch.float16 --> F16, shape = {2304, 2048}\n",
+ "INFO:hf-to-gguf:blk.7.attn_k.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.7.attn_v.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.7.attn_output.weight, torch.float16 --> F16, shape = {2048, 2304}\n",
+ "INFO:hf-to-gguf:blk.7.ffn_gate.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.7.ffn_up.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.7.ffn_down.weight, torch.float16 --> F16, shape = {9216, 2304}\n",
+ "INFO:hf-to-gguf:blk.7.attn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.7.post_attention_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.7.ffn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.7.post_ffw_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.8.attn_q.weight, torch.float16 --> F16, shape = {2304, 2048}\n",
+ "INFO:hf-to-gguf:blk.8.attn_k.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.8.attn_v.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.8.attn_output.weight, torch.float16 --> F16, shape = {2048, 2304}\n",
+ "INFO:hf-to-gguf:blk.8.ffn_gate.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.8.ffn_up.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.8.ffn_down.weight, torch.float16 --> F16, shape = {9216, 2304}\n",
+ "INFO:hf-to-gguf:blk.8.attn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.8.post_attention_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.8.ffn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.8.post_ffw_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.9.attn_q.weight, torch.float16 --> F16, shape = {2304, 2048}\n",
+ "INFO:hf-to-gguf:blk.9.attn_k.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.9.attn_v.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.9.attn_output.weight, torch.float16 --> F16, shape = {2048, 2304}\n",
+ "INFO:hf-to-gguf:blk.9.ffn_gate.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.9.ffn_up.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.9.ffn_down.weight, torch.float16 --> F16, shape = {9216, 2304}\n",
+ "INFO:hf-to-gguf:blk.9.attn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.9.post_attention_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.9.ffn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.9.post_ffw_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.10.attn_q.weight, torch.float16 --> F16, shape = {2304, 2048}\n",
+ "INFO:hf-to-gguf:blk.10.attn_k.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.10.attn_v.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.10.attn_output.weight, torch.float16 --> F16, shape = {2048, 2304}\n",
+ "INFO:hf-to-gguf:blk.10.ffn_gate.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.10.ffn_up.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.10.ffn_down.weight, torch.float16 --> F16, shape = {9216, 2304}\n",
+ "INFO:hf-to-gguf:blk.10.attn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.10.post_attention_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.10.ffn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.10.post_ffw_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.11.attn_q.weight, torch.float16 --> F16, shape = {2304, 2048}\n",
+ "INFO:hf-to-gguf:blk.11.attn_k.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.11.attn_v.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.11.attn_output.weight, torch.float16 --> F16, shape = {2048, 2304}\n",
+ "INFO:hf-to-gguf:blk.11.ffn_gate.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.11.ffn_up.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.11.ffn_down.weight, torch.float16 --> F16, shape = {9216, 2304}\n",
+ "INFO:hf-to-gguf:blk.11.attn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.11.post_attention_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.11.ffn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.11.post_ffw_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.12.attn_q.weight, torch.float16 --> F16, shape = {2304, 2048}\n",
+ "INFO:hf-to-gguf:blk.12.attn_k.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.12.attn_v.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.12.attn_output.weight, torch.float16 --> F16, shape = {2048, 2304}\n",
+ "INFO:hf-to-gguf:blk.12.ffn_gate.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.12.ffn_up.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.12.ffn_down.weight, torch.float16 --> F16, shape = {9216, 2304}\n",
+ "INFO:hf-to-gguf:blk.12.attn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.12.post_attention_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.12.ffn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.12.post_ffw_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.13.attn_q.weight, torch.float16 --> F16, shape = {2304, 2048}\n",
+ "INFO:hf-to-gguf:blk.13.attn_k.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.13.attn_v.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.13.attn_output.weight, torch.float16 --> F16, shape = {2048, 2304}\n",
+ "INFO:hf-to-gguf:blk.13.ffn_gate.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.13.ffn_up.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.13.ffn_down.weight, torch.float16 --> F16, shape = {9216, 2304}\n",
+ "INFO:hf-to-gguf:blk.13.attn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.13.post_attention_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.13.ffn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.13.post_ffw_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.14.attn_q.weight, torch.float16 --> F16, shape = {2304, 2048}\n",
+ "INFO:hf-to-gguf:blk.14.attn_k.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.14.attn_v.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.14.attn_output.weight, torch.float16 --> F16, shape = {2048, 2304}\n",
+ "INFO:hf-to-gguf:blk.14.ffn_gate.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.14.ffn_up.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.14.ffn_down.weight, torch.float16 --> F16, shape = {9216, 2304}\n",
+ "INFO:hf-to-gguf:blk.14.attn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.14.post_attention_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.14.ffn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.14.post_ffw_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.15.attn_q.weight, torch.float16 --> F16, shape = {2304, 2048}\n",
+ "INFO:hf-to-gguf:blk.15.attn_k.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.15.attn_v.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.15.attn_output.weight, torch.float16 --> F16, shape = {2048, 2304}\n",
+ "INFO:hf-to-gguf:blk.15.ffn_gate.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.15.ffn_up.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.15.ffn_down.weight, torch.float16 --> F16, shape = {9216, 2304}\n",
+ "INFO:hf-to-gguf:blk.15.attn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.15.post_attention_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.15.ffn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.15.post_ffw_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.16.attn_q.weight, torch.float16 --> F16, shape = {2304, 2048}\n",
+ "INFO:hf-to-gguf:blk.16.attn_k.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.16.attn_v.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.16.attn_output.weight, torch.float16 --> F16, shape = {2048, 2304}\n",
+ "INFO:hf-to-gguf:blk.16.ffn_gate.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.16.ffn_up.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.16.ffn_down.weight, torch.float16 --> F16, shape = {9216, 2304}\n",
+ "INFO:hf-to-gguf:blk.16.attn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.16.post_attention_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.16.ffn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.16.post_ffw_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.17.attn_q.weight, torch.float16 --> F16, shape = {2304, 2048}\n",
+ "INFO:hf-to-gguf:blk.17.attn_k.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.17.attn_v.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.17.attn_output.weight, torch.float16 --> F16, shape = {2048, 2304}\n",
+ "INFO:hf-to-gguf:blk.17.ffn_gate.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.17.ffn_up.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.17.ffn_down.weight, torch.float16 --> F16, shape = {9216, 2304}\n",
+ "INFO:hf-to-gguf:blk.17.attn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.17.post_attention_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.17.ffn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.17.post_ffw_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.18.attn_q.weight, torch.float16 --> F16, shape = {2304, 2048}\n",
+ "INFO:hf-to-gguf:blk.18.attn_k.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.18.attn_v.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.18.attn_output.weight, torch.float16 --> F16, shape = {2048, 2304}\n",
+ "INFO:hf-to-gguf:blk.18.ffn_gate.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.18.ffn_up.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.18.ffn_down.weight, torch.float16 --> F16, shape = {9216, 2304}\n",
+ "INFO:hf-to-gguf:blk.18.attn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.18.post_attention_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.18.ffn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.18.post_ffw_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.19.attn_q.weight, torch.float16 --> F16, shape = {2304, 2048}\n",
+ "INFO:hf-to-gguf:blk.19.attn_k.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.19.attn_v.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.19.attn_output.weight, torch.float16 --> F16, shape = {2048, 2304}\n",
+ "INFO:hf-to-gguf:blk.19.ffn_gate.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.19.ffn_up.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.19.ffn_down.weight, torch.float16 --> F16, shape = {9216, 2304}\n",
+ "INFO:hf-to-gguf:blk.19.attn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.19.post_attention_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.19.ffn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.19.post_ffw_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.20.attn_q.weight, torch.float16 --> F16, shape = {2304, 2048}\n",
+ "INFO:hf-to-gguf:blk.20.attn_k.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.20.attn_v.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.20.attn_output.weight, torch.float16 --> F16, shape = {2048, 2304}\n",
+ "INFO:hf-to-gguf:blk.20.ffn_gate.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.20.ffn_up.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.20.ffn_down.weight, torch.float16 --> F16, shape = {9216, 2304}\n",
+ "INFO:hf-to-gguf:blk.20.attn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.20.post_attention_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.20.ffn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.20.post_ffw_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.21.attn_q.weight, torch.float16 --> F16, shape = {2304, 2048}\n",
+ "INFO:hf-to-gguf:blk.21.attn_k.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.21.attn_v.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.21.attn_output.weight, torch.float16 --> F16, shape = {2048, 2304}\n",
+ "INFO:hf-to-gguf:blk.21.ffn_gate.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.21.ffn_up.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.21.ffn_down.weight, torch.float16 --> F16, shape = {9216, 2304}\n",
+ "INFO:hf-to-gguf:blk.21.attn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.21.post_attention_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.21.ffn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.21.post_ffw_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.22.attn_q.weight, torch.float16 --> F16, shape = {2304, 2048}\n",
+ "INFO:hf-to-gguf:blk.22.attn_k.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.22.attn_v.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.22.attn_output.weight, torch.float16 --> F16, shape = {2048, 2304}\n",
+ "INFO:hf-to-gguf:blk.22.ffn_gate.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.22.ffn_up.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.22.ffn_down.weight, torch.float16 --> F16, shape = {9216, 2304}\n",
+ "INFO:hf-to-gguf:blk.22.attn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.22.post_attention_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.22.ffn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.22.post_ffw_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.23.attn_q.weight, torch.float16 --> F16, shape = {2304, 2048}\n",
+ "INFO:hf-to-gguf:blk.23.attn_k.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.23.attn_v.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.23.attn_output.weight, torch.float16 --> F16, shape = {2048, 2304}\n",
+ "INFO:hf-to-gguf:blk.23.ffn_gate.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.23.ffn_up.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.23.ffn_down.weight, torch.float16 --> F16, shape = {9216, 2304}\n",
+ "INFO:hf-to-gguf:blk.23.attn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.23.post_attention_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.23.ffn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.23.post_ffw_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.24.attn_q.weight, torch.float16 --> F16, shape = {2304, 2048}\n",
+ "INFO:hf-to-gguf:blk.24.attn_k.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.24.attn_v.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.24.attn_output.weight, torch.float16 --> F16, shape = {2048, 2304}\n",
+ "INFO:hf-to-gguf:blk.24.ffn_gate.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:gguf: loading model part 'pytorch_model-00002-of-00002.bin'\n",
+ "INFO:hf-to-gguf:blk.24.ffn_up.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.24.ffn_down.weight, torch.float16 --> F16, shape = {9216, 2304}\n",
+ "INFO:hf-to-gguf:blk.24.attn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.24.post_attention_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.24.ffn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.24.post_ffw_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.25.attn_q.weight, torch.float16 --> F16, shape = {2304, 2048}\n",
+ "INFO:hf-to-gguf:blk.25.attn_k.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.25.attn_v.weight, torch.float16 --> F16, shape = {2304, 1024}\n",
+ "INFO:hf-to-gguf:blk.25.attn_output.weight, torch.float16 --> F16, shape = {2048, 2304}\n",
+ "INFO:hf-to-gguf:blk.25.ffn_gate.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.25.ffn_up.weight, torch.float16 --> F16, shape = {2304, 9216}\n",
+ "INFO:hf-to-gguf:blk.25.ffn_down.weight, torch.float16 --> F16, shape = {9216, 2304}\n",
+ "INFO:hf-to-gguf:blk.25.attn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.25.post_attention_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.25.ffn_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:blk.25.post_ffw_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:output_norm.weight, torch.float16 --> F32, shape = {2304}\n",
+ "INFO:hf-to-gguf:Set meta model\n",
+ "INFO:hf-to-gguf:Set model parameters\n",
+ "INFO:hf-to-gguf:Set model tokenizer\n",
+ "INFO:gguf.vocab:Setting special token type bos to 2\n",
+ "INFO:gguf.vocab:Setting special token type eos to 1\n",
+ "INFO:gguf.vocab:Setting special token type unk to 3\n",
+ "INFO:gguf.vocab:Setting special token type pad to 0\n",
+ "INFO:gguf.vocab:Setting add_bos_token to True\n",
+ "INFO:gguf.vocab:Setting add_eos_token to False\n",
+ "INFO:gguf.vocab:Setting chat_template to {% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '' + role + '\n",
+ "' + message['content'] | trim + '\n",
+ "' }}{% endfor %}{% if add_generation_prompt %}{{'model\n",
+ "'}}{% endif %}\n",
+ "INFO:hf-to-gguf:Set model quantization version\n",
+ "INFO:gguf.gguf_writer:Writing the following files:\n",
+ "INFO:gguf.gguf_writer:bastienp/Gemma-2-2B-it-JSON-data-extration/unsloth.F16.gguf: n_tensors = 288, total_size = 5.2G\n",
+ "Writing: 100%|██████████| 5.23G/5.23G [01:00<00:00, 86.7Mbyte/s]\n",
+ "INFO:hf-to-gguf:Model successfully exported to bastienp/Gemma-2-2B-it-JSON-data-extration/unsloth.F16.gguf\n",
+ "Unsloth: Conversion completed! Output location: ./bastienp/Gemma-2-2B-it-JSON-data-extration/unsloth.F16.gguf\n",
+ "Unsloth: [2] Converting GGUF 16bit into q4_k_m. This will take 20 minutes...\n",
+ "main: build = 3605 (cfac111e)\n",
+ "main: built with cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 for x86_64-linux-gnu\n",
+ "main: quantizing './bastienp/Gemma-2-2B-it-JSON-data-extration/unsloth.F16.gguf' to './bastienp/Gemma-2-2B-it-JSON-data-extration/unsloth.Q4_K_M.gguf' as Q4_K_M using 4 threads\n",
+ "llama_model_loader: loaded meta data with 34 key-value pairs and 288 tensors from ./bastienp/Gemma-2-2B-it-JSON-data-extration/unsloth.F16.gguf (version GGUF V3 (latest))\n",
+ "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
+ "llama_model_loader: - kv 0: general.architecture str = gemma2\n",
+ "llama_model_loader: - kv 1: general.type str = model\n",
+ "llama_model_loader: - kv 2: general.name str = Gemma 2 2b It\n",
+ "llama_model_loader: - kv 3: general.organization str = Unsloth\n",
+ "llama_model_loader: - kv 4: general.finetune str = it\n",
+ "llama_model_loader: - kv 5: general.basename str = gemma-2\n",
+ "llama_model_loader: - kv 6: general.size_label str = 2B\n",
+ "llama_model_loader: - kv 7: gemma2.context_length u32 = 8192\n",
+ "llama_model_loader: - kv 8: gemma2.embedding_length u32 = 2304\n",
+ "llama_model_loader: - kv 9: gemma2.block_count u32 = 26\n",
+ "llama_model_loader: - kv 10: gemma2.feed_forward_length u32 = 9216\n",
+ "llama_model_loader: - kv 11: gemma2.attention.head_count u32 = 8\n",
+ "llama_model_loader: - kv 12: gemma2.attention.head_count_kv u32 = 4\n",
+ "llama_model_loader: - kv 13: gemma2.attention.layer_norm_rms_epsilon f32 = 0.000001\n",
+ "llama_model_loader: - kv 14: gemma2.attention.key_length u32 = 256\n",
+ "llama_model_loader: - kv 15: gemma2.attention.value_length u32 = 256\n",
+ "llama_model_loader: - kv 16: general.file_type u32 = 1\n",
+ "llama_model_loader: - kv 17: gemma2.attn_logit_softcapping f32 = 50.000000\n",
+ "llama_model_loader: - kv 18: gemma2.final_logit_softcapping f32 = 30.000000\n",
+ "llama_model_loader: - kv 19: gemma2.attention.sliding_window u32 = 4096\n",
+ "llama_model_loader: - kv 20: tokenizer.ggml.model str = llama\n",
+ "llama_model_loader: - kv 21: tokenizer.ggml.pre str = default\n",
+ "llama_model_loader: - kv 22: tokenizer.ggml.tokens arr[str,256000] = [\"\", \"\", \"\", \"\", ...\n",
+ "llama_model_loader: - kv 23: tokenizer.ggml.scores arr[f32,256000] = [-1000.000000, -1000.000000, -1000.00...\n",
+ "llama_model_loader: - kv 24: tokenizer.ggml.token_type arr[i32,256000] = [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, ...\n",
+ "llama_model_loader: - kv 25: tokenizer.ggml.bos_token_id u32 = 2\n",
+ "llama_model_loader: - kv 26: tokenizer.ggml.eos_token_id u32 = 1\n",
+ "llama_model_loader: - kv 27: tokenizer.ggml.unknown_token_id u32 = 3\n",
+ "llama_model_loader: - kv 28: tokenizer.ggml.padding_token_id u32 = 0\n",
+ "llama_model_loader: - kv 29: tokenizer.ggml.add_bos_token bool = true\n",
+ "llama_model_loader: - kv 30: tokenizer.ggml.add_eos_token bool = false\n",
+ "llama_model_loader: - kv 31: tokenizer.chat_template str = {% if messages[0]['role'] == 'system'...\n",
+ "llama_model_loader: - kv 32: tokenizer.ggml.add_space_prefix bool = false\n",
+ "llama_model_loader: - kv 33: general.quantization_version u32 = 2\n",
+ "llama_model_loader: - type f32: 105 tensors\n",
+ "llama_model_loader: - type f16: 183 tensors\n",
+ "[ 1/ 288] token_embd.weight - [ 2304, 256000, 1, 1], type = f16, converting to q6_K .. size = 1125.00 MiB -> 461.43 MiB\n",
+ "[ 2/ 288] blk.0.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 3/ 288] blk.0.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q4_K .. size = 4.50 MiB -> 1.27 MiB\n",
+ "[ 4/ 288] blk.0.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q6_K .. size = 4.50 MiB -> 1.85 MiB\n",
+ "[ 5/ 288] blk.0.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 6/ 288] blk.0.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 7/ 288] blk.0.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 8/ 288] blk.0.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q6_K .. size = 40.50 MiB -> 16.61 MiB\n",
+ "[ 9/ 288] blk.0.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 10/ 288] blk.0.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 11/ 288] blk.0.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 12/ 288] blk.0.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 13/ 288] blk.1.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 14/ 288] blk.1.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q4_K .. size = 4.50 MiB -> 1.27 MiB\n",
+ "[ 15/ 288] blk.1.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q6_K .. size = 4.50 MiB -> 1.85 MiB\n",
+ "[ 16/ 288] blk.1.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 17/ 288] blk.1.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 18/ 288] blk.1.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 19/ 288] blk.1.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q6_K .. size = 40.50 MiB -> 16.61 MiB\n",
+ "[ 20/ 288] blk.1.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 21/ 288] blk.1.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 22/ 288] blk.1.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 23/ 288] blk.1.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 24/ 288] blk.2.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 25/ 288] blk.2.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q4_K .. size = 4.50 MiB -> 1.27 MiB\n",
+ "[ 26/ 288] blk.2.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q6_K .. size = 4.50 MiB -> 1.85 MiB\n",
+ "[ 27/ 288] blk.2.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 28/ 288] blk.2.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 29/ 288] blk.2.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 30/ 288] blk.2.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q6_K .. size = 40.50 MiB -> 16.61 MiB\n",
+ "[ 31/ 288] blk.2.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 32/ 288] blk.2.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 33/ 288] blk.2.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 34/ 288] blk.2.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 35/ 288] blk.3.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 36/ 288] blk.3.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q4_K .. size = 4.50 MiB -> 1.27 MiB\n",
+ "[ 37/ 288] blk.3.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q4_K .. size = 4.50 MiB -> 1.27 MiB\n",
+ "[ 38/ 288] blk.3.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 39/ 288] blk.3.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 40/ 288] blk.3.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 41/ 288] blk.3.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 42/ 288] blk.3.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 43/ 288] blk.3.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 44/ 288] blk.3.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 45/ 288] blk.3.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 46/ 288] blk.4.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 47/ 288] blk.4.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q4_K .. size = 4.50 MiB -> 1.27 MiB\n",
+ "[ 48/ 288] blk.4.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q4_K .. size = 4.50 MiB -> 1.27 MiB\n",
+ "[ 49/ 288] blk.4.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 50/ 288] blk.4.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 51/ 288] blk.4.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 52/ 288] blk.4.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 53/ 288] blk.4.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 54/ 288] blk.4.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 55/ 288] blk.4.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 56/ 288] blk.4.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 57/ 288] blk.5.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 58/ 288] blk.5.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q4_K .. size = 4.50 MiB -> 1.27 MiB\n",
+ "[ 59/ 288] blk.5.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q6_K .. size = 4.50 MiB -> 1.85 MiB\n",
+ "[ 60/ 288] blk.5.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 61/ 288] blk.5.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 62/ 288] blk.5.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 63/ 288] blk.5.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q6_K .. size = 40.50 MiB -> 16.61 MiB\n",
+ "[ 64/ 288] blk.5.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 65/ 288] blk.5.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 66/ 288] blk.5.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 67/ 288] blk.5.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 68/ 288] blk.6.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 69/ 288] blk.6.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q4_K .. size = 4.50 MiB -> 1.27 MiB\n",
+ "[ 70/ 288] blk.6.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q4_K .. size = 4.50 MiB -> 1.27 MiB\n",
+ "[ 71/ 288] blk.6.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 72/ 288] blk.6.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 73/ 288] blk.6.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 74/ 288] blk.6.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 75/ 288] blk.6.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 76/ 288] blk.6.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 77/ 288] blk.6.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 78/ 288] blk.6.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 79/ 288] blk.7.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 80/ 288] blk.7.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q4_K .. size = 4.50 MiB -> 1.27 MiB\n",
+ "[ 81/ 288] blk.7.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q4_K .. size = 4.50 MiB -> 1.27 MiB\n",
+ "[ 82/ 288] blk.7.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 83/ 288] blk.7.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 84/ 288] blk.7.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 85/ 288] blk.7.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 86/ 288] blk.7.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 87/ 288] blk.7.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 88/ 288] blk.7.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 89/ 288] blk.7.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 90/ 288] blk.8.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 91/ 288] blk.8.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q4_K .. size = 4.50 MiB -> 1.27 MiB\n",
+ "[ 92/ 288] blk.8.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q6_K .. size = 4.50 MiB -> 1.85 MiB\n",
+ "[ 93/ 288] blk.8.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 94/ 288] blk.8.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 95/ 288] blk.8.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 96/ 288] blk.8.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q6_K .. size = 40.50 MiB -> 16.61 MiB\n",
+ "[ 97/ 288] blk.8.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 98/ 288] blk.8.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 99/ 288] blk.8.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 100/ 288] blk.8.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 101/ 288] blk.9.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 102/ 288] blk.9.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q4_K .. size = 4.50 MiB -> 1.27 MiB\n",
+ "[ 103/ 288] blk.9.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q4_K .. size = 4.50 MiB -> 1.27 MiB\n",
+ "[ 104/ 288] blk.9.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 105/ 288] blk.9.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 106/ 288] blk.9.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 107/ 288] blk.9.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 108/ 288] blk.9.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 109/ 288] blk.9.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 110/ 288] blk.9.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 111/ 288] blk.9.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 112/ 288] blk.10.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 113/ 288] blk.10.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q4_K .. size = 4.50 MiB -> 1.27 MiB\n",
+ "[ 114/ 288] blk.10.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q4_K .. size = 4.50 MiB -> 1.27 MiB\n",
+ "[ 115/ 288] blk.10.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 116/ 288] blk.10.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 117/ 288] blk.10.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 118/ 288] blk.10.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 119/ 288] blk.10.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 120/ 288] blk.10.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 121/ 288] blk.10.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 122/ 288] blk.10.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 123/ 288] blk.11.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 124/ 288] blk.11.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q4_K .. size = 4.50 MiB -> 1.27 MiB\n",
+ "[ 125/ 288] blk.11.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q6_K .. size = 4.50 MiB -> 1.85 MiB\n",
+ "[ 126/ 288] blk.11.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 127/ 288] blk.11.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 128/ 288] blk.11.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 129/ 288] blk.11.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q6_K .. size = 40.50 MiB -> 16.61 MiB\n",
+ "[ 130/ 288] blk.11.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 131/ 288] blk.11.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 132/ 288] blk.11.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 133/ 288] blk.11.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 134/ 288] blk.12.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 135/ 288] blk.12.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q4_K .. size = 4.50 MiB -> 1.27 MiB\n",
+ "[ 136/ 288] blk.12.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q4_K .. size = 4.50 MiB -> 1.27 MiB\n",
+ "[ 137/ 288] blk.12.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 138/ 288] blk.12.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 139/ 288] blk.12.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 140/ 288] blk.12.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 141/ 288] blk.12.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 142/ 288] blk.12.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 143/ 288] blk.12.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 144/ 288] blk.12.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 145/ 288] blk.13.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 146/ 288] blk.13.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q4_K .. size = 4.50 MiB -> 1.27 MiB\n",
+ "[ 147/ 288] blk.13.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q4_K .. size = 4.50 MiB -> 1.27 MiB\n",
+ "[ 148/ 288] blk.13.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 149/ 288] blk.13.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 150/ 288] blk.13.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 151/ 288] blk.13.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 152/ 288] blk.13.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 153/ 288] blk.13.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 154/ 288] blk.13.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 155/ 288] blk.13.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 156/ 288] blk.14.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 157/ 288] blk.14.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q4_K .. size = 4.50 MiB -> 1.27 MiB\n",
+ "[ 158/ 288] blk.14.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q6_K .. size = 4.50 MiB -> 1.85 MiB\n",
+ "[ 159/ 288] blk.14.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 160/ 288] blk.14.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 161/ 288] blk.14.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 162/ 288] blk.14.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q6_K .. size = 40.50 MiB -> 16.61 MiB\n",
+ "[ 163/ 288] blk.14.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 164/ 288] blk.14.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 165/ 288] blk.14.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 166/ 288] blk.14.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 167/ 288] blk.15.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 168/ 288] blk.15.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q4_K .. size = 4.50 MiB -> 1.27 MiB\n",
+ "[ 169/ 288] blk.15.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q4_K .. size = 4.50 MiB -> 1.27 MiB\n",
+ "[ 170/ 288] blk.15.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 171/ 288] blk.15.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 172/ 288] blk.15.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 173/ 288] blk.15.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 174/ 288] blk.15.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 175/ 288] blk.15.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 176/ 288] blk.15.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 177/ 288] blk.15.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 178/ 288] blk.16.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 179/ 288] blk.16.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q4_K .. size = 4.50 MiB -> 1.27 MiB\n",
+ "[ 180/ 288] blk.16.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q4_K .. size = 4.50 MiB -> 1.27 MiB\n",
+ "[ 181/ 288] blk.16.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 182/ 288] blk.16.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 183/ 288] blk.16.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 184/ 288] blk.16.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 185/ 288] blk.16.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 186/ 288] blk.16.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 187/ 288] blk.16.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 188/ 288] blk.16.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 189/ 288] blk.17.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 190/ 288] blk.17.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q4_K .. size = 4.50 MiB -> 1.27 MiB\n",
+ "[ 191/ 288] blk.17.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q6_K .. size = 4.50 MiB -> 1.85 MiB\n",
+ "[ 192/ 288] blk.17.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 193/ 288] blk.17.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 194/ 288] blk.17.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 195/ 288] blk.17.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q6_K .. size = 40.50 MiB -> 16.61 MiB\n",
+ "[ 196/ 288] blk.17.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 197/ 288] blk.17.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 198/ 288] blk.17.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 199/ 288] blk.17.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 200/ 288] blk.18.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 201/ 288] blk.18.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q4_K .. size = 4.50 MiB -> 1.27 MiB\n",
+ "[ 202/ 288] blk.18.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q4_K .. size = 4.50 MiB -> 1.27 MiB\n",
+ "[ 203/ 288] blk.18.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 204/ 288] blk.18.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 205/ 288] blk.18.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 206/ 288] blk.18.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 207/ 288] blk.18.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 208/ 288] blk.18.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 209/ 288] blk.18.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 210/ 288] blk.18.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 211/ 288] blk.19.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 212/ 288] blk.19.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q4_K .. size = 4.50 MiB -> 1.27 MiB\n",
+ "[ 213/ 288] blk.19.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q4_K .. size = 4.50 MiB -> 1.27 MiB\n",
+ "[ 214/ 288] blk.19.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 215/ 288] blk.19.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 216/ 288] blk.19.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 217/ 288] blk.19.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 218/ 288] blk.19.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 219/ 288] blk.19.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 220/ 288] blk.19.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 221/ 288] blk.19.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 222/ 288] blk.20.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 223/ 288] blk.20.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q4_K .. size = 4.50 MiB -> 1.27 MiB\n",
+ "[ 224/ 288] blk.20.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q6_K .. size = 4.50 MiB -> 1.85 MiB\n",
+ "[ 225/ 288] blk.20.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 226/ 288] blk.20.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 227/ 288] blk.20.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 228/ 288] blk.20.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q6_K .. size = 40.50 MiB -> 16.61 MiB\n",
+ "[ 229/ 288] blk.20.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 230/ 288] blk.20.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 231/ 288] blk.20.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 232/ 288] blk.20.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 233/ 288] blk.21.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 234/ 288] blk.21.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q4_K .. size = 4.50 MiB -> 1.27 MiB\n",
+ "[ 235/ 288] blk.21.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q4_K .. size = 4.50 MiB -> 1.27 MiB\n",
+ "[ 236/ 288] blk.21.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 237/ 288] blk.21.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 238/ 288] blk.21.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 239/ 288] blk.21.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 240/ 288] blk.21.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 241/ 288] blk.21.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 242/ 288] blk.21.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 243/ 288] blk.21.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 244/ 288] blk.22.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 245/ 288] blk.22.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q4_K .. size = 4.50 MiB -> 1.27 MiB\n",
+ "[ 246/ 288] blk.22.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q6_K .. size = 4.50 MiB -> 1.85 MiB\n",
+ "[ 247/ 288] blk.22.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 248/ 288] blk.22.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 249/ 288] blk.22.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 250/ 288] blk.22.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q6_K .. size = 40.50 MiB -> 16.61 MiB\n",
+ "[ 251/ 288] blk.22.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 252/ 288] blk.22.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 253/ 288] blk.22.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 254/ 288] blk.22.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 255/ 288] blk.23.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 256/ 288] blk.23.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q4_K .. size = 4.50 MiB -> 1.27 MiB\n",
+ "[ 257/ 288] blk.23.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q6_K .. size = 4.50 MiB -> 1.85 MiB\n",
+ "[ 258/ 288] blk.23.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 259/ 288] blk.23.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 260/ 288] blk.23.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 261/ 288] blk.23.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q6_K .. size = 40.50 MiB -> 16.61 MiB\n",
+ "[ 262/ 288] blk.23.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 263/ 288] blk.23.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 264/ 288] blk.23.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 265/ 288] blk.23.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 266/ 288] blk.24.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 267/ 288] blk.24.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q4_K .. size = 4.50 MiB -> 1.27 MiB\n",
+ "[ 268/ 288] blk.24.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q6_K .. size = 4.50 MiB -> 1.85 MiB\n",
+ "[ 269/ 288] blk.24.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 270/ 288] blk.24.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 271/ 288] blk.24.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 272/ 288] blk.24.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q6_K .. size = 40.50 MiB -> 16.61 MiB\n",
+ "[ 273/ 288] blk.24.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 274/ 288] blk.24.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 275/ 288] blk.24.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 276/ 288] blk.24.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 277/ 288] blk.25.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 278/ 288] blk.25.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q4_K .. size = 4.50 MiB -> 1.27 MiB\n",
+ "[ 279/ 288] blk.25.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q6_K .. size = 4.50 MiB -> 1.85 MiB\n",
+ "[ 280/ 288] blk.25.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q4_K .. size = 9.00 MiB -> 2.53 MiB\n",
+ "[ 281/ 288] blk.25.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 282/ 288] blk.25.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q4_K .. size = 40.50 MiB -> 11.39 MiB\n",
+ "[ 283/ 288] blk.25.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q6_K .. size = 40.50 MiB -> 16.61 MiB\n",
+ "[ 284/ 288] blk.25.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 285/ 288] blk.25.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 286/ 288] blk.25.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 287/ 288] blk.25.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 288/ 288] output_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "llama_model_quantize_internal: model size = 4986.92 MB\n",
+ "llama_model_quantize_internal: quant size = 1623.67 MB\n",
+ "\n",
+ "main: quantize time = 266446.72 ms\n",
+ "main: total time = 266446.72 ms\n",
+ "Unsloth: Conversion completed! Output location: ./bastienp/Gemma-2-2B-it-JSON-data-extration/unsloth.Q4_K_M.gguf\n",
+ "Unsloth: [2] Converting GGUF 16bit into q8_0. This will take 20 minutes...\n",
+ "main: build = 3605 (cfac111e)\n",
+ "main: built with cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 for x86_64-linux-gnu\n",
+ "main: quantizing './bastienp/Gemma-2-2B-it-JSON-data-extration/unsloth.F16.gguf' to './bastienp/Gemma-2-2B-it-JSON-data-extration/unsloth.Q8_0.gguf' as Q8_0 using 4 threads\n",
+ "llama_model_loader: loaded meta data with 34 key-value pairs and 288 tensors from ./bastienp/Gemma-2-2B-it-JSON-data-extration/unsloth.F16.gguf (version GGUF V3 (latest))\n",
+ "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
+ "llama_model_loader: - kv 0: general.architecture str = gemma2\n",
+ "llama_model_loader: - kv 1: general.type str = model\n",
+ "llama_model_loader: - kv 2: general.name str = Gemma 2 2b It\n",
+ "llama_model_loader: - kv 3: general.organization str = Unsloth\n",
+ "llama_model_loader: - kv 4: general.finetune str = it\n",
+ "llama_model_loader: - kv 5: general.basename str = gemma-2\n",
+ "llama_model_loader: - kv 6: general.size_label str = 2B\n",
+ "llama_model_loader: - kv 7: gemma2.context_length u32 = 8192\n",
+ "llama_model_loader: - kv 8: gemma2.embedding_length u32 = 2304\n",
+ "llama_model_loader: - kv 9: gemma2.block_count u32 = 26\n",
+ "llama_model_loader: - kv 10: gemma2.feed_forward_length u32 = 9216\n",
+ "llama_model_loader: - kv 11: gemma2.attention.head_count u32 = 8\n",
+ "llama_model_loader: - kv 12: gemma2.attention.head_count_kv u32 = 4\n",
+ "llama_model_loader: - kv 13: gemma2.attention.layer_norm_rms_epsilon f32 = 0.000001\n",
+ "llama_model_loader: - kv 14: gemma2.attention.key_length u32 = 256\n",
+ "llama_model_loader: - kv 15: gemma2.attention.value_length u32 = 256\n",
+ "llama_model_loader: - kv 16: general.file_type u32 = 1\n",
+ "llama_model_loader: - kv 17: gemma2.attn_logit_softcapping f32 = 50.000000\n",
+ "llama_model_loader: - kv 18: gemma2.final_logit_softcapping f32 = 30.000000\n",
+ "llama_model_loader: - kv 19: gemma2.attention.sliding_window u32 = 4096\n",
+ "llama_model_loader: - kv 20: tokenizer.ggml.model str = llama\n",
+ "llama_model_loader: - kv 21: tokenizer.ggml.pre str = default\n",
+ "llama_model_loader: - kv 22: tokenizer.ggml.tokens arr[str,256000] = [\"\", \"\", \"\", \"\", ...\n",
+ "llama_model_loader: - kv 23: tokenizer.ggml.scores arr[f32,256000] = [-1000.000000, -1000.000000, -1000.00...\n",
+ "llama_model_loader: - kv 24: tokenizer.ggml.token_type arr[i32,256000] = [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, ...\n",
+ "llama_model_loader: - kv 25: tokenizer.ggml.bos_token_id u32 = 2\n",
+ "llama_model_loader: - kv 26: tokenizer.ggml.eos_token_id u32 = 1\n",
+ "llama_model_loader: - kv 27: tokenizer.ggml.unknown_token_id u32 = 3\n",
+ "llama_model_loader: - kv 28: tokenizer.ggml.padding_token_id u32 = 0\n",
+ "llama_model_loader: - kv 29: tokenizer.ggml.add_bos_token bool = true\n",
+ "llama_model_loader: - kv 30: tokenizer.ggml.add_eos_token bool = false\n",
+ "llama_model_loader: - kv 31: tokenizer.chat_template str = {% if messages[0]['role'] == 'system'...\n",
+ "llama_model_loader: - kv 32: tokenizer.ggml.add_space_prefix bool = false\n",
+ "llama_model_loader: - kv 33: general.quantization_version u32 = 2\n",
+ "llama_model_loader: - type f32: 105 tensors\n",
+ "llama_model_loader: - type f16: 183 tensors\n",
+ "[ 1/ 288] token_embd.weight - [ 2304, 256000, 1, 1], type = f16, converting to q8_0 .. size = 1125.00 MiB -> 597.66 MiB\n",
+ "[ 2/ 288] blk.0.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 3/ 288] blk.0.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 4/ 288] blk.0.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 5/ 288] blk.0.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 6/ 288] blk.0.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 7/ 288] blk.0.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 8/ 288] blk.0.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 9/ 288] blk.0.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 10/ 288] blk.0.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 11/ 288] blk.0.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 12/ 288] blk.0.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 13/ 288] blk.1.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 14/ 288] blk.1.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 15/ 288] blk.1.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 16/ 288] blk.1.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 17/ 288] blk.1.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 18/ 288] blk.1.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 19/ 288] blk.1.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 20/ 288] blk.1.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 21/ 288] blk.1.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 22/ 288] blk.1.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 23/ 288] blk.1.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 24/ 288] blk.2.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 25/ 288] blk.2.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 26/ 288] blk.2.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 27/ 288] blk.2.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 28/ 288] blk.2.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 29/ 288] blk.2.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 30/ 288] blk.2.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 31/ 288] blk.2.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 32/ 288] blk.2.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 33/ 288] blk.2.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 34/ 288] blk.2.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 35/ 288] blk.3.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 36/ 288] blk.3.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 37/ 288] blk.3.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 38/ 288] blk.3.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 39/ 288] blk.3.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 40/ 288] blk.3.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 41/ 288] blk.3.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 42/ 288] blk.3.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 43/ 288] blk.3.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 44/ 288] blk.3.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 45/ 288] blk.3.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 46/ 288] blk.4.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 47/ 288] blk.4.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 48/ 288] blk.4.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 49/ 288] blk.4.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 50/ 288] blk.4.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 51/ 288] blk.4.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 52/ 288] blk.4.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 53/ 288] blk.4.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 54/ 288] blk.4.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 55/ 288] blk.4.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 56/ 288] blk.4.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 57/ 288] blk.5.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 58/ 288] blk.5.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 59/ 288] blk.5.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 60/ 288] blk.5.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 61/ 288] blk.5.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 62/ 288] blk.5.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 63/ 288] blk.5.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 64/ 288] blk.5.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 65/ 288] blk.5.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 66/ 288] blk.5.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 67/ 288] blk.5.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 68/ 288] blk.6.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 69/ 288] blk.6.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 70/ 288] blk.6.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 71/ 288] blk.6.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 72/ 288] blk.6.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 73/ 288] blk.6.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 74/ 288] blk.6.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 75/ 288] blk.6.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 76/ 288] blk.6.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 77/ 288] blk.6.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 78/ 288] blk.6.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 79/ 288] blk.7.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 80/ 288] blk.7.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 81/ 288] blk.7.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 82/ 288] blk.7.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 83/ 288] blk.7.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 84/ 288] blk.7.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 85/ 288] blk.7.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 86/ 288] blk.7.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 87/ 288] blk.7.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 88/ 288] blk.7.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 89/ 288] blk.7.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 90/ 288] blk.8.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 91/ 288] blk.8.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 92/ 288] blk.8.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 93/ 288] blk.8.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 94/ 288] blk.8.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 95/ 288] blk.8.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 96/ 288] blk.8.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 97/ 288] blk.8.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 98/ 288] blk.8.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 99/ 288] blk.8.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 100/ 288] blk.8.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 101/ 288] blk.9.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 102/ 288] blk.9.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 103/ 288] blk.9.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 104/ 288] blk.9.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 105/ 288] blk.9.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 106/ 288] blk.9.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 107/ 288] blk.9.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 108/ 288] blk.9.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 109/ 288] blk.9.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 110/ 288] blk.9.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 111/ 288] blk.9.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 112/ 288] blk.10.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 113/ 288] blk.10.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 114/ 288] blk.10.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 115/ 288] blk.10.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 116/ 288] blk.10.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 117/ 288] blk.10.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 118/ 288] blk.10.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 119/ 288] blk.10.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 120/ 288] blk.10.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 121/ 288] blk.10.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 122/ 288] blk.10.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 123/ 288] blk.11.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 124/ 288] blk.11.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 125/ 288] blk.11.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 126/ 288] blk.11.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 127/ 288] blk.11.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 128/ 288] blk.11.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 129/ 288] blk.11.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 130/ 288] blk.11.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 131/ 288] blk.11.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 132/ 288] blk.11.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 133/ 288] blk.11.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 134/ 288] blk.12.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 135/ 288] blk.12.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 136/ 288] blk.12.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 137/ 288] blk.12.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 138/ 288] blk.12.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 139/ 288] blk.12.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 140/ 288] blk.12.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 141/ 288] blk.12.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 142/ 288] blk.12.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 143/ 288] blk.12.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 144/ 288] blk.12.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 145/ 288] blk.13.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 146/ 288] blk.13.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 147/ 288] blk.13.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 148/ 288] blk.13.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 149/ 288] blk.13.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 150/ 288] blk.13.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 151/ 288] blk.13.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 152/ 288] blk.13.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 153/ 288] blk.13.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 154/ 288] blk.13.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 155/ 288] blk.13.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 156/ 288] blk.14.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 157/ 288] blk.14.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 158/ 288] blk.14.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 159/ 288] blk.14.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 160/ 288] blk.14.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 161/ 288] blk.14.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 162/ 288] blk.14.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 163/ 288] blk.14.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 164/ 288] blk.14.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 165/ 288] blk.14.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 166/ 288] blk.14.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 167/ 288] blk.15.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 168/ 288] blk.15.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 169/ 288] blk.15.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 170/ 288] blk.15.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 171/ 288] blk.15.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 172/ 288] blk.15.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 173/ 288] blk.15.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 174/ 288] blk.15.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 175/ 288] blk.15.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 176/ 288] blk.15.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 177/ 288] blk.15.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 178/ 288] blk.16.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 179/ 288] blk.16.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 180/ 288] blk.16.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 181/ 288] blk.16.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 182/ 288] blk.16.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 183/ 288] blk.16.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 184/ 288] blk.16.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 185/ 288] blk.16.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 186/ 288] blk.16.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 187/ 288] blk.16.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 188/ 288] blk.16.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 189/ 288] blk.17.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 190/ 288] blk.17.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 191/ 288] blk.17.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 192/ 288] blk.17.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 193/ 288] blk.17.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 194/ 288] blk.17.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 195/ 288] blk.17.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 196/ 288] blk.17.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 197/ 288] blk.17.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 198/ 288] blk.17.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 199/ 288] blk.17.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 200/ 288] blk.18.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 201/ 288] blk.18.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 202/ 288] blk.18.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 203/ 288] blk.18.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 204/ 288] blk.18.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 205/ 288] blk.18.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 206/ 288] blk.18.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 207/ 288] blk.18.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 208/ 288] blk.18.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 209/ 288] blk.18.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 210/ 288] blk.18.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 211/ 288] blk.19.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 212/ 288] blk.19.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 213/ 288] blk.19.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 214/ 288] blk.19.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 215/ 288] blk.19.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 216/ 288] blk.19.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 217/ 288] blk.19.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 218/ 288] blk.19.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 219/ 288] blk.19.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 220/ 288] blk.19.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 221/ 288] blk.19.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 222/ 288] blk.20.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 223/ 288] blk.20.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 224/ 288] blk.20.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 225/ 288] blk.20.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 226/ 288] blk.20.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 227/ 288] blk.20.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 228/ 288] blk.20.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 229/ 288] blk.20.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 230/ 288] blk.20.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 231/ 288] blk.20.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 232/ 288] blk.20.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 233/ 288] blk.21.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 234/ 288] blk.21.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 235/ 288] blk.21.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 236/ 288] blk.21.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 237/ 288] blk.21.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 238/ 288] blk.21.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 239/ 288] blk.21.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 240/ 288] blk.21.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 241/ 288] blk.21.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 242/ 288] blk.21.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 243/ 288] blk.21.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 244/ 288] blk.22.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 245/ 288] blk.22.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 246/ 288] blk.22.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 247/ 288] blk.22.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 248/ 288] blk.22.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 249/ 288] blk.22.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 250/ 288] blk.22.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 251/ 288] blk.22.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 252/ 288] blk.22.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 253/ 288] blk.22.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 254/ 288] blk.22.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 255/ 288] blk.23.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 256/ 288] blk.23.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 257/ 288] blk.23.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 258/ 288] blk.23.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 259/ 288] blk.23.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 260/ 288] blk.23.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 261/ 288] blk.23.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 262/ 288] blk.23.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 263/ 288] blk.23.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 264/ 288] blk.23.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 265/ 288] blk.23.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 266/ 288] blk.24.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 267/ 288] blk.24.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 268/ 288] blk.24.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 269/ 288] blk.24.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 270/ 288] blk.24.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 271/ 288] blk.24.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 272/ 288] blk.24.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 273/ 288] blk.24.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 274/ 288] blk.24.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 275/ 288] blk.24.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 276/ 288] blk.24.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 277/ 288] blk.25.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 278/ 288] blk.25.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 279/ 288] blk.25.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q8_0 .. size = 4.50 MiB -> 2.39 MiB\n",
+ "[ 280/ 288] blk.25.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q8_0 .. size = 9.00 MiB -> 4.78 MiB\n",
+ "[ 281/ 288] blk.25.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 282/ 288] blk.25.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 283/ 288] blk.25.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q8_0 .. size = 40.50 MiB -> 21.52 MiB\n",
+ "[ 284/ 288] blk.25.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 285/ 288] blk.25.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 286/ 288] blk.25.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 287/ 288] blk.25.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 288/ 288] output_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "llama_model_quantize_internal: model size = 4986.92 MB\n",
+ "llama_model_quantize_internal: quant size = 2649.74 MB\n",
+ "\n",
+ "main: quantize time = 54438.41 ms\n",
+ "main: total time = 54438.41 ms\n",
+ "Unsloth: Conversion completed! Output location: ./bastienp/Gemma-2-2B-it-JSON-data-extration/unsloth.Q8_0.gguf\n",
+ "Unsloth: [2] Converting GGUF 16bit into q5_k_m. This will take 20 minutes...\n",
+ "main: build = 3605 (cfac111e)\n",
+ "main: built with cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 for x86_64-linux-gnu\n",
+ "main: quantizing './bastienp/Gemma-2-2B-it-JSON-data-extration/unsloth.F16.gguf' to './bastienp/Gemma-2-2B-it-JSON-data-extration/unsloth.Q5_K_M.gguf' as Q5_K_M using 4 threads\n",
+ "llama_model_loader: loaded meta data with 34 key-value pairs and 288 tensors from ./bastienp/Gemma-2-2B-it-JSON-data-extration/unsloth.F16.gguf (version GGUF V3 (latest))\n",
+ "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
+ "llama_model_loader: - kv 0: general.architecture str = gemma2\n",
+ "llama_model_loader: - kv 1: general.type str = model\n",
+ "llama_model_loader: - kv 2: general.name str = Gemma 2 2b It\n",
+ "llama_model_loader: - kv 3: general.organization str = Unsloth\n",
+ "llama_model_loader: - kv 4: general.finetune str = it\n",
+ "llama_model_loader: - kv 5: general.basename str = gemma-2\n",
+ "llama_model_loader: - kv 6: general.size_label str = 2B\n",
+ "llama_model_loader: - kv 7: gemma2.context_length u32 = 8192\n",
+ "llama_model_loader: - kv 8: gemma2.embedding_length u32 = 2304\n",
+ "llama_model_loader: - kv 9: gemma2.block_count u32 = 26\n",
+ "llama_model_loader: - kv 10: gemma2.feed_forward_length u32 = 9216\n",
+ "llama_model_loader: - kv 11: gemma2.attention.head_count u32 = 8\n",
+ "llama_model_loader: - kv 12: gemma2.attention.head_count_kv u32 = 4\n",
+ "llama_model_loader: - kv 13: gemma2.attention.layer_norm_rms_epsilon f32 = 0.000001\n",
+ "llama_model_loader: - kv 14: gemma2.attention.key_length u32 = 256\n",
+ "llama_model_loader: - kv 15: gemma2.attention.value_length u32 = 256\n",
+ "llama_model_loader: - kv 16: general.file_type u32 = 1\n",
+ "llama_model_loader: - kv 17: gemma2.attn_logit_softcapping f32 = 50.000000\n",
+ "llama_model_loader: - kv 18: gemma2.final_logit_softcapping f32 = 30.000000\n",
+ "llama_model_loader: - kv 19: gemma2.attention.sliding_window u32 = 4096\n",
+ "llama_model_loader: - kv 20: tokenizer.ggml.model str = llama\n",
+ "llama_model_loader: - kv 21: tokenizer.ggml.pre str = default\n",
+ "llama_model_loader: - kv 22: tokenizer.ggml.tokens arr[str,256000] = [\"\", \"\", \"\", \"\", ...\n",
+ "llama_model_loader: - kv 23: tokenizer.ggml.scores arr[f32,256000] = [-1000.000000, -1000.000000, -1000.00...\n",
+ "llama_model_loader: - kv 24: tokenizer.ggml.token_type arr[i32,256000] = [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, ...\n",
+ "llama_model_loader: - kv 25: tokenizer.ggml.bos_token_id u32 = 2\n",
+ "llama_model_loader: - kv 26: tokenizer.ggml.eos_token_id u32 = 1\n",
+ "llama_model_loader: - kv 27: tokenizer.ggml.unknown_token_id u32 = 3\n",
+ "llama_model_loader: - kv 28: tokenizer.ggml.padding_token_id u32 = 0\n",
+ "llama_model_loader: - kv 29: tokenizer.ggml.add_bos_token bool = true\n",
+ "llama_model_loader: - kv 30: tokenizer.ggml.add_eos_token bool = false\n",
+ "llama_model_loader: - kv 31: tokenizer.chat_template str = {% if messages[0]['role'] == 'system'...\n",
+ "llama_model_loader: - kv 32: tokenizer.ggml.add_space_prefix bool = false\n",
+ "llama_model_loader: - kv 33: general.quantization_version u32 = 2\n",
+ "llama_model_loader: - type f32: 105 tensors\n",
+ "llama_model_loader: - type f16: 183 tensors\n",
+ "[ 1/ 288] token_embd.weight - [ 2304, 256000, 1, 1], type = f16, converting to q6_K .. size = 1125.00 MiB -> 461.43 MiB\n",
+ "[ 2/ 288] blk.0.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 3/ 288] blk.0.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q5_K .. size = 4.50 MiB -> 1.55 MiB\n",
+ "[ 4/ 288] blk.0.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q6_K .. size = 4.50 MiB -> 1.85 MiB\n",
+ "[ 5/ 288] blk.0.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 6/ 288] blk.0.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 7/ 288] blk.0.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 8/ 288] blk.0.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q6_K .. size = 40.50 MiB -> 16.61 MiB\n",
+ "[ 9/ 288] blk.0.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 10/ 288] blk.0.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 11/ 288] blk.0.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 12/ 288] blk.0.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 13/ 288] blk.1.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 14/ 288] blk.1.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q5_K .. size = 4.50 MiB -> 1.55 MiB\n",
+ "[ 15/ 288] blk.1.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q6_K .. size = 4.50 MiB -> 1.85 MiB\n",
+ "[ 16/ 288] blk.1.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 17/ 288] blk.1.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 18/ 288] blk.1.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 19/ 288] blk.1.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q6_K .. size = 40.50 MiB -> 16.61 MiB\n",
+ "[ 20/ 288] blk.1.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 21/ 288] blk.1.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 22/ 288] blk.1.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 23/ 288] blk.1.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 24/ 288] blk.2.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 25/ 288] blk.2.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q5_K .. size = 4.50 MiB -> 1.55 MiB\n",
+ "[ 26/ 288] blk.2.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q6_K .. size = 4.50 MiB -> 1.85 MiB\n",
+ "[ 27/ 288] blk.2.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 28/ 288] blk.2.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 29/ 288] blk.2.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 30/ 288] blk.2.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q6_K .. size = 40.50 MiB -> 16.61 MiB\n",
+ "[ 31/ 288] blk.2.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 32/ 288] blk.2.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 33/ 288] blk.2.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 34/ 288] blk.2.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 35/ 288] blk.3.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 36/ 288] blk.3.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q5_K .. size = 4.50 MiB -> 1.55 MiB\n",
+ "[ 37/ 288] blk.3.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q5_K .. size = 4.50 MiB -> 1.55 MiB\n",
+ "[ 38/ 288] blk.3.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 39/ 288] blk.3.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 40/ 288] blk.3.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 41/ 288] blk.3.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 42/ 288] blk.3.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 43/ 288] blk.3.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 44/ 288] blk.3.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 45/ 288] blk.3.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 46/ 288] blk.4.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 47/ 288] blk.4.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q5_K .. size = 4.50 MiB -> 1.55 MiB\n",
+ "[ 48/ 288] blk.4.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q5_K .. size = 4.50 MiB -> 1.55 MiB\n",
+ "[ 49/ 288] blk.4.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 50/ 288] blk.4.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 51/ 288] blk.4.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 52/ 288] blk.4.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 53/ 288] blk.4.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 54/ 288] blk.4.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 55/ 288] blk.4.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 56/ 288] blk.4.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 57/ 288] blk.5.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 58/ 288] blk.5.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q5_K .. size = 4.50 MiB -> 1.55 MiB\n",
+ "[ 59/ 288] blk.5.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q6_K .. size = 4.50 MiB -> 1.85 MiB\n",
+ "[ 60/ 288] blk.5.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 61/ 288] blk.5.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 62/ 288] blk.5.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 63/ 288] blk.5.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q6_K .. size = 40.50 MiB -> 16.61 MiB\n",
+ "[ 64/ 288] blk.5.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 65/ 288] blk.5.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 66/ 288] blk.5.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 67/ 288] blk.5.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 68/ 288] blk.6.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 69/ 288] blk.6.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q5_K .. size = 4.50 MiB -> 1.55 MiB\n",
+ "[ 70/ 288] blk.6.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q5_K .. size = 4.50 MiB -> 1.55 MiB\n",
+ "[ 71/ 288] blk.6.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 72/ 288] blk.6.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 73/ 288] blk.6.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 74/ 288] blk.6.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 75/ 288] blk.6.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 76/ 288] blk.6.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 77/ 288] blk.6.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 78/ 288] blk.6.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 79/ 288] blk.7.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 80/ 288] blk.7.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q5_K .. size = 4.50 MiB -> 1.55 MiB\n",
+ "[ 81/ 288] blk.7.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q5_K .. size = 4.50 MiB -> 1.55 MiB\n",
+ "[ 82/ 288] blk.7.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 83/ 288] blk.7.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 84/ 288] blk.7.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 85/ 288] blk.7.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 86/ 288] blk.7.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 87/ 288] blk.7.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 88/ 288] blk.7.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 89/ 288] blk.7.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 90/ 288] blk.8.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 91/ 288] blk.8.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q5_K .. size = 4.50 MiB -> 1.55 MiB\n",
+ "[ 92/ 288] blk.8.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q6_K .. size = 4.50 MiB -> 1.85 MiB\n",
+ "[ 93/ 288] blk.8.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 94/ 288] blk.8.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 95/ 288] blk.8.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 96/ 288] blk.8.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q6_K .. size = 40.50 MiB -> 16.61 MiB\n",
+ "[ 97/ 288] blk.8.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 98/ 288] blk.8.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 99/ 288] blk.8.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 100/ 288] blk.8.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 101/ 288] blk.9.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 102/ 288] blk.9.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q5_K .. size = 4.50 MiB -> 1.55 MiB\n",
+ "[ 103/ 288] blk.9.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q5_K .. size = 4.50 MiB -> 1.55 MiB\n",
+ "[ 104/ 288] blk.9.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 105/ 288] blk.9.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 106/ 288] blk.9.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 107/ 288] blk.9.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 108/ 288] blk.9.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 109/ 288] blk.9.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 110/ 288] blk.9.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 111/ 288] blk.9.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 112/ 288] blk.10.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 113/ 288] blk.10.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q5_K .. size = 4.50 MiB -> 1.55 MiB\n",
+ "[ 114/ 288] blk.10.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q5_K .. size = 4.50 MiB -> 1.55 MiB\n",
+ "[ 115/ 288] blk.10.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 116/ 288] blk.10.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 117/ 288] blk.10.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 118/ 288] blk.10.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 119/ 288] blk.10.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 120/ 288] blk.10.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 121/ 288] blk.10.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 122/ 288] blk.10.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 123/ 288] blk.11.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 124/ 288] blk.11.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q5_K .. size = 4.50 MiB -> 1.55 MiB\n",
+ "[ 125/ 288] blk.11.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q6_K .. size = 4.50 MiB -> 1.85 MiB\n",
+ "[ 126/ 288] blk.11.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 127/ 288] blk.11.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 128/ 288] blk.11.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 129/ 288] blk.11.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q6_K .. size = 40.50 MiB -> 16.61 MiB\n",
+ "[ 130/ 288] blk.11.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 131/ 288] blk.11.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 132/ 288] blk.11.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 133/ 288] blk.11.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 134/ 288] blk.12.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 135/ 288] blk.12.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q5_K .. size = 4.50 MiB -> 1.55 MiB\n",
+ "[ 136/ 288] blk.12.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q5_K .. size = 4.50 MiB -> 1.55 MiB\n",
+ "[ 137/ 288] blk.12.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 138/ 288] blk.12.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 139/ 288] blk.12.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 140/ 288] blk.12.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 141/ 288] blk.12.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 142/ 288] blk.12.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 143/ 288] blk.12.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 144/ 288] blk.12.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 145/ 288] blk.13.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 146/ 288] blk.13.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q5_K .. size = 4.50 MiB -> 1.55 MiB\n",
+ "[ 147/ 288] blk.13.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q5_K .. size = 4.50 MiB -> 1.55 MiB\n",
+ "[ 148/ 288] blk.13.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 149/ 288] blk.13.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 150/ 288] blk.13.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 151/ 288] blk.13.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 152/ 288] blk.13.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 153/ 288] blk.13.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 154/ 288] blk.13.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 155/ 288] blk.13.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 156/ 288] blk.14.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 157/ 288] blk.14.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q5_K .. size = 4.50 MiB -> 1.55 MiB\n",
+ "[ 158/ 288] blk.14.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q6_K .. size = 4.50 MiB -> 1.85 MiB\n",
+ "[ 159/ 288] blk.14.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 160/ 288] blk.14.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 161/ 288] blk.14.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 162/ 288] blk.14.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q6_K .. size = 40.50 MiB -> 16.61 MiB\n",
+ "[ 163/ 288] blk.14.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 164/ 288] blk.14.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 165/ 288] blk.14.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 166/ 288] blk.14.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 167/ 288] blk.15.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 168/ 288] blk.15.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q5_K .. size = 4.50 MiB -> 1.55 MiB\n",
+ "[ 169/ 288] blk.15.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q5_K .. size = 4.50 MiB -> 1.55 MiB\n",
+ "[ 170/ 288] blk.15.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 171/ 288] blk.15.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 172/ 288] blk.15.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 173/ 288] blk.15.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 174/ 288] blk.15.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 175/ 288] blk.15.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 176/ 288] blk.15.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 177/ 288] blk.15.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 178/ 288] blk.16.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 179/ 288] blk.16.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q5_K .. size = 4.50 MiB -> 1.55 MiB\n",
+ "[ 180/ 288] blk.16.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q5_K .. size = 4.50 MiB -> 1.55 MiB\n",
+ "[ 181/ 288] blk.16.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 182/ 288] blk.16.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 183/ 288] blk.16.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 184/ 288] blk.16.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 185/ 288] blk.16.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 186/ 288] blk.16.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 187/ 288] blk.16.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 188/ 288] blk.16.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 189/ 288] blk.17.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 190/ 288] blk.17.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q5_K .. size = 4.50 MiB -> 1.55 MiB\n",
+ "[ 191/ 288] blk.17.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q6_K .. size = 4.50 MiB -> 1.85 MiB\n",
+ "[ 192/ 288] blk.17.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 193/ 288] blk.17.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 194/ 288] blk.17.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 195/ 288] blk.17.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q6_K .. size = 40.50 MiB -> 16.61 MiB\n",
+ "[ 196/ 288] blk.17.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 197/ 288] blk.17.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 198/ 288] blk.17.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 199/ 288] blk.17.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 200/ 288] blk.18.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 201/ 288] blk.18.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q5_K .. size = 4.50 MiB -> 1.55 MiB\n",
+ "[ 202/ 288] blk.18.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q5_K .. size = 4.50 MiB -> 1.55 MiB\n",
+ "[ 203/ 288] blk.18.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 204/ 288] blk.18.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 205/ 288] blk.18.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 206/ 288] blk.18.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 207/ 288] blk.18.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 208/ 288] blk.18.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 209/ 288] blk.18.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 210/ 288] blk.18.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 211/ 288] blk.19.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 212/ 288] blk.19.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q5_K .. size = 4.50 MiB -> 1.55 MiB\n",
+ "[ 213/ 288] blk.19.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q5_K .. size = 4.50 MiB -> 1.55 MiB\n",
+ "[ 214/ 288] blk.19.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 215/ 288] blk.19.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 216/ 288] blk.19.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 217/ 288] blk.19.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 218/ 288] blk.19.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 219/ 288] blk.19.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 220/ 288] blk.19.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 221/ 288] blk.19.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 222/ 288] blk.20.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 223/ 288] blk.20.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q5_K .. size = 4.50 MiB -> 1.55 MiB\n",
+ "[ 224/ 288] blk.20.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q6_K .. size = 4.50 MiB -> 1.85 MiB\n",
+ "[ 225/ 288] blk.20.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 226/ 288] blk.20.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 227/ 288] blk.20.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 228/ 288] blk.20.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q6_K .. size = 40.50 MiB -> 16.61 MiB\n",
+ "[ 229/ 288] blk.20.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 230/ 288] blk.20.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 231/ 288] blk.20.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 232/ 288] blk.20.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 233/ 288] blk.21.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 234/ 288] blk.21.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q5_K .. size = 4.50 MiB -> 1.55 MiB\n",
+ "[ 235/ 288] blk.21.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q5_K .. size = 4.50 MiB -> 1.55 MiB\n",
+ "[ 236/ 288] blk.21.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 237/ 288] blk.21.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 238/ 288] blk.21.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 239/ 288] blk.21.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 240/ 288] blk.21.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 241/ 288] blk.21.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 242/ 288] blk.21.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 243/ 288] blk.21.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 244/ 288] blk.22.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 245/ 288] blk.22.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q5_K .. size = 4.50 MiB -> 1.55 MiB\n",
+ "[ 246/ 288] blk.22.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q6_K .. size = 4.50 MiB -> 1.85 MiB\n",
+ "[ 247/ 288] blk.22.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 248/ 288] blk.22.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 249/ 288] blk.22.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 250/ 288] blk.22.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q6_K .. size = 40.50 MiB -> 16.61 MiB\n",
+ "[ 251/ 288] blk.22.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 252/ 288] blk.22.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 253/ 288] blk.22.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 254/ 288] blk.22.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 255/ 288] blk.23.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 256/ 288] blk.23.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q5_K .. size = 4.50 MiB -> 1.55 MiB\n",
+ "[ 257/ 288] blk.23.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q6_K .. size = 4.50 MiB -> 1.85 MiB\n",
+ "[ 258/ 288] blk.23.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 259/ 288] blk.23.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 260/ 288] blk.23.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 261/ 288] blk.23.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q6_K .. size = 40.50 MiB -> 16.61 MiB\n",
+ "[ 262/ 288] blk.23.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 263/ 288] blk.23.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 264/ 288] blk.23.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 265/ 288] blk.23.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 266/ 288] blk.24.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 267/ 288] blk.24.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q5_K .. size = 4.50 MiB -> 1.55 MiB\n",
+ "[ 268/ 288] blk.24.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q6_K .. size = 4.50 MiB -> 1.85 MiB\n",
+ "[ 269/ 288] blk.24.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 270/ 288] blk.24.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 271/ 288] blk.24.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 272/ 288] blk.24.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q6_K .. size = 40.50 MiB -> 16.61 MiB\n",
+ "[ 273/ 288] blk.24.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 274/ 288] blk.24.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 275/ 288] blk.24.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 276/ 288] blk.24.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 277/ 288] blk.25.attn_q.weight - [ 2304, 2048, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 278/ 288] blk.25.attn_k.weight - [ 2304, 1024, 1, 1], type = f16, converting to q5_K .. size = 4.50 MiB -> 1.55 MiB\n",
+ "[ 279/ 288] blk.25.attn_v.weight - [ 2304, 1024, 1, 1], type = f16, converting to q6_K .. size = 4.50 MiB -> 1.85 MiB\n",
+ "[ 280/ 288] blk.25.attn_output.weight - [ 2048, 2304, 1, 1], type = f16, converting to q5_K .. size = 9.00 MiB -> 3.09 MiB\n",
+ "[ 281/ 288] blk.25.ffn_gate.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 282/ 288] blk.25.ffn_up.weight - [ 2304, 9216, 1, 1], type = f16, converting to q5_K .. size = 40.50 MiB -> 13.92 MiB\n",
+ "[ 283/ 288] blk.25.ffn_down.weight - [ 9216, 2304, 1, 1], type = f16, converting to q6_K .. size = 40.50 MiB -> 16.61 MiB\n",
+ "[ 284/ 288] blk.25.attn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 285/ 288] blk.25.post_attention_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 286/ 288] blk.25.ffn_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 287/ 288] blk.25.post_ffw_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "[ 288/ 288] output_norm.weight - [ 2304, 1, 1, 1], type = f32, size = 0.009 MB\n",
+ "llama_model_quantize_internal: model size = 4986.92 MB\n",
+ "llama_model_quantize_internal: quant size = 1828.42 MB\n",
+ "\n",
+ "main: quantize time = 219018.34 ms\n",
+ "main: total time = 219018.34 ms\n",
+ "Unsloth: Conversion completed! Output location: ./bastienp/Gemma-2-2B-it-JSON-data-extration/unsloth.Q5_K_M.gguf\n",
+ "Unsloth: Uploading GGUF to Huggingface Hub...\n"
+ ]
+ },
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "431941f18de6491aa1d7c7b2899800fa",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ " 0%| | 0/1 [00:00 Iterator["FlaskClient"]:
+ """Fixture to return a test client for the Flask app."""
+ app.config["TESTING"] = True
+
+ with app.test_client() as client:
+ yield client
+
+
+def test_health(client: FlaskClient) -> None:
+ """Test the health endpoint."""
+ response = client.get("/api/v1/health")
+
+ assert response.status_code == SUCCESS
+ assert response.json.get("status") == "OK"
+ assert datetime.fromisoformat(response.json.get("lastChecked"))
+
+
+def test_chat_completions(client: FlaskClient) -> None:
+ """Test the chat completions endpoint."""
+ response = client.post(
+ "/api/v1/chat/completions",
+ json={"query": "Hello, how are you?"},
+ )
+
+ assert response.status_code == SUCCESS
+ assert response.json.get("response") != ""
diff --git a/src/web/__init__.py b/src/web/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/src/web/app.py b/src/web/app.py
new file mode 100644
index 0000000..f29bd8b
--- /dev/null
+++ b/src/web/app.py
@@ -0,0 +1,60 @@
+"""Flask app to serve the API endpoints."""
+
+import datetime
+
+from flask import Blueprint, Flask, abort, jsonify, request
+from llama_cpp import Llama
+
+app = Flask(__name__)
+
+api_v1 = Blueprint("api_v1", __name__)
+
+llm = Llama.from_pretrained(
+ "bastienp/Gemma-2-2B-it-JSON-data-extration",
+ filename="*Q4_K_M.gguf",
+ verbose=False,
+)
+
+
+@api_v1.route("/health", methods=["GET"])
+def ready() -> tuple[dict, int]:
+ """GET request to check if the server is ready."""
+ return jsonify(
+ {
+ "status": "OK",
+ "lastChecked": datetime.datetime.now(tz=datetime.UTC).isoformat(),
+ },
+ ), 200
+
+
+@api_v1.route("/chat/completions", methods=["POST"])
+def completions() -> tuple[dict, int]:
+ """POST request to get completions for a given prompt."""
+ if not request.is_json:
+ abort(400, "Request must be JSON in json format")
+
+ # Check if the request is a valid JSON
+ try:
+ data = request.get_json(silent=True)
+ except ValueError as e:
+ abort(400, f"Invalid JSON: {e}")
+
+ if "query" not in data:
+ abort(400, "Request must contain a query field")
+
+ start = datetime.datetime.now(tz=datetime.UTC)
+
+ res = llm.create_completion(prompt=data["query"], max_tokens=248)
+
+ end = datetime.datetime.now(tz=datetime.UTC)
+
+ return jsonify(
+ {
+ "response": str.strip(res["choices"][0]["text"]),
+ "duration": (end - start).total_seconds(),
+ "usage_metadata": res["usage"],
+ }
+ ), 200
+
+
+app.register_blueprint(api_v1, url_prefix="/api/v1")