inferless.yaml

version: 1.0.0

name: OpenHermes-2-5-Mistral-7B
import_source: GIT

# you can choose the options between ONNX, TENSORFLOW, PYTORCH
source_framework_type: PYTORCH

configuration:
  # if you want to use a custom runtime, add the runtime id and name below,
  # you can find it by running `inferless runtime list` or create one with `inferless runtime upload`
  # NOTE: this is not yet supported for Serverless
  custom_runtime_id: ''
  custom_runtime_url: ''

  # if you want to use a custom volume, add the volume id and name below,
  # you can find it by running `inferless volume list` or create one with `inferless volume create -n {VOLUME_NAME}`
  # NOTE: this is not yet supported for Serverless
  custom_volume_id: ''
  custom_volume_name: ''

  gpu_type: T4
  inference_time: '180'
  is_dedicated: true
  is_serverless: false
  max_replica: '1'
  min_replica: '0'
  scale_down_delay: '600'
env:
  # Add your environment variables here
  # ENV: 'PROD'
secrets:
  # Add your secret ids here you can find it by running `inferless secrets list`
  # - 65723205-ce21-4392-a10b-3tf00c58988c
optional:
  # you can update file names here
  input_file_name: input.json
  output_file_name: output.json
  runtime_file_name: inferless-runtime-config.yaml
model_url: https://github.com/inferless/OpenHermes-2-5-Mistral-7B.git
provider: GITHUB
io_schema: true