diff --git a/.github/workflows/llm.yml b/.github/workflows/llm.yml index 8f0752b607..d37ba9b24d 100644 --- a/.github/workflows/llm.yml +++ b/.github/workflows/llm.yml @@ -33,7 +33,7 @@ jobs: matrix: os: [ macos, ubuntu ] model: [ OpenELM-270M, OpenELM-1_1B, TinyLlama_v1.1, phi-1_5 ] - q: [ f16f16, f32f32, q40f16, q40f32 ] + q: [ f16f16, f32f32, q40f16, q40f32, q40ef16, q40ef32 ] fail-fast: false permissions: id-token: write diff --git a/.travis/test-llm.sh b/.travis/test-llm.sh index d85dfabfab..419271646f 100755 --- a/.travis/test-llm.sh +++ b/.travis/test-llm.sh @@ -30,24 +30,23 @@ esac case $q in q40f16) approx=ultra;; + q40ef16) approx=ultra;; f16f16) approx=ultra;; q40f32) approx=very;; + q40ef32) approx=very;; f32f32) approx=approximate;; esac nnef=llm/$generation/$id/$id.nnef.tgz -pp=llm/$generation/$id/$id.pp.io.npz -tg=llm/$generation/$id/$id.tg.io.npz set -x -$CACHE_FILE $nnef $pp $tg - -$TRACT_RUN -v --nnef-tract-core $MODELS/$nnef -O run \ - --input-from-npz $MODELS/$pp \ - --assert-output-bundle $MODELS/$pp \ - --approx $approx --allow-float-casts - -$TRACT_RUN -v --nnef-tract-core $MODELS/$nnef -O run \ - --input-from-npz $MODELS/$tg \ - --assert-output-bundle $MODELS/$tg \ - --approx $approx --allow-float-casts +$CACHE_FILE $nnef +for t in p0s100 p50s50 p99s1 +do + npz=llm/$generation/$id/$id.$t.io.npz + $CACHE_FILE $npz + $TRACT_RUN -v --nnef-tract-core $MODELS/$nnef -O run \ + --input-from-npz $MODELS/$npz \ + --assert-output-bundle $MODELS/$npz \ + --approx $approx --allow-float-casts +done