diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 85d1ff7..4c8c037 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -146,6 +146,44 @@ jobs:
           token: ${{ secrets.CODECOV_TOKEN }}
           slug: sigstore/sigstore-ruby
 
+  smoketest:
+    needs: ruby-versions
+    runs-on: ubuntu-latest
+    name: Smoketest
+    permissions:
+      id-token: write
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: ${{ fromJson(needs.ruby-versions.outputs.versions) }}
+        os: [ubuntu-latest]
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
+        with:
+          egress-policy: audit
+
+      - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
+        with:
+          persist-credentials: false
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@7bae1d00b5db9166f4f0fc47985a3a5702cb58f0 # v1.197.0
+        with:
+          ruby-version: ${{ fromJson(needs.ruby-versions.outputs.latest) }}
+          bundler-cache: true
+      - name: Build the gem
+        run: bin/rake build
+      - name: Run the smoketest
+        run: |
+          # we smoke-test sigstore by installing each of the distributions
+          # we've built in a fresh environment and using each to sign and
+          # verify for itself, using the ambient OIDC identity
+          for dist in pkg/*; do
+            ./bin/smoketest "${dist}"
+          done
+        env:
+          WORKFLOW_NAME: ci
+
   all-tests-pass:
     if: always()
 
diff --git a/bin/sigstore-ruby b/bin/sigstore-ruby
index f3809df..87b4762 100755
--- a/bin/sigstore-ruby
+++ b/bin/sigstore-ruby
@@ -178,7 +178,7 @@ module Sigstore
 
     def collect_verification_state(files)
       if (options[:certificate] || options[:signature] || options[:bundle]) && files.size > 1
-        raise Thor::InvocationError, "Too many arguments"
+        raise Thor::InvocationError, "Too many files specified: #{files.inspect}"
       end
 
       if options[:bundle] && (options[:certificate] || options[:signature])
diff --git a/bin/smoketest b/bin/smoketest
index 95ff8bb..1a11401 100755
--- a/bin/smoketest
+++ b/bin/smoketest
@@ -42,18 +42,20 @@ sh(env, File.expand_path("sigstore-ruby", __dir__),
    exception: true)
 
 cert_identity = "#{ENV.fetch("GITHUB_SERVER_URL")}/#{ENV.fetch("GITHUB_REPOSITORY")}" \
-                "/.github/workflows/release-with-provenance.yml@#{ENV.fetch("GITHUB_REF")}"
+                "/.github/workflows/#{ENV.fetch("WORKFLOW_NAME", "release")}.yml@#{ENV.fetch("GITHUB_REF")}"
 
 sh(env, File.expand_path("sigstore-ruby", __dir__),
-   "verify", dist,
+   "verify",
    "--signature=smoketest-artifacts/#{File.basename(dist)}.sig",
    "--certificate=smoketest-artifacts/#{File.basename(dist)}.crt",
-   "--cert-oidc-issuer=https://token.actions.githubusercontent.com",
-   "--cert-identity=#{cert_identity}",
+   "--certificate-oidc-issuer=https://token.actions.githubusercontent.com",
+   "--certificate-identity=#{cert_identity}",
+   dist,
    exception: true)
 sh(env, File.expand_path("sigstore-ruby", __dir__),
-   "verify", dist,
+   "verify",
    "--bundle=smoketest-artifacts/#{File.basename(dist)}.sigstore.json",
-   "--cert-oidc-issuer=https://token.actions.githubusercontent.com",
-   "--cert-identity=#{cert_identity}",
+   "--certificate-oidc-issuer=https://token.actions.githubusercontent.com",
+   "--certificate-identity=#{cert_identity}",
+   dist,
    exception: true)