diff --git a/oras/auth/__init__.py b/oras/auth/__init__.py
index d913ad3..6f828fc 100644
--- a/oras/auth/__init__.py
+++ b/oras/auth/__init__.py
@@ -6,6 +6,12 @@
 auth_backends = {"token": TokenAuth, "basic": BasicAuth}
 
 
+class AuthenticationException(Exception):
+    """
+    An exception to traise with Authentication errors are fatal
+    """
+
+
 def get_auth_backend(name="token", session=None, **kwargs):
     backend = auth_backends.get(name)
     if not backend:
diff --git a/oras/decorator.py b/oras/decorator.py
index 885ec52..7730438 100644
--- a/oras/decorator.py
+++ b/oras/decorator.py
@@ -5,6 +5,7 @@
 import time
 from functools import partial, update_wrapper
 
+import oras.auth
 from oras.logger import logger
 
 
@@ -52,6 +53,8 @@ def __call__(self, cls, *args, **kwargs):
         while attempt < attempts:
             try:
                 return self.func(cls, *args, **kwargs)
+            except oras.auth.AuthenticationException as e:
+                raise e
             except Exception as e:
                 sleep = timeout + 3**attempt
                 logger.info(f"Retrying in {sleep} seconds - error: {e}")
@@ -71,6 +74,8 @@ def inner(*args, **kwargs):
             while attempt < attempts:
                 try:
                     return func(*args, **kwargs)
+                except oras.auth.AuthenticationException as e:
+                    raise e
                 except Exception as e:
                     sleep = timeout + 3**attempt
                     logger.info(f"Retrying in {sleep} seconds - error: {e}")