diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py
index 919f836..01b45d6 100644
--- a/pydantic_settings/sources.py
+++ b/pydantic_settings/sources.py
@@ -1917,7 +1917,15 @@ def _read_files(self, files: PathType | None) -> dict[str, Any]:
         for file in files:
             file_path = Path(file).expanduser()
             if file_path.is_file():
-                vars.update(self._read_file(file_path))
+                try:
+                    settings = self._read_file(file_path)
+                except ValueError as e:
+                    raise SettingsError(f'Failed to parse settings from {file_path}, {e}')
+                if not isinstance(settings, dict):
+                    raise SettingsError(
+                        f'Failed to parse settings from {file_path}, expecting an object (valid dictionnary)'
+                    )
+                vars.update(settings)
         return vars
 
     @abstractmethod
diff --git a/tests/test_source_json.py b/tests/test_source_json.py
index e348a6b..7a6ba5d 100644
--- a/tests/test_source_json.py
+++ b/tests/test_source_json.py
@@ -5,6 +5,7 @@
 import json
 from typing import Tuple, Type, Union
 
+import pytest
 from pydantic import BaseModel
 
 from pydantic_settings import (
@@ -12,6 +13,7 @@
     JsonConfigSettingsSource,
     PydanticBaseSettingsSource,
     SettingsConfigDict,
+    SettingsError,
 )
 
 
@@ -67,6 +69,33 @@ def settings_customise_sources(
     assert s.model_dump() == {}
 
 
+def test_nondict_json_file(tmp_path):
+    p = tmp_path / '.env'
+    p.write_text(
+        """
+        "noway"
+    """
+    )
+
+    class Settings(BaseSettings):
+        foobar: str
+        model_config = SettingsConfigDict(json_file=p)
+
+        @classmethod
+        def settings_customise_sources(
+            cls,
+            settings_cls: Type[BaseSettings],
+            init_settings: PydanticBaseSettingsSource,
+            env_settings: PydanticBaseSettingsSource,
+            dotenv_settings: PydanticBaseSettingsSource,
+            file_secret_settings: PydanticBaseSettingsSource,
+        ) -> Tuple[PydanticBaseSettingsSource, ...]:
+            return (JsonConfigSettingsSource(settings_cls),)
+
+    with pytest.raises(SettingsError, match='Failed to parse settings from .*, expecting an object'):
+        Settings()
+
+
 def test_multiple_file_json(tmp_path):
     p5 = tmp_path / '.env.json5'
     p6 = tmp_path / '.env.json6'
diff --git a/tests/test_source_toml.py b/tests/test_source_toml.py
index 2918601..0d2f70e 100644
--- a/tests/test_source_toml.py
+++ b/tests/test_source_toml.py
@@ -12,6 +12,7 @@
     BaseSettings,
     PydanticBaseSettingsSource,
     SettingsConfigDict,
+    SettingsError,
     TomlConfigSettingsSource,
 )
 
@@ -77,6 +78,30 @@ def settings_customise_sources(
     assert s.model_dump() == {}
 
 
+@pytest.mark.skipif(sys.version_info <= (3, 11) and tomli is None, reason='tomli/tomllib is not installed')
+def test_pyproject_nondict_toml(cd_tmp_path):
+    pyproject = cd_tmp_path / 'pyproject.toml'
+    pyproject.write_text(
+        """
+    [tool.pydantic-settings]
+    foobar
+    """
+    )
+
+    class Settings(BaseSettings):
+        foobar: str
+        model_config = SettingsConfigDict()
+
+        @classmethod
+        def settings_customise_sources(
+            cls, settings_cls: Type[BaseSettings], **_kwargs: PydanticBaseSettingsSource
+        ) -> Tuple[PydanticBaseSettingsSource, ...]:
+            return (TomlConfigSettingsSource(settings_cls, pyproject),)
+
+    with pytest.raises(SettingsError, match='Failed to parse settings from'):
+        Settings()
+
+
 @pytest.mark.skipif(sys.version_info <= (3, 11) and tomli is None, reason='tomli/tomllib is not installed')
 def test_multiple_file_toml(tmp_path):
     p1 = tmp_path / '.env.toml1'
diff --git a/tests/test_source_yaml.py b/tests/test_source_yaml.py
index fd25de6..10f5fd1 100644
--- a/tests/test_source_yaml.py
+++ b/tests/test_source_yaml.py
@@ -11,6 +11,7 @@
     BaseSettings,
     PydanticBaseSettingsSource,
     SettingsConfigDict,
+    SettingsError,
     YamlConfigSettingsSource,
 )
 
@@ -85,6 +86,30 @@ def settings_customise_sources(
     assert s.nested.nested_field == 'world!'
 
 
+@pytest.mark.skipif(yaml is None, reason='pyYaml is not installed')
+def test_nondict_yaml_file(tmp_path):
+    p = tmp_path / '.env'
+    p.write_text('test invalid yaml')
+
+    class Settings(BaseSettings):
+        foobar: str
+        model_config = SettingsConfigDict(yaml_file=p)
+
+        @classmethod
+        def settings_customise_sources(
+            cls,
+            settings_cls: Type[BaseSettings],
+            init_settings: PydanticBaseSettingsSource,
+            env_settings: PydanticBaseSettingsSource,
+            dotenv_settings: PydanticBaseSettingsSource,
+            file_secret_settings: PydanticBaseSettingsSource,
+        ) -> Tuple[PydanticBaseSettingsSource, ...]:
+            return (YamlConfigSettingsSource(settings_cls),)
+
+    with pytest.raises(SettingsError, match='Failed to parse settings from .*, expecting an object'):
+        Settings()
+
+
 @pytest.mark.skipif(yaml is None, reason='pyYaml is not installed')
 def test_yaml_no_file():
     class Settings(BaseSettings):