Generate API (#19530)

* API Generator for Keras

* API Generator for Keras

* Generates API Gen via api_gen.sh

* Remove recursive import of _tf_keras

* Generate API Files via api_gen.sh

.github/workflows/actions.yml

@@ -24,13 +24,13 @@ jobs:
       KERAS_HOME: .github/workflows/config/${{ matrix.backend }}
     steps:
       - uses: actions/checkout@v4
-      - name: Check for changes in keras/applications
+      - name: Check for changes in keras/src/applications
         uses: dorny/paths-filter@v3
         id: filter
         with:
           filters: |
             applications:
-              - 'keras/applications/**'
+              - 'keras/src/applications/**'
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
@@ -49,13 +49,13 @@ jobs:
         run: |
           pip install -r requirements.txt --progress-bar off --upgrade
           pip uninstall -y keras keras-nightly
-          pip install tf_keras==2.16.0rc0 --progress-bar off --upgrade
+          pip install tf_keras==2.16.0 --progress-bar off --upgrade
           pip install -e "." --progress-bar off --upgrade
       - name: Test applications with pytest
         if: ${{ steps.filter.outputs.applications == 'true' }}
         run: |
-          pytest keras/applications --cov=keras/applications
-          coverage xml --include='keras/applications/*' -o apps-coverage.xml
+          pytest keras/src/applications --cov=keras/src/applications
+          coverage xml --include='keras/src/applications/*' -o apps-coverage.xml
       - name: Codecov keras.applications
         if: ${{ steps.filter.outputs.applications == 'true' }}
         uses: codecov/codecov-action@v4
@@ -80,8 +80,8 @@ jobs:
           pytest integration_tests/torch_workflow_test.py
       - name: Test with pytest
         run: |
-          pytest keras --ignore keras/applications --cov=keras
-          coverage xml --omit='keras/applications/*' -o core-coverage.xml
+          pytest keras --ignore keras/src/applications --cov=keras
+          coverage xml --omit='keras/src/applications/*,keras/api' -o core-coverage.xml
       - name: Codecov keras
         uses: codecov/codecov-action@v4
         with:
@@ -115,5 +115,14 @@ jobs:
           pip install -r requirements.txt --progress-bar off --upgrade
           pip uninstall -y keras keras-nightly
           pip install -e "." --progress-bar off --upgrade
+      - name: Check for API changes
+        run: |
+          bash shell/api_gen.sh
+          git status
+          clean=$(git status | grep "nothing to commit")
+          if [ -z "$clean" ]; then
+            echo "Please run shell/api_gen.sh to generate API."
+            exit 1
+          fi
       - name: Lint
         run: bash shell/lint.sh

.github/workflows/nightly.yml

@@ -55,7 +55,7 @@ jobs:
           pytest integration_tests/torch_workflow_test.py
       - name: Test with pytest
         run: |
-          pytest keras --ignore keras/applications --cov=keras
+          pytest keras --ignore keras/src/applications --cov=keras
 
   format:
     name: Check the code format
@@ -81,6 +81,15 @@ jobs:
           pip install -r requirements.txt --progress-bar off --upgrade
           pip uninstall -y keras keras-nightly
           pip install -e "." --progress-bar off --upgrade
+      - name: Check for API changes
+        run: |
+          bash shell/api_gen.sh
+          git status
+          clean=$(git status | grep "nothing to commit")
+          if [ -z "$clean" ]; then
+            echo "Please run shell/api_gen.sh to generate API."
+            exit 1
+          fi
       - name: Lint
         run: bash shell/lint.sh
 
@@ -108,4 +117,4 @@ jobs:
         with:
           password: ${{ secrets.PYPI_NIGHTLY_API_TOKEN }}
           packages-dir: dist/
-          verbose: true
+          verbose: true

api_gen.py

@@ -0,0 +1,175 @@
+"""Script to generate keras public API in `keras/api` directory.
+
+Usage:
+
+Run via `./shell/api_gen.sh`.
+It generates API and formats user and generated APIs.
+"""
+
+import os
+import shutil
+
+import namex
+
+package = "keras"
+
+
+def ignore_files(_, filenames):
+    return [f for f in filenames if f.endswith("_test.py")]
+
+
+def create_legacy_directory():
+    API_DIR = os.path.join(package, "api")
+    # Make keras/_tf_keras/ by copying keras/
+    tf_keras_dirpath_parent = os.path.join(API_DIR, "_tf_keras")
+    tf_keras_dirpath = os.path.join(tf_keras_dirpath_parent, "keras")
+    os.makedirs(tf_keras_dirpath, exist_ok=True)
+    with open(os.path.join(tf_keras_dirpath_parent, "__init__.py"), "w") as f:
+        f.write("from keras.api._tf_keras import keras\n")
+    with open(os.path.join(API_DIR, "__init__.py")) as f:
+        init_file = f.read()
+        init_file = init_file.replace(
+            "from keras.api import _legacy",
+            "from keras.api import _tf_keras",
+        )
+    with open(os.path.join(API_DIR, "__init__.py"), "w") as f:
+        f.write(init_file)
+    # Remove the import of `_tf_keras` in `keras/_tf_keras/keras/__init__.py`
+    init_file = init_file.replace("from keras.api import _tf_keras\n", "\n")
+    with open(os.path.join(tf_keras_dirpath, "__init__.py"), "w") as f:
+        f.write(init_file)
+    for dirname in os.listdir(API_DIR):
+        dirpath = os.path.join(API_DIR, dirname)
+        if os.path.isdir(dirpath) and dirname not in (
+            "_legacy",
+            "_tf_keras",
+            "src",
+        ):
+            destpath = os.path.join(tf_keras_dirpath, dirname)
+            if os.path.exists(destpath):
+                shutil.rmtree(destpath)
+            shutil.copytree(
+                dirpath,
+                destpath,
+                ignore=ignore_files,
+            )
+
+    # Copy keras/_legacy/ file contents to keras/_tf_keras/keras
+    legacy_submodules = [
+        path[:-3]
+        for path in os.listdir(os.path.join(package, "src", "legacy"))
+        if path.endswith(".py")
+    ]
+    legacy_submodules += [
+        path
+        for path in os.listdir(os.path.join(package, "src", "legacy"))
+        if os.path.isdir(os.path.join(package, "src", "legacy", path))
+    ]
+
+    for root, _, fnames in os.walk(os.path.join(package, "_legacy")):
+        for fname in fnames:
+            if fname.endswith(".py"):
+                legacy_fpath = os.path.join(root, fname)
+                tf_keras_root = root.replace("/_legacy", "/_tf_keras/keras")
+                core_api_fpath = os.path.join(
+                    root.replace("/_legacy", ""), fname
+                )
+                if not os.path.exists(tf_keras_root):
+                    os.makedirs(tf_keras_root)
+                tf_keras_fpath = os.path.join(tf_keras_root, fname)
+                with open(legacy_fpath) as f:
+                    legacy_contents = f.read()
+                    legacy_contents = legacy_contents.replace(
+                        "keras.api._legacy", "keras.api._tf_keras.keras"
+                    )
+                if os.path.exists(core_api_fpath):
+                    with open(core_api_fpath) as f:
+                        core_api_contents = f.read()
+                    core_api_contents = core_api_contents.replace(
+                        "from keras.api import _tf_keras\n", ""
+                    )
+                    for legacy_submodule in legacy_submodules:
+                        core_api_contents = core_api_contents.replace(
+                            f"from keras.api import {legacy_submodule}\n",
+                            "",
+                        )
+                        core_api_contents = core_api_contents.replace(
+                            f"keras.api.{legacy_submodule}",
+                            f"keras.api._tf_keras.keras.{legacy_submodule}",
+                        )
+                    legacy_contents = core_api_contents + "\n" + legacy_contents
+                with open(tf_keras_fpath, "w") as f:
+                    f.write(legacy_contents)
+
+    # Delete keras/api/_legacy/
+    shutil.rmtree(os.path.join(API_DIR, "_legacy"))
+
+
+def export_version_string():
+    API_INIT = os.path.join(package, "api", "__init__.py")
+    with open(API_INIT) as f:
+        contents = f.read()
+    with open(API_INIT, "w") as f:
+        contents += "from keras.src.version import __version__\n"
+        f.write(contents)
+
+
+def update_package_init():
+    contents = """
+# Import everything from /api/ into keras.
+from keras.api import *  # noqa: F403
+from keras.api import __version__  # Import * ignores names start with "_".
+
+import os
+
+# Add everything in /api/ to the module search path.
+__path__.append(os.path.join(os.path.dirname(__file__), "api"))  # noqa: F405
+
+# Don't pollute namespace.
+del os
+
+# Never autocomplete `.src` or `.api` on an imported keras object.
+def __dir__():
+    keys = dict.fromkeys((globals().keys()))
+    keys.pop("src")
+    keys.pop("api")
+    return list(keys)
+
+
+# Don't import `.src` or `.api` during `from keras import *`.
+__all__ = [
+    name
+    for name in globals().keys()
+    if not (name.startswith("_") or name in ("src", "api"))
+]"""
+    with open(os.path.join(package, "__init__.py")) as f:
+        init_contents = f.read()
+    with open(os.path.join(package, "__init__.py"), "w") as f:
+        f.write(init_contents.replace("\nfrom keras import api", contents))
+
+
+if __name__ == "__main__":
+    # Backup the `keras/__init__.py` and restore it on error in api gen.
+    os.makedirs(os.path.join(package, "api"), exist_ok=True)
+    init_fname = os.path.join(package, "__init__.py")
+    backup_init_fname = os.path.join(package, "__init__.py.bak")
+    try:
+        if os.path.exists(init_fname):
+            shutil.move(init_fname, backup_init_fname)
+        # Generates `keras/api` directory.
+        namex.generate_api_files(
+            "keras", code_directory="src", target_directory="api"
+        )
+        # Creates `keras/__init__.py` importing from `keras/api`
+        update_package_init()
+    except Exception as e:
+        if os.path.exists(backup_init_fname):
+            shutil.move(backup_init_fname, init_fname)
+        raise e
+    finally:
+        if os.path.exists(backup_init_fname):
+            os.remove(backup_init_fname)
+    # Add __version__ to keras package
+    export_version_string()
+    # Creates `_tf_keras` with full keras API
+    create_legacy_directory()

conftest.py

@@ -14,7 +14,7 @@
 
 import pytest  # noqa: E402
 
-from keras.backend import backend  # noqa: E402
+from keras.src.backend import backend  # noqa: E402
 
 
 def pytest_configure(config):

integration_tests/basic_full_flow.py

@@ -2,11 +2,11 @@
 import pytest
 
 import keras
-from keras import layers
-from keras import losses
-from keras import metrics
-from keras import optimizers
-from keras import testing
+from keras.src import layers
+from keras.src import losses
+from keras.src import metrics
+from keras.src import optimizers
+from keras.src import testing
 
 
 class MyModel(keras.Model):

integration_tests/dataset_tests/boston_housing_test.py

@@ -1,5 +1,5 @@
-from keras import testing
-from keras.datasets import boston_housing
+from keras.src import testing
+from keras.src.datasets import boston_housing
 
 
 class BostonHousingTest(testing.TestCase):

integration_tests/dataset_tests/california_housing_test.py

@@ -1,5 +1,5 @@
-from keras import testing
-from keras.datasets import california_housing
+from keras.src import testing
+from keras.src.datasets import california_housing
 
 
 class CaliforniaHousingTest(testing.TestCase):

integration_tests/dataset_tests/cifar100_test.py

@@ -1,7 +1,7 @@
 import numpy as np
 
-from keras import testing
-from keras.datasets import cifar100
+from keras.src import testing
+from keras.src.datasets import cifar100
 
 
 class Cifar100LoadDataTest(testing.TestCase):

integration_tests/dataset_tests/cifar10_test.py

@@ -1,7 +1,7 @@
 import numpy as np
 
-from keras import testing
-from keras.datasets import cifar10
+from keras.src import testing
+from keras.src.datasets import cifar10
 
 
 class Cifar10LoadDataTest(testing.TestCase):

integration_tests/dataset_tests/fashion_mnist_test.py

@@ -1,7 +1,7 @@
 import numpy as np
 
-from keras import testing
-from keras.datasets import fashion_mnist
+from keras.src import testing
+from keras.src.datasets import fashion_mnist
 
 
 class FashionMnistLoadDataTest(testing.TestCase):

integration_tests/dataset_tests/imdb_test.py

@@ -1,7 +1,7 @@
 import numpy as np
 
-from keras import testing
-from keras.datasets import imdb
+from keras.src import testing
+from keras.src.datasets import imdb
 
 
 class ImdbLoadDataTest(testing.TestCase):

integration_tests/dataset_tests/mnist_test.py

@@ -1,7 +1,7 @@
 import numpy as np
 
-from keras import testing
-from keras.datasets import mnist
+from keras.src import testing
+from keras.src.datasets import mnist
 
 
 class MnistLoadDataTest(testing.TestCase):

integration_tests/dataset_tests/reuters_test.py

@@ -1,7 +1,7 @@
 import numpy as np
 
-from keras import testing
-from keras.datasets import reuters
+from keras.src import testing
+from keras.src.datasets import reuters
 
 
 class ReutersLoadDataTest(testing.TestCase):