Combine changes for py3.7 +unittests. (#165)

Summary: Pull Request resolved: #165

Reviewed By: PaliC

Differential Revision: D39388436

Pulled By: anirbanr-fb-r2p

fbshipit-source-id: 05650cf1f60d93ba1eb0e9d8e547d94cbe687f51

.github/workflows/runtime_nightly.yaml

@@ -8,7 +8,8 @@ jobs:
   unittest:
     strategy:
       matrix:
-        python-version: [3.8, 3.9, '3.10']
+        python-major-version: [3]
+        python-minor-version: [7,8,9,10]
         platform: [ubuntu-18.04]
       fail-fast: false
     runs-on: ${{ matrix.platform }}
@@ -26,14 +27,15 @@ jobs:
       - name: Build
         env:
           DOCKER_BUILDKIT: 1
-        run: docker build -t multipy --progress=plain --build-arg PYTHON_VERSION=${{ matrix.python-version }} .
+        run: docker build -t multipy --progress=plain --build-arg PYTHON_MAJOR_VERSION=${{ matrix.python-major-version }} --build-arg PYTHON_MINOR_VERSION=${{ matrix.python-minor-version }} .
 
       - name: Test
         run: |
-          docker run --rm multipy multipy/runtime/build/test_deploy
+          docker run --rm multipy bash -c "if [[ ${{ matrix.python-minor-version }} -lt 8 ]]; then source ~/venvs/multipy/bin/activate; fi && multipy/runtime/build/test_deploy"
+
       - name: Compat Tests
         run: |
-          docker run --rm multipy bash -c "pip install -r compat-requirements.txt && multipy/runtime/build/interactive_embedded_interpreter --pyscript multipy/runtime/test_compat.py"
+          docker run --rm multipy bash -c "if [[ ${{ matrix.python-minor-version }} -gt 7 ]]; then pip install -r compat-requirements.txt && multipy/runtime/build/interactive_embedded_interpreter --pyscript multipy/runtime/test_compat.py; fi"
 
       - name: Create Tarball
         run: |

.github/workflows/runtime_tests.yaml

@@ -10,7 +10,8 @@ jobs:
   unittest:
     strategy:
       matrix:
-        python-version: [3.8, 3.9, '3.10']
+        python-major-version: [3]
+        python-minor-version: [7,8,9,10]
         platform: [ubuntu-18.04]
       fail-fast: false
     runs-on: ${{ matrix.platform }}
@@ -28,12 +29,12 @@ jobs:
       - name: Build
         env:
           DOCKER_BUILDKIT: 1
-        run: docker build -t multipy --progress=plain --build-arg PYTHON_VERSION=${{ matrix.python-version }} .
+        run: docker build -t multipy --progress=plain --build-arg PYTHON_MAJOR_VERSION=${{ matrix.python-major-version }} --build-arg PYTHON_MINOR_VERSION=${{ matrix.python-minor-version }} .
 
       - name: Test
         run: |
-          docker run --rm multipy multipy/runtime/build/test_deploy
+          docker run --rm multipy bash -c "if [[ ${{ matrix.python-minor-version }} -lt 8 ]]; then source ~/venvs/multipy/bin/activate; fi && multipy/runtime/build/test_deploy"
 
       - name: Compat Tests
         run: |
-          docker run --rm multipy bash -c "pip install -r compat-requirements.txt && multipy/runtime/build/interactive_embedded_interpreter --pyscript multipy/runtime/test_compat.py"
+          docker run --rm multipy bash -c "if [[ ${{ matrix.python-minor-version }} -gt 7 ]]; then pip install -r compat-requirements.txt && multipy/runtime/build/interactive_embedded_interpreter --pyscript multipy/runtime/test_compat.py; fi"

Dockerfile

@@ -2,6 +2,8 @@ ARG BASE_IMAGE=nvidia/cuda:11.3.1-devel-ubuntu18.04
 
 FROM ${BASE_IMAGE} as dev-base
 
+SHELL ["/bin/bash", "-c"]
+
 # Install system dependencies
 RUN --mount=type=cache,id=apt-dev,target=/var/cache/apt \
         apt update && DEBIAN_FRONTEND=noninteractive apt install -yq --no-install-recommends \
@@ -39,7 +41,9 @@ RUN --mount=type=cache,id=apt-dev,target=/var/cache/apt \
         apt-transport-https \
         ca-certificates \
         gnupg \
-        software-properties-common && \
+        software-properties-common \
+        python-pip \
+        python3-pip && \
         wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | apt-key add - && \
         apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main' && \
         echo "deb http://security.ubuntu.com/ubuntu focal-security main" >> /etc/apt/sources.list && \
@@ -56,43 +60,51 @@ WORKDIR /opt/multipy
 COPY . .
 RUN git submodule update --init --recursive --jobs 0
 
-# install pyenv
-FROM dev-base as pyenv-install
-RUN git clone https://github.com/pyenv/pyenv.git ~/.pyenv && \
-    export PYENV_ROOT="~/.pyenv" && \
-    export PATH="$PYENV_ROOT/bin:$PATH"
-# dummy cmd to verify installation.
-RUN pyenv install --list
-
-# Install conda + neccessary python dependencies
-FROM dev-base as conda
-ARG PYTHON_VERSION=3.8
-RUN curl -fsSL -v -o ~/miniconda.sh -O  https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh  && \
+# Install conda/pyenv + necessary python dependencies
+FROM dev-base as conda-pyenv
+ARG PYTHON_MAJOR_VERSION=3
+ARG PYTHON_MINOR_VERSION=8
+ENV PYTHON_MINOR_VERSION=${PYTHON_MINOR_VERSION}
+ENV PYTHON_VERSION=${PYTHON_MAJOR_VERSION}.${PYTHON_MINOR_VERSION}
+RUN if [[ ${PYTHON_MINOR_VERSION} -gt 7 ]]; then \
+    curl -fsSL -v -o ~/miniconda.sh -O  https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh  && \
     chmod +x ~/miniconda.sh && \
     ~/miniconda.sh -b -p /opt/conda && \
     rm ~/miniconda.sh && \
     /opt/conda/bin/conda install -y python=${PYTHON_VERSION} mkl mkl-include conda-build pyyaml numpy ipython && \
     /opt/conda/bin/conda install -y -c conda-forge libpython-static=${PYTHON_VERSION} && \
     /opt/conda/bin/conda install -y pytorch torchvision torchaudio cudatoolkit=11.3 -c pytorch-nightly && \
-    /opt/conda/bin/conda clean -ya
-
+    /opt/conda/bin/conda clean -ya; \
+    else \
+    pip3 install virtualenv && \
+    git clone https://github.com/pyenv/pyenv.git ~/.pyenv && \
+    export CFLAGS="-fPIC -g" && \
+    ~/.pyenv/bin/pyenv install --force 3.7.10 && \
+    virtualenv -p ~/.pyenv/versions/3.7.10/bin/python3 ~/venvs/multipy; \
+    fi
 
 # Build/Install pytorch with post-cxx11 ABI
-FROM conda as build
+FROM conda-pyenv as build
 WORKDIR /opt/multipy/multipy/runtime/third-party/pytorch
-COPY --from=conda /opt/conda /opt/conda
+COPY --from=conda-pyenv /opt/conda* /opt/conda
 COPY --from=submodule-update /opt/multipy /opt/multipy
 
 WORKDIR /opt/multipy
 
 # Build Multipy
 RUN mkdir multipy/runtime/build && \
-   cd multipy/runtime/build && \
-   cmake .. && \
-   cmake --build . --config Release && \
-   cmake --install . --prefix "."
+    cd multipy/runtime/build && \
+    if [[ ${PYTHON_MINOR_VERSION} -lt 8 ]]; then \
+    source ~/venvs/multipy/bin/activate && \
+    pip3 install --pre torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/nightly/cu113 && \
+    cmake -DLEGACY_PYTHON_PRE_3_8=ON ..; \
+    else \
+    cmake -DLEGACY_PYTHON_PRE_3_8=OFF ..; \
+    fi && \
+    cmake --build . --config Release && \
+    cmake --install . --prefix "." && \
+    cd ../example && python generate_examples.py
 
-RUN cd multipy/runtime/example && python generate_examples.py
 ENV PYTHONPATH=. LIBTEST_DEPLOY_LIB=multipy/runtime/build/libtest_deploy_lib.so
 
 RUN mkdir /opt/dist && cp -r multipy/runtime/build/dist/* /opt/dist/

multipy/runtime/CMakeLists.txt

@@ -12,6 +12,8 @@ project(MultipyRuntime)
 option(ABI_EQUALS_1 "Set ABI value to 1, by default it is set to 0. Pytorch by default builds with ABI set to 1." OFF)
 option(BUILD_CUDA_TESTS "Set to ON in order to build cuda tests. By default we do not" OFF)
 
+OPTION(LEGACY_PYTHON_PRE_3_8 "Whether to use Python 3.7 codepaths." OFF)
+
 if(ABI_EQUALS_1)
   set(ABI_VALUE 1)
 else()
@@ -105,7 +107,11 @@ set(INTERPRETER_TEST_SOURCES_GPU
 
 LINK_DIRECTORIES("${PYTORCH_ROOT}/torch/lib")
 add_executable(test_deploy ${INTERPRETER_TEST_SOURCES})
-target_compile_definitions(test_deploy PUBLIC TEST_CUSTOM_LIBRARY)
+# target_compile_definitions(test_deploy PUBLIC TEST_CUSTOM_LIBRARY)
+if (${LEGACY_PYTHON_PRE_3_8})
+  message(STATUS "LEGACY_PYTHON_PRE_3_8 set in parent cmakefile.")
+  target_compile_definitions(test_deploy PUBLIC LEGACY_PYTHON_PRE_3_8)
+endif()
 target_include_directories(test_deploy PRIVATE ${PYTORCH_ROOT}/torch)
 target_link_libraries(test_deploy
   PUBLIC "-Wl,--no-as-needed -rdynamic" gtest dl torch_deploy_interface c10 torch_cpu

multipy/runtime/interpreter/CMakeLists.txt

@@ -4,6 +4,7 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 
 SET(INTERPRETER_DIR "${DEPLOY_DIR}/interpreter" )
 SET(INTERPRETER_DIR "${DEPLOY_DIR}/interpreter" PARENT_SCOPE)
@@ -13,6 +14,8 @@ include_directories(BEFORE "${PYTORCH_ROOT}/torch/include")
 include_directories(BEFORE "${PYTORCH_ROOT}/torch/include/torch/csrc/api/include/")
 SET(MULTIPY_UTILS "${CMAKE_CURRENT_SOURCE_DIR}/../../utils")
 
+OPTION(LEGACY_PYTHON_PRE_3_8 "Whether to use Python 3.7 codepaths." OFF)
+
 find_package (Python3 COMPONENTS Interpreter Development)
 
 message(STATUS "Python3_EXECUTABLE - ${Python3_EXECUTABLE}" )
@@ -32,6 +35,11 @@ link_directories(BEFORE "${Python3_SITELIB}/torch/lib")
 include_directories(BEFORE "${Python3_INCLUDE_DIRS}")
 include_directories(BEFORE ${CMAKE_SOURCE_DIR}/../..)
 
+if (${LEGACY_PYTHON_PRE_3_8})
+  message(STATUS "LEGACY_PYTHON_PRE_3_8 set")
+  add_compile_definitions(LEGACY_PYTHON_PRE_3_8)
+endif()
+
 # add gtest dependency
 include(FetchContent)
 FetchContent_Declare(
@@ -80,3 +88,5 @@ target_link_libraries(torch_deployinterpreter PRIVATE fmt::fmt-header-only)
 target_link_libraries(torch_deployinterpreter PRIVATE torch_python)
 target_link_libraries(torch_deployinterpreter PRIVATE gtest)
 target_link_libraries(torch_deployinterpreter PRIVATE multipy_torch)
+
+unset(LEGACY_PYTHON_PRE_3_8)

multipy/runtime/interpreter/builtin_registry.cpp

@@ -64,6 +64,7 @@ void BuiltinRegistry::runPreInitialization() {
   appendCPythonInittab();
 }
 
+#ifndef LEGACY_PYTHON_PRE_3_8
 const char* metaPathSetupTemplate = R"PYTHON(
 import sys
 from importlib.metadata import DistributionFinder, Distribution
@@ -107,6 +108,20 @@ class DummyDistribution(Distribution):
 
 sys.meta_path.insert(0, F())
 )PYTHON";
+#else
+const char* metaPathSetupTemplate = R"PYTHON(
+import sys
+class F:
+    MODULES = set({<<<DEPLOY_BUILTIN_MODULES_CSV>>>})
+    def find_spec(self, fullname, path, target=None):
+        if fullname in self.MODULES:
+            # Load this module using `BuiltinImporter`, but set `path` to None
+            # in order to trick it into loading our module.
+            return sys.meta_path[1].find_spec(fullname, path=None, target=None)
+        return None
+sys.meta_path.insert(0, F())
+)PYTHON";
+#endif
 
 void BuiltinRegistry::runPostInitialization() {
   TORCH_INTERNAL_ASSERT(Py_IsInitialized());

multipy/runtime/interpreter/interpreter_impl.cpp

@@ -247,6 +247,8 @@ ConcreteInterpreterImplConstructorCommon(
     const std::vector<std::string>& extra_python_paths,
     const std::vector<std::string>& plugin_paths) {
   BuiltinRegistry::runPreInitialization();
+
+#ifndef LEGACY_PYTHON_PRE_3_8
   PyPreConfig preconfig;
   PyPreConfig_InitIsolatedConfig(&preconfig);
   PyStatus status = Py_PreInitialize(&preconfig);
@@ -283,6 +285,12 @@ ConcreteInterpreterImplConstructorCommon(
   status = Py_InitializeFromConfig(&config);
   PyConfig_Clear(&config);
   TORCH_INTERNAL_ASSERT(!PyStatus_Exception(status))
+
+#else
+  Py_InitializeEx(1);
+  TORCH_INTERNAL_ASSERT(Py_IsInitialized);
+#endif
+
 #ifdef FBCODE_CAFFE2
   auto sys_path = global_impl("sys", "path");
   for (const auto& entry : extra_python_paths) {

multipy/runtime/test_deploy.cpp

@@ -459,6 +459,7 @@ result = torch.Tensor([1,2,3])
   EXPECT_TRUE(w_grad0.equal(w_grad1));
 }
 
+#ifndef LEGACY_PYTHON_PRE_3_8
 TEST(TorchpyTest, ImportlibMetadata) {
   torch::deploy::InterpreterManager m(1);
   m.registerModuleSource("importlib_test", R"PYTHON(
@@ -470,6 +471,7 @@ result = version("torch")
   auto ver = I.global("importlib_test", "result").toIValue().toString();
   ASSERT_EQ(ver->string(), "0.0.1+fake_multipy");
 }
+#endif
 
 // OSS build does not have bultin numpy support yet. Use this flag to guard the
 // test case.