diff --git a/CMakeLists.txt b/CMakeLists.txt index ddb2a9f43c..c54ef2825e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,6 +19,38 @@ cmake_minimum_required(VERSION 3.14) SET(CMAKE_NINJA_FORCE_RESPONSE_FILE 1 CACHE INTERNAL "") include(SelectLibraryConfigurations) +# Passing of variables to vcpkg +# +# vcpkg runs cmake scripts in an isolated environment, see this for details: +# https://github.com/Microsoft/vcpkg/issues/3712 +# +# Here's how this works and how we work around this issue: +# +# 1. This file (CMakeLists.txt) runs first and is authoritative. It is the one +# that reads the environment, sets variables and sets a default value. +# 2. It writes the contents of the variables to +# $CMAKE_CURRENT_BINARY_DIR/_env/$VARNAME +# 3. hifi_vcpkg.py takes the _env directory, and copies it to the vcpkg dir. +# This solves the issue of CMakeLists.txt not knowing where the vcpkg dir is. +# 4. cmake/ports/*/portfile.cmake does know where the vcpkg dir is, and can +# read the _env that was copied there to obtain the variable's name. +# +# To ensure no old data could be accidentally read, the _env directories are +# deleted on each execution and fully recreated. + +# Ensure nothing is kept from any previous run +file(REMOVE_RECURSE "${CMAKE_CURRENT_BINARY_DIR}/_env") +file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/_env") + +# Base URL for externally downloaded files +set(EXTERNAL_BUILD_ASSETS "https://build-deps.overte.org") + +if( DEFINED ENV{EXTERNAL_BUILD_ASSETS} ) + set(EXTERNAL_BUILD_ASSETS "$ENV{EXTERNAL_BUILD_ASSETS}") +endif() + +file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/_env/EXTERNAL_BUILD_ASSETS.txt" "${EXTERNAL_BUILD_ASSETS}") +MESSAGE(STATUS "EXTERNAL_BUILD_ASSETS: ${EXTERNAL_BUILD_ASSETS}") # read USE_GLES enviroment variable and sets it as GLES option # TODO still gets overwritten by "use GLES on linux aarch64" @@ -35,6 +67,18 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarc set(GLES_OPTION ON) endif() +# Will affect VCPKG dependencies +file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/_env/USE_GLES.txt" "${GLES_OPTION}") +MESSAGE(STATUS "GLES_OPTION: ${GLES_OPTION}") + +include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/macros/TargetPython.cmake") +target_python() + +if (WIN32 AND NOT HIFI_ANDROID AND NOT (CMAKE_GENERATOR STREQUAL "Ninja")) + # Force x64 toolset + set(CMAKE_GENERATOR_TOOLSET "host=x64" CACHE STRING "64-bit toolset" FORCE) +endif() + # set our OS X deployment target # (needs to be set before first project() call and before prebuild.py) # Will affect VCPKG dependencies @@ -42,8 +86,6 @@ if (APPLE) set(ENV{MACOSX_DEPLOYMENT_TARGET} 10.11) endif() -set(EXTERNAL_BUILD_ASSETS "https://build-deps.overte.org") - set(RELEASE_TYPE "$ENV{RELEASE_TYPE}") if ((NOT "${RELEASE_TYPE}" STREQUAL "PRODUCTION") AND (NOT "${RELEASE_TYPE}" STREQUAL "PR")) set(RELEASE_TYPE "DEV") @@ -67,6 +109,8 @@ if( NOT WIN32 ) set(OVERTE_OPTIMIZE_FLAGS "") if(OVERTE_OPTIMIZE) + + file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/_env/OVERTE_OPTIMIZE.txt" "${OVERTE_OPTIMIZE}") if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") message("Clang compiler detected, adding -O3 -fPIC -g flags") set(OVERTE_OPTIMIZE_FLAGS "-O3 -fPIC -g") @@ -92,6 +136,7 @@ if( NOT WIN32 ) endif() if(DEFINED OVERTE_CPU_ARCHITECTURE) + file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/_env/OVERTE_CPU_ARCHITECTURE.txt" "${OVERTE_CPU_ARCHITECTURE}") set(OVERTE_OPTIMIZE_FLAGS "${OVERTE_OPTIMIZE_FLAGS} ${OVERTE_CPU_ARCHITECTURE}") message("Adding CPU architecture flags: ${OVERTE_CPU_ARCHITECTURE}") MESSAGE(STATUS "OVERTE_CPU_ARCHITECTURE: ${OVERTE_CPU_ARCHITECTURE}") @@ -156,9 +201,53 @@ if(OVERTE_WARNINGS_AS_ERRORS) endif() +if (HIFI_ANDROID) + execute_process( + COMMAND ${HIFI_PYTHON_EXEC} ${CMAKE_CURRENT_SOURCE_DIR}/prebuild.py --release-type ${RELEASE_TYPE} --android ${HIFI_ANDROID_APP} --build-root ${CMAKE_BINARY_DIR} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} RESULTS_VARIABLE PREBUILD_RET + ) +else() + set(VCPKG_BUILD_TYPE_PARAM "") + if (VCPKG_BUILD_TYPE) + set(VCPKG_BUILD_TYPE_PARAM --vcpkg-build-type ${VCPKG_BUILD_TYPE}) + endif() + execute_process( + COMMAND ${HIFI_PYTHON_EXEC} ${CMAKE_CURRENT_SOURCE_DIR}/prebuild.py --release-type ${RELEASE_TYPE} --build-root ${CMAKE_BINARY_DIR} ${VCPKG_BUILD_TYPE_PARAM} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} RESULTS_VARIABLE PREBUILD_RET + ) +endif() + +if (PREBUILD_RET GREATER 0) + message(FATAL_ERROR "prebuild.py failed with error ${PREBUILD_RET}") +endif() +if(NOT EXISTS "${CMAKE_BINARY_DIR}/vcpkg.cmake") + message(FATAL_ERROR "vcpkg configuration missing.") +endif() +include("${CMAKE_BINARY_DIR}/vcpkg.cmake") + +if (HIFI_ANDROID) + set(QT_CMAKE_PREFIX_PATH "$ENV{HIFI_ANDROID_PRECOMPILED}/qt/lib/cmake") +else() + if ("$ENV{OVERTE_USE_SYSTEM_QT}" STREQUAL "") + if(NOT EXISTS "${CMAKE_BINARY_DIR}/qt.cmake") + message(FATAL_ERROR "qt configuration missing.") + endif() + include("${CMAKE_BINARY_DIR}/qt.cmake") + message(STATUS "${CMAKE_BINARY_DIR}/qt.cmake included!") + else() + message(STATUS "System Qt in use, not including qt.cmake!") + endif() +endif() + +option(VCPKG_APPLOCAL_DEPS OFF) + project(overte) include("cmake/init.cmake") include("cmake/compiler.cmake") +option(VCPKG_APPLOCAL_DEPS OFF) + +add_paths_to_fixup_libs(${VCPKG_INSTALL_ROOT}/bin) +add_paths_to_fixup_libs(${VCPKG_INSTALL_ROOT}/debug/bin) if (NOT DEFINED CLIENT_ONLY) set(CLIENT_ONLY 0) @@ -335,6 +424,12 @@ endif() set_packaging_parameters() +# Locate the required Qt build on the filesystem +setup_qt() + +if ("$ENV{OVERTE_USE_SYSTEM_QT}" STREQUAL "") + list(APPEND CMAKE_PREFIX_PATH "${QT_CMAKE_PREFIX_PATH}") +endif() find_package( Threads ) diff --git a/hifi_android.py b/hifi_android.py new file mode 100644 index 0000000000..46e985b05c --- /dev/null +++ b/hifi_android.py @@ -0,0 +1,337 @@ +import hifi_utils +import json +import os +import platform +import re +import shutil +import xml.etree.ElementTree as ET +import functools +import zipfile + +print = functools.partial(print, flush=True) + +ANDROID_PACKAGE_URL = 'https://build-deps.overte.org/dependencies/android/' + +ANDROID_PACKAGES = { + 'qt' : { + 'file': 'qt-5.11.1_linux_armv8-libcpp_openssl_patched.tgz', + 'checksum': 'aa449d4bfa963f3bc9a9dfe558ba29df', + }, + 'bullet': { + 'file': 'bullet-2.88_armv8-libcpp.tgz', + 'checksum': '81642779ccb110f8c7338e8739ac38a0', + }, + 'draco': { + 'file': 'draco_armv8-libcpp.tgz', + 'checksum': '617a80d213a5ec69fbfa21a1f2f738cd', + }, + 'glad': { + 'file': 'glad_armv8-libcpp.zip', + 'checksum': 'a8ee8584cf1ccd34766c7ddd9d5e5449', + }, + 'gvr': { + 'file': 'gvrsdk_v1.101.0.tgz', + 'checksum': '57fd02baa069176ba18597a29b6b4fc7', + }, + 'nvtt': { + 'file': 'nvtt_armv8-libcpp.zip', + 'checksum': 'eb46d0b683e66987190ed124aabf8910', + 'sharedLibFolder': 'lib', + 'includeLibs': ['libnvtt.so', 'libnvmath.so', 'libnvimage.so', 'libnvcore.so'] + }, + 'ovr_sdk_mobile_1.37.0': { + 'file': 'ovr_sdk_mobile_1.37.0.zip', + 'checksum': '6040e1966f335a3e5015295154cd7383', + 'sharedLibFolder': 'VrApi/Libs/Android/arm64-v8a/Release', + 'includeLibs': ['libvrapi.so'] + }, + 'ovr_platform_sdk_23.0.0': { + 'file': 'ovr_platform_sdk_23.0.0.zip', + 'checksum': '29d02b560f60d0fa7b8a64cd965dd55b', + 'sharedLibFolder': 'Android/libs/arm64-v8a', + 'includeLibs': ['libovrplatformloader.so'] + }, + 'openssl': { + 'file': 'openssl-1.1.0g_armv8.tgz', + 'checksum': 'cabb681fbccd79594f65fcc266e02f32' + }, + 'polyvox': { + 'file': 'polyvox_armv8-libcpp.tgz', + 'checksum': 'dba88b3a098747af4bb169e9eb9af57e', + 'sharedLibFolder': 'lib', + 'includeLibs': ['Release/libPolyVoxCore.so', 'libPolyVoxUtil.so'], + }, + 'tbb': { + 'file': 'tbb-2018_U1_armv8_libcpp.tgz', + 'checksum': '20768f298f53b195e71b414b0ae240c4', + 'sharedLibFolder': 'lib/release', + 'includeLibs': ['libtbb.so', 'libtbbmalloc.so'], + }, + 'etc2comp': { + 'file': 'etc2comp-patched-armv8-libcpp.tgz', + 'checksum': '14b02795d774457a33bbc60e00a786bc' + }, + 'breakpad': { + 'file': 'breakpad.tgz', + 'checksum': 'ddcb23df336b08017042ba4786db1d9e', + 'sharedLibFolder': 'lib', + 'includeLibs': {'libbreakpad_client.a'} + }, + 'webrtc': { + 'file': 'webrtc-20190626-android.tar.gz', + 'checksum': 'e2dccd3d8efdcba6d428c87ba7fb2a53' + } +} + +ANDROID_PLATFORM_PACKAGES = { + 'Darwin' : { + 'qt': { + 'file': 'qt-5.11.1_osx_armv8-libcpp_openssl_patched.tgz', + 'checksum': 'c83cc477c08a892e00c71764dca051a0' + }, + }, + 'Windows' : { + 'qt': { + 'file': 'qt-5.11.1_win_armv8-libcpp_openssl_patched.tgz', + 'checksum': '0582191cc55431aa4f660848a542883e' + }, + } +} + +QT5_DEPS = [ + 'Qt5Concurrent', + 'Qt5Core', + 'Qt5Gui', + 'Qt5Multimedia', + 'Qt5Network', + 'Qt5OpenGL', + 'Qt5Qml', + 'Qt5Quick', + 'Qt5QuickControls2', + 'Qt5QuickTemplates2', + 'Qt5Script', + 'Qt5ScriptTools', + 'Qt5Svg', + 'Qt5WebChannel', + 'Qt5WebSockets', + 'Qt5Widgets', + 'Qt5XmlPatterns', + # Android specific + 'Qt5AndroidExtras', + 'Qt5WebView', +] + +def getPlatformPackages(): + result = ANDROID_PACKAGES.copy() + system = platform.system() + if system in ANDROID_PLATFORM_PACKAGES: + platformPackages = ANDROID_PLATFORM_PACKAGES[system] + result = { **result, **platformPackages } + return result + +def getPackageUrl(package): + url = ANDROID_PACKAGE_URL + if 'baseUrl' in package: + url = package['baseUrl'] + url += package['file'] + if 'versionId' in package: + url += '?versionId=' + package['versionId'] + return url + +def copyAndroidLibs(packagePath, appPath): + androidPackages = getPlatformPackages() + jniPath = os.path.join(appPath, 'src/main/jniLibs/arm64-v8a') + if not os.path.isdir(jniPath): + os.makedirs(jniPath) + for packageName in androidPackages: + package = androidPackages[packageName] + if 'sharedLibFolder' in package: + sharedLibFolder = os.path.join(packagePath, packageName, package['sharedLibFolder']) + if 'includeLibs' in package: + for lib in package['includeLibs']: + sourceFile = os.path.join(sharedLibFolder, lib) + destFile = os.path.join(jniPath, os.path.split(lib)[1]) + if not os.path.isfile(destFile): + print("Copying {}".format(lib)) + shutil.copy(sourceFile, destFile) + + gvrLibFolder = os.path.join(packagePath, 'gvr/gvr-android-sdk-1.101.0/libraries') + audioSoOut = os.path.join(gvrLibFolder, 'libgvr_audio.so') + if not os.path.isfile(audioSoOut): + audioAar = os.path.join(gvrLibFolder, 'sdk-audio-1.101.0.aar') + with zipfile.ZipFile(audioAar) as z: + with z.open('jni/arm64-v8a/libgvr_audio.so') as f: + with open(audioSoOut, 'wb') as of: + shutil.copyfileobj(f, of) + + audioSoOut2 = os.path.join(jniPath, 'libgvr_audio.so') + if not os.path.isfile(audioSoOut2): + shutil.copy(audioSoOut, audioSoOut2) + + baseSoOut = os.path.join(gvrLibFolder, 'libgvr.so') + if not os.path.isfile(baseSoOut): + baseAar = os.path.join(gvrLibFolder, 'sdk-base-1.101.0.aar') + with zipfile.ZipFile(baseAar) as z: + with z.open('jni/arm64-v8a/libgvr.so') as f: + with open(baseSoOut, 'wb') as of: + shutil.copyfileobj(f, of) + + baseSoOut2 = os.path.join(jniPath, 'libgvr.so') + if not os.path.isfile(baseSoOut2): + shutil.copy(baseSoOut, baseSoOut2) + +class QtPackager: + def __init__(self, appPath, qtRootPath): + self.appPath = appPath + self.qtRootPath = qtRootPath + self.jniPath = os.path.join(self.appPath, 'src/main/jniLibs/arm64-v8a') + self.assetPath = os.path.join(self.appPath, 'src/main/assets') + self.qtAssetPath = os.path.join(self.assetPath, '--Added-by-androiddeployqt--') + self.qtAssetCacheList = os.path.join(self.qtAssetPath, 'qt_cache_pregenerated_file_list') + # Jars go into the qt library + self.jarPath = os.path.realpath(os.path.join(self.appPath, '../../libraries/qt/libs')) + self.xmlFile = os.path.join(self.appPath, 'src/main/res/values/libs.xml') + self.files = [] + self.features = [] + self.permissions = [] + + def copyQtDeps(self): + for lib in QT5_DEPS: + libfile = os.path.join(self.qtRootPath, "lib/lib{}.so".format(lib)) + if not os.path.exists(libfile): + continue + self.files.append(libfile) + androidDeps = os.path.join(self.qtRootPath, "lib/{}-android-dependencies.xml".format(lib)) + if not os.path.exists(androidDeps): + continue + + tree = ET.parse(androidDeps) + root = tree.getroot() + for item in root.findall('./dependencies/lib/depends/*'): + if (item.tag == 'lib') or (item.tag == 'bundled'): + relativeFilename = item.attrib['file'] + if (relativeFilename.startswith('qml')): + continue + filename = os.path.join(self.qtRootPath, relativeFilename) + self.files.extend(hifi_utils.recursiveFileList(filename, excludeNamePattern=r"^\.")) + elif item.tag == 'jar' and 'bundling' in item.attrib and item.attrib['bundling'] == "1": + self.files.append(os.path.join(self.qtRootPath, item.attrib['file'])) + elif item.tag == 'permission': + self.permissions.append(item.attrib['name']) + elif item.tag == 'feature': + self.features.append(item.attrib['name']) + + def scanQmlImports(self): + qmlImportCommandFile = os.path.join(self.qtRootPath, 'bin/qmlimportscanner') + system = platform.system() + if 'Windows' == system: + qmlImportCommandFile += ".exe" + if not os.path.isfile(qmlImportCommandFile): + raise RuntimeError("Couldn't find qml import scanner") + qmlRootPath = hifi_utils.scriptRelative('interface/resources/qml') + qmlImportPath = os.path.join(self.qtRootPath, 'qml') + commandResult = hifi_utils.executeSubprocessCapture([ + qmlImportCommandFile, + '-rootPath', qmlRootPath, + '-importPath', qmlImportPath + ]) + qmlImportResults = json.loads(commandResult) + for item in qmlImportResults: + if 'path' not in item: + continue + path = os.path.realpath(item['path']) + if not os.path.exists(path): + continue + basePath = path + if os.path.isfile(basePath): + basePath = os.path.dirname(basePath) + basePath = os.path.normcase(basePath) + if basePath.startswith(qmlRootPath): + continue + self.files.extend(hifi_utils.recursiveFileList(path, excludeNamePattern=r"^\.")) + + def processFiles(self): + self.files = list(set(self.files)) + self.files.sort() + libsXmlRoot = ET.Element('resources') + qtLibsNode = ET.SubElement(libsXmlRoot, 'array', {'name':'qt_libs'}) + bundledLibsNode = ET.SubElement(libsXmlRoot, 'array', {'name':'bundled_in_lib'}) + bundledAssetsNode = ET.SubElement(libsXmlRoot, 'array', {'name':'bundled_in_assets'}) + libPrefix = 'lib' + for sourceFile in self.files: + if not os.path.isfile(sourceFile): + raise RuntimeError("Unable to find dependency file " + sourceFile) + relativePath = os.path.relpath(sourceFile, self.qtRootPath).replace('\\', '/') + destinationFile = None + if relativePath.endswith('.so'): + garbledFileName = None + if relativePath.startswith(libPrefix): + garbledFileName = relativePath[4:] + p = re.compile(r'lib(Qt5.*).so') + m = p.search(garbledFileName) + if not m: + raise RuntimeError("Huh?") + libName = m.group(1) + ET.SubElement(qtLibsNode, 'item').text = libName + else: + garbledFileName = 'lib' + relativePath.replace('/', '_'[0]) + value = "{}:{}".format(garbledFileName, relativePath).replace('\\', '/') + ET.SubElement(bundledLibsNode, 'item').text = value + destinationFile = os.path.join(self.jniPath, garbledFileName) + elif relativePath.startswith('jar'): + destinationFile = os.path.join(self.jarPath, relativePath[4:]) + else: + value = "--Added-by-androiddeployqt--/{}:{}".format(relativePath,relativePath).replace('\\', '/') + ET.SubElement(bundledAssetsNode, 'item').text = value + destinationFile = os.path.join(self.qtAssetPath, relativePath) + + destinationParent = os.path.realpath(os.path.dirname(destinationFile)) + if not os.path.isdir(destinationParent): + os.makedirs(destinationParent) + if not os.path.isfile(destinationFile): + shutil.copy(sourceFile, destinationFile) + + tree = ET.ElementTree(libsXmlRoot) + tree.write(self.xmlFile, 'UTF-8', True) + + def generateAssetsFileList(self): + print("Implement asset file list") + # outputFilename = os.path.join(self.qtAssetPath, "qt_cache_pregenerated_file_list") + # fileList = hifi_utils.recursiveFileList(self.qtAssetPath) + # fileMap = {} + # for fileName in fileList: + # relativeFileName = os.path.relpath(fileName, self.assetPath) + # dirName, localFileName = os.path.split(relativeFileName) + # if not dirName in fileMap: + # fileMap[dirName] = [] + # fileMap[dirName].append(localFileName) + + # for dirName in fileMap: + # for localFileName in fileMap[dirName]: + # ???? + + # + # Gradle version + # + # DataOutputStream fos = new DataOutputStream(new FileOutputStream(outputFile)); + # for (Map.Entry> e: directoryContents.entrySet()) { + # def entryList = e.getValue() + # fos.writeInt(e.key.length()*2); // 2 bytes per char + # fos.writeChars(e.key); + # fos.writeInt(entryList.size()); + # for (String entry: entryList) { + # fos.writeInt(entry.length()*2); + # fos.writeChars(entry); + # } + # } + + def bundle(self): + if not os.path.isfile(self.xmlFile): + print("Bundling Qt info into {}".format(self.xmlFile)) + self.copyQtDeps() + self.scanQmlImports() + self.processFiles() + # if not os.path.isfile(self.qtAssetCacheList): + # self.generateAssetsFileList() + + diff --git a/hifi_qt.py b/hifi_qt.py new file mode 100644 index 0000000000..053ff997b7 --- /dev/null +++ b/hifi_qt.py @@ -0,0 +1,257 @@ +# Copyright 2013-2019 High Fidelity, Inc. +# Copyright 2020-2022 Vircadia contributors. +# Copyright 2020-2022 Overte e.V. +# SPDX-License-Identifier: Apache-2.0 + +import hifi_utils +import hifi_android +import hashlib +import os +import platform +import re +import shutil +import tempfile +import json +import xml.etree.ElementTree as ET +import functools + +# The way Qt is handled is a bit complicated, so I'm documenting it here. +# +# 1. User runs cmake +# 2. cmake calls prebuild.py, which is referenced in /CMakeLists.txt +# 3. prebuild.py calls this code. +# 4. hifi_qt.py determines how to handle cmake: do we need to download a package, and which? +# 4.a - Using system Qt +# No download, most special paths are turned off. +# We build in the same way a normal Qt program would. +# 4.b - Using an user-provided Qt build in a custom directory. +# We just need to set the cmakePath to the right dir (qt5-install/lib/cmake) +# 4.c - Using a premade package. +# We check the OS and distro and set qtUrl to the URL to download. +# After this, it works on the same pathway as 4.b. +# 5. We write /qt.cmake, which contains paths that are passed down to SetupQt.cmake +# The template for this file is in CMAKE_TEMPLATE just below this comment +# and it sets the QT_CMAKE_PREFIX_PATH variable used by SetupQt.cmake. +# 6. cmake includes /qt.cmake receiving our information +# In the case of system Qt, this step is skipped. +# 7. cmake runs SetupQt.cmake which takes care of the cmake parts of the Qt configuration. +# In the case of system Qt, SetupQt.cmake is a no-op. It runs but exits immediately. +# +# The format for a prebuilt qt is a package containing a top-level directory named +# 'qt5-install', which contains the result of a "make install" from a build of the Qt source. + +print = functools.partial(print, flush=True) + +# Encapsulates the vcpkg system +class QtDownloader: + CMAKE_TEMPLATE = """ +# this file auto-generated by hifi_qt.py +get_filename_component(QT_CMAKE_PREFIX_PATH "{}" ABSOLUTE CACHE) +get_filename_component(QT_CMAKE_PREFIX_PATH_UNCACHED "{}" ABSOLUTE) + +# If the cached cmake toolchain path is different from the computed one, exit +if(NOT (QT_CMAKE_PREFIX_PATH_UNCACHED STREQUAL QT_CMAKE_PREFIX_PATH)) + message(FATAL_ERROR "QT_CMAKE_PREFIX_PATH has changed, please wipe the build directory and rerun cmake") +endif() +""" + def __init__(self, args): + self.args = args + self.configFilePath = os.path.join(args.build_root, 'qt.cmake') + self.version = os.getenv('OVERTE_USE_QT_VERSION', '5.15.2') + self.assets_url = hifi_utils.readEnviromentVariableFromFile(args.build_root, 'EXTERNAL_BUILD_ASSETS') + + # OS dependent information + system = platform.system() + + qt_found = False + system_qt = False + + # Here we handle the 3 possible cases of dealing with Qt: + if bool(os.getenv('OVERTE_USE_SYSTEM_QT', False)): + # 1. Using the system provided Qt. This is only recommended for Qt 5.15.0 and above, + # as it includes a required fix on Linux. + # + # This path only works on Linux as neither Windows nor OSX ship Qt. + + if system != "Linux": + raise Exception("Using the system Qt is only supported on Linux") + + self.path = None + self.cmakePath = None + + qt_found = True + system_qt = True + + if not self.args.quiet: + print("Using system Qt") + + elif os.getenv('OVERTE_QT_PATH', "") != "": + # 2. Using an user-provided directory. + # OVERTE_QT_PATH must point to a directory with a Qt install in it. + + self.path = os.getenv('OVERTE_QT_PATH') + self.fullPath = self.path + self.cmakePath = os.path.join(self.fullPath, 'lib', 'cmake') + + qt_found = True + + if not self.args.quiet: + print("Using Qt from " + self.fullPath) + + else: + # 3. Using a pre-built Qt. + # + # This works somewhat differently from above, notice how path and fullPath are + # used differently in this case. + # + # In the case of an user-provided directory, we just use the user-supplied directory. + # + # For a pre-built qt, however, we have to unpack it. The archive is required to contain + # a qt5-install directory in it. + + self.path = os.path.expanduser("~/overte-files/qt") + self.fullPath = os.path.join(self.path, 'qt5-install') + self.cmakePath = os.path.join(self.fullPath, 'lib', 'cmake') + + if (not os.path.isdir(self.path)): + os.makedirs(self.path) + + qt_found = os.path.isdir(self.fullPath) + print("Using a packaged Qt") + + + if not system_qt: + if qt_found: + # Sanity check, ensure we have a good cmake directory + qt5_dir = os.path.join(self.cmakePath, "Qt5") + if not os.path.isdir(qt5_dir): + raise Exception("Failed to find Qt5 directory under " + self.cmakePath + ". There should be a " + qt5_dir) + else: + print("Qt5 check passed, found " + qt5_dir) + + # I'm not sure why this is needed. It's used by hifi_singleton. + # Perhaps it stops multiple build processes from interferring? + lockDir, lockName = os.path.split(self.path) + lockName += '.lock' + if not os.path.isdir(lockDir): + os.makedirs(lockDir) + + self.lockFile = os.path.join(lockDir, lockName) + + if qt_found: + if not self.args.quiet: + print("Found pre-built Qt5") + return + + if 'Windows' == system: + self.qtUrl = self.assets_url + '/dependencies/qt5/qt5-install-5.15.10-2023.10.02-windows-x86_64.tar.xz' + elif 'Darwin' == system: + self.qtUrl = self.assets_url + '/dependencies/vcpkg/qt5-install-5.15.2-macos.tar.gz' + elif 'Linux' == system: + import distro + cpu_architecture = platform.machine() + + if 'x86_64' == cpu_architecture: + # `major_version()` can return blank string on rolling release distros like arch + # The `or 0` conditional assignment prevents the int parsing error from hiding the useful Qt package error + u_major = int( distro.major_version() or '0' ) + if distro.id() == 'ubuntu' or distro.id() == 'linuxmint': + if (distro.id() == 'ubuntu' and u_major == 20) or distro.id() == 'linuxmint' and u_major == 20: + self.qtUrl = self.assets_url + '/dependencies/qt5/qt5-install-5.15.16-2024.12.14-kde_32be154325bfba3ad2ba8bf75dad702f3588e8d3-ubuntu-20.04-amd64.tar.xz' + elif (distro.id() == 'ubuntu' and u_major > 20) or (distro.id() == 'linuxmint' and u_major > 20): + self.__no_qt_package_error() + else: + self.__unsupported_error() + else: + self.__no_qt_package_error() + + + elif 'aarch64' == cpu_architecture: + if distro.id() == 'ubuntu': + u_major = int( distro.major_version() ) + + if u_major == 20: + self.qtUrl = self.assets_url + '/dependencies/qt5/qt5-install-5.15.9-2023.05.21-kde_fb3ec282151b1ee281a24f0545a40ac6438537c2-ubuntu-20.04-aarch64.tar.xz' + elif u_major > 20: + self.__no_qt_package_error() + else: + self.__unsupported_error() + + elif distro.id() == 'debian': + u_major = int( distro.major_version() ) + + if u_major > 10: + self.__no_qt_package_error() + else: + self.__unsupported_error() + + else: + self.__no_qt_package_error() + else: + raise Exception('UNKNOWN CPU ARCHITECTURE!!!') + + else: + print("System : " + platform.system()) + print("Architecture: " + platform.architecture()) + print("Machine : " + platform.machine()) + raise Exception('UNKNOWN OPERATING SYSTEM!!!') + + def showQtBuildInfo(self): + print("") + print("It's also possible to build Qt for your distribution, please see the documentation at:") + print("https://github.com/overte-org/overte/tree/master/tools/qt-builder") + print("") + print("Alternatively, you can try building against the system Qt by setting the OVERTE_USE_SYSTEM_QT environment variable.") + print("You'll need to install the development packages, and to have Qt 5.15.0 or later.") + + def writeConfig(self): + print("Writing cmake config to {}".format(self.configFilePath)) + # Write out the configuration for use by CMake + cmakeConfig = QtDownloader.CMAKE_TEMPLATE.format(self.cmakePath, self.cmakePath).replace('\\', '/') + with open(self.configFilePath, 'w') as f: + f.write(cmakeConfig) + + def installQt(self): + if not os.path.isdir(self.fullPath): + print ('Downloading Qt package') + print('Extracting ' + self.qtUrl + ' to ' + self.path) + hifi_utils.downloadAndExtract(self.qtUrl, self.path) + else: + print ('Qt has already been downloaded') + + + def __unsupported_error(self): + import distro + cpu_architecture = platform.machine() + + print('') + hifi_utils.color('red') + print("Sorry, " + distro.name(pretty=True) + " on " + cpu_architecture + " is too old and won't be officially supported.") + hifi_utils.color('white') + print("Please upgrade to a more recent Linux distribution.") + hifi_utils.color('clear') + print('') + raise hifi_utils.SilentFatalError(3) + + def __no_qt_package_error(self): + import distro + cpu_architecture = platform.machine() + + print('') + hifi_utils.color('red') + print("Sorry, we don't have a prebuilt Qt package for " + distro.name(pretty=True) + " on " + cpu_architecture + ".") + hifi_utils.color('white') + print('') + print("If this is a recent distribution, dating from 2021 or so, you can try building") + print("against the system Qt by running this command, and trying again:") + print(" export OVERTE_USE_SYSTEM_QT=1") + print("") + hifi_utils.color('clear') + print("If you'd like to try to build Qt from source either for building Overte, or") + print("to contribute a prebuilt package for your distribution, please see the") + print("documentation at: ", end='') + hifi_utils.color('blue') + print("https://github.com/overte-org/overte/tree/master/tools/qt-builder") + hifi_utils.color('clear') + print('') + raise hifi_utils.SilentFatalError(2) diff --git a/hifi_singleton.py b/hifi_singleton.py new file mode 100644 index 0000000000..f020378d7d --- /dev/null +++ b/hifi_singleton.py @@ -0,0 +1,160 @@ +import json +import logging +import os +import platform +import time + +try: + import fcntl +except ImportError: + fcntl = None + +try: + import msvcrt +except ImportError: + msvcrt = None + + +logger = logging.getLogger(__name__) + + +# Used to ensure only one instance of the script runs at a time +class Singleton: + def __init__(self, path): + self.fh = None + self.windows = 'Windows' == platform.system() + self.path = path + + def __enter__(self): + success = False + while not success: + try: + if self.windows: + if os.path.exists(self.path): + os.unlink(self.path) + self.fh = os.open(self.path, os.O_CREAT | os.O_EXCL | os.O_RDWR) + else: + self.fh = open(self.path, 'x') + fcntl.lockf(self.fh, fcntl.LOCK_EX | fcntl.LOCK_NB) + success = True + except EnvironmentError as err: + if self.fh is not None: + if self.windows: + os.close(self.fh) + else: + self.fh.close() + self.fh = None + # print is horked here so write directly to stdout. + with open(1, mode="w", closefd=False) as _stdout: + _stdout.write(f"Couldn't aquire lock {self.path}, retrying in 10 seconds\n") + _stdout.flush() + time.sleep(10) + return self + + def __exit__(self, type, value, traceback): + if self.windows: + os.close(self.fh) + else: + fcntl.lockf(self.fh, fcntl.LOCK_UN) + self.fh.close() + os.unlink(self.path) + + +class FLock: + """ + File locking context manager + + >> with FLock("/tmp/foo.lock"): + >> do_something_that_must_be_synced() + + The lock file must stick around forever. The author is not aware of a no cross platform way to clean it up w/o introducting race conditions. + """ + def __init__(self, path): + self.fh = os.open(path, os.O_CREAT | os.O_RDWR) + self.path = path + + def _lock_posix(self): + try: + fcntl.lockf(self.fh, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + # Windows sleeps for 10 seconds before giving up on a lock. + # Lets mimic that behavior. + time.sleep(10) + return False + else: + return True + + def _lock_windows(self): + try: + msvcrt.locking(self.fh, msvcrt.LK_LOCK, 1) + except OSError: + return False + else: + return True + + if fcntl is not None: + _lock = _lock_posix + elif msvcrt is not None: + _lock = _lock_windows + else: + raise RuntimeError("No locking library found") + + def read_stats(self): + data = {} + with open(self.fh, mode="r", closefd=False) as stats_file: + stats_file.seek(0) + try: + data = json.loads(stats_file.read()) + except json.decoder.JSONDecodeError: + logger.warning("couldn't decode json in lock file") + except PermissionError: + # Can't read a locked file on Windows :( + pass + + lock_age = time.time() - os.fstat(self.fh).st_mtime + if lock_age > 0: + data["Age"] = "%0.2f" % lock_age + + with open(1, mode="w", closefd=False) as _stdout: + _stdout.write("Lock stats:\n") + for key, value in sorted(data.items()): + _stdout.write("* %s: %s\n" % (key, value)) + _stdout.flush() + + def write_stats(self): + stats = { + "Owner PID": os.getpid(), + } + flock_env_vars = os.getenv("FLOCK_ENV_VARS") + if flock_env_vars: + for env_var_name in flock_env_vars.split(":"): + stats[env_var_name] = os.getenv(env_var_name) + + with open(self.fh, mode="w", closefd=False) as stats_file: + stats_file.truncate() + return stats_file.write(json.dumps(stats, indent=2)) + + def __enter__(self): + while not self._lock(): + try: + self.read_stats() + except (IOError, ValueError) as exc: + logger.exception("couldn't read stats") + time.sleep(3.33) # don't hammer the file + + self.write_stats() + + return self + + def __exit__(self, type, value, traceback): + os.close(self.fh) + # WARNING: `os.close` gives up the lock on `fh` then we attempt the `os.unlink`. On posix platforms this can lead to us deleting a lock file that another process owns. This step is required to maintain compatablity with Singleton. When and if FLock is completely rolled out to the build fleet this unlink should be removed. + try: + os.unlink(self.path) + except (FileNotFoundError, PermissionError): + logger.exception("couldn't unlink lock file") + + +if os.getenv("USE_FLOCK_CLS") is not None: + logger.warning("Using FLock locker") + Singleton = FLock diff --git a/hifi_utils.py b/hifi_utils.py new file mode 100644 index 0000000000..58492eecb5 --- /dev/null +++ b/hifi_utils.py @@ -0,0 +1,172 @@ +import os +import hashlib +import platform +import shutil +import ssl +import subprocess +import sys +import tarfile +import re +import urllib +import urllib.request +import zipfile +import tempfile +import time +import functools + +print = functools.partial(print, flush=True) + +ansi_colors = { + 'black' : 30, + 'red': 31, + 'green': 32, + 'yellow': 33, + 'blue': 34, + 'magenta': 35, + 'cyan': 36, + 'white': 37, + 'clear': 0 +} + +def scriptRelative(*paths): + scriptdir = os.path.dirname(os.path.realpath(sys.argv[0])) + result = os.path.join(scriptdir, *paths) + result = os.path.realpath(result) + result = os.path.normcase(result) + return result + + +def recursiveFileList(startPath, excludeNamePattern=None ): + result = [] + if os.path.isfile(startPath): + result.append(startPath) + elif os.path.isdir(startPath): + for dirName, subdirList, fileList in os.walk(startPath): + for fname in fileList: + if excludeNamePattern and re.match(excludeNamePattern, fname): + continue + result.append(os.path.realpath(os.path.join(startPath, dirName, fname))) + result.sort() + return result + + +def executeSubprocessCapture(processArgs): + processResult = subprocess.run(processArgs, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if (0 != processResult.returncode): + raise RuntimeError('Call to "{}" failed.\n\narguments:\n{}\n\nstdout:\n{}\n\nstderr:\n{}'.format( + processArgs[0], + ' '.join(processArgs[1:]), + processResult.stdout.decode('utf-8'), + processResult.stderr.decode('utf-8'))) + return processResult.stdout.decode('utf-8') + +def executeSubprocess(processArgs, folder=None, env=None): + restoreDir = None + if folder != None: + restoreDir = os.getcwd() + os.chdir(folder) + + process = subprocess.Popen( + processArgs, stdout=sys.stdout, stderr=sys.stderr, env=env) + process.wait() + + if (0 != process.returncode): + raise RuntimeError('Call to "{}" failed.\n\narguments:\n{}\n'.format( + processArgs[0], + ' '.join(processArgs[1:]), + )) + + if restoreDir != None: + os.chdir(restoreDir) + + +def hashFile(file, hasher = hashlib.sha512()): + with open(file, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + hasher.update(chunk) + return hasher.hexdigest() + +# Assumes input files are in deterministic order +def hashFiles(filenames): + hasher = hashlib.sha256() + for filename in filenames: + with open(filename, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + hasher.update(chunk) + return hasher.hexdigest() + +def hashFolder(folder): + filenames = recursiveFileList(folder) + return hashFiles(filenames) + +def downloadFile(url, hash=None, hasher=hashlib.sha512(), retries=3): + for i in range(retries): + tempFileName = None + # OSX Python doesn't support SSL, so we need to bypass it. + # However, we still validate the downloaded file's sha512 hash + if 'Darwin' == platform.system(): + tempFileDescriptor, tempFileName = tempfile.mkstemp() + context = ssl._create_unverified_context() + with urllib.request.urlopen(url, context=context) as response, open(tempFileDescriptor, 'wb') as tempFile: + shutil.copyfileobj(response, tempFile) + else: + tempFileName, headers = urllib.request.urlretrieve(url) + + downloadHash = hashFile(tempFileName, hasher) + # Verify the hash + if hash is not None and hash != downloadHash: + print("Try {}: Downloaded file {} hash {} does not match expected hash {} for url {}".format(i + 1, tempFileName, downloadHash, hash, url)) + os.remove(tempFileName) + continue + return tempFileName + + raise RuntimeError("Downloaded file hash {} does not match expected hash {} for\n{}".format(downloadHash, hash, url)) + + +def downloadAndExtract(url, destPath, hash=None, hasher=hashlib.sha512(), isZip=False): + tempFileName = downloadFile(url, hash, hasher) + if isZip or ".zip" in url: + with zipfile.ZipFile(tempFileName) as zip: + zip.extractall(destPath) + else: + # Extract the archive + with tarfile.open(tempFileName, 'r:*') as tgz: + def is_within_directory(directory, target): + + abs_directory = os.path.abspath(directory) + abs_target = os.path.abspath(target) + + prefix = os.path.commonprefix([abs_directory, abs_target]) + + return prefix == abs_directory + + def safe_extract(tar, path=".", members=None, *, numeric_owner=False): + + for member in tar.getmembers(): + member_path = os.path.join(path, member.name) + if not is_within_directory(path, member_path): + raise Exception("Attempted Path Traversal in Tar File") + + tar.extractall(path, members, numeric_owner=numeric_owner) + + + safe_extract(tgz, destPath) + os.remove(tempFileName) + +def readEnviromentVariableFromFile(buildRootDir, var): + with open(os.path.join(buildRootDir, '_env', var + ".txt")) as fp: + return fp.read() + +class SilentFatalError(Exception): + """Thrown when some sort of fatal condition happened, and we already reported it to the user. + This excecption exists to give a chance to run any cleanup needed before exiting. + + It should be handled at the bottom of the call stack, where the only action is to call + sys.exit(ex.exit_code) + """ + def __init__(self, exit_code): + self.exit_code = exit_code + +def color(color_name): + # Ideally we'd use the termcolor module, but this avoids adding it as a dependency. + print("\033[1;{}m".format(ansi_colors[color_name]), end='') \ No newline at end of file diff --git a/hifi_vcpkg.py b/hifi_vcpkg.py new file mode 100644 index 0000000000..ff1549ab1e --- /dev/null +++ b/hifi_vcpkg.py @@ -0,0 +1,365 @@ +import hifi_utils +import hifi_android +import hashlib +import os +import platform +import re +import shutil +import tempfile +import json +import xml.etree.ElementTree as ET +import functools +from os import path + +print = functools.partial(print, flush=True) + +# Encapsulates the vcpkg system +class VcpkgRepo: + CMAKE_TEMPLATE = """ +# this file auto-generated by hifi_vcpkg.py +get_filename_component(CMAKE_TOOLCHAIN_FILE "{}" ABSOLUTE CACHE) +get_filename_component(CMAKE_TOOLCHAIN_FILE_UNCACHED "{}" ABSOLUTE) +set(VCPKG_INSTALL_ROOT "{}") +set(VCPKG_TOOLS_DIR "{}") +set(VCPKG_TARGET_TRIPLET "{}") +""" + + CMAKE_TEMPLATE_NON_ANDROID = """ +# If the cached cmake toolchain path is different from the computed one, exit +if(NOT (CMAKE_TOOLCHAIN_FILE_UNCACHED STREQUAL CMAKE_TOOLCHAIN_FILE)) + message(FATAL_ERROR "CMAKE_TOOLCHAIN_FILE has changed, please wipe the build directory and rerun cmake") +endif() +""" + + def __init__(self, args): + self.args = args + # our custom ports, relative to the script location + self.sourcePortsPath = args.ports_path + self.vcpkgBuildType = args.vcpkg_build_type + if (self.vcpkgBuildType): + self.id = hifi_utils.hashFolder(self.sourcePortsPath)[:8] + "-" + self.vcpkgBuildType + else: + self.id = hifi_utils.hashFolder(self.sourcePortsPath)[:8] + self.configFilePath = os.path.join(args.build_root, 'vcpkg.cmake') + + if args.get_vcpkg_id or args.get_vcpkg_path: + # With these arguments no assets will be downloaded, and they may be used in conditions + # where the _env hack doesn't work. + self.assets_url = "http://no_assets.invalid" + else: + self.assets_url = self.readVar('EXTERNAL_BUILD_ASSETS') + + # The noClean flag indicates we're doing weird dependency maintenance stuff + # i.e. we've got an explicit checkout of vcpkg and we don't want the script to + # do stuff it might otherwise do. It typically indicates that we're using our + # own git checkout of vcpkg and manually managing it + self.noClean = False + + # OS dependent information + system = platform.system() + machine = platform.machine() + + if 'HIFI_VCPKG_PATH' in os.environ: + self.path = os.environ['HIFI_VCPKG_PATH'] + self.noClean = True + elif self.args.vcpkg_root is not None: + self.path = args.vcpkg_root + self.noClean = True + else: + defaultBasePath = os.path.expanduser('~/overte-files/vcpkg') + if 'CI_WORKSPACE' in os.environ: + self.basePath = os.path.join(os.getenv('CI_WORKSPACE'), 'overte-files/vcpkg') + else: + self.basePath = os.getenv('HIFI_VCPKG_BASE', defaultBasePath) + if self.args.android: + self.basePath = os.path.join(self.basePath, 'android') + if (not os.path.isdir(self.basePath)): + os.makedirs(self.basePath) + self.path = os.path.join(self.basePath, self.id) + + if not self.args.quiet: + print("Using vcpkg path {}".format(self.path)) + lockDir, lockName = os.path.split(self.path) + lockName += '.lock' + if not os.path.isdir(lockDir): + os.makedirs(lockDir) + + self.lockFile = os.path.join(lockDir, lockName) + self.tagFile = os.path.join(self.path, '.id') + self.prebuildTagFile = os.path.join(self.path, '.prebuild') + # A format version attached to the tag file... increment when you want to force the build systems to rebuild + # without the contents of the ports changing + self.version = 1 + self.tagContents = "{}_{}".format(self.id, self.version) + self.bootstrapEnv = os.environ.copy() + self.buildEnv = os.environ.copy() + self.prebuiltArchive = None + usePrebuilt = False + # usePrebuild Disabled, to re-enabled using the prebuilt archives for GitHub action builds uncomment the following line: + # usePrebuilt = ('CI_BUILD' in os.environ) and os.environ["CI_BUILD"] == "Github" and (not self.noClean) + + if 'Windows' == system: + self.exe = os.path.join(self.path, 'vcpkg.exe') + self.bootstrapCmds = [ os.path.join(self.path, 'bootstrap-vcpkg.bat'), '-disableMetrics' ] + self.vcpkgUrl = self.assets_url + '/dependencies/vcpkg/vcpkg-windows_x86_64_2024.06.15.zip' + self.vcpkgHash = 'f335234f0722c15376fb10747f558c18c83a3e1e3b6565cf0dabfb18c9625a99234d054457fd05190c0ecd7a59ca43305bc93b50dbf764a4e1f567a15168d051' + self.hostTriplet = 'x64-windows' + if usePrebuilt: + self.prebuiltArchive = self.assets_url + "/dependencies/vcpkg/builds/vcpkg-win32.zip%3FversionId=3SF3mDC8dkQH1JP041m88xnYmWNzZflx" + elif 'Darwin' == system: + self.exe = os.path.join(self.path, 'vcpkg') + self.bootstrapCmds = [ os.path.join(self.path, 'bootstrap-vcpkg.sh'), '-disableMetrics' ] + self.vcpkgUrl = self.assets_url + '/dependencies/vcpkg/vcpkg-osx-x86_64-2022.06.16.1.tar.xz' + self.vcpkgHash = '4dfdb3d8c40440330d50b54073e59218334379d0cb3ca437252a29d3c8674c0eeeec9acd3927488a5ce96cbcdf411632d3576baf1e279181233209ec01761d1d' + self.hostTriplet = 'x64-osx' + elif 'Linux' == system and 'aarch64' == machine: + self.exe = os.path.join(self.path, 'vcpkg') + self.bootstrapCmds = [ os.path.join(self.path, 'bootstrap-vcpkg.sh'), '-disableMetrics' ] + self.vcpkgUrl = self.assets_url + '/dependencies/vcpkg/vcpkg-linux_aarch64_2023.11.20.tar.xz' + self.vcpkgHash = 'f38efba40bd4b0b6df47986e373d5535d3e787e257cf19d66ee8ee00e670a6fb95b3e824020024f3edbdcf86a0548e5bbddcc0ac7bd2ff6352a245efac8402fe' + self.hostTriplet = 'arm64-linux' + else: + self.exe = os.path.join(self.path, 'vcpkg') + self.bootstrapCmds = [ os.path.join(self.path, 'bootstrap-vcpkg.sh'), '-disableMetrics' ] + self.vcpkgUrl = self.assets_url + '/dependencies/vcpkg/vcpkg-linux_amd64_2023.10.19.tar.xz' + self.vcpkgHash = '6c26ff73d6348e121cca47e90d5358587bf83ba22852acb195b76fbf0473070b24512c8fdd3216d26f03515a79c085f239272ef87c7020cc578cc79abbbd338d' + self.hostTriplet = 'x64-linux' + + if self.args.android: + self.triplet = 'arm64-android' + self.androidPackagePath = os.getenv('HIFI_ANDROID_PRECOMPILED', os.path.join(self.path, 'android')) + else: + self.triplet = self.hostTriplet + + def readVar(self, var): + with open(os.path.join(self.args.build_root, '_env', var + ".txt")) as fp: + return fp.read() + + def writeVar(self, var, value): + with open(os.path.join(self.args.build_root, '_env', var + ".txt"), 'w') as fp: + fp.write(value) + + def upToDate(self): + # Prevent doing a clean if we've explcitly set a directory for vcpkg + if self.noClean: + return True + + if self.args.force_build: + print("Force build, out of date") + return False + if not os.path.isfile(self.exe): + print("Exe file {} not found, out of date".format(self.exe)) + return False + if not os.path.isfile(self.tagFile): + print("Tag file {} not found, out of date".format(self.tagFile)) + return False + with open(self.tagFile, 'r') as f: + storedTag = f.read() + if storedTag != self.tagContents: + print("Tag file {} contents don't match computed tag {}, out of date".format(self.tagFile, self.tagContents)) + return False + return True + + def copyEnv(self): + print("Passing on variables to vcpkg") + srcEnv = os.path.join(self.args.build_root, "_env") + destEnv = os.path.join(self.path, "_env") + + if path.exists(destEnv): + shutil.rmtree(destEnv) + + shutil.copytree(srcEnv, destEnv) + + + def clean(self): + print("Cleaning vcpkg installation at {}".format(self.path)) + if os.path.isdir(self.path): + print("Removing {}".format(self.path)) + shutil.rmtree(self.path, ignore_errors=True) + + # Make sure the VCPKG prerequisites are all there. + def bootstrap(self): + if self.upToDate(): + self.copyEnv() + return + + if self.prebuiltArchive is not None: + return + + self.clean() + downloadVcpkg = False + if self.args.force_bootstrap: + print("Forcing bootstrap") + downloadVcpkg = True + + if not downloadVcpkg and not os.path.isfile(self.exe): + print("Missing executable, boot-strapping") + downloadVcpkg = True + + # Make sure we have a vcpkg executable + testFile = os.path.join(self.path, '.vcpkg-root') + if not downloadVcpkg and not os.path.isfile(testFile): + print("Missing {}, bootstrapping".format(testFile)) + downloadVcpkg = True + + if downloadVcpkg: + if "HIFI_VCPKG_BOOTSTRAP" in os.environ: + print("Cloning vcpkg from github to {}".format(self.path)) + hifi_utils.executeSubprocess(['git', 'clone', 'https://github.com/microsoft/vcpkg', self.path]) + print("Bootstrapping vcpkg") + hifi_utils.executeSubprocess(self.bootstrapCmds, folder=self.path, env=self.bootstrapEnv) + else: + print("Fetching vcpkg from {} to {}".format(self.vcpkgUrl, self.path)) + hifi_utils.downloadAndExtract(self.vcpkgUrl, self.path) + + print("Replacing port files") + portsPath = os.path.join(self.path, 'ports') + if (os.path.islink(portsPath)): + os.unlink(portsPath) + if (os.path.isdir(portsPath)): + shutil.rmtree(portsPath, ignore_errors=True) + shutil.copytree(self.sourcePortsPath, portsPath) + self.copyEnv() + + def run(self, commands): + actualCommands = [self.exe, '--vcpkg-root', self.path] + actualCommands.extend(commands) + print("Running command") + print(actualCommands) + hifi_utils.executeSubprocess(actualCommands, folder=self.path, env=self.buildEnv) + + def copyTripletForBuildType(self, triplet): + print('Copying triplet ' + triplet + ' to have build type ' + self.vcpkgBuildType) + tripletPath = os.path.join(self.path, 'triplets', triplet + '.cmake') + tripletForBuildTypePath = os.path.join(self.path, 'triplets', self.getTripletWithBuildType(triplet) + '.cmake') + shutil.copy(tripletPath, tripletForBuildTypePath) + with open(tripletForBuildTypePath, "a") as tripletForBuildTypeFile: + tripletForBuildTypeFile.write("set(VCPKG_BUILD_TYPE " + self.vcpkgBuildType + ")\n") + + def getTripletWithBuildType(self, triplet): + if (not self.vcpkgBuildType): + return triplet + return triplet + '-' + self.vcpkgBuildType + + def setupDependencies(self, qt=None): + if self.prebuiltArchive: + if not os.path.isfile(self.prebuildTagFile): + print('Extracting ' + self.prebuiltArchive + ' to ' + self.path) + hifi_utils.downloadAndExtract(self.prebuiltArchive, self.path) + self.writePrebuildTag() + return + + if qt is not None: + self.buildEnv['QT_CMAKE_PREFIX_PATH'] = qt + + # Special case for android, grab a bunch of binaries + # FIXME remove special casing for android builds eventually + if self.args.android: + print("Installing Android binaries") + self.setupAndroidDependencies() + + print("Installing host tools") + if (self.vcpkgBuildType): + self.copyTripletForBuildType(self.hostTriplet) + self.run(['install', '--triplet', self.getTripletWithBuildType(self.hostTriplet), 'hifi-host-tools']) + + # If not android, install the hifi-client-deps libraries + if not self.args.android: + print("Installing build dependencies") + if (self.vcpkgBuildType): + self.copyTripletForBuildType(self.triplet) + self.run(['install', '--triplet', self.getTripletWithBuildType(self.triplet), 'hifi-client-deps']) + + def cleanBuilds(self): + if self.noClean: + return + # Remove temporary build artifacts + builddir = os.path.join(self.path, 'buildtrees') + if os.path.isdir(builddir): + print("Wiping build trees") + shutil.rmtree(builddir, ignore_errors=True) + + # Removes large files used to build the vcpkg, for CI purposes. + def cleanupDevelopmentFiles(self): + shutil.rmtree(os.path.join(self.path, "downloads"), ignore_errors=True) + shutil.rmtree(os.path.join(self.path, "packages"), ignore_errors=True) + + + def setupAndroidDependencies(self): + # vcpkg prebuilt + if not os.path.isdir(os.path.join(self.path, 'installed', 'arm64-android')): + dest = os.path.join(self.path, 'installed') + url = self.assets_url + "/dependencies/vcpkg/vcpkg-arm64-android.tar.gz" + # FIXME I don't know why the hash check frequently fails here. If you examine the file later it has the right hash + #hash = "832f82a4d090046bdec25d313e20f56ead45b54dd06eee3798c5c8cbdd64cce4067692b1c3f26a89afe6ff9917c10e4b601c118bea06d23f8adbfe5c0ec12bc3" + #hifi_utils.downloadAndExtract(url, dest, hash) + hifi_utils.downloadAndExtract(url, dest) + + print("Installing additional android archives") + androidPackages = hifi_android.getPlatformPackages() + for packageName in androidPackages: + package = androidPackages[packageName] + dest = os.path.join(self.androidPackagePath, packageName) + if os.path.isdir(dest): + continue + url = hifi_android.getPackageUrl(package) + zipFile = package['file'].endswith('.zip') + print("Android archive {}".format(package['file'])) + hifi_utils.downloadAndExtract(url, dest, isZip=zipFile, hash=package['checksum'], hasher=hashlib.md5()) + + def writeTag(self): + if self.noClean: + return + print("Writing tag {} to {}".format(self.tagContents, self.tagFile)) + if not os.path.isdir(self.path): + os.makedirs(self.path) + with open(self.tagFile, 'w') as f: + f.write(self.tagContents) + + def writePrebuildTag(self): + print("Writing tag {} to {}".format(self.tagContents, self.tagFile)) + with open(self.prebuildTagFile, 'w') as f: + f.write(self.tagContents) + + def fixupCmakeScript(self): + cmakeScript = os.path.join(self.path, 'scripts/buildsystems/vcpkg.cmake') + newCmakeScript = cmakeScript + '.new' + isFileChanged = False + removalPrefix = "set(VCPKG_TARGET_TRIPLET " + # Open original file in read only mode and dummy file in write mode + with open(cmakeScript, 'r') as read_obj, open(newCmakeScript, 'w') as write_obj: + # Line by line copy data from original file to dummy file + for line in read_obj: + if not line.startswith(removalPrefix): + write_obj.write(line) + else: + isFileChanged = True + + if isFileChanged: + shutil.move(newCmakeScript, cmakeScript) + else: + os.remove(newCmakeScript) + + + def writeConfig(self): + print("Writing cmake config to {}".format(self.configFilePath)) + # Write out the configuration for use by CMake + cmakeScript = os.path.join(self.path, 'scripts/buildsystems/vcpkg.cmake') + installPath = os.path.join(self.path, 'installed', self.getTripletWithBuildType(self.triplet)) + toolsPath = os.path.join(self.path, 'installed', self.getTripletWithBuildType(self.hostTriplet), 'tools') + + cmakeTemplate = VcpkgRepo.CMAKE_TEMPLATE + if self.args.android: + precompiled = os.path.realpath(self.androidPackagePath) + cmakeTemplate += 'set(HIFI_ANDROID_PRECOMPILED "{}")\n'.format(precompiled) + else: + cmakeTemplate += VcpkgRepo.CMAKE_TEMPLATE_NON_ANDROID + cmakeConfig = cmakeTemplate.format(cmakeScript, cmakeScript, installPath, toolsPath, self.getTripletWithBuildType(self.hostTriplet)).replace('\\', '/') + with open(self.configFilePath, 'w') as f: + f.write(cmakeConfig) + + def cleanOldBuilds(self): + # FIXME because we have the base directory, and because a build will + # update the tag file on every run, we can scan the base dir for sub directories containing + # a tag file that is older than N days, and if found, delete the directory, recovering space + print("Not implemented") diff --git a/prebuild.py b/prebuild.py new file mode 100644 index 0000000000..4a567989b5 --- /dev/null +++ b/prebuild.py @@ -0,0 +1,220 @@ +#!python + +# The prebuild script is intended to simplify life for developers and dev-ops. It's repsonsible for acquiring +# tools required by the build as well as dependencies on which we rely. +# +# By using this script, we can reduce the requirements for a developer getting started to: +# +# * A working C++ dev environment like visual studio, xcode, gcc, or clang +# * Qt +# * CMake +# * Python 3.x +# +# The function of the build script is to acquire, if not already present, all the other build requirements +# The build script should be idempotent. If you run it with the same arguments multiple times, that should +# have no negative impact on the subsequent build times (i.e. re-running the prebuild script should not +# trigger a header change that causes files to be rebuilt). Subsequent runs after the first run should +# execute quickly, determining that no work is to be done + +import hifi_singleton +import hifi_utils +import hifi_android +import hifi_vcpkg +import hifi_qt + +import argparse +import concurrent +import hashlib +import importlib +import json +import os +import platform +import shutil +import ssl +import sys +import re +import tempfile +import time +import functools +import subprocess +import logging + +from uuid import uuid4 +from contextlib import contextmanager + +print = functools.partial(print, flush=True) + +class TrackableLogger(logging.Logger): + guid = str(uuid4()) + + def _log(self, msg, *args, **kwargs): + x = {'guid': self.guid} + if 'extra' in kwargs: + kwargs['extra'].update(x) + else: + kwargs['extra'] = x + super()._log(msg, *args, **kwargs) + +logging.setLoggerClass(TrackableLogger) +logger = logging.getLogger('prebuild') + +@contextmanager +def timer(name): + ''' Print the elapsed time a context's execution takes to execute ''' + start = time.time() + yield + # Please take care when modifiying this print statement. + # Log parsing logic may depend on it. + logger.info('%s took %.3f secs' % (name, time.time() - start)) + +def parse_args(): + # our custom ports, relative to the script location + defaultPortsPath = hifi_utils.scriptRelative('cmake', 'ports') + from argparse import ArgumentParser + parser = ArgumentParser(description='Prepare build dependencies.') + parser.add_argument('--android', type=str) + parser.add_argument('--debug', action='store_true') + parser.add_argument('--force-bootstrap', action='store_true') + parser.add_argument('--force-build', action='store_true') + parser.add_argument('--release-type', type=str, default="DEV", help="DEV, PR, or PRODUCTION") + parser.add_argument('--vcpkg-root', type=str, help='The location of the vcpkg distribution') + parser.add_argument('--vcpkg-build-type', type=str, help='Could be `release` or `debug`. By default it doesn`t set the build-type') + parser.add_argument('--vcpkg-skip-clean', action='store_true', help='Skip the cleanup of vcpkg downloads and packages folders after vcpkg build completition.') + parser.add_argument('--build-root', required=True, type=str, help='The location of the cmake build') + parser.add_argument('--ports-path', type=str, default=defaultPortsPath) + parser.add_argument('--ci-build', action='store_true', default=os.getenv('CI_BUILD') is not None) + parser.add_argument('--get-vcpkg-id', action='store_true', help='Get the VCPKG ID, the hash path of the full VCPKG path') + parser.add_argument('--get-vcpkg-path', action='store_true', help='Get the full VCPKG path, ID included.') + parser.add_argument('--quiet', action='store_true', default=False, help='Quiet mode with less output') + + if True: + args = parser.parse_args() + else: + args = parser.parse_args(['--android', 'questInterface', '--build-root', 'C:/git/overte/android/apps/questInterface/.externalNativeBuild/cmake/debug/arm64-v8a']) + return args + +def main(): + # Fixup env variables. Leaving `USE_CCACHE` on will cause scribe to fail to build + # VCPKG_ROOT seems to cause confusion on Windows systems that previously used it for + # building OpenSSL + removeEnvVars = ['VCPKG_ROOT', 'USE_CCACHE'] + for var in removeEnvVars: + if var in os.environ: + del os.environ[var] + + args = parse_args() + + if args.get_vcpkg_id or args.get_vcpkg_path: + # These arguments need quiet mode to avoid confusing scripts that use them. + args.quiet = True + + if not args.quiet: + print(sys.argv) + + if args.ci_build: + logging.basicConfig(datefmt='%H:%M:%S', format='%(asctime)s %(guid)s %(message)s', level=logging.INFO) + + logger.info('start') + + pm = hifi_vcpkg.VcpkgRepo(args) + + if args.get_vcpkg_id: + print(pm.id) + exit(0) + + if args.get_vcpkg_path: + print(pm.path) + exit(0) + + assets_url = hifi_utils.readEnviromentVariableFromFile(args.build_root, 'EXTERNAL_BUILD_ASSETS') + + # OS dependent information + system = platform.system() + if 'Windows' == system and 'CI_BUILD' in os.environ and os.environ["CI_BUILD"] == "Github": + logger.info("Downloading NSIS") + with timer('NSIS'): + hifi_utils.downloadAndExtract(assets_url + '/dependencies/NSIS-hifi-plugins-1.0.tgz', "C:/Program Files (x86)") + + qtInstallPath = None + # If not android, install our Qt build + if not args.android: + qt = hifi_qt.QtDownloader(args) + qtInstallPath = qt.cmakePath + + if qtInstallPath is not None: + # qtInstallPath is None when we're doing a system Qt build + print("cmake path: " + qtInstallPath) + + with hifi_singleton.Singleton(qt.lockFile) as lock: + with timer('Qt'): + qt.installQt() + qt.writeConfig() + else: + if (os.environ["OVERTE_USE_SYSTEM_QT"]): + if not args.quiet: + print("System Qt selected") + + else: + raise Exception("Internal error: System Qt not selected, but hifi_qt.py failed to return a cmake path") + + if qtInstallPath is not None: + pm.writeVar('QT_CMAKE_PREFIX_PATH', qtInstallPath) + + # Only allow one instance of the program to run at a time + + if qtInstallPath is not None: + pm.writeVar('QT_CMAKE_PREFIX_PATH', qtInstallPath) + + # Only allow one instance of the program to run at a time + with hifi_singleton.Singleton(pm.lockFile) as lock: + + with timer('Bootstraping'): + if not pm.upToDate(): + pm.bootstrap() + + # Always write the tag, even if we changed nothing. This + # allows vcpkg to reclaim disk space by identifying directories with + # tags that haven't been touched in a long time + pm.writeTag() + + # Grab our required dependencies: + # * build host tools, like spirv-cross and scribe + # * build client dependencies like openssl and nvtt + with timer('Setting up dependencies'): + pm.setupDependencies(qt=qtInstallPath) + + # wipe out the build directories (after writing the tag, since failure + # here shouldn't invalidate the vcpkg install) + if not args.vcpkg_skip_clean: + with timer('Cleaning builds'): + pm.cleanBuilds() + + # If we're running in android mode, we also need to grab a bunch of additional binaries + # (this logic is all migrated from the old setupDependencies tasks in gradle) + if args.android: + # Find the target location + appPath = hifi_utils.scriptRelative('android/apps/' + args.android) + # Copy the non-Qt libraries specified in the config in hifi_android.py + hifi_android.copyAndroidLibs(pm.androidPackagePath, appPath) + # Determine the Qt package path + qtPath = os.path.join(pm.androidPackagePath, 'qt') + hifi_android.QtPackager(appPath, qtPath).bundle() + + # Fixup the vcpkg cmake to not reset VCPKG_TARGET_TRIPLET + pm.fixupCmakeScript() + + if not args.vcpkg_skip_clean: + # Cleanup downloads and packages folders in vcpkg to make it smaller for CI + pm.cleanupDevelopmentFiles() + + # Write the vcpkg config to the build directory last + with timer('Writing configuration'): + pm.writeConfig() + + logger.info('end') + + +try: + main() +except hifi_utils.SilentFatalError as fatal_ex: + sys.exit(fatal_ex.exit_code)