Merge pull request ydb-platform#14325 from ydb-platform/merge-libs-25…

…0207-1355

build/conf/java.conf

@@ -456,14 +456,9 @@ macro _HASH_HELPER(Args...) {
     .SEM=${hash:Args}
 }
 
-macro _HASH_GENTAR_HELPER(HASH, OUT_DIR[]) {
-    .CMD=${cwd:BINDIR} $YMAKE_PYTHON ${input:"build/scripts/autotar_gendirs.py"} --pack ${OUT_DIR} --outs ${output;tared;suf=.$HASH.gentar:OUT_DIR} ${hide;kv:"tared_kind nodir"}
-    .SEM=${hide;output;tared;suf=.$HASH.gentar:OUT_DIR}
-}
-
-macro _GENTAR_HELPER(OUT_DIR[], Args...) {
-    .CMD=$_HASH_GENTAR_HELPER(${hash:Args} OUT_DIR $OUT_DIR)
-    .SEM=$_HASH_GENTAR_HELPER(${hash:Args} OUT_DIR $OUT_DIR)
+macro _GENTAR_HELPER(HASH_SUF="hash_suf", OUT_DIR[]) {
+    .CMD=${cwd:BINDIR} $YMAKE_PYTHON ${input:"build/scripts/autotar_gendirs.py"} --pack ${OUT_DIR} --outs ${output;tared;suf=$HASH_SUF:OUT_DIR} ${hide;kv:"tared_kind nodir"}
+    .SEM=${hide;suf=$HASH_SUF;tared;output:OUT_DIR}
 }
 
 # tag:java-specific
@@ -489,26 +484,25 @@ _SEM_RUN_JAVA_PROGRAM=runs-ITEM \
     && runs-in_dirs_inputs ${IN_DIRS_INPUTS} \
     && runs-in_noparse ${IN_NOPARSE} ${hide;context=TEXT;input:IN_NOPARSE} \
     && runs-out ${OUT} ${hide;output:OUT} ${OUT_NOAUTO} ${hide;noauto;output:OUT_NOAUTO} \
-    $_GENTAR_HELPER($CLASSPATH $IN_DIR $IN $IN_NOPARSE $TOOL $Args OUT_DIR $OUT_DIR) \
+    $_GENTAR_HELPER(HASH_SUF $HASH_SUF OUT_DIR $OUT_DIR) \
     && runs-out_dir ${OUT_DIR} \
     && runs-tool ${TOOL} ${hide;tool:TOOL}
 
-macro _DO_2_RUN_JAR_PROGRAM(IN_DIRS_VAR="uniq_", IN_DIRS_INPUTS[], IN{input}[], IN_NOPARSE{input}[], IN_DIR[], OUT_NOAUTO{output}[], OUT{output}[], TOOL{tool}[], OUT_DIR[], CLASSPATH[], ADD_SRCS_TO_CLASSPATH?"yes":"no", CWD="${ARCADIA_BUILD_ROOT}", STDOUT="", STDOUT_NOAUTO="", Args...) {
+macro _DO_2_RUN_JAR_PROGRAM(IN_DIRS_VAR="uniq_", IN_DIRS_INPUTS[], IN{input}[], IN_NOPARSE{input}[], IN_DIR[], OUT_NOAUTO{output}[], OUT{output}[], TOOL{tool}[], OUT_DIR[], CLASSPATH[], ADD_SRCS_TO_CLASSPATH?"yes":"no", CWD="${ARCADIA_BUILD_ROOT}", STDOUT="", STDOUT_NOAUTO="", HASH_SUF="hash_suf", Args...) {
     _LATE_GLOB(${IN_DIRS_VAR} ${suf=/**/*:IN_DIR})
     _CHECK_RUN_JAVA_PROG_CLASSPATH($CLASSPATH)
     .PEERDIR=build/platform/java/jdk $JDK_RESOURCE_PEERDIR
-    .CMD=${hide;kv:"p RJ"} ${hide;kv:"pc blue"} ${hide:JAVA_FAKEID} ${cwd:BINDIR} $YMAKE_PYTHON ${input:"build/scripts/mkdir.py"} ${OUT_DIR} && ${cwd:CWD} $YMAKE_PYTHON ${input:"build/scripts/setup_java_tmpdir.py"} $YMAKE_PYTHON ${input:"build/scripts/stdout2stderr.py"} ${pre=--file=:STDOUT} ${hide;output:STDOUT} ${pre=--file=:STDOUT_NOAUTO} ${hide;noauto;output:STDOUT_NOAUTO} $YMAKE_PYTHON ${input:"build/scripts/fix_java_command_file_cp.py"} --build-root ${ARCADIA_BUILD_ROOT} $JDK_RESOURCE/bin/java -Dfile.encoding=utf8 -classpath ${RUN_JAR_PROG_CP_PRE}${tool:CLASSPATH}${RUN_JAR_PROG_CP_SUF} ${Args} && $_GENTAR_HELPER($CLASSPATH $IN_DIR $IN $IN_NOPARSE $TOOL $Args OUT_DIR $OUT_DIR) ${hide;input:IN} ${hide;context=TEXT;input:IN_NOPARSE} ${hide;noauto;output:OUT_NOAUTO} ${hide;output:OUT} ${hide;tool:TOOL} ${IN_DIRS_INPUTS}
+    .CMD=${hide;kv:"p RJ"} ${hide;kv:"pc blue"} ${hide:JAVA_FAKEID} ${cwd:BINDIR} $YMAKE_PYTHON ${input:"build/scripts/mkdir.py"} ${OUT_DIR} && ${cwd:CWD} $YMAKE_PYTHON ${input:"build/scripts/setup_java_tmpdir.py"} $YMAKE_PYTHON ${input:"build/scripts/stdout2stderr.py"} ${pre=--file=:STDOUT} ${hide;output:STDOUT} ${pre=--file=:STDOUT_NOAUTO} ${hide;noauto;output:STDOUT_NOAUTO} $YMAKE_PYTHON ${input:"build/scripts/fix_java_command_file_cp.py"} --build-root ${ARCADIA_BUILD_ROOT} $JDK_RESOURCE/bin/java -Dfile.encoding=utf8 -classpath ${RUN_JAR_PROG_CP_PRE}${tool:CLASSPATH}${RUN_JAR_PROG_CP_SUF} ${Args} && $_GENTAR_HELPER(HASH_SUF $HASH_SUF OUT_DIR $OUT_DIR) ${hide;input:IN} ${hide;context=TEXT;input:IN_NOPARSE} ${hide;noauto;output:OUT_NOAUTO} ${hide;output:OUT} ${hide;tool:TOOL} ${IN_DIRS_INPUTS}
     .SEM=$_SEM_RUN_JAVA_PROGRAM
 }
 
 # tag:java-specific
 macro _DO_1_RUN_JAR_PROGRAM(IN_DIRS_VAR="uniq", Args...) {
-    _DO_2_RUN_JAR_PROGRAM($Args IN_DIRS_VAR $IN_DIRS_VAR IN_DIRS_INPUTS ${"$"}{hide;input:$IN_DIRS_VAR})
+    _DO_2_RUN_JAR_PROGRAM($Args IN_DIRS_VAR $IN_DIRS_VAR IN_DIRS_INPUTS ${"$"}{hide;input:$IN_DIRS_VAR} HASH_SUF .${hash:Args}.gentar)
 }
 
 # tag:java-specific
 macro RUN_JAVA_PROGRAM(Args...) {
-    .STRUCT_SEM=yes
     _DO_1_RUN_JAR_PROGRAM($Args IN_DIRS_VAR uniq_${hash:Args})
 }
 

build/export_generators/ide-gradle/a.yaml

@@ -0,0 +1,78 @@
+service: ya_make
+title: ya ide gradle export testing
+
+ci:
+  secret: sec-01hmxvzfvzwhj0k6qe9vh9w9yw
+  runtime:
+    sandbox:
+      owner: YATOOL
+      notifications:
+      - statuses: [FAILURE, EXCEPTION]
+        transport: email
+        recipients: dimdim11
+  autocheck:
+    fast-targets:
+      - devtools/ya
+      - devtools/ymake
+      - devtools/yexport
+
+  actions:
+    my-action:
+      flow: gradle-tests
+      tags: ci-action
+      triggers:
+        - on: pr
+          on-demand: true
+          filters:
+            - abs-paths:
+                - build/export_generators/ide-gradle/**
+                - build/conf/**
+                - build/internal/conf/**
+                - devtools/ya/ide/gradle/**
+                - devtools/ya/build/sem_graph/**
+
+  on-demand-auto:
+    - abs-paths: build/export_generators/ide-gradle/**
+      tags: ci-action
+
+  flows:
+    gradle-tests:
+      jobs:
+        run_gradle_tests:
+          description: Build results of ya ide gradle in Gradle
+          task: common/misc/run_command
+          requirements:
+            sandbox:
+              client_tags: GENERIC & LINUX & SSD & LXC
+              container_resource: 3493534654
+              privileged: true
+              dns: dns64
+          input:
+            config:
+              arc_mount_config:
+                enabled: true
+              secret_environment_variables:
+                - key: DOCKER_OAUTH
+                  secret_spec:
+                    key: docker.token
+                - key: YA_TOKEN
+                  secret_spec:
+                    key: ya.token
+                - key: BUCKET_PASSWORD
+                  secret_spec:
+                    key: bucket.token
+                - key: YT_TOKEN
+                  secret_spec:
+                    key: YT_TOKEN
+              environment_variables:
+                - key: DOCKER_LOGIN
+                  value: robot-ya-devops
+                - key: YA_USER
+                  value: robot-ya-devops
+                - key: BUCKET_LOGIN
+                  value: robot-ya-devops
+              cmd_line: |
+                set -ex
+                . ci/tasklet/registry/common/misc/run_command/docker/docker_setup_0.sh
+                docker login -u $DOCKER_LOGIN -p $DOCKER_OAUTH registry.yandex.net
+                docker run -e YA_USER="$YA_USER" -e YA_TOKEN="$YA_TOKEN" -e BUCKET_LOGIN="$BUCKET_LOGIN" -e BUCKET_PASSWORD="$BUCKET_PASSWORD" -e YT_TOKEN="$YT_TOKEN" --network=bridge6 -v "$ARCADIA_PATH:/root/arcadia" --entrypoint /root/arcadia/devtools/yexport/ci/gradle_cmake_docker/gradle_tests.sh registry.yandex.net/yexported-cmake-build

build/export_generators/ide-gradle/generator.toml

@@ -95,6 +95,7 @@ test=[
     "/src/test",
     "/src/test/java",
     "/src/test-integration",
+    "/src/integration-test",
     "/src/testFixtures",
     "/src/intTest",
 ]

build/export_generators/ide-gradle/source_sets.jinja

@@ -32,6 +32,8 @@ sourceSets {
         resources.srcDir("ut/resources")
         java.srcDir("src/test-integration/java")
         resources.srcDir("src/test-integration/resources")
+        java.srcDir("src/integration-test/java")
+        resources.srcDir("src/integration-test/resources")
         java.srcDir("src/testFixtures/java")
         resources.srcDir("src/testFixtures/resources")
         java.srcDir("src/intTest/java")

build/plugins/java.py

@@ -58,13 +58,6 @@ def onjava_module(unit, *args):
         # TODO remove when java test dart is in prod
         'UNITTEST_DIR': unit.get('UNITTEST_DIR'),
         'JVM_ARGS': extract_macro_calls(unit, 'JVM_ARGS_VALUE', args_delim),
-        'TEST_CWD': extract_macro_calls(unit, 'TEST_CWD_VALUE', args_delim),
-        'TEST_FORK_MODE': extract_macro_calls(unit, 'TEST_FORK_MODE', args_delim),
-        'SPLIT_FACTOR': extract_macro_calls(unit, 'TEST_SPLIT_FACTOR', args_delim),
-        'TIMEOUT': extract_macro_calls(unit, 'TEST_TIMEOUT', args_delim),
-        'TAG': extract_macro_calls(unit, 'TEST_TAGS_VALUE', args_delim),
-        'SIZE': extract_macro_calls(unit, 'TEST_SIZE_NAME', args_delim),
-        'DEPENDS': extract_macro_calls(unit, 'TEST_DEPENDS_VALUE', args_delim),
         'IDEA_EXCLUDE': extract_macro_calls(unit, 'IDEA_EXCLUDE_DIRS_VALUE', args_delim),
         'IDEA_RESOURCE': extract_macro_calls(unit, 'IDEA_RESOURCE_DIRS_VALUE', args_delim),
         'IDEA_MODULE_NAME': extract_macro_calls(unit, 'IDEA_MODULE_NAME_VALUE', args_delim),
@@ -81,7 +74,6 @@ def onjava_module(unit, *args):
         data['ENABLE_PREVIEW'] = extract_macro_calls(unit, 'ENABLE_PREVIEW_VALUE', args_delim)
 
     if unit.get('SAVE_JAVAC_GENERATED_SRCS_DIR') and unit.get('SAVE_JAVAC_GENERATED_SRCS_TAR'):
-        data['SAVE_JAVAC_GENERATED_SRCS_DIR'] = extract_macro_calls(unit, 'SAVE_JAVAC_GENERATED_SRCS_DIR', args_delim)
         data['SAVE_JAVAC_GENERATED_SRCS_TAR'] = extract_macro_calls(unit, 'SAVE_JAVAC_GENERATED_SRCS_TAR', args_delim)
 
     if unit.get('JAVA_ADD_DLLS_VALUE') == 'yes':
@@ -91,17 +83,9 @@ def onjava_module(unit, *args):
         data['ERROR_PRONE'] = extract_macro_calls(unit, 'ERROR_PRONE_VALUE', args_delim)
 
     if unit.get('WITH_KOTLIN_VALUE') == 'yes':
-        data['WITH_KOTLIN'] = extract_macro_calls(unit, 'WITH_KOTLIN_VALUE', args_delim)
-        if unit.get('KOTLIN_JVM_TARGET'):
-            data['KOTLIN_JVM_TARGET'] = extract_macro_calls(unit, 'KOTLIN_JVM_TARGET', args_delim)
-        if unit.get('KOTLINC_FLAGS_VALUE'):
-            data['KOTLINC_FLAGS'] = extract_macro_calls(unit, 'KOTLINC_FLAGS_VALUE', args_delim)
         if unit.get('KOTLINC_OPTS_VALUE'):
             data['KOTLINC_OPTS'] = extract_macro_calls(unit, 'KOTLINC_OPTS_VALUE', args_delim)
 
-    if unit.get('DIRECT_DEPS_ONLY_VALUE') == 'yes':
-        data['DIRECT_DEPS_ONLY'] = extract_macro_calls(unit, 'DIRECT_DEPS_ONLY_VALUE', args_delim)
-
     if unit.get('JAVA_EXTERNAL_DEPENDENCIES_VALUE'):
         valid = []
         for dep in sum(extract_macro_calls(unit, 'JAVA_EXTERNAL_DEPENDENCIES_VALUE', args_delim), []):
@@ -118,13 +102,6 @@ def onjava_module(unit, *args):
         if valid:
             data['EXTERNAL_DEPENDENCIES'] = [valid]
 
-    if unit.get('WITH_JDK_VALUE') == 'yes':
-        if unit.get('MODULE_TYPE') != 'JAVA_PROGRAM':
-            ymake.report_configure_error(
-                '{}: JDK export supported only for JAVA_PROGRAM module type'.format(unit.path())
-            )
-        data['WITH_JDK'] = extract_macro_calls(unit, 'WITH_JDK_VALUE', args_delim)
-
     # IMPORTANT before switching vcs_info.py to python3 the value was always evaluated to $YMAKE_PYTHON but no
     # code in java dart parser extracts its value only checks this key for existance.
     data['EMBED_VCS'] = [['yes']]

build/plugins/lib/nots/package_manager/base/node_modules_bundler.py

@@ -78,8 +78,9 @@ def extract_node_modules(build_root, node_modules_path, bundle_path):
                 if os.path.exists(bundled_nm_path):
                     os.rename(bundled_nm_path, nm_path)
             except FileNotFoundError as e:
-                logging.error(f"exists={os.path.exists(bundled_nm_path)} : {bundled_nm_path}")
-                logging.error(f"exists={os.path.exists(nm_path)} : {nm_path}")
+                parent_directory = os.path.dirname(nm_path)
+                logging.error(f"bundled_nm exists={os.path.exists(bundled_nm_path)} : {bundled_nm_path}")
+                logging.error(f"nm_path/.. exists={os.path.exists(parent_directory)} : {parent_directory}")
                 raise e
 
     return True

build/scripts/link_lib.py

@@ -52,6 +52,68 @@ def get_opts(args):
     return Opts(args)
 
 
+def run(*args):
+    return subprocess.check_output(list(args), shell=False).strip()
+
+
+def gen_renames_1(d):
+    for l in d.split('\n'):
+        l = l.strip()
+
+        if ' ' in l:
+            yield l.split(' ')[-1]
+
+
+def have_prefix(l, p):
+    for x in l:
+        if not x.startswith(p):
+            return False
+
+    return True
+
+
+def gen_renames_2(p, d):
+    l = list(gen_renames_1(d))
+    a = have_prefix(l, '_')
+
+    for s in l:
+        if 'asan_globals' in s:
+            continue
+
+        if s in ['HMAC', 'SHA1', 'SHA256', 'SHA256', 'SHA512', 'RC4', 'MD5', 'SHA384']:
+            continue
+
+        if a and s[1:] in ['HMAC', 'SHA1', 'SHA256', 'SHA256', 'SHA512', 'RC4', 'MD5', 'SHA384']:
+            continue
+
+        if a:
+            yield s + ' _' + p + s[1:]
+        else:
+            yield s + ' ' + p + s
+
+
+def gen_renames(p, d):
+    return '\n'.join(gen_renames_2(p, d)).strip() + '\n'
+
+
+def rename_syms(where, ret):
+    p = 'v1_'
+
+    # find symbols to rename
+    syms = run(where + 'llvm-nm', '--extern-only', '--defined-only', '-A', ret)
+
+    # prepare rename plan
+    renames = gen_renames(p, syms)
+    tmp = ret + '.syms'
+
+    with open(tmp, 'w') as f:
+        f.write(renames)
+
+    # rename symbols
+    run(where + 'llvm-objcopy', '--redefine-syms=' + tmp, ret)
+    os.unlink(tmp)
+
+
 if __name__ == "__main__":
     opts = get_opts(sys.argv[1:])
 
@@ -99,3 +161,6 @@ def call():
 
     if exit_code != 0:
         raise Exception('{0} returned non-zero exit code {1}. Stop.'.format(' '.join(cmd), exit_code))
+
+    if os.path.basename(opts.output) in ['libcontrib-libs-openssl.a', 'liblibs-openssl-crypto.a']:
+        rename_syms(os.path.dirname(opts.archiver) + '/', opts.output)

contrib/libs/openjpeg/.yandex_meta/override.nix

@@ -1,11 +1,11 @@
 pkgs: attrs: with pkgs; with attrs; rec {
-  version = "2.5.2";
+  version = "2.5.3";
 
   src = fetchFromGitHub {
     owner = "uclouvain";
     repo = "openjpeg";
     rev = "v${version}";
-    sha256 = "sha256-mQ9B3MJY2/bg0yY/7jUJrAXM6ozAHT5fmwES5Q1SGxw=";
+    sha256 = "sha256-ONPahcQ80e3ahYRQU+Tu8Z7ZTARjRlpXqPAYpUlX5sY=";
   };
 
   patches = [];