From 0c4716623afac4d443819248be269bf6c26f0f71 Mon Sep 17 00:00:00 2001 From: "Ansari, Mujammil" Date: Thu, 7 Nov 2024 18:37:43 +0530 Subject: [PATCH 1/2] Adjust ExportToFiles compiler for re-usability Signed-off-by: Ansari, Mujammil --- .../jobs/steps/compiler/ExportToFiles.java | 55 ++++++++++--------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/xyz-jobs/xyz-job-service/src/main/java/com/here/xyz/jobs/steps/compiler/ExportToFiles.java b/xyz-jobs/xyz-job-service/src/main/java/com/here/xyz/jobs/steps/compiler/ExportToFiles.java index 961e333c9..9e169d676 100644 --- a/xyz-jobs/xyz-job-service/src/main/java/com/here/xyz/jobs/steps/compiler/ExportToFiles.java +++ b/xyz-jobs/xyz-job-service/src/main/java/com/here/xyz/jobs/steps/compiler/ExportToFiles.java @@ -24,6 +24,7 @@ import com.here.xyz.jobs.datasets.Files; import com.here.xyz.jobs.datasets.files.GeoJson; +import com.here.xyz.jobs.datasets.filters.Filters; import com.here.xyz.jobs.steps.CompilationStepGraph; import com.here.xyz.jobs.steps.Config; import com.here.xyz.jobs.steps.JobCompiler; @@ -48,41 +49,45 @@ public boolean chooseMe(Job job) { @Override public CompilationStepGraph compile(Job job) { Space source = (Space) job.getSource(); - resolveVersionRef(source); - String spaceId = source.getId(); - - ExportSpaceToFiles exportToFilesStep = new ExportSpaceToFiles() - .withSpaceId(spaceId) - .withJobId(job.getId()) - .withSpatialFilter(source.getFilters() != null ? source.getFilters().getSpatialFilter() : null) - .withPropertyFilter(source.getFilters() != null ? source.getFilters().getPropertyFilter() : null) - .withContext(source.getFilters() != null ? source.getFilters().getContext() : null) - .withVersionRef(source.getVersionRef() != null ? source.getVersionRef() : null); - - return compileImportSteps(exportToFilesStep); + return compileSteps(source.getId(), job.getId(), source.getFilters(), source.getVersionRef()); } - public static CompilationStepGraph compileImportSteps(ExportSpaceToFiles exportToFilesStep) { + public static CompilationStepGraph compileSteps(String spaceId, String jobId, Filters filters, Ref versionRef) { return (CompilationStepGraph) new CompilationStepGraph() - .addExecution(exportToFilesStep); + .addExecution(compileExportStep(spaceId, jobId, filters, versionRef, false, true)); + } + + public static ExportSpaceToFiles compileExportStep(String spaceId, String jobId, Filters filters, Ref versionRef, boolean useSystemOutput, boolean addStatistics) { + versionRef = resolveVersionRef(spaceId, versionRef); + return new ExportSpaceToFiles() + .withSpaceId(spaceId) + .withJobId(jobId) + .withSpatialFilter(filters != null ? filters.getSpatialFilter() : null) + .withPropertyFilter(filters != null ? filters.getPropertyFilter() : null) + .withContext(filters != null ? filters.getContext() : null) + .withVersionRef(versionRef) + .withUseSystemOutput(useSystemOutput) + .withAddStatisticsToUserOutput(addStatistics); } - private void resolveVersionRef(Space sourceSpace) { - if(sourceSpace.getVersionRef() == null || sourceSpace.getVersionRef().isRange()) - return; + private static Ref resolveVersionRef(String spaceId, Ref versionRef) { + if(versionRef == null || versionRef.isRange()) + return versionRef; try { - if (sourceSpace.getVersionRef().isHead()) { - StatisticsResponse statisticsResponse = HubWebClient.getInstance(Config.instance.HUB_ENDPOINT).loadSpaceStatistics(sourceSpace.getId()); - sourceSpace.setVersionRef(new Ref(statisticsResponse.getMaxVersion().getValue())); - } else if (sourceSpace.getVersionRef().isTag()) { - long version = HubWebClient.getInstance(Config.instance.HUB_ENDPOINT).loadTag(sourceSpace.getId(), sourceSpace.getVersionRef().getTag()).getVersion(); + Long resolvedVersion = null; + if (versionRef.isHead()) { + StatisticsResponse statisticsResponse = HubWebClient.getInstance(Config.instance.HUB_ENDPOINT).loadSpaceStatistics(spaceId); + resolvedVersion = statisticsResponse.getMaxVersion().getValue(); + } else if (versionRef.isTag()) { + long version = HubWebClient.getInstance(Config.instance.HUB_ENDPOINT).loadTag(spaceId, versionRef.getTag()).getVersion(); if (version >= 0) { - sourceSpace.setVersionRef(new Ref(version)); + resolvedVersion = version; } } - }catch (Exception e) { - throw new JobCompiler.CompilationError("Unable to resolve '" + sourceSpace.getVersionRef() + "' VersionRef!"); + return resolvedVersion == null ? versionRef : new Ref(resolvedVersion); + } catch (Exception e) { + throw new JobCompiler.CompilationError("Unable to resolve '" + versionRef + "' VersionRef!"); } } } From dfa2c0afab2ebf25a7dfaf99a1870825fddbb16c Mon Sep 17 00:00:00 2001 From: "Ansari, Mujammil" Date: Mon, 11 Nov 2024 14:34:49 +0530 Subject: [PATCH 2/2] Resolve resourceKey for export jobs Signed-off-by: Ansari, Mujammil --- .../src/main/java/com/here/xyz/jobs/Job.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/xyz-jobs/xyz-job-service/src/main/java/com/here/xyz/jobs/Job.java b/xyz-jobs/xyz-job-service/src/main/java/com/here/xyz/jobs/Job.java index f11a63a39..9bc3ce7d6 100644 --- a/xyz-jobs/xyz-job-service/src/main/java/com/here/xyz/jobs/Job.java +++ b/xyz-jobs/xyz-job-service/src/main/java/com/here/xyz/jobs/Job.java @@ -41,6 +41,7 @@ import com.here.xyz.jobs.RuntimeInfo.State; import com.here.xyz.jobs.config.JobConfigClient; import com.here.xyz.jobs.datasets.DatasetDescription; +import com.here.xyz.jobs.datasets.Files; import com.here.xyz.jobs.datasets.streams.DynamicStream; import com.here.xyz.jobs.steps.JobCompiler; import com.here.xyz.jobs.steps.Step; @@ -520,10 +521,9 @@ public Job withId(String id) { @JsonView(Static.class) public String getResourceKey() { - //TODO: Identify when to use the key from source vs from target - if (getTarget() == null) - return null; - return getTarget().getKey(); + //Always use key from the source except when the source is Files + if(getSource() == null) return null; + return getSource() instanceof Files ? getTarget().getKey() : getSource().getKey(); } public String getDescription() {