From b3f639292f718987db3e596a63422102240f8010 Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Thu, 9 Jan 2025 18:45:24 +0530 Subject: [PATCH 1/4] Use policy update time as last refresh time --- .../policytransformer/CachePolicyTransformerImpl.java | 2 ++ .../main/java/org/apache/atlas/web/rest/AuthREST.java | 9 ++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index 71e798091b..b60d9fa602 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -174,6 +174,8 @@ public ServicePolicies getPoliciesDelta(String serviceName, Map policyGuids = new ArrayList<>(policyChanges.keySet()); List allAtlasPolicies = getAtlasPolicies(serviceName, POLICY_BATCH_SIZE, policyGuids); + Date latestUpdateTime = allAtlasPolicies.stream().map(AtlasEntityHeader::getUpdateTime).max(Date::compareTo).orElse(null); + servicePolicies.setPolicyUpdateTime(latestUpdateTime); List atlasServicePolicies = allAtlasPolicies.stream().filter(x -> serviceName.equals(x.getAttribute(ATTR_POLICY_SERVICE_NAME))).collect(Collectors.toList()); List policiesDelta = getRangerPolicyDelta(service, policyChanges, atlasServicePolicies); diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java index d50ac8d0ea..37b6dd2cb1 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java @@ -158,13 +158,12 @@ public ServicePolicies downloadPolicies(@PathParam("serviceName") final String s ServicePolicies ret; if (usePolicyDelta) { List auditEvents = getPolicyAuditLogs(serviceName, lastUpdatedTime); - long lastEventTime = auditEvents.isEmpty() ? 0 : auditEvents.get(auditEvents.size() - 1).getCreated(); - LOG.info("PolicyDelta: serviceName={}, lastUpdatedTime={}, audit events found={}", serviceName, lastEventTime, auditEvents.size()); + LOG.info("PolicyDelta: serviceName={}, lastUpdatedTime={}, audit events found={}", serviceName, lastUpdatedTime, auditEvents.size()); if (auditEvents.isEmpty()) { return null; } Map policyChanges = policyTransformer.createPolicyChangeMap(serviceName, auditEvents); - ret = policyTransformer.getPoliciesDelta(serviceName, policyChanges, lastEventTime); + ret = policyTransformer.getPoliciesDelta(serviceName, policyChanges, lastUpdatedTime); } else { if (!isPolicyUpdated(serviceName, lastUpdatedTime)) { return null; @@ -196,12 +195,12 @@ private List getPolicyAuditLogs(String serviceName, long las mustClauseList.add(getMap("terms", getMap("typeName", entityUpdateToWatch))); lastUpdatedTime = lastUpdatedTime == -1 ? 0 : lastUpdatedTime; - mustClauseList.add(getMap("range", getMap("created", getMap("gt", lastUpdatedTime)))); + mustClauseList.add(getMap("range", getMap("timestamp", getMap("gt", lastUpdatedTime)))); dsl.put("query", getMap("bool", getMap("must", mustClauseList))); List> sortClause = new ArrayList<>(); - sortClause.add(getMap("created", getMap("order", "asc"))); + sortClause.add(getMap("timestamp", getMap("order", "asc"))); dsl.put("sort", sortClause); int from = 0; From 55d6e7ccdf38a6490a3ec245356e3f94adf0e7b1 Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Tue, 14 Jan 2025 19:17:11 +0530 Subject: [PATCH 2/4] Throw exception if policy engine is null This will allow lastrefesh time to reflect the fact that policies are not yet updated --- .../java/org/apache/atlas/plugin/service/RangerBasePlugin.java | 1 + 1 file changed, 1 insertion(+) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java index af0aa716e1..0c86b8b89d 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java @@ -324,6 +324,7 @@ public void setPolicies(ServicePolicies policies) { if (defaultSvcPolicies == null) { LOG.error("Could not get default Service Policies. Keeping old policy-engine! This is a FATAL error as the old policy-engine is null!"); isNewEngineNeeded = false; + throw new RuntimeException("PolicyRefresher("+policies.getServiceName()+").setPolicies: fetched service policies contains no policies or delta and current policy engine is null"); } else { defaultSvcPolicies.setPolicyVersion(policies.getPolicyVersion()); policies = defaultSvcPolicies; From 974c50fb1318ab5d95c70bee7288a797bb50a396 Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Fri, 17 Jan 2025 16:21:01 +0530 Subject: [PATCH 3/4] Add logs for successful policy engine update --- .../java/org/apache/atlas/plugin/service/RangerBasePlugin.java | 1 + .../main/java/org/apache/atlas/plugin/util/PolicyRefresher.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java index 0c86b8b89d..fdf689c396 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java @@ -420,6 +420,7 @@ public void setPolicies(ServicePolicies policies) { if (this.refresher != null) { this.refresher.saveToCache(usePolicyDeltas ? servicePolicies : policies); } + LOG.info("New RangerPolicyEngine created with policy count:"+ (usePolicyDeltas? servicePolicies.getPolicies().size() : policies.getPolicies().size())); } } else { diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java index 99deb21d2d..ecb68f0301 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java @@ -295,7 +295,7 @@ private void loadPolicy() { lastUpdatedTimeInMillis = -1; } } catch (Exception excp) { - LOG.error("Encountered unexpected exception!!!!!!!!!!!", excp); + LOG.error("Encountered unexpected exception!!!!!!!!!!! Message:" + excp.getMessage() + "Stacktrace: " + excp.getStackTrace().toString(), excp); } RangerPerfTracer.log(perf); From 49f8eaead166c5cff6675ce459fae2bc40731a3a Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Fri, 17 Jan 2025 16:21:42 +0530 Subject: [PATCH 4/4] Update timestamp and updated by on delete entity operation as well --- .../atlas/repository/store/graph/v2/AtlasEntityStoreV2.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index f48d206653..69e03d0d5e 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -1974,6 +1974,8 @@ private EntityMutationResponse deleteVertices(Collection deletionCa MetricRecorder metric = RequestContext.get().startMetricRecord("filterCategoryVertices"); for (AtlasVertex vertex : deletionCandidates) { + updateModificationMetadata(vertex); + String typeName = getTypeName(vertex); List preProcessors = getPreProcessor(typeName);