Skip to content

Commit

Permalink
Shell lookups must return shells ordered in a deterministic way
Browse files Browse the repository at this point in the history 
- Modifies repository and service implementation to fetch AAS Ids ordered by the createdDate (and then the AAS Id as secondary criteria)
- Adds new tests to cover the changes better
  • Loading branch information
@istvan-nagy-epam
istvan-nagy-epam committed Feb 21, 2024
1 parent e778924 commit 017c19b
Show file tree
Hide file tree
Showing 5 changed files with 450 additions and 59 deletions.
54 changes: 41 additions & 13 deletions ...in/java/org/eclipse/tractusx/semantics/registry/repository/ShellIdentifierRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,15 @@

package org.eclipse.tractusx.semantics.registry.repository;

import java.time.Instant;
import java.util.List;
import java.util.Set;
import java.util.UUID;

import org.eclipse.tractusx.semantics.registry.model.Shell;
import org.eclipse.tractusx.semantics.registry.model.ShellIdentifier;
import org.eclipse.tractusx.semantics.registry.model.projection.ShellIdentifierMinimal;
import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
Expand All @@ -41,19 +43,36 @@ public interface ShellIdentifierRepository extends JpaRepository<ShellIdentifier
Set<ShellIdentifier> findByShellId( Shell shellId );

@Query( value = """
SELECT NEW org.eclipse.tractusx.semantics.registry.model.projection.ShellIdentifierMinimal(sid.shellId.idExternal, sid.key, sid.value)
SELECT s.id
FROM ShellIdentifier sid
JOIN sid.shellId s
WHERE
CONCAT( sid.key, sid.value ) IN ( :keyValueCombinations )
AND (
s.createdDate > :cutoffDate
OR ( s.createdDate = :cutoffDate AND s.idExternal > :cursorValue )
)
GROUP BY s.id, s.createdDate, s.idExternal
HAVING COUNT(*) = :keyValueCombinationsSize
ORDER BY s.createdDate ASC, s.idExternal ASC
""" )
List<UUID> findAPageOfShellIdsBySpecificAssetIds(
List<String> keyValueCombinations, int keyValueCombinationsSize, Instant cutoffDate, String cursorValue, Pageable pageable );

@Query( value = """
SELECT NEW org.eclipse.tractusx.semantics.registry.model.projection.ShellIdentifierMinimal(s.idExternal, sid.key, sid.value)
FROM ShellIdentifier sid
JOIN sid.shellId s
WHERE
sid.shellId.id IN (
SELECT filtersid.shellId.id
FROM ShellIdentifier filtersid
WHERE
CONCAT(filtersid.key, filtersid.value) IN (:keyValueCombinations)
GROUP BY filtersid.shellId.id
HAVING COUNT(*) = :keyValueCombinationsSize
)
s.id IN ( :shellIds )
AND (
s.createdDate > :cutoffDate
OR ( s.createdDate = :cutoffDate AND s.idExternal > :cursorValue )
)
ORDER BY s.createdDate ASC, s.idExternal ASC
""" )
List<ShellIdentifierMinimal> findMinimalShellIdsBySpecificAssetIds( List<String> keyValueCombinations, int keyValueCombinationsSize );
List<ShellIdentifierMinimal> findMinimalShellIdsByShellIds(
List<UUID> shellIds, Instant cutoffDate, String cursorValue );

/**
* Returns external shell ids for the given keyValueCombinations.
Expand All @@ -74,7 +93,11 @@ HAVING COUNT(*) = :keyValueCombinationsSize
FROM shell s
JOIN shell_identifier si ON s.id = si.fk_shell_id
WHERE
CONCAT(si.namespace, si.identifier) IN (:keyValueCombinations)
CONCAT( si.namespace, si.identifier ) IN ( :keyValueCombinations )
AND (
s.created_date > :cutoffDate
OR ( s.created_date = :cutoffDate AND s.id_external > :cursorValue )
)
AND (
:tenantId = :owningTenantId
OR si.namespace = :globalAssetId
Expand All @@ -90,14 +113,19 @@ OR EXISTS (
AND sies.FK_SHELL_IDENTIFIER_EXTERNAL_SUBJECT_ID = si.id
)
)
GROUP BY s.id_external
GROUP BY s.id_external, s.created_date
HAVING COUNT(*) = :keyValueCombinationsSize
ORDER BY s.created_date, s.id_external
LIMIT :pageSize
""", nativeQuery = true )
List<String> findExternalShellIdsByIdentifiersByExactMatch( @Param( "keyValueCombinations" ) List<String> keyValueCombinations,
@Param( "keyValueCombinationsSize" ) int keyValueCombinationsSize,
@Param( "tenantId" ) String tenantId,
@Param( "publicWildcardPrefix" ) String publicWildcardPrefix,
@Param( "publicWildcardAllowedTypes" ) List<String> publicWildcardAllowedTypes,
@Param( "owningTenantId" ) String owningTenantId,
@Param( "globalAssetId" ) String globalAssetId );
@Param( "globalAssetId" ) String globalAssetId,
@Param( "cutoffDate" ) Instant cutoffDate,
@Param( "cursorValue" ) String cursorValue,
@Param( "pageSize" ) int pageSize);
}
4 changes: 4 additions & 0 deletions ...end/src/main/java/org/eclipse/tractusx/semantics/registry/repository/ShellRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

package org.eclipse.tractusx.semantics.registry.repository;

import java.time.Instant;
import java.util.List;
import java.util.Optional;
import java.util.Set;
Expand Down Expand Up @@ -142,4 +143,7 @@ s.id IN (
)
""")
List<Shell> findAllBySubmodelEndpointAddress( String endpointAddress );

@Query("SELECT s.createdDate FROM Shell s WHERE s.idExternal = :idExternal")
Optional<Instant> getCreatedDateByIdExternal( String idExternal );
}
125 changes: 79 additions & 46 deletions backend/src/main/java/org/eclipse/tractusx/semantics/registry/service/ShellService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@

import static org.springframework.data.domain.PageRequest.ofSize;

import java.time.Instant;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.Base64;
import java.util.Collection;
Expand Down Expand Up @@ -56,6 +59,7 @@
import org.eclipse.tractusx.semantics.registry.utils.ShellSpecification;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.jpa.domain.Specification;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Service;
Expand All @@ -73,6 +77,9 @@ public class ShellService {
public static final String DUPLICATE_SUBMODEL_ID_SHORT_EXCEPTION = "An AssetAdministration Submodel for the given IdShort does already exists.";
private static final String SORT_FIELD_NAME_SHELL = "createdDate";
private static final String SORT_FIELD_NAME_SUBMODEL = "id";
private static final String DEFAULT_EXTERNAL_ID = "00000000-0000-0000-0000-000000000000";
private static final Instant MINIMUM_SQL_DATETIME = OffsetDateTime
.of( 1800, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC ).toInstant();
private static final int MAXIMUM_RECORDS = 1000;

private final ShellRepository shellRepository;
Expand Down Expand Up @@ -261,35 +268,17 @@ public GetAllAssetAdministrationShellIdsByAssetLink200Response findExternalShell
Integer pageSize, String cursor, String externalSubjectId ) {

pageSize = getPageSize( pageSize );
final String cursorValue = getCursorDecoded( cursor ).orElse( DEFAULT_EXTERNAL_ID );
try {
List<String> keyValueCombinations = shellIdentifiers.stream().map( shellIdentifier -> shellIdentifier.getKey() + shellIdentifier.getValue() ).toList();

//TODO: if we can define a sorting order, we should consider doing it on the database side
final List<String> assetIdList = new ArrayList<>();
String nextCursor;
final List<String> visibleAssetIds;
if ( shellAccessHandler.supportsGranularAccessControl() ) {
List<ShellIdentifierMinimal> queryResults = shellIdentifierRepository
.findMinimalShellIdsBySpecificAssetIds( keyValueCombinations, keyValueCombinations.size() );
Set<SpecificAssetId> userQuery = shellIdentifiers.stream()
.map( id -> new SpecificAssetId( id.getKey(), id.getValue() ) )
.collect( Collectors.toSet() );
List<String> allVisible = shellAccessHandler.filterToVisibleShellIdsForLookup( userQuery, queryResults, externalSubjectId );
allVisible.stream()
.skip( getCursorDecoded( cursor, allVisible ) )
.limit( pageSize )
.forEach( assetIdList::add );
nextCursor = getCursorEncoded( allVisible, assetIdList );
visibleAssetIds = fetchAPageOfAasIdsUsingGranularAccessControl( shellIdentifiers, externalSubjectId, cursorValue, pageSize );
} else {
List<String> queryResult = shellIdentifierRepository.findExternalShellIdsByIdentifiersByExactMatch( keyValueCombinations,
keyValueCombinations.size(), externalSubjectId, externalSubjectIdWildcardPrefix, externalSubjectIdWildcardAllowedTypes, owningTenantId,
ShellIdentifier.GLOBAL_ASSET_ID_KEY );
pageSize = getPageSize( pageSize );

int startIndex = getCursorDecoded( cursor, queryResult );
queryResult.stream().skip( startIndex ).limit( pageSize ).forEach( assetIdList::add );

nextCursor = getCursorEncoded( queryResult, assetIdList );
visibleAssetIds = fetchAPageOfAasIdsUsingLegacyAccessControl( shellIdentifiers, externalSubjectId, cursorValue, pageSize );
}

final var assetIdList = visibleAssetIds.stream().limit( pageSize ).toList();
final String nextCursor = getCursorEncoded( visibleAssetIds, assetIdList );
final var response = new GetAllAssetAdministrationShellIdsByAssetLink200Response();
response.setResult( assetIdList );
response.setPagingMetadata( new PagedResultPagingMetadata().cursor( nextCursor ) );
Expand All @@ -301,32 +290,50 @@ public GetAllAssetAdministrationShellIdsByAssetLink200Response findExternalShell
}
}

private String getCursorEncoded( List<String> queryResult, List<String> assetIdList ) {
if ( !queryResult.isEmpty() ) {
if ( !assetIdList.get( assetIdList.size() - 1 ).equals( queryResult.get( queryResult.size() - 1 ) ) ) {
String lastEle = assetIdList.get( assetIdList.size() - 1 );
return Base64.getEncoder().encodeToString( lastEle.getBytes() );
private List<String> fetchAPageOfAasIdsUsingLegacyAccessControl(
Set<ShellIdentifier> shellIdentifiers, String externalSubjectId, String cursorValue, int pageSize ) {
final var fetchSize = pageSize + 1;
final Instant cutoffDate = shellRepository.getCreatedDateByIdExternal( cursorValue )
.orElse( MINIMUM_SQL_DATETIME );
List<String> keyValueCombinations = toKeyValueCombinations( shellIdentifiers );
return shellIdentifierRepository.findExternalShellIdsByIdentifiersByExactMatch( keyValueCombinations,
keyValueCombinations.size(), externalSubjectId, externalSubjectIdWildcardPrefix, externalSubjectIdWildcardAllowedTypes, owningTenantId,
ShellIdentifier.GLOBAL_ASSET_ID_KEY, cutoffDate, cursorValue, fetchSize );
}

private List<String> fetchAPageOfAasIdsUsingGranularAccessControl(
Set<ShellIdentifier> shellIdentifiers, String externalSubjectId, String cursorValue, int pageSize )
throws DenyAccessException {
Set<SpecificAssetId> userQuery = shellIdentifiers.stream()
.map( id -> new SpecificAssetId( id.getKey(), id.getValue() ) )
.collect( Collectors.toSet() );
List<String> keyValueCombinations = toKeyValueCombinations( shellIdentifiers );
final var fetchSize = pageSize + pageSize;

String currentCursorValue = cursorValue;
final List<String> visibleAssetIds = new ArrayList<>();
while ( visibleAssetIds.size() < fetchSize ) {
final Instant currentCutoffDate = shellRepository.getCreatedDateByIdExternal( currentCursorValue )
.orElse( MINIMUM_SQL_DATETIME );
List<UUID> shellIds = shellIdentifierRepository.findAPageOfShellIdsBySpecificAssetIds(
keyValueCombinations, keyValueCombinations.size(), currentCutoffDate, currentCursorValue, PageRequest.ofSize( fetchSize ) );
if ( shellIds.isEmpty() ) {
break;
}
}
return null;
}
List<ShellIdentifierMinimal> queryResults = shellIdentifierRepository
.findMinimalShellIdsByShellIds( shellIds, currentCutoffDate, currentCursorValue );

private String getCursorDecoded( String cursor ) {
return Optional.ofNullable( cursor )
.map( Base64.getDecoder()::decode )
.map( String::new )
.orElse( null );
}

private int getCursorDecoded( String cursor, List<String> queryResult ) {
return Optional.ofNullable( getCursorDecoded( cursor ) )
.map( decodedValue -> queryResult.indexOf( decodedValue ) + 1 )
.orElse( 0 );
shellAccessHandler.filterToVisibleShellIdsForLookup( userQuery, queryResults, externalSubjectId ).stream()
.limit( fetchSize - visibleAssetIds.size() )
.forEach( visibleAssetIds::add );
currentCursorValue = lastItemOf( queryResults ).shellId();
}
return visibleAssetIds;
}

@Transactional( readOnly = true )
public List<String> findExternalShellIdsByIdentifiersByAnyMatch( Set<ShellIdentifier> shellIdentifiers, String externalSubjectId ) {
List<String> keyValueCombinations = shellIdentifiers.stream().map( shellIdentifier -> shellIdentifier.getKey() + shellIdentifier.getValue() ).toList();
List<String> keyValueCombinations = toKeyValueCombinations( shellIdentifiers );

return shellRepository.findExternalShellIdsByIdentifiersByAnyMatch(
keyValueCombinations,
Expand Down Expand Up @@ -386,7 +393,7 @@ public Set<ShellIdentifier> save( String externalShellId, Set<ShellIdentifier> s
return ImmutableSet.copyOf( shellIdentifierRepository.saveAll( identifiersToUpdate ) );
}

private static void mapShellIdentifier( Stream<ShellIdentifier> identifiersToUpdate ) {
private void mapShellIdentifier( Stream<ShellIdentifier> identifiersToUpdate ) {
identifiersToUpdate.filter( identifiers -> !identifiers.getKey().equalsIgnoreCase( "globalAssetId" ) ).forEach(
identifier -> {
if ( identifier.getSemanticId() != null ) {
Expand Down Expand Up @@ -508,4 +515,30 @@ private Shell doFindShellByExternalIdWithoutFiltering( String externalShellId )
return shellRepository.findByIdExternal( externalShellId )
.orElseThrow( () -> new EntityNotFoundException( String.format( "Shell for identifier %s not found", externalShellId ) ) );
}

private <T> T lastItemOf( List<T> list ) {
return list.get( list.size() - 1 );
}

private List<String> toKeyValueCombinations( Set<ShellIdentifier> shellIdentifiers ) {
return shellIdentifiers.stream()
.map( shellIdentifier -> shellIdentifier.getKey() + shellIdentifier.getValue() )
.toList();
}

private String getCursorEncoded( List<String> queryResult, List<String> assetIdList ) {
if ( !queryResult.isEmpty() ) {
//don't return a cursor if the last item is reached
if ( !lastItemOf( assetIdList ).equals( lastItemOf( queryResult ) ) ) {
return Base64.getEncoder().encodeToString( lastItemOf( assetIdList ).getBytes() );
}
}
return null;
}

private Optional<String> getCursorDecoded( String cursor ) {
return Optional.ofNullable( cursor )
.map( Base64.getDecoder()::decode )
.map( String::new );
}
}
Loading

0 comments on commit 017c19b

Please sign in to comment.