diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index 68f1d851420258..c142dbf7eeb229 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -3141,13 +3141,26 @@ private void configureDataProcessInstanceResolvers(final RuntimeWiring.Builder b "DataProcessInstance", typeWiring -> typeWiring + .dataFetcher("exists", new EntityExistsResolver(entityService)) + .dataFetcher( + "platform", + new LoadableTypeResolver<>( + dataPlatformType, + (env) -> { + final DataProcessInstance dataProcessInstance = env.getSource(); + return dataProcessInstance != null + && dataProcessInstance.getPlatform() != null + ? dataProcessInstance.getPlatform().getUrn() + : null; + })) .dataFetcher( "dataPlatformInstance", new LoadableTypeResolver<>( dataPlatformInstanceType, (env) -> { final DataProcessInstance dataProcessInstance = env.getSource(); - return dataProcessInstance.getDataPlatformInstance() != null + return dataProcessInstance != null + && dataProcessInstance.getDataPlatformInstance() != null ? dataProcessInstance.getDataPlatformInstance().getUrn() : null; })) @@ -3160,6 +3173,11 @@ private void configureDataProcessInstanceResolvers(final RuntimeWiring.Builder b final DataProcessInstance dpi = env.getSource(); return dpi.getContainer() != null ? dpi.getContainer().getUrn() : null; })) + .dataFetcher( + "parentTemplate", + new EntityTypeResolver( + entityTypes, + (env) -> ((DataProcessInstance) env.getSource()).getParentTemplate())) .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)) .dataFetcher( "lineage", diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/DataPlatformInstanceAspectMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/DataPlatformInstanceAspectMapper.java index ab3127a3ae232b..7355b19762924c 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/DataPlatformInstanceAspectMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/DataPlatformInstanceAspectMapper.java @@ -29,6 +29,7 @@ public DataPlatformInstance apply( result.setType(EntityType.DATA_PLATFORM_INSTANCE); result.setUrn(input.getInstance().toString()); } + // Warning: This often cannot be read properly: overwritten by LoadableTypeResolver result.setPlatform( DataPlatform.builder() .setUrn(input.getPlatform().toString()) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataprocessinst/DataProcessInstanceType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataprocessinst/DataProcessInstanceType.java index eeaaaa96f51704..0b79d6ee220ebf 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataprocessinst/DataProcessInstanceType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataprocessinst/DataProcessInstanceType.java @@ -41,7 +41,8 @@ public class DataProcessInstanceType DATA_PROCESS_INSTANCE_RELATIONSHIPS_ASPECT_NAME, ML_TRAINING_RUN_PROPERTIES_ASPECT_NAME, SUB_TYPES_ASPECT_NAME, - CONTAINER_ASPECT_NAME); + CONTAINER_ASPECT_NAME, + STATUS_ASPECT_NAME); private final EntityClient _entityClient; private final FeatureFlags _featureFlags; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataprocessinst/mappers/DataProcessInstanceMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataprocessinst/mappers/DataProcessInstanceMapper.java index d721f5a5fb522d..66aae8d555c431 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataprocessinst/mappers/DataProcessInstanceMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataprocessinst/mappers/DataProcessInstanceMapper.java @@ -3,6 +3,7 @@ import static com.linkedin.metadata.Constants.*; import com.linkedin.common.DataPlatformInstance; +import com.linkedin.common.Status; import com.linkedin.common.SubTypes; import com.linkedin.common.urn.Urn; import com.linkedin.data.DataMap; @@ -13,12 +14,15 @@ import com.linkedin.datahub.graphql.types.common.mappers.AuditStampMapper; import com.linkedin.datahub.graphql.types.common.mappers.CustomPropertiesMapper; import com.linkedin.datahub.graphql.types.common.mappers.DataPlatformInstanceAspectMapper; +import com.linkedin.datahub.graphql.types.common.mappers.StatusMapper; import com.linkedin.datahub.graphql.types.common.mappers.SubTypesMapper; +import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper; import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper; import com.linkedin.datahub.graphql.types.mappers.ModelMapper; import com.linkedin.datahub.graphql.types.mlmodel.mappers.MLHyperParamMapper; import com.linkedin.datahub.graphql.types.mlmodel.mappers.MLMetricMapper; import com.linkedin.dataprocess.DataProcessInstanceProperties; +import com.linkedin.dataprocess.DataProcessInstanceRelationships; import com.linkedin.entity.EntityResponse; import com.linkedin.entity.EnvelopedAspectMap; import com.linkedin.ml.metadata.MLTrainingRunProperties; @@ -68,7 +72,7 @@ public DataProcessInstance apply( mappingHelper.mapToResult( DATA_PROCESS_INSTANCE_PROPERTIES_ASPECT_NAME, (dataProcessInstance, dataMap) -> - mapDataProcessProperties(context, dataProcessInstance, dataMap, entityUrn)); + mapDataProcessInstanceProperties(context, dataProcessInstance, dataMap, entityUrn)); mappingHelper.mapToResult( ML_TRAINING_RUN_PROPERTIES_ASPECT_NAME, (dataProcessInstance, dataMap) -> @@ -77,8 +81,10 @@ public DataProcessInstance apply( DATA_PLATFORM_INSTANCE_ASPECT_NAME, (dataProcessInstance, dataMap) -> { DataPlatformInstance dataPlatformInstance = new DataPlatformInstance(dataMap); - dataProcessInstance.setDataPlatformInstance( - DataPlatformInstanceAspectMapper.map(context, dataPlatformInstance)); + com.linkedin.datahub.graphql.generated.DataPlatformInstance value = + DataPlatformInstanceAspectMapper.map(context, dataPlatformInstance); + dataProcessInstance.setPlatform(value.getPlatform()); + dataProcessInstance.setDataPlatformInstance(value); }); mappingHelper.mapToResult( SUB_TYPES_ASPECT_NAME, @@ -87,6 +93,14 @@ public DataProcessInstance apply( mappingHelper.mapToResult( CONTAINER_ASPECT_NAME, (dataProcessInstance, dataMap) -> mapContainers(context, dataProcessInstance, dataMap)); + mappingHelper.mapToResult( + STATUS_ASPECT_NAME, + (dataProcessInstance, dataMap) -> + dataProcessInstance.setStatus(StatusMapper.map(context, new Status(dataMap)))); + mappingHelper.mapToResult( + DATA_PROCESS_INSTANCE_RELATIONSHIPS_ASPECT_NAME, + (dataProcessInstance, dataMap) -> + mapDataProcessInstanceRelationships(context, dataProcessInstance, dataMap)); return mappingHelper.getResult(); } @@ -124,8 +138,8 @@ private void mapTrainingRunProperties( dpi.setMlTrainingRunProperties(properties); } - private void mapDataProcessProperties( - @Nonnull QueryContext context, + private void mapDataProcessInstanceProperties( + @Nullable QueryContext context, @Nonnull DataProcessInstance dpi, @Nonnull DataMap dataMap, @Nonnull Urn entityUrn) { @@ -146,9 +160,20 @@ private void mapDataProcessProperties( CustomPropertiesMapper.map( dataProcessInstanceProperties.getCustomProperties(), entityUrn)); } - if (dataProcessInstanceProperties.hasCreated()) { - dpi.setCreated(AuditStampMapper.map(context, dataProcessInstanceProperties.getCreated())); - } + dpi.setCreated(AuditStampMapper.map(context, dataProcessInstanceProperties.getCreated())); + properties.setCreated( + AuditStampMapper.map(context, dataProcessInstanceProperties.getCreated())); dpi.setProperties(properties); } + + private void mapDataProcessInstanceRelationships( + @Nullable QueryContext context, @Nonnull DataProcessInstance dpi, @Nonnull DataMap dataMap) { + DataProcessInstanceRelationships dataProcessInstanceRelationships = + new DataProcessInstanceRelationships(dataMap); + + if (dataProcessInstanceRelationships.getParentTemplate() != null) { + dpi.setParentTemplate( + UrnToEntityMapper.map(context, dataProcessInstanceRelationships.getParentTemplate())); + } + } } diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index f6adf884b2badc..25615d9b6aa4fe 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -6733,6 +6733,16 @@ type DataProcessInstance implements EntityWithRelationships & Entity { """ type: EntityType! + """ + Whether or not this entity exists on DataHub + """ + exists: Boolean + + """ + Status metadata of the data process instance + """ + status: Status + """ The history of state changes for the run """ @@ -6741,12 +6751,12 @@ type DataProcessInstance implements EntityWithRelationships & Entity { """ When the run was kicked off """ - created: AuditStamp + created: AuditStamp @deprecated(reason: "Use `properties.created`") """ The name of the data process """ - name: String + name: String @deprecated(reason: "Use `properties.name`") """ Edges extending from this entity. @@ -13154,7 +13164,7 @@ type DataProcessInstanceProperties { """ When this process instance was created """ - created: AuditStamp + created: AuditStamp! """ Additional custom properties specific to this process instance @@ -13188,9 +13198,8 @@ type MLTrainingRunProperties { } extend type DataProcessInstance { - """ - Additional read only properties associated with the Data Job + Additional read only properties associated with the Data Process Instance """ properties: DataProcessInstanceProperties @@ -13209,6 +13218,11 @@ extend type DataProcessInstance { """ container: Container + """ + Standardized platform urn where the data process instance is defined + """ + platform: DataPlatform + """ Recursively get the lineage of containers for this entity """ @@ -13218,4 +13232,9 @@ extend type DataProcessInstance { Additional properties when subtype is Training Run """ mlTrainingRunProperties: MLTrainingRunProperties + + """ + The parent entity whose run instance it is + """ + parentTemplate: Entity } diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/dataprocessinst/DataProcessInstanceTypeTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/dataprocessinst/DataProcessInstanceTypeTest.java index 437c74ab669146..9535830407117f 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/dataprocessinst/DataProcessInstanceTypeTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/dataprocessinst/DataProcessInstanceTypeTest.java @@ -7,6 +7,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.linkedin.common.AuditStamp; import com.linkedin.common.DataPlatformInstance; import com.linkedin.common.FabricType; import com.linkedin.common.Status; @@ -61,7 +62,11 @@ public class DataProcessInstanceTypeTest { private static final DataProcessInstanceKey TEST_DPI_1_KEY = new DataProcessInstanceKey().setId("id-1"); private static final DataProcessInstanceProperties TEST_DPI_1_PROPERTIES = - new DataProcessInstanceProperties().setName("Test DPI").setType(DataProcessType.STREAMING); + new DataProcessInstanceProperties() + .setName("Test DPI") + .setType(DataProcessType.STREAMING) + .setCreated( + new AuditStamp().setTime(1234L).setActor(UrnUtils.getUrn("urn:li:corpuser:1"))); private static final DataProcessInstanceInput TEST_DPI_1_DPI_INPUT = new DataProcessInstanceInput().setInputs(new UrnArray(ImmutableList.of(DATASET_URN))); private static final DataProcessInstanceOutput TEST_DPI_1_DPI_OUTPUT = diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/dataprocessinst/mappers/DataProcessInstanceMapperTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/dataprocessinst/mappers/DataProcessInstanceMapperTest.java index cd9d58b54e6b3a..cac3c6f594932c 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/dataprocessinst/mappers/DataProcessInstanceMapperTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/dataprocessinst/mappers/DataProcessInstanceMapperTest.java @@ -3,6 +3,7 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; +import com.linkedin.common.AuditStamp; import com.linkedin.common.DataPlatformInstance; import com.linkedin.common.url.Url; import com.linkedin.common.urn.Urn; @@ -12,6 +13,7 @@ import com.linkedin.datahub.graphql.generated.DataProcessInstance; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.dataprocess.DataProcessInstanceProperties; +import com.linkedin.dataprocess.DataProcessInstanceRelationships; import com.linkedin.entity.Aspect; import com.linkedin.entity.EntityResponse; import com.linkedin.entity.EnvelopedAspect; @@ -28,6 +30,7 @@ public class DataProcessInstanceMapperTest { private static final String TEST_INSTANCE_URN = "urn:li:dataProcessInstance:(test-workflow,test-instance)"; private static final String TEST_CONTAINER_URN = "urn:li:container:testContainer"; + private static final String TEST_USER_URN = "urn:li:corpuser:test"; private static final String TEST_EXTERNAL_URL = "https://example.com/process"; private static final String TEST_NAME = "Test Process Instance"; @@ -52,11 +55,15 @@ public void testMapBasicFields() throws Exception { } @Test - public void testMapDataProcessProperties() throws Exception { + public void testMapDataProcessInstanceProperties() throws Exception { // Create DataProcessInstanceProperties DataProcessInstanceProperties properties = new DataProcessInstanceProperties(); properties.setName(TEST_NAME); properties.setExternalUrl(new Url(TEST_EXTERNAL_URL)); + AuditStamp created = new AuditStamp(); + created.setTime(123456789L); + created.setActor(Urn.createFromString(TEST_USER_URN)); + properties.setCreated(created); // Add properties aspect addAspect(Constants.DATA_PROCESS_INSTANCE_PROPERTIES_ASPECT_NAME, properties); @@ -66,6 +73,20 @@ public void testMapDataProcessProperties() throws Exception { assertNotNull(instance.getProperties()); assertEquals(instance.getName(), TEST_NAME); assertEquals(instance.getExternalUrl(), TEST_EXTERNAL_URL); + assertEquals(instance.getCreated().getTime(), 123456789L); + assertEquals(instance.getCreated().getActor(), TEST_USER_URN); + } + + @Test + public void testMapDataProcessInstanceRelationships() throws Exception { + DataProcessInstanceRelationships relationships = new DataProcessInstanceRelationships(); + relationships.setParentTemplate(Urn.createFromString(TEST_INSTANCE_URN)); + + addAspect(Constants.DATA_PROCESS_INSTANCE_RELATIONSHIPS_ASPECT_NAME, relationships); + + DataProcessInstance instance = DataProcessInstanceMapper.map(null, entityResponse); + assertNotNull(instance.getParentTemplate()); + assertEquals(instance.getParentTemplate().getUrn(), TEST_INSTANCE_URN); } @Test diff --git a/datahub-web-react/src/app/entity/dataProcessInstance/DataProcessInstanceEntity.tsx b/datahub-web-react/src/app/entity/dataProcessInstance/DataProcessInstanceEntity.tsx index bdf77959e97c7f..239681609323eb 100644 --- a/datahub-web-react/src/app/entity/dataProcessInstance/DataProcessInstanceEntity.tsx +++ b/datahub-web-react/src/app/entity/dataProcessInstance/DataProcessInstanceEntity.tsx @@ -5,7 +5,10 @@ import { DataProcessInstance, EntityType, OwnershipType, SearchResult } from '.. import { Preview } from './preview/Preview'; import { Entity, EntityCapabilityType, IconStyleType, PreviewType } from '../Entity'; import { EntityProfile } from '../shared/containers/profile/EntityProfile'; -import { useGetDataProcessInstanceQuery } from '../../../graphql/dataProcessInstance.generated'; +import { + GetDataProcessInstanceQuery, + useGetDataProcessInstanceQuery, +} from '../../../graphql/dataProcessInstance.generated'; import { PropertiesTab } from '../shared/tabs/Properties/PropertiesTab'; import { LineageTab } from '../shared/tabs/Lineage/LineageTab'; import { SidebarAboutSection } from '../shared/containers/profile/sidebar/AboutSection/SidebarAboutSection'; @@ -15,7 +18,6 @@ import { GenericEntityProperties } from '../shared/types'; import { getDataForEntityType } from '../shared/containers/profile/utils'; import { SidebarDomainSection } from '../shared/containers/profile/sidebar/Domain/SidebarDomainSection'; import { EntityMenuItems } from '../shared/EntityDropdown/EntityDropdown'; -import { capitalizeFirstLetterOnly } from '../../shared/textUtil'; import DataProductSection from '../shared/containers/profile/sidebar/DataProduct/DataProductSection'; import { getDataProduct } from '../shared/utils'; import SummaryTab from './profile/DataProcessInstanceSummary'; @@ -58,9 +60,9 @@ export class DataProcessInstanceEntity implements Entity { ); }; - isSearchEnabled = () => true; + isSearchEnabled = () => false; - isBrowseEnabled = () => true; + isBrowseEnabled = () => false; isLineageEnabled = () => true; @@ -127,12 +129,9 @@ export class DataProcessInstanceEntity implements Entity { ]; getOverridePropertiesFromEntity = (processInstance?: DataProcessInstance | null): GenericEntityProperties => { - const name = processInstance?.name; - const externalUrl = processInstance?.externalUrl; return { - name, - externalUrl, - platform: processInstance?.dataPlatformInstance?.platform, + name: processInstance && this.displayName(processInstance), + platform: (processInstance as GetDataProcessInstanceQuery['dataProcessInstance'])?.optionalPlatform, }; }; @@ -142,14 +141,11 @@ export class DataProcessInstanceEntity implements Entity { return ( { return ( { }; getLineageVizConfig = (entity: DataProcessInstance) => { + const properties = this.getGenericEntityProperties(entity); return { urn: entity?.urn, name: this.displayName(entity), type: EntityType.DataProcessInstance, subtype: entity?.subTypes?.typeNames?.[0], - icon: entity?.dataPlatformInstance?.platform?.properties?.logoUrl || undefined, - platform: entity?.dataPlatformInstance?.platform, + icon: properties?.platform?.properties?.logoUrl ?? undefined, + platform: properties?.platform ?? undefined, container: entity?.container, }; }; diff --git a/datahub-web-react/src/app/entityV2/dataProcessInstance/DataProcessInstanceEntity.tsx b/datahub-web-react/src/app/entityV2/dataProcessInstance/DataProcessInstanceEntity.tsx index 3ab8c2b268aac3..09301b101b259c 100644 --- a/datahub-web-react/src/app/entityV2/dataProcessInstance/DataProcessInstanceEntity.tsx +++ b/datahub-web-react/src/app/entityV2/dataProcessInstance/DataProcessInstanceEntity.tsx @@ -1,36 +1,18 @@ +import { GenericEntityProperties } from '@app/entity/shared/types'; +import { globalEntityRegistryV2 } from '@app/EntityRegistryProvider'; +import { Entity, EntityCapabilityType, IconStyleType, PreviewType } from '@app/entityV2/Entity'; +import { EntityProfile } from '@app/entityV2/shared/containers/profile/EntityProfile'; +import SidebarEntityHeader from '@app/entityV2/shared/containers/profile/sidebar/SidebarEntityHeader'; +import { getDataForEntityType } from '@app/entityV2/shared/containers/profile/utils'; +import { EntityMenuItems } from '@app/entityV2/shared/EntityDropdown/EntityMenuActions'; +import { LineageTab } from '@app/entityV2/shared/tabs/Lineage/LineageTab'; +import { PropertiesTab } from '@app/entityV2/shared/tabs/Properties/PropertiesTab'; +import { getDataProduct } from '@app/entityV2/shared/utils'; +import { GetDataProcessInstanceQuery, useGetDataProcessInstanceQuery } from '@graphql/dataProcessInstance.generated'; +import { ArrowsClockwise } from 'phosphor-react'; import React from 'react'; -import { ApiOutlined } from '@ant-design/icons'; -import { GenericEntityProperties } from '@src/app/entity/shared/types'; -import { - DataProcessInstance, - Entity as GeneratedEntity, - EntityType, - OwnershipType, - SearchResult, -} from '../../../types.generated'; -import { Preview } from './preview/Preview'; -import { Entity, EntityCapabilityType, IconStyleType, PreviewType } from '../Entity'; -import { EntityProfile } from '../shared/containers/profile/EntityProfile'; -import { useGetDataProcessInstanceQuery } from '../../../graphql/dataProcessInstance.generated'; -import { PropertiesTab } from '../shared/tabs/Properties/PropertiesTab'; -import { LineageTab } from '../shared/tabs/Lineage/LineageTab'; -import { SidebarAboutSection } from '../shared/containers/profile/sidebar/AboutSection/SidebarAboutSection'; -import { SidebarTagsSection } from '../shared/containers/profile/sidebar/SidebarTagsSection'; -import { SidebarOwnerSection } from '../shared/containers/profile/sidebar/Ownership/sidebar/SidebarOwnerSection'; -import { getDataForEntityType } from '../shared/containers/profile/utils'; -import { SidebarDomainSection } from '../shared/containers/profile/sidebar/Domain/SidebarDomainSection'; -import { capitalizeFirstLetterOnly } from '../../shared/textUtil'; -import DataProductSection from '../shared/containers/profile/sidebar/DataProduct/DataProductSection'; -import { getDataProduct } from '../shared/utils'; -// import SummaryTab from './profile/DataProcessInstaceSummary'; - -// const getProcessPlatformName = (data?: DataProcessInstance): string => { -// return ( -// data?.dataPlatformInstance?.platform?.properties?.displayName || -// capitalizeFirstLetterOnly(data?.dataPlatformInstance?.platform?.name) || -// '' -// ); -// }; +import { DataProcessInstance, Entity as GeneratedEntity, EntityType, SearchResult } from '../../../types.generated'; +import Preview from './preview/Preview'; const getParentEntities = (data: DataProcessInstance): GeneratedEntity[] => { const parentEntity = data?.relationships?.relationships?.find( @@ -48,6 +30,7 @@ const getParentEntities = (data: DataProcessInstance): GeneratedEntity[] => { }, ]; }; + /** * Definition of the DataHub DataProcessInstance entity. */ @@ -56,15 +39,15 @@ export class DataProcessInstanceEntity implements Entity { icon = (fontSize?: number, styleType?: IconStyleType, color?: string) => { if (styleType === IconStyleType.TAB_VIEW) { - return ; + return ; } if (styleType === IconStyleType.HIGHLIGHT) { - return ; + return ; } return ( - { ); }; - isSearchEnabled = () => true; + isSearchEnabled = () => false; - isBrowseEnabled = () => true; + isBrowseEnabled = () => false; isLineageEnabled = () => true; @@ -98,15 +81,10 @@ export class DataProcessInstanceEntity implements Entity { useEntityQuery={this.useEntityQuery} // useUpdateQuery={useUpdateDataProcessInstanceMutation} getOverrideProperties={this.getOverridePropertiesFromEntity} + headerDropdownItems={ + new Set([EntityMenuItems.UPDATE_DEPRECATION, EntityMenuItems.RAISE_INCIDENT, EntityMenuItems.SHARE]) + } tabs={[ - // { - // name: 'Documentation', - // component: DocumentationTab, - // }, - // { - // name: 'Summary', - // component: SummaryTab, - // }, { name: 'Lineage', component: LineageTab, @@ -115,51 +93,26 @@ export class DataProcessInstanceEntity implements Entity { name: 'Properties', component: PropertiesTab, }, - // { - // name: 'Incidents', - // component: IncidentTab, - // getDynamicName: (_, processInstance) => { - // const activeIncidentCount = processInstance?.dataProcessInstance?.activeIncidents.total; - // return `Incidents${(activeIncidentCount && ` (${activeIncidentCount})`) || ''}`; - // }, - // }, ]} sidebarSections={this.getSidebarSections()} /> ); - getSidebarSections = () => [ - { - component: SidebarAboutSection, - }, - { - component: SidebarOwnerSection, - properties: { - defaultOwnerType: OwnershipType.TechnicalOwner, - }, - }, - { - component: SidebarTagsSection, - properties: { - hasTags: true, - hasTerms: true, - }, - }, - { - component: SidebarDomainSection, - }, - { - component: DataProductSection, - }, - ]; + getSidebarSections = () => [{ component: SidebarEntityHeader }]; getOverridePropertiesFromEntity = (processInstance?: DataProcessInstance | null): GenericEntityProperties => { - const name = processInstance?.name; - const externalUrl = processInstance?.externalUrl; + const parent = + processInstance?.parentTemplate && + globalEntityRegistryV2.getGenericEntityProperties( + processInstance.parentTemplate.type, + processInstance.parentTemplate, + ); return { - name, - externalUrl, - platform: processInstance?.dataPlatformInstance?.platform, + name: processInstance && this.displayName(processInstance), + platform: + (processInstance as GetDataProcessInstanceQuery['dataProcessInstance'])?.optionalPlatform || + parent?.platform, + parent, }; }; @@ -169,73 +122,35 @@ export class DataProcessInstanceEntity implements Entity { return ( ); }; - renderSearch = (result: SearchResult) => { - const data = result.entity as DataProcessInstance; - const genericProperties = this.getGenericEntityProperties(data); - const parentEntities = getParentEntities(data); - return ( - - ); - }; + renderSearch = (result: SearchResult) => + this.renderPreview(PreviewType.SEARCH, result.entity as DataProcessInstance); getLineageVizConfig = (entity: DataProcessInstance) => { + const properties = this.getGenericEntityProperties(entity); return { urn: entity?.urn, name: this.displayName(entity), type: EntityType.DataProcessInstance, subtype: entity?.subTypes?.typeNames?.[0], - icon: entity?.dataPlatformInstance?.platform?.properties?.logoUrl || undefined, - platform: entity?.dataPlatformInstance?.platform, + icon: properties?.platform?.properties?.logoUrl ?? undefined, + platform: properties?.platform ?? undefined, container: entity?.container, - // health: entity?.health || undefined, }; }; diff --git a/datahub-web-react/src/app/entityV2/dataProcessInstance/preview/Preview.tsx b/datahub-web-react/src/app/entityV2/dataProcessInstance/preview/Preview.tsx index 3a3b0340695d96..4f95d13600374a 100644 --- a/datahub-web-react/src/app/entityV2/dataProcessInstance/preview/Preview.tsx +++ b/datahub-web-react/src/app/entityV2/dataProcessInstance/preview/Preview.tsx @@ -1,3 +1,5 @@ +import { GenericEntityProperties } from '@app/entity/shared/types'; +import DefaultPreviewCard from '@app/previewV2/DefaultPreviewCard'; import React from 'react'; import { DataProduct, @@ -11,15 +13,14 @@ import { Owner, SearchInsight, Container, - ParentContainersResult, } from '../../../../types.generated'; -import DefaultPreviewCard from '../../../preview/DefaultPreviewCard'; import { useEntityRegistry } from '../../../useEntityRegistry'; import { IconStyleType } from '../../Entity'; -export const Preview = ({ +export default function Preview({ urn, name, + data, subType, description, platformName, @@ -38,13 +39,10 @@ export const Preview = ({ paths, health, parentEntities, - parentContainers, -}: // duration, -// status, -// startTime, -{ +}: { urn: string; name: string; + data: GenericEntityProperties | null; subType?: string | null; description?: string | null; platformName?: string; @@ -63,17 +61,15 @@ export const Preview = ({ paths?: EntityPath[]; health?: Health[] | null; parentEntities?: Array | null; - parentContainers?: ParentContainersResult | null; - // duration?: number | null; - // status?: string | null; - // startTime?: number | null; -}): JSX.Element => { +}): JSX.Element { const entityRegistry = useEntityRegistry(); return ( ); -}; +} diff --git a/datahub-web-react/src/app/entityV2/shared/components/styled/search/EmbeddedListSearchSection.tsx b/datahub-web-react/src/app/entityV2/shared/components/styled/search/EmbeddedListSearchSection.tsx index a9601cc12df30b..2dd17f21e8e0eb 100644 --- a/datahub-web-react/src/app/entityV2/shared/components/styled/search/EmbeddedListSearchSection.tsx +++ b/datahub-web-react/src/app/entityV2/shared/components/styled/search/EmbeddedListSearchSection.tsx @@ -4,7 +4,7 @@ import { useHistory, useLocation } from 'react-router'; import { ApolloError } from '@apollo/client'; import useSortInput from '@src/app/searchV2/sorting/useSortInput'; import { useSelectedSortOption } from '@src/app/search/context/SearchContext'; -import { FacetFilterInput } from '../../../../../../types.generated'; +import { EntityType, FacetFilterInput } from '../../../../../../types.generated'; import useFilters from '../../../../../search/utils/useFilters'; import { navigateToEntitySearchUrl } from './navigateToEntitySearchUrl'; import { FilterSet, GetSearchResultsParams, SearchResultsInterface } from './types'; @@ -19,6 +19,30 @@ import { import { decodeComma } from '../../../utils'; const FILTER = 'filter'; +const SEARCH_ENTITY_TYPES = [ + EntityType.Dataset, + EntityType.Dashboard, + EntityType.Chart, + EntityType.Mlmodel, + EntityType.MlmodelGroup, + EntityType.MlfeatureTable, + EntityType.Mlfeature, + EntityType.MlprimaryKey, + EntityType.DataFlow, + EntityType.DataJob, + EntityType.GlossaryTerm, + EntityType.GlossaryNode, + EntityType.Tag, + EntityType.Role, + EntityType.CorpUser, + EntityType.CorpGroup, + EntityType.Container, + EntityType.Domain, + EntityType.DataProduct, + EntityType.Notebook, + EntityType.BusinessAttribute, + EntityType.DataProcessInstance, +]; function getParamsWithoutFilters(params: QueryString.ParsedQuery) { const paramsCopy = { ...params }; @@ -146,6 +170,7 @@ export const EmbeddedListSearchSection = ({ return ( @@ -222,11 +231,7 @@ export const DefaultEntityHeader = ({ type={displayedEntityType} entityType={entityType} browsePaths={entityData?.browsePathV2} - parentEntities={ - entityData?.parentContainers?.containers || - entityData?.parentDomains?.domains || - entityData?.parentNodes?.nodes - } + parentEntities={parentEntities} contentRef={contentRef} isContentTruncated={isContentTruncated} /> diff --git a/datahub-web-react/src/app/entityV2/shared/containers/profile/sidebar/SidebarEntityHeader.tsx b/datahub-web-react/src/app/entityV2/shared/containers/profile/sidebar/SidebarEntityHeader.tsx index 4d8eeba24408c7..eaef45ad00dcbd 100644 --- a/datahub-web-react/src/app/entityV2/shared/containers/profile/sidebar/SidebarEntityHeader.tsx +++ b/datahub-web-react/src/app/entityV2/shared/containers/profile/sidebar/SidebarEntityHeader.tsx @@ -1,7 +1,7 @@ import React from 'react'; import styled from 'styled-components'; -import { Container, DataPlatform, EntityType, Post } from '../../../../../../types.generated'; import { useEntityData, useRefetch } from '../../../../../entity/shared/EntityContext'; +import { Container, DataPlatform, EntityType, Post, Entity } from '../../../../../../types.generated'; import ContextPath from '../../../../../previewV2/ContextPath'; import HealthIcon from '../../../../../previewV2/HealthIcon'; import NotesIcon from '../../../../../previewV2/NotesIcon'; @@ -56,6 +56,14 @@ const SidebarEntityHeader = () => { const platforms = entityType === EntityType.SchemaField ? entityData?.parent?.siblingPlatforms : entityData?.siblingPlatforms; + const containerPath = + entityData?.parentContainers?.containers || + entityData?.parentDomains?.domains || + entityData?.parentNodes?.nodes || + []; + const parentPath: Entity[] = entityData?.parent ? [entityData.parent as Entity] : []; + const parentEntities = containerPath.length ? containerPath : parentPath; + if (loading) { return ; } @@ -90,11 +98,7 @@ const SidebarEntityHeader = () => { type={displayedEntityType} entityType={entityType} browsePaths={entityData?.browsePathV2} - parentEntities={ - entityData?.parentContainers?.containers || - entityData?.parentDomains?.domains || - entityData?.parentNodes?.nodes - } + parentEntities={parentEntities} contentRef={contentRef} isContentTruncated={isContentTruncated} /> diff --git a/datahub-web-react/src/app/ingest/source/builder/constants.ts b/datahub-web-react/src/app/ingest/source/builder/constants.ts index 58525b3e88f975..9711dfe2f03d73 100644 --- a/datahub-web-react/src/app/ingest/source/builder/constants.ts +++ b/datahub-web-react/src/app/ingest/source/builder/constants.ts @@ -29,7 +29,7 @@ import powerbiLogo from '../../../../images/powerbilogo.png'; import modeLogo from '../../../../images/modelogo.png'; import databricksLogo from '../../../../images/databrickslogo.png'; import verticaLogo from '../../../../images/verticalogo.png'; -import mlflowLogo from '../../../../images/mlflowlogo.png'; +import mlflowLogo2 from '../../../../images/mlflowlogo2.png'; import dynamodbLogo from '../../../../images/dynamodblogo.png'; import fivetranLogo from '../../../../images/fivetranlogo.png'; import csvLogo from '../../../../images/csv-logo.png'; @@ -159,7 +159,7 @@ export const PLATFORM_URN_TO_LOGO = { [LOOKER_URN]: lookerLogo, [MARIA_DB_URN]: mariadbLogo, [METABASE_URN]: metabaseLogo, - [MLFLOW_URN]: mlflowLogo, + [MLFLOW_URN]: mlflowLogo2, [MODE_URN]: modeLogo, [MONGO_DB_URN]: mongodbLogo, [MSSQL_URN]: mssqlLogo, diff --git a/datahub-web-react/src/app/lineageV2/LineageExplorer.tsx b/datahub-web-react/src/app/lineageV2/LineageExplorer.tsx index d9f64696625ca6..ff30a54dc36db9 100644 --- a/datahub-web-react/src/app/lineageV2/LineageExplorer.tsx +++ b/datahub-web-react/src/app/lineageV2/LineageExplorer.tsx @@ -36,10 +36,11 @@ export default function LineageExplorer(props: Props) { const [columnEdgeVersion, setColumnEdgeVersion] = useState(0); const [displayVersion, setDisplayVersion] = useState<[number, string[]]>([0, []]); const [hideTransformations, setHideTransformations] = useShouldHideTransformations(); + const [showDataProcessInstances, setShowDataProcessInstances] = useState(false); const [showGhostEntities, setShowGhostEntities] = useState(false); - const context = { + const context: NodeContext = { rootUrn: urn, rootType: type, nodes, @@ -55,6 +56,8 @@ export default function LineageExplorer(props: Props) { setColumnEdgeVersion, hideTransformations, setHideTransformations, + showDataProcessInstances, + setShowDataProcessInstances, showGhostEntities, setShowGhostEntities, }; diff --git a/datahub-web-react/src/app/lineageV2/LineageFilterNode/computeOrFilters.ts b/datahub-web-react/src/app/lineageV2/LineageFilterNode/computeOrFilters.ts index 252f1d2054ea38..44b523efe49acb 100644 --- a/datahub-web-react/src/app/lineageV2/LineageFilterNode/computeOrFilters.ts +++ b/datahub-web-react/src/app/lineageV2/LineageFilterNode/computeOrFilters.ts @@ -13,10 +13,27 @@ import { AndFilterInput, EntityType, FacetFilterInput } from '@types'; export default function computeOrFilters( defaultFilters: FacetFilterInput[], hideTransformations = true, + hideDataProcessInstances = true, ): AndFilterInput[] { - if (!hideTransformations) { + if (!hideTransformations && !hideDataProcessInstances) { return [{ and: defaultFilters }]; } + + if (!hideTransformations) { + return [ + { + and: [ + ...defaultFilters, + { + field: ENTITY_FILTER_NAME, + values: [EntityType.DataProcessInstance], + negated: true, + }, + ], + }, + ]; + } + return [ { and: [ diff --git a/datahub-web-react/src/app/lineageV2/LineageFilterNode/useFetchFilterNodeContents.ts b/datahub-web-react/src/app/lineageV2/LineageFilterNode/useFetchFilterNodeContents.ts index bab6e1dc4705bf..cdfd7f66f3a1ff 100644 --- a/datahub-web-react/src/app/lineageV2/LineageFilterNode/useFetchFilterNodeContents.ts +++ b/datahub-web-react/src/app/lineageV2/LineageFilterNode/useFetchFilterNodeContents.ts @@ -20,9 +20,13 @@ interface Return { export default function useFetchFilterNodeContents(parent: string, direction: LineageDirection, skip: boolean): Return { const { startTimeMillis, endTimeMillis } = useGetLineageTimeParams(); - const { hideTransformations } = useContext(LineageNodesContext); + const { hideTransformations, showDataProcessInstances } = useContext(LineageNodesContext); - const orFilters = computeOrFilters([{ field: DEGREE_FILTER_NAME, values: ['1'] }], hideTransformations); + const orFilters = computeOrFilters( + [{ field: DEGREE_FILTER_NAME, values: ['1'] }], + hideTransformations, + showDataProcessInstances, + ); const { data } = useAggregateAcrossLineageQuery({ skip, fetchPolicy: 'cache-first', diff --git a/datahub-web-react/src/app/lineageV2/LineageTransformationNode/LineageTransformationNode.tsx b/datahub-web-react/src/app/lineageV2/LineageTransformationNode/LineageTransformationNode.tsx index 84e4b8fb39a883..aef9ecf9833f28 100644 --- a/datahub-web-react/src/app/lineageV2/LineageTransformationNode/LineageTransformationNode.tsx +++ b/datahub-web-react/src/app/lineageV2/LineageTransformationNode/LineageTransformationNode.tsx @@ -2,6 +2,7 @@ import { ConsoleSqlOutlined, HomeOutlined, LoadingOutlined } from '@ant-design/i import LineageVisualizationContext from '@app/lineageV2/LineageVisualizationContext'; import { Skeleton, Spin } from 'antd'; import { Tooltip } from '@components'; +import { useEntityRegistryV2 } from '@app/useEntityRegistry'; import React, { useContext } from 'react'; import { Handle, NodeProps, Position } from 'reactflow'; import styled from 'styled-components'; @@ -68,6 +69,7 @@ const NodeWrapper = styled.div<{ `; const IconWrapper = styled.div<{ isGhost: boolean }>` + display: flex; opacity: ${({ isGhost }) => (isGhost ? 0.5 : 1)}; `; @@ -92,7 +94,9 @@ const CustomIcon = styled.img` export default function LineageTransformationNode(props: NodeProps) { const { data, selected, dragging } = props; const { urn, type, entity, fetchStatus } = data; + const entityRegistry = useEntityRegistryV2(); const isQuery = type === EntityType.Query; + const isDataProcessInstance = type === EntityType.DataProcessInstance; const { rootUrn } = useContext(LineageNodesContext); const { cllHighlightedNodes, setHoveredNode } = useContext(LineageDisplayContext); @@ -131,8 +135,11 @@ export default function LineageTransformationNode(props: NodeProps {icon && } + {!icon && isDataProcessInstance && entityRegistry.getIcon(EntityType.DataProcessInstance, 18)} {!icon && isQuery && } - {!icon && !isQuery && } + {!icon && !isQuery && !isDataProcessInstance && ( + + )} {fetchStatus[LineageDirection.Upstream] === FetchStatus.LOADING && ( diff --git a/datahub-web-react/src/app/lineageV2/NodeBuilder.ts b/datahub-web-react/src/app/lineageV2/NodeBuilder.ts index f369f95ceefc4f..07eacabc974f1e 100644 --- a/datahub-web-react/src/app/lineageV2/NodeBuilder.ts +++ b/datahub-web-react/src/app/lineageV2/NodeBuilder.ts @@ -180,7 +180,6 @@ export default class NodeBuilder { createEdges(edges: NodeContext['edges']): Edge[] { const baseEdges = new Map>(); edges.forEach((edge, edgeId) => { - if (!edge.isDisplayed) return; const [upstream, downstream] = parseEdgeId(edgeId); if (upstream in this.nodeInformation && downstream in this.nodeInformation) { const upstreamDirection = this.nodeInformation[upstream].direction; diff --git a/datahub-web-react/src/app/lineageV2/common.ts b/datahub-web-react/src/app/lineageV2/common.ts index 1110460a6a0537..65ae9736f2cbb2 100644 --- a/datahub-web-react/src/app/lineageV2/common.ts +++ b/datahub-web-react/src/app/lineageV2/common.ts @@ -75,7 +75,7 @@ export interface LineageFilter extends NodeBase { export type LineageNode = LineageEntity | LineageFilter; -const TRANSFORMATION_TYPES: string[] = [EntityType.Query, EntityType.DataJob]; +const TRANSFORMATION_TYPES: string[] = [EntityType.Query, EntityType.DataJob, EntityType.DataProcessInstance]; export function useIgnoreSchemaFieldStatus(): boolean { return useAppConfig().config.featureFlags.schemaFieldLineageIgnoreStatus; @@ -128,6 +128,11 @@ export function isUrnQuery(urn: string, entityRegistry: EntityRegistry): boolean return type === EntityType.Query; } +export function isUrnDataProcessInstance(urn: string, entityRegistry: EntityRegistry): boolean { + const type = getEntityTypeFromEntityUrn(urn, entityRegistry); + return type === EntityType.DataProcessInstance; +} + export function isUrnTransformational(urn: string, entityRegistry: EntityRegistry): boolean { const type = getEntityTypeFromEntityUrn(urn, entityRegistry); return (!!type && TRANSFORMATION_TYPES.includes(type)) || isUrnDbt(urn, entityRegistry); @@ -211,6 +216,8 @@ export interface NodeContext { setColumnEdgeVersion: Dispatch>; hideTransformations: boolean; setHideTransformations: (hide: boolean) => void; + showDataProcessInstances: boolean; + setShowDataProcessInstances: (hide: boolean) => void; showGhostEntities: boolean; setShowGhostEntities: (hide: boolean) => void; } @@ -234,6 +241,8 @@ export const LineageNodesContext = React.createContext({ setColumnEdgeVersion: () => {}, hideTransformations: false, setHideTransformations: () => {}, + showDataProcessInstances: false, + setShowDataProcessInstances: () => {}, showGhostEntities: false, setShowGhostEntities: () => {}, }); @@ -342,6 +351,7 @@ export function onClickPreventSelect(event: React.MouseEvent): true { const DATA_STORE_COLOR = '#ffd279'; const BI_TOOL_COLOR = '#8682a2'; +const ML_COLOR = '#206de8'; const DEFAULT_COLOR = '#ff7979'; export function getNodeColor(type?: EntityType): [string, string] { @@ -351,5 +361,14 @@ export function getNodeColor(type?: EntityType): [string, string] { if (type === EntityType.Dataset) { return [DATA_STORE_COLOR, 'Column']; } - return [DEFAULT_COLOR, 'Column']; + if ( + type === EntityType.Mlmodel || + type === EntityType.MlmodelGroup || + type === EntityType.Mlfeature || + type === EntityType.MlfeatureTable || + type === EntityType.MlprimaryKey + ) { + return [ML_COLOR, '']; + } + return [DEFAULT_COLOR, '']; } diff --git a/datahub-web-react/src/app/lineageV2/controls/LineageControls.tsx b/datahub-web-react/src/app/lineageV2/controls/LineageControls.tsx index 4dfde23ea3427c..df306d5759869b 100644 --- a/datahub-web-react/src/app/lineageV2/controls/LineageControls.tsx +++ b/datahub-web-react/src/app/lineageV2/controls/LineageControls.tsx @@ -56,7 +56,8 @@ const ControlsColumn = styled.div``; type PanelType = 'filters' | 'timeRange'; export default function LineageControls() { - const { rootUrn, hideTransformations, showGhostEntities } = useContext(LineageNodesContext); + const { rootUrn, hideTransformations, showDataProcessInstances, showGhostEntities } = + useContext(LineageNodesContext); const { isTabFullsize, setTabFullsize } = useContext(TabFullsizedContext); const { isDefault: isLineageTimeUnchanged } = useGetLineageTimeParams(); const { fitView } = useReactFlow(); @@ -104,7 +105,10 @@ export default function LineageControls() { > {showExpandedText ? 'Filter' : null} diff --git a/datahub-web-react/src/app/lineageV2/controls/LineageSearchFilters.tsx b/datahub-web-react/src/app/lineageV2/controls/LineageSearchFilters.tsx index 59d4a801a5147c..25a34748bf9c43 100644 --- a/datahub-web-react/src/app/lineageV2/controls/LineageSearchFilters.tsx +++ b/datahub-web-react/src/app/lineageV2/controls/LineageSearchFilters.tsx @@ -41,6 +41,8 @@ export default function LineageSearchFilters() { nodeVersion, hideTransformations, setHideTransformations, + showDataProcessInstances, + setShowDataProcessInstances, showGhostEntities, setShowGhostEntities, } = useContext(LineageNodesContext); @@ -75,6 +77,17 @@ export default function LineageSearchFilters() { /> + + + + Show Process Instances + Show task runs. Will not hide home node.} + /> + + + + diff --git a/datahub-web-react/src/app/lineageV2/pruneAllDuplicateEdges.ts b/datahub-web-react/src/app/lineageV2/pruneAllDuplicateEdges.ts new file mode 100644 index 00000000000000..9873b5906ac44a --- /dev/null +++ b/datahub-web-react/src/app/lineageV2/pruneAllDuplicateEdges.ts @@ -0,0 +1,135 @@ +import EntityRegistry from '@app/entityV2/EntityRegistry'; +import { + createEdgeId, + getEdgeId, + isUrnDataProcessInstance, + isUrnTransformational, + NodeContext, +} from '@app/lineageV2/common'; +import { LineageDirection } from '@types'; + +enum HideOption { + TRANSFORMATIONS = 'transformations', + DATA_PROCESS_INSTANCES = 'dataProcessInstances', +} + +const hideOptionIncludeUrnFunctions: Record boolean> = { + [HideOption.TRANSFORMATIONS]: isUrnTransformational, + [HideOption.DATA_PROCESS_INSTANCES]: isUrnDataProcessInstance, +}; + +/** + * Remove direct edges between non-transformational nodes, if there is a path between them through a transformational node. + * Remove direct edges between non-data process instances, if there is a path between them through data process instances. + * This prevents the graph from being cluttered with effectively duplicate edges. + * @param urn Urn for which to remove parent edges. + * @param direction Direction to look for parents. + * @param context Lineage node context. + * @param entityRegistry EntityRegistry, used to get EntityType from an urn. + */ +export default function pruneAllDuplicateEdges( + urn: string, + direction: LineageDirection | null, + context: Pick, + entityRegistry: EntityRegistry, +) { + let changed = false; + Object.values(HideOption).forEach((hideOption) => { + changed ||= pruneDuplicateEdges(urn, direction, hideOption, context, entityRegistry); + }); + if (changed) { + context.setDisplayVersion(([version, nodes]) => [version + 1, nodes]); + } +} + +/** + * Remove direct edges between a certain set of "excluded" nodes, if there is a path between them through only "included" nodes. + */ +export function pruneDuplicateEdges( + urn: string, + direction: LineageDirection | null, + hideOption: HideOption, + context: Pick, + entityRegistry: EntityRegistry, +): boolean { + const { edges } = context; + const neighbors: Record> = { + [LineageDirection.Downstream]: new Set(), + [LineageDirection.Upstream]: new Set(), + }; + + const includeUrn = hideOptionIncludeUrnFunctions[hideOption]; + const isUrnIncluded = includeUrn(urn, entityRegistry); + + function getNeighbors(d: LineageDirection) { + return getNeighborsByFunction(urn, d, includeUrn, context, entityRegistry); + } + + if (direction) { + neighbors[direction] = getNeighbors(direction); + } else { + neighbors[LineageDirection.Upstream] = getNeighbors(LineageDirection.Upstream); + neighbors[LineageDirection.Downstream] = getNeighbors(LineageDirection.Downstream); + } + + let changed = false; + if (isUrnIncluded) { + neighbors[LineageDirection.Upstream].forEach((source) => { + neighbors[LineageDirection.Downstream].forEach((destination) => { + const edge = edges.get(createEdgeId(source, destination)); + if (edge?.isDisplayed) { + edge.isDisplayed = false; + changed = true; + } + }); + }); + } else { + Object.values(LineageDirection).forEach((d) => { + neighbors[d].forEach((source) => { + const edge = edges.get(getEdgeId(urn, source, d)); + if (edge?.isDisplayed) { + edge.isDisplayed = false; + changed = true; + } + }); + }); + } + + return changed; +} + +/** + * Get the non-transformational nodes that are reachable from `urn` in `direction` via a transformational path. + * @param urn Urn for which to get neighbors. + * @param direction Direction to look for neighbors. + * @param includeUrn Function to determine if a node should be included, based on its urn. + * @param adjacencyList Adjacency list of the lineage graph. + * @param entityRegistry EntityRegistry, used to get EntityType from an urn. + */ +function getNeighborsByFunction( + urn: string, + direction: LineageDirection, + includeUrn: (urn: string, entityRegistry: EntityRegistry) => boolean, + { adjacencyList }: Pick, + entityRegistry: EntityRegistry, +) { + const neighbors = new Set(); + // If urn is included, then direct neighbors can be included + const stack = includeUrn(urn, entityRegistry) + ? [urn] + : Array.from(adjacencyList[direction].get(urn) || []).filter((p) => includeUrn(p, entityRegistry)); + const seen = new Set(stack); + for (let u = stack.pop(); u; u = stack.pop()) { + Array.from(adjacencyList[direction].get(u) || []).forEach((parent) => { + if (includeUrn(parent, entityRegistry)) { + if (!seen.has(parent)) { + stack.push(parent); + seen.add(parent); + } + } else { + neighbors.add(parent); + } + }); + } + return neighbors; +} diff --git a/datahub-web-react/src/app/lineageV2/useBulkEntityLineage.ts b/datahub-web-react/src/app/lineageV2/useBulkEntityLineage.ts index 1cfc6c892fc3e4..54769a0a1243fe 100644 --- a/datahub-web-react/src/app/lineageV2/useBulkEntityLineage.ts +++ b/datahub-web-react/src/app/lineageV2/useBulkEntityLineage.ts @@ -1,3 +1,4 @@ +import pruneAllDuplicateEdges from '@app/lineageV2/pruneAllDuplicateEdges'; import { useAppConfig } from '@app/useAppConfig'; import { useCallback, useContext, useEffect, useState } from 'react'; import { useGetBulkEntityLineageV2Query } from '../../graphql/lineage.generated'; @@ -18,7 +19,7 @@ import { useIgnoreSchemaFieldStatus, } from './common'; import { FetchedEntityV2Relationship } from './types'; -import { addQueryNodes, pruneDuplicateEdges, setEntityNodeDefault } from './useSearchAcrossLineage'; +import { addQueryNodes, setEntityNodeDefault } from './useSearchAcrossLineage'; const BATCH_SIZE = 10; @@ -100,7 +101,7 @@ export default function useBulkEntityLineage(shownUrns: string[]): (urn: string) const entityRegistry = useEntityRegistryV2(); useEffect(() => { - const smallContext = { nodes, edges, adjacencyList }; + const smallContext = { nodes, edges, adjacencyList, setDisplayVersion }; let changed = false; data?.entities?.forEach((rawEntity) => { if (!rawEntity) return; @@ -122,7 +123,7 @@ export default function useBulkEntityLineage(shownUrns: string[]): (urn: string) entity.upstreamRelationships?.forEach((relationship) => { processEdge(node, relationship, LineageDirection.Upstream, smallContext); }); - pruneDuplicateEdges(node.urn, null, smallContext, entityRegistry); + pruneAllDuplicateEdges(node.urn, null, smallContext, entityRegistry); } } }); diff --git a/datahub-web-react/src/app/lineageV2/useColumnHighlighting.ts b/datahub-web-react/src/app/lineageV2/useColumnHighlighting.ts index cd44609e0630df..82de5330346429 100644 --- a/datahub-web-react/src/app/lineageV2/useColumnHighlighting.ts +++ b/datahub-web-react/src/app/lineageV2/useColumnHighlighting.ts @@ -33,8 +33,17 @@ export default function useColumnHighlighting( } { const entityRegistry = useEntityRegistryV2(); const { setEdges } = useReactFlow(); - const { nodes, adjacencyList, edges, rootUrn, rootType, nodeVersion, columnEdgeVersion, hideTransformations } = - useContext(LineageNodesContext); + const { + nodes, + adjacencyList, + edges, + rootUrn, + rootType, + nodeVersion, + columnEdgeVersion, + hideTransformations, + showDataProcessInstances, + } = useContext(LineageNodesContext); const { cllHighlightedNodes, highlightedColumns, columnEdges } = useMemo(() => { const displayedNodeIds = new Set(shownUrns); @@ -73,7 +82,7 @@ export default function useColumnHighlighting( }), 0, ); - }, [nodeVersion, hideTransformations, columnEdges, setEdges]); + }, [nodeVersion, hideTransformations, showDataProcessInstances, columnEdges, setEdges]); return { cllHighlightedNodes, highlightedColumns }; } diff --git a/datahub-web-react/src/app/lineageV2/useComputeGraph/filterNodes.ts b/datahub-web-react/src/app/lineageV2/useComputeGraph/filterNodes.ts index 224986962b89aa..edfd36dd53e0ea 100644 --- a/datahub-web-react/src/app/lineageV2/useComputeGraph/filterNodes.ts +++ b/datahub-web-react/src/app/lineageV2/useComputeGraph/filterNodes.ts @@ -1,19 +1,22 @@ +import { globalEntityRegistryV2 } from '@app/EntityRegistryProvider'; import { addToAdjacencyList, EdgeId, getEdgeId, isGhostEntity, isTransformational, + isUrnQuery, LineageAuditStamp, LineageEdge, NodeContext, parseEdgeId, setDefault, } from '@app/lineageV2/common'; -import { LineageDirection } from '@types'; +import { EntityType, LineageDirection } from '@types'; export interface HideNodesConfig { hideTransformations: boolean; + hideDataProcessInstances: boolean; hideGhostEntities: boolean; ignoreSchemaFieldStatus: boolean; } @@ -25,7 +28,7 @@ type ContextSubset = Pick; */ export default function hideNodes( rootUrn: string, - { hideTransformations, hideGhostEntities, ignoreSchemaFieldStatus }: HideNodesConfig, + { hideTransformations, hideDataProcessInstances, hideGhostEntities, ignoreSchemaFieldStatus }: HideNodesConfig, { nodes, edges, adjacencyList }: ContextSubset, ): ContextSubset { let newNodes = nodes; @@ -51,6 +54,25 @@ export default function hideNodes( adjacencyList: newAdjacencyList, })); } + if (hideDataProcessInstances) { + // Note: Will only pick one query node if there is lineage t1 -> q1 -> dpi1 -> q2 -> t2 + // Currently data process instances can't have lineage to queries so this is fine + newNodes = new Map( + Array.from(newNodes).filter( + ([urn, node]) => urn === rootUrn || node?.entity?.type !== EntityType.DataProcessInstance, + ), + ); + ({ newEdges, newAdjacencyList } = connectEdges(rootUrn, { + nodes: newNodes, + edges: newEdges, + adjacencyList: newAdjacencyList, + })); + } + ({ newEdges, newAdjacencyList } = removeHiddenEdges({ + nodes: newNodes, + adjacencyList: newAdjacencyList, + edges: newEdges, + })); return { nodes: newNodes, edges: newEdges, adjacencyList: newAdjacencyList }; } @@ -99,24 +121,30 @@ function connectEdges(rootUrn: string, { nodes, edges, adjacencyList }: ContextS seen.add(id); adjacencyList[direction].get(id)?.forEach((neighbor) => { + if (isUrnQuery(neighbor, globalEntityRegistryV2)) { + return; + } if (nodes.has(neighbor)) { addToAdjacencyList(newAdjacencyList, direction, id, neighbor); const edgeId = getEdgeId(id, neighbor, direction); - // isDisplayed always true -- only set to false right now to deduplicate edges through dbt - newEdges.set(edgeId, { ...edges.get(edgeId), via: undefined, isDisplayed: true }); + const existingEdge = newEdges.get(edgeId); + newEdges.set(edgeId, mergeEdges(edges.get(edgeId), existingEdge)); buildNewAdjacencyList(neighbor, direction); } else { buildNewAdjacencyList(neighbor, direction)?.forEach((child) => { addToAdjacencyList(newAdjacencyList, direction, id, child); const edgeId = getEdgeId(id, child, direction); const firstEdge = edges.get(getEdgeId(id, neighbor, direction)); - const secondEdge = edges.get(getEdgeId(neighbor, child, direction)); - newEdges.set(edgeId, { + const secondEdge = newEdges.get(getEdgeId(neighbor, child, direction)); + const existingEdge = newEdges.get(edgeId); + const newEdge = { isManual: (firstEdge?.isManual || secondEdge?.isManual) ?? false, created: getLatestTimestamp(firstEdge?.created, secondEdge?.created), updated: getLatestTimestamp(firstEdge?.updated, secondEdge?.updated), - isDisplayed: true, - }); + isDisplayed: (firstEdge?.isDisplayed && secondEdge?.isDisplayed) ?? false, + via: firstEdge?.via || secondEdge?.via, + }; + newEdges.set(edgeId, mergeEdges(newEdge, existingEdge)); }); } }); @@ -126,9 +154,50 @@ function connectEdges(rootUrn: string, { nodes, edges, adjacencyList }: ContextS buildNewAdjacencyList(rootUrn, LineageDirection.Upstream); seen.clear(); buildNewAdjacencyList(rootUrn, LineageDirection.Downstream); + + newEdges.forEach((edge, edgeId) => { + const [upstream, downstream] = parseEdgeId(edgeId); + if (edge.via && nodes.has(edge.via) && nodes.get(edge.via)?.type === EntityType.Query) { + setDefault(newAdjacencyList[LineageDirection.Upstream], edge.via, new Set()).add(upstream); + setDefault(newAdjacencyList[LineageDirection.Downstream], edge.via, new Set()).add(downstream); + } + }); + return { newAdjacencyList, newEdges }; } +/** Merge two edges, each representing a different path between two nodes. */ +function mergeEdges(edgeA?: LineageEdge, edgeB?: LineageEdge): LineageEdge { + return { + isManual: edgeA?.isManual && edgeB?.isManual, + created: getLatestTimestamp(edgeA?.created, edgeB?.created), + updated: getLatestTimestamp(edgeA?.updated, edgeB?.updated), + isDisplayed: (edgeA?.isDisplayed || edgeB?.isDisplayed) ?? false, + via: edgeA?.via || edgeB?.via, + }; +} + +function removeHiddenEdges({ edges }: ContextSubset) { + const newEdges = new Map(); + const newAdjacencyList: NodeContext['adjacencyList'] = { + [LineageDirection.Upstream]: new Map(), + [LineageDirection.Downstream]: new Map(), + }; + + edges.forEach((edge, edgeId) => { + const [upstream, downstream] = parseEdgeId(edgeId); + if (edge.isDisplayed) { + addToAdjacencyList(newAdjacencyList, LineageDirection.Upstream, downstream, upstream); + newEdges.set(edgeId, edge); + if (edge.via) { + setDefault(newAdjacencyList[LineageDirection.Upstream], edge.via, new Set()).add(upstream); + setDefault(newAdjacencyList[LineageDirection.Downstream], edge.via, new Set()).add(downstream); + } + } + }); + return { newEdges, newAdjacencyList }; +} + function getLatestTimestamp( a: LineageAuditStamp | undefined, b: LineageAuditStamp | undefined, diff --git a/datahub-web-react/src/app/lineageV2/useComputeGraph/getDisplayedNodes.ts b/datahub-web-react/src/app/lineageV2/useComputeGraph/getDisplayedNodes.ts index 81d213a158cd53..cfe4e442884b0b 100644 --- a/datahub-web-react/src/app/lineageV2/useComputeGraph/getDisplayedNodes.ts +++ b/datahub-web-react/src/app/lineageV2/useComputeGraph/getDisplayedNodes.ts @@ -254,7 +254,7 @@ function getTransformationalNodes( } if (nodesInBetween.has(child) || leafUrns.has(child)) { const edge = edges.get(getEdgeId(urn, child, direction)); - if (edge?.isDisplayed && edge?.via) { + if (edge?.via) { const queryNode = nodes.get(edge.via); if (queryNode) { result.push(queryNode); diff --git a/datahub-web-react/src/app/lineageV2/useComputeGraph/useComputeGraph.tsx b/datahub-web-react/src/app/lineageV2/useComputeGraph/useComputeGraph.tsx index eebcc84b3d9bda..4d1040ded3896d 100644 --- a/datahub-web-react/src/app/lineageV2/useComputeGraph/useComputeGraph.tsx +++ b/datahub-web-react/src/app/lineageV2/useComputeGraph/useComputeGraph.tsx @@ -27,6 +27,7 @@ export default function useComputeGraph(urn: string, type: EntityType): Processe dataVersion, displayVersion, hideTransformations, + showDataProcessInstances, showGhostEntities, } = useContext(LineageNodesContext); const entityRegistry = useEntityRegistryV2(); @@ -56,6 +57,7 @@ export default function useComputeGraph(urn: string, type: EntityType): Processe const config: HideNodesConfig = { hideTransformations, + hideDataProcessInstances: !showDataProcessInstances, hideGhostEntities: !showGhostEntities, ignoreSchemaFieldStatus, }; @@ -88,6 +90,7 @@ export default function useComputeGraph(urn: string, type: EntityType): Processe displayVersionNumber, hideTransformations, prevHideTransformations, + showDataProcessInstances, showGhostEntities, ignoreSchemaFieldStatus, dataVersion, diff --git a/datahub-web-react/src/app/lineageV2/useSearchAcrossLineage.ts b/datahub-web-react/src/app/lineageV2/useSearchAcrossLineage.ts index f114eac6288ee3..a1eba9ba01e5f3 100644 --- a/datahub-web-react/src/app/lineageV2/useSearchAcrossLineage.ts +++ b/datahub-web-react/src/app/lineageV2/useSearchAcrossLineage.ts @@ -1,20 +1,18 @@ +import pruneAllDuplicateEdges from '@app/lineageV2/pruneAllDuplicateEdges'; import { useEffect, useState } from 'react'; import { useSearchAcrossLineageStructureLazyQuery } from '../../graphql/search.generated'; import { Entity, EntityType, LineageDirection, Maybe, SearchAcrossLineageInput } from '../../types.generated'; -import EntityRegistry from '../entityV2/EntityRegistry'; import { DBT_URN } from '../ingest/source/builder/constants'; import { useGetLineageTimeParams } from '../lineage/utils/useGetLineageTimeParams'; import { DEGREE_FILTER_NAME } from '../search/utils/constants'; import { useEntityRegistryV2 } from '../useEntityRegistry'; import { addToAdjacencyList, - createEdgeId, FetchStatus, Filters, getEdgeId, isQuery, isTransformational, - isUrnTransformational, LINEAGE_FILTER_PAGINATION, LineageEntity, NodeContext, @@ -80,6 +78,7 @@ export default function useSearchAcrossLineage( platforms: [DBT_URN], }, { entityType: EntityType.DataJob }, + { entityType: EntityType.DataProcessInstance }, ], }, searchFlags: { @@ -101,7 +100,7 @@ export default function useSearchAcrossLineage( }, [fetchLineage, lazy]); useEffect(() => { - const smallContext = { nodes, edges, adjacencyList }; + const smallContext = { nodes, edges, adjacencyList, setDisplayVersion }; let addedNode = false; data?.searchAcrossLineage?.searchResults.forEach((result) => { @@ -137,7 +136,7 @@ export default function useSearchAcrossLineage( } if (data) { - pruneDuplicateEdges(urn, direction, smallContext, entityRegistry); + pruneAllDuplicateEdges(urn, direction, smallContext, entityRegistry); processed.add(urn); if (addedNode) setNodeVersion((version) => version + 1); @@ -162,94 +161,6 @@ export default function useSearchAcrossLineage( return { fetchLineage, processed: processed.has(urn) }; } -/** - * Remove direct edges between non-transformational nodes, if there is a path between them through only transformational nodes. - * This prevents the graph from being cluttered with effectively duplicate edges. - * @param urn Urn for which to remove parent edges. - * @param direction Direction to look for parents. - * @param context Lineage node context. - * @param entityRegistry EntityRegistry, used to get EntityType from an urn. - */ -export function pruneDuplicateEdges( - urn: string, - direction: LineageDirection | null, - context: Pick, - entityRegistry: EntityRegistry, -) { - const { edges } = context; - const neighbors: Record> = { - [LineageDirection.Downstream]: new Set(), - [LineageDirection.Upstream]: new Set(), - }; - - const urnIsTransformational = isUrnTransformational(urn, entityRegistry); - function getNeighbors(d: LineageDirection) { - return getNonTransformationalNeighbors(urn, d, urnIsTransformational, context, entityRegistry); - } - - if (direction) { - neighbors[direction] = getNeighbors(direction); - } else { - neighbors[LineageDirection.Upstream] = getNeighbors(LineageDirection.Upstream); - neighbors[LineageDirection.Downstream] = getNeighbors(LineageDirection.Downstream); - } - - if (urnIsTransformational) { - neighbors[LineageDirection.Upstream].forEach((source) => { - neighbors[LineageDirection.Downstream].forEach((destination) => { - const edge = edges.get(createEdgeId(source, destination)); - if (edge?.isDisplayed) { - edge.isDisplayed = false; - } - }); - }); - } else { - Object.values(LineageDirection).forEach((d) => { - neighbors[d].forEach((source) => { - const edge = edges.get(getEdgeId(urn, source, d)); - if (edge?.isDisplayed) { - edge.isDisplayed = false; - } - }); - }); - } -} - -/** - * Get the non-transformational nodes that are reachable from `urn` in `direction` via a transformational path. - * @param urn Urn for which to get neighbors. - * @param direction Direction to look for neighbors. - * @param includeDirect If false, only include non-transformational neighbors through a transformational node. - * @param adjacencyList Adjacency list of the lineage graph. - * @param entityRegistry EntityRegistry, used to get EntityType from an urn. - */ -function getNonTransformationalNeighbors( - urn: string, - direction: LineageDirection, - includeDirect: boolean, - { adjacencyList }: Pick, - entityRegistry: EntityRegistry, -) { - const neighbors = new Set(); - const stack = includeDirect - ? [urn] - : Array.from(adjacencyList[direction].get(urn) || []).filter((p) => isUrnTransformational(p, entityRegistry)); - const seen = new Set(stack); - for (let u = stack.pop(); u; u = stack.pop()) { - Array.from(adjacencyList[direction].get(u) || []).forEach((parent) => { - if (isUrnTransformational(parent, entityRegistry)) { - if (!seen.has(parent)) { - stack.push(parent); - seen.add(parent); - } - } else { - neighbors.add(parent); - } - }); - } - return neighbors; -} - export function setEntityNodeDefault( urn: string, type: EntityType, diff --git a/datahub-web-react/src/app/previewV2/EntityHeader.tsx b/datahub-web-react/src/app/previewV2/EntityHeader.tsx index be986d6293eac2..9ebf30732b9d7b 100644 --- a/datahub-web-react/src/app/previewV2/EntityHeader.tsx +++ b/datahub-web-react/src/app/previewV2/EntityHeader.tsx @@ -96,12 +96,12 @@ const EntityHeader: React.FC = ({ {previewType === PreviewType.HOVER_CARD ? ( - {name || ''} + {name || urn} ) : ( - + )} diff --git a/datahub-web-react/src/graphql/dataProcessInstance.graphql b/datahub-web-react/src/graphql/dataProcessInstance.graphql index 442f8db0a933b2..548aa746b3a85b 100644 --- a/datahub-web-react/src/graphql/dataProcessInstance.graphql +++ b/datahub-web-react/src/graphql/dataProcessInstance.graphql @@ -67,6 +67,10 @@ fragment processInstanceRelationshipResults on EntityRelationshipsResult { fragment dataProcessInstanceFields on DataProcessInstance { urn type + exists + status { + removed + } parentContainers { ...parentContainersFields } @@ -78,7 +82,7 @@ fragment dataProcessInstanceFields on DataProcessInstance { } properties { name - createdTS: created { + created { time actor } @@ -88,6 +92,7 @@ fragment dataProcessInstanceFields on DataProcessInstance { } } mlTrainingRunProperties { + id outputUrls trainingMetrics { name @@ -100,67 +105,79 @@ fragment dataProcessInstanceFields on DataProcessInstance { value } } + optionalPlatform: platform { + ...platformFields + } dataPlatformInstance { ...dataPlatformInstanceFields } - state(startTimeMillis: null, endTimeMillis: null, limit: 1) { - status - attempt - result { - resultType - nativeResultType - } - timestampMillis - durationMillis - } - relationships(input: { types: ["InstanceOf", "Consumes", "Produces"], direction: OUTGOING, start: 0, count: 50 }) { - ...processInstanceRelationshipResults - } -} - -query getDataProcessInstance($urn: String!) { - dataProcessInstance(urn: $urn) { + parentTemplate { urn type - parentContainers { - ...parentContainersFields - } - subTypes { - typeNames - } - container { - ...entityContainer - } - name - properties { + # TODO: Clean up fields below; support DataFlow + ... on Dataset { name - created { - time - actor + properties { + name + description + qualifiedName + } + editableProperties { + description + } + platform { + ...platformFields + } + subTypes { + typeNames + } + status { + removed } } - mlTrainingRunProperties { - id - outputUrls - trainingMetrics { + ... on DataJob { + urn + type + dataFlow { + ...nonRecursiveDataFlowFields + } + jobId + properties { name description - value + externalUrl + customProperties { + key + value + } } - hyperParams { - name + deprecation { + ...deprecationFields + } + dataPlatformInstance { + ...dataPlatformInstanceFields + } + subTypes { + typeNames + } + editableProperties { description - value + } + status { + removed } } + } +} + +query getDataProcessInstance($urn: String!) { + dataProcessInstance(urn: $urn) { + ...dataProcessInstanceFields relationships( input: { types: ["InstanceOf", "Consumes", "Produces"], direction: OUTGOING, start: 0, count: 50 } ) { ...processInstanceRelationshipResults } - dataPlatformInstance { - ...dataPlatformInstanceFields - } state(startTimeMillis: null, endTimeMillis: null, limit: 1) { status attempt diff --git a/datahub-web-react/src/graphql/lineage.graphql b/datahub-web-react/src/graphql/lineage.graphql index 2a89789f1c50d1..5daf200fef5fc7 100644 --- a/datahub-web-react/src/graphql/lineage.graphql +++ b/datahub-web-react/src/graphql/lineage.graphql @@ -716,6 +716,9 @@ fragment entityLineageV2 on Entity { ...platformFields } } + ... on DataProcessInstance { + ...dataProcessInstanceFields + } ... on Domain { id properties { diff --git a/datahub-web-react/src/images/mlflowlogo2.png b/datahub-web-react/src/images/mlflowlogo2.png new file mode 100644 index 00000000000000..d2b7dbea783415 Binary files /dev/null and b/datahub-web-react/src/images/mlflowlogo2.png differ diff --git a/metadata-ingestion/tests/integration/openapi/openapi_mces_golden.json b/metadata-ingestion/tests/integration/openapi/openapi_mces_golden.json index ffa8d43a6a5ab4..f74a0bb9e0102d 100755 --- a/metadata-ingestion/tests/integration/openapi/openapi_mces_golden.json +++ b/metadata-ingestion/tests/integration/openapi/openapi_mces_golden.json @@ -20,7 +20,7 @@ "com.linkedin.pegasus2avro.common.InstitutionalMemory": { "elements": [ { - "url": "https://raw.githubusercontent.com/OAI/OpenAPI-Specification/main/tests/", + "url": "https://raw.githubusercontent.com/OAI/learn.openapis.org/refs/heads/main/examples/", "description": "Link to call for the dataset.", "createStamp": { "time": 1586847600, @@ -96,7 +96,7 @@ "com.linkedin.pegasus2avro.common.InstitutionalMemory": { "elements": [ { - "url": "https://raw.githubusercontent.com/OAI/OpenAPI-Specification/main/tests/v2", + "url": "https://raw.githubusercontent.com/OAI/learn.openapis.org/refs/heads/main/examples/v2", "description": "Link to call for the dataset.", "createStamp": { "time": 1586847600, @@ -183,4 +183,4 @@ "lastRunId": "no-run-id-provided" } } -] \ No newline at end of file +] diff --git a/metadata-ingestion/tests/integration/openapi/openapi_to_file.yml b/metadata-ingestion/tests/integration/openapi/openapi_to_file.yml index 8575c23239461e..937ec54e9f1c27 100644 --- a/metadata-ingestion/tests/integration/openapi/openapi_to_file.yml +++ b/metadata-ingestion/tests/integration/openapi/openapi_to_file.yml @@ -2,12 +2,11 @@ source: type: openapi config: name: test_openapi - url: https://raw.githubusercontent.com/OAI/OpenAPI-Specification/main/tests/ - swagger_file: v3.0/pass/api-with-examples.yaml + url: https://raw.githubusercontent.com/OAI/learn.openapis.org/refs/heads/main/examples/ + swagger_file: v3.0/api-with-examples.yaml sink: type: file config: filename: "/tmp/openapi_mces.json" - diff --git a/metadata-service/configuration/src/main/resources/bootstrap_mcps/data-platforms.yaml b/metadata-service/configuration/src/main/resources/bootstrap_mcps/data-platforms.yaml index 2230d552ed4c0e..4c4a7c2183073c 100644 --- a/metadata-service/configuration/src/main/resources/bootstrap_mcps/data-platforms.yaml +++ b/metadata-service/configuration/src/main/resources/bootstrap_mcps/data-platforms.yaml @@ -417,7 +417,7 @@ name: mlflow displayName: MLflow type: OTHERS - logoUrl: "/assets/platforms/mlflowlogo.png" + logoUrl: "/assets/platforms/mlflowlogo2.png" - entityUrn: urn:li:dataPlatform:glue entityType: dataPlatform aspectName: dataPlatformInfo diff --git a/smoke-test/tests/cypress/cypress/e2e/lineageV2/v2_lineage_column_level.js b/smoke-test/tests/cypress/cypress/e2e/lineageV2/v2_lineage_column_level.js index a82ac125bf820b..10fcce58f8d2b9 100644 --- a/smoke-test/tests/cypress/cypress/e2e/lineageV2/v2_lineage_column_level.js +++ b/smoke-test/tests/cypress/cypress/e2e/lineageV2/v2_lineage_column_level.js @@ -5,7 +5,7 @@ const DATASET_URN = const clickDownAndUpArrow = (asset, arrow) => { cy.contains(".react-flow__node-lineage-entity", asset) .find(`svg[data-testid="${arrow}"]`) - .click(); + .click({ force: true }); }; describe("column-level lineage graph test", () => {