Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enable/disable preview for all the datasets when publishing Kedro-Viz from UI #1895

Merged
merged 52 commits into from
Jun 12, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
52 commits
Select commit Hold shift + click to select a range
74e0371
Add options for cli common with disable preview of datasets
jitu5 May 3, 2024
5398a11
New approach for disable preview
jitu5 May 7, 2024
4da365f
Disable preview for cli and lint fix
jitu5 May 7, 2024
9f2afa2
deploy api with preview
jitu5 May 8, 2024
35f1436
Removing UI changes
jitu5 May 8, 2024
85e1a2d
Removing UI changes
jitu5 May 8, 2024
f8f96cf
Disable preview from UI
jitu5 May 8, 2024
dc51032
Code review suggestions
jitu5 May 10, 2024
f3b8007
temp code removed
jitu5 May 10, 2024
d8ef988
refactor disable preview cli
ravi-kumar-pilla May 15, 2024
3377832
fix lint
ravi-kumar-pilla May 15, 2024
7a31729
fix lint
ravi-kumar-pilla May 15, 2024
028a7f3
Merge branch 'main' of https://github.com/kedro-org/kedro-viz into fe…
ravi-kumar-pilla May 16, 2024
78f44bb
testing 1
ravi-kumar-pilla May 16, 2024
663924c
change help text
ravi-kumar-pilla May 16, 2024
defbf66
change help text
ravi-kumar-pilla May 16, 2024
9195150
update tests
ravi-kumar-pilla May 16, 2024
c4c7375
format help text
ravi-kumar-pilla May 16, 2024
6eaaaeb
change default preview for run
ravi-kumar-pilla May 16, 2024
8372ec3
update release note
ravi-kumar-pilla May 16, 2024
c854392
merged cli version to avoid test failures
ravi-kumar-pilla May 17, 2024
2f67e78
address vale suggestion
ravi-kumar-pilla May 17, 2024
ffdce46
Merge branch 'feature/disable-preview' of https://github.com/kedro-or…
ravi-kumar-pilla May 17, 2024
e465101
fix lint
ravi-kumar-pilla May 17, 2024
12eabd1
fix pytests and linter
ravi-kumar-pilla May 17, 2024
57b68b3
adjust file permission
ravi-kumar-pilla May 17, 2024
ec92a7f
merged new changes
ravi-kumar-pilla May 17, 2024
50ef2a4
update tests
ravi-kumar-pilla May 17, 2024
e523f23
update release note
ravi-kumar-pilla May 17, 2024
4f5c446
update release note
ravi-kumar-pilla May 17, 2024
a4b564b
resolve conflicts
ravi-kumar-pilla May 17, 2024
c815a9b
--include-preview flag added for CLI
jitu5 May 20, 2024
5a43fda
Merge branch 'main' into feature/disable-preview
jitu5 May 20, 2024
0998f7a
Merge branch 'main' into feature/disable-preview
jitu5 May 21, 2024
d06f36e
are_datasets_previewable renamed to is_all_previews_enabled
jitu5 May 21, 2024
73b1944
lint fix
jitu5 May 21, 2024
2294cc6
are_datasets_previewable renamed to is_all_previews_enabled
jitu5 May 21, 2024
148e92d
Merge branch 'feature/disable-preview' into feature/disable-preview-ui
jitu5 May 21, 2024
94e3501
type ignore add for TRANSCODING_SEPARATOR
jitu5 May 21, 2024
036c89e
Merge branch 'feature/disable-preview' into feature/disable-preview-ui
jitu5 May 21, 2024
46dbf28
Merge branch 'main' into feature/disable-preview
jitu5 May 22, 2024
20bbe47
Merge branch 'main' into feature/disable-preview
jitu5 May 29, 2024
6f088b5
File permissions updated
jitu5 May 29, 2024
b80c799
Merge branch 'main' into feature/disable-preview
jitu5 May 30, 2024
788a491
Merge branch 'main' into feature/disable-preview
jitu5 May 31, 2024
57aec4e
Merge branch 'main' into feature/disable-preview
jitu5 May 31, 2024
2f54122
Merge branch 'feature/disable-preview' into feature/disable-preview-ui
jitu5 May 31, 2024
80ad5e7
Merge branch 'main' into feature/disable-preview-ui
jitu5 May 31, 2024
c706625
fix
jitu5 May 31, 2024
0ac6b1b
Merge branch 'main' into feature/disable-preview-ui
jitu5 Jun 7, 2024
9d50abb
All dataset preview toggle works with deploy api
jitu5 Jun 7, 2024
4a82230
CLI flag renamed to --include-previews
jitu5 Jun 11, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions package/kedro_viz/api/rest/requests.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,5 +7,6 @@ class DeployerConfiguration(BaseModel):
"""Credentials for Deployers."""

platform: str
preview: bool
jitu5 marked this conversation as resolved.
Show resolved Hide resolved
endpoint: str
bucket_name: str
12 changes: 6 additions & 6 deletions package/kedro_viz/api/rest/responses.py
Original file line number Diff line number Diff line change
Expand Up @@ -320,7 +320,7 @@ def get_default_response() -> GraphAPIResponse:
)


def get_node_metadata_response(node_id: str):
def get_node_metadata_response(node_id: str, preview: bool = True):
"""API response for `/api/nodes/node_id`."""
node = data_access_manager.nodes.get_node_by_id(node_id)
if not node:
Expand All @@ -333,7 +333,7 @@ def get_node_metadata_response(node_id: str):
return TaskNodeMetadata(task_node=node)

if isinstance(node, DataNode):
return DataNodeMetadata(data_node=node)
return DataNodeMetadata(data_node=node, is_preview_enabled_for_all_nodes=preview)
jitu5 marked this conversation as resolved.
Show resolved Hide resolved

if isinstance(node, TranscodedDataNode):
return TranscodedDataNodeMetadata(transcoded_data_node=node)
Expand Down Expand Up @@ -419,12 +419,12 @@ def save_api_main_response_to_fs(main_path: str, remote_fs: Any):
raise exc


def save_api_node_response_to_fs(nodes_path: str, remote_fs: Any):
def save_api_node_response_to_fs(nodes_path: str, remote_fs: Any, preview: bool):
ravi-kumar-pilla marked this conversation as resolved.
Show resolved Hide resolved
"""Saves API /nodes/{node} response to a directory."""
for nodeId in data_access_manager.nodes.get_node_ids():
try:
write_api_response_to_fs(
f"{nodes_path}/{nodeId}", get_node_metadata_response(nodeId), remote_fs
f"{nodes_path}/{nodeId}", get_node_metadata_response(nodeId, preview), remote_fs
ravi-kumar-pilla marked this conversation as resolved.
Show resolved Hide resolved
)
except Exception as exc: # pragma: no cover
logger.exception(
Expand All @@ -451,7 +451,7 @@ def save_api_pipeline_response_to_fs(pipelines_path: str, remote_fs: Any):
raise exc


def save_api_responses_to_fs(path: str, remote_fs: Any):
def save_api_responses_to_fs(path: str, remote_fs: Any, preview: bool):
"""Saves all Kedro Viz API responses to a directory."""
try:
logger.debug(
Expand All @@ -469,7 +469,7 @@ def save_api_responses_to_fs(path: str, remote_fs: Any):
remote_fs.makedirs(pipelines_path, exist_ok=True)

save_api_main_response_to_fs(main_path, remote_fs)
save_api_node_response_to_fs(nodes_path, remote_fs)
save_api_node_response_to_fs(nodes_path, remote_fs, preview)
save_api_pipeline_response_to_fs(pipelines_path, remote_fs)

except Exception as exc: # pragma: no cover
Expand Down
2 changes: 1 addition & 1 deletion package/kedro_viz/api/rest/router.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ async def get_single_pipeline_data(registered_pipeline_id: str):
async def deploy_kedro_viz(input_values: DeployerConfiguration):
try:
deployer = DeployerFactory.create_deployer(
input_values.platform, input_values.endpoint, input_values.bucket_name
input_values.platform, input_values.preview,input_values.endpoint, input_values.bucket_name
ravi-kumar-pilla marked this conversation as resolved.
Show resolved Hide resolved
)
deployer.deploy()
response = {
Expand Down
3 changes: 2 additions & 1 deletion package/kedro_viz/integrations/deployment/aws_deployer.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,14 @@ class AWSDeployer(BaseDeployer):
_fs (fsspec.filesystem): Filesystem for S3 protocol.
"""

def __init__(self, endpoint, bucket_name):
def __init__(self, preview, endpoint, bucket_name):
jitu5 marked this conversation as resolved.
Show resolved Hide resolved
"""Initialize S3Deployer with endpoint and bucket name.
Args:
endpoint (str): AWS endpoint of the hosted site.
bucket_name (str): Name of the S3 bucket.
"""
super().__init__()
self.preview = preview
self._endpoint = endpoint
self._bucket_name = bucket_name
self._path = f"{_S3_PROTOCOL}://{bucket_name}"
Expand Down
3 changes: 2 additions & 1 deletion package/kedro_viz/integrations/deployment/azure_deployer.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,14 +30,15 @@ class AzureDeployer(BaseDeployer):
_fs (fsspec.filesystem): Filesystem for Azure protocol.
"""

def __init__(self, endpoint, bucket_name):
def __init__(self, preview, endpoint, bucket_name):
"""Initialize AzureBlobStorage with endpoint and bucket name.

Args:
endpoint (str): Azure endpoint of the hosted site.
bucket_name (str): Name of the AzureBlobStorage account.
"""
super().__init__()
self.preview = preview
self._endpoint = endpoint
self._bucket_name = bucket_name
self._path = f"{_AZ_PROTOCOL}://$web"
Expand Down
3 changes: 2 additions & 1 deletion package/kedro_viz/integrations/deployment/base_deployer.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,13 @@ class BaseDeployer(abc.ABC):
"""

def __init__(self):
self.preview = False
self._path = None
self._fs = None

def _upload_api_responses(self):
"""Write API responses to the build."""
save_api_responses_to_fs(self._path, self._fs)
save_api_responses_to_fs(self._path, self._fs, self.preview)

def _ingest_heap_analytics(self):
"""Ingest heap analytics to index file in the build."""
Expand Down
8 changes: 4 additions & 4 deletions package/kedro_viz/integrations/deployment/deployer_factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,15 +12,15 @@ class DeployerFactory:
"""A class to handle creation of deployer class instances."""

@staticmethod
def create_deployer(platform, endpoint=None, bucket_name=None):
def create_deployer(platform, preview, endpoint=None, bucket_name=None):
"""Instantiate Kedro-viz deployer classes"""
platform_name = platform.lower()
if platform_name == "aws":
return AWSDeployer(endpoint, bucket_name)
return AWSDeployer(preview, endpoint, bucket_name)
if platform_name == "azure":
return AzureDeployer(endpoint, bucket_name)
return AzureDeployer(preview, endpoint, bucket_name)
if platform_name == "gcp":
return GCPDeployer(endpoint, bucket_name)
return GCPDeployer(preview, endpoint, bucket_name)
if platform_name == "local":
return LocalDeployer()
raise ValueError(
Expand Down
17 changes: 17 additions & 0 deletions package/kedro_viz/models/flowchart.py
Original file line number Diff line number Diff line change
Expand Up @@ -769,6 +769,17 @@ class DataNodeMetadata(GraphNodeMetadata):
description="The statistics for the data node.",
)

@model_validator(mode="before")
@classmethod
def check_is_preview_enabled_for_all_nodes_exists(cls, values):
assert "is_preview_enabled_for_all_nodes" in values
cls.set_is_preview_enabled_for_all_nodes(values["is_preview_enabled_for_all_nodes"])
return values

@classmethod
def set_is_preview_enabled_for_all_nodes(cls, is_preview_enabled_for_all_nodes):
cls.is_preview_enabled_for_all_nodes = is_preview_enabled_for_all_nodes

@model_validator(mode="before")
@classmethod
def check_data_node_exists(cls, values):
Expand Down Expand Up @@ -806,6 +817,9 @@ def set_run_command(cls, _):
@field_validator("preview")
@classmethod
def set_preview(cls, _):
if not cls.is_preview_enabled_for_all_nodes:
return None

if cls.data_node.is_preview_disabled() or not hasattr(cls.dataset, "preview"):
return None

Expand All @@ -829,6 +843,9 @@ def set_preview(cls, _):
@field_validator("preview_type")
@classmethod
def set_preview_type(cls, _):
if not cls.is_preview_enabled_for_all_nodes:
return None

if cls.data_node.is_preview_disabled() or not hasattr(cls.dataset, "preview"):
return None

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ const ShareableUrlModal = ({ onToggleModal, visible }) => {
'Content-Type': 'application/json',
},
method: 'POST',
body: JSON.stringify(inputValues),
body: JSON.stringify({ ...inputValues, preview: false }), // preview appended to the body to disable the preview data
});
const response = await request.json();

Expand Down
Loading