Skip to content

Commit

Permalink
Revert "Revert "fix wrong merge""
Browse files Browse the repository at this point in the history
This reverts commit f0ceb10.
  • Loading branch information
eric-wang-1990 committed Jan 28, 2025
1 parent f0ceb10 commit dacf1cf
Show file tree
Hide file tree
Showing 49 changed files with 524 additions and 1,149 deletions.
25 changes: 5 additions & 20 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,25 +10,7 @@

## dbt-databricks 1.9.2 (TBD)

### Under the Hood

- Refactor global state reading ([888](https://github.com/databricks/dbt-databricks/pull/888))
- Switch to relation.render() for string interpolation ([903](https://github.com/databricks/dbt-databricks/pull/903))
- Ensure retry defaults for PySQL ([907](https://github.com/databricks/dbt-databricks/pull/907))

## dbt-databricks 1.9.1 (December 16, 2024)

### Features

- Merge strategy now supports the `update set ...` action with the explicit list of updates for `when not matched by source` ([866](https://github.com/databricks/dbt-databricks/pull/866)) (thanks @mi-volodin).

### Under the Hood

- Removed pins for pandas and pydantic to ease user burdens ([874](https://github.com/databricks/dbt-databricks/pull/874))
- Add more relation types to make codegen happy ([875](https://github.com/databricks/dbt-databricks/pull/875))
- add UP ruleset ([865](https://github.com/databricks/dbt-databricks/pull/865))

## dbt-databricks 1.9.0 (December 9, 2024)
## dbt-databricks 1.9.0 (TBD)

### Features

Expand All @@ -53,7 +35,6 @@

- Replace array indexing with 'get' in split_part so as not to raise exception when indexing beyond bounds ([839](https://github.com/databricks/dbt-databricks/pull/839))
- Set queue enabled for Python notebook jobs ([856](https://github.com/databricks/dbt-databricks/pull/856))
- Ensure columns that are added get backticked ([859](https://github.com/databricks/dbt-databricks/pull/859))

### Under the Hood

Expand All @@ -64,7 +45,11 @@
- Prepare for python typing deprecations ([837](https://github.com/databricks/dbt-databricks/pull/837))
- Fix behavior flag use in init of DatabricksAdapter (thanks @VersusFacit!) ([836](https://github.com/databricks/dbt-databricks/pull/836))
- Restrict pydantic to V1 per dbt Labs' request ([843](https://github.com/databricks/dbt-databricks/pull/843))
<<<<<<< HEAD
- Switching to Ruff for formatting and linting ([847](https://github.com/databricks/dbt-databricks/pull/847)
=======
- Switching to Ruff for formatting and linting ([847](https://github.com/databricks/dbt-databricks/pull/847))
>>>>>>> parent of f1065d27 (resolve conflict)
- Switching to Hatch and pyproject.toml for project config ([853](https://github.com/databricks/dbt-databricks/pull/853))

## dbt-databricks 1.8.7 (October 10, 2024)
Expand Down
57 changes: 1 addition & 56 deletions dbt/adapters/databricks/api_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,7 @@ def _poll_api(


@dataclass(frozen=True, eq=True, unsafe_hash=True)
class CommandExecution:
class CommandExecution(object):
command_id: str
context_id: str
cluster_id: str
Expand Down Expand Up @@ -459,60 +459,6 @@ def run(self, job_id: str, enable_queueing: bool = True) -> str:
return response_json["run_id"]


class DltPipelineApi(PollableApi):
def __init__(self, session: Session, host: str, polling_interval: int):
super().__init__(session, host, "/api/2.0/pipelines", polling_interval, 60 * 60)

def poll_for_completion(self, pipeline_id: str) -> None:
self._poll_api(
url=f"/{pipeline_id}",
params={},
get_state_func=lambda response: response.json()["state"],
terminal_states={"IDLE", "FAILED", "DELETED"},
expected_end_state="IDLE",
unexpected_end_state_func=self._get_exception,
)

def _get_exception(self, response: Response) -> None:
response_json = response.json()
cause = response_json.get("cause")
if cause:
raise DbtRuntimeError(f"Pipeline {response_json.get('pipeline_id')} failed: {cause}")
else:
latest_update = response_json.get("latest_updates")[0]
last_error = self.get_update_error(response_json.get("pipeline_id"), latest_update)
raise DbtRuntimeError(
f"Pipeline {response_json.get('pipeline_id')} failed: {last_error}"
)

def get_update_error(self, pipeline_id: str, update_id: str) -> str:
response = self.session.get(f"/{pipeline_id}/events")
if response.status_code != 200:
raise DbtRuntimeError(
f"Error getting pipeline event info for {pipeline_id}: {response.text}"
)

events = response.json().get("events", [])
update_events = [
e
for e in events
if e.get("event_type", "") == "update_progress"
and e.get("origin", {}).get("update_id") == update_id
]

error_events = [
e
for e in update_events
if e.get("details", {}).get("update_progress", {}).get("state", "") == "FAILED"
]

msg = ""
if error_events:
msg = error_events[0].get("message", "")

return msg


class DatabricksApiClient:
def __init__(
self,
Expand All @@ -534,7 +480,6 @@ def __init__(
self.job_runs = JobRunsApi(session, host, polling_interval, timeout)
self.workflows = WorkflowJobApi(session, host)
self.workflow_permissions = JobPermissionsApi(session, host)
self.dlt_pipelines = DltPipelineApi(session, host, polling_interval)

@staticmethod
def create(
Expand Down
18 changes: 2 additions & 16 deletions dbt/adapters/databricks/column.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
from dataclasses import dataclass
from typing import Any, ClassVar, Optional
from typing import ClassVar, Optional

from dbt.adapters.databricks.utils import quote
from dbt.adapters.spark.column import SparkColumn


Expand Down Expand Up @@ -50,17 +49,4 @@ def render_for_create(self) -> str:
return column_str

def __repr__(self) -> str:
return f"<DatabricksColumn {self.name} ({self.data_type})>"

@staticmethod
def get_name(column: dict[str, Any]) -> str:
name = column["name"]
return quote(name) if column.get("quote", False) else name

@staticmethod
def format_remove_column_list(columns: list["DatabricksColumn"]) -> str:
return ", ".join([quote(c.name) for c in columns])

@staticmethod
def format_add_column_list(columns: list["DatabricksColumn"]) -> str:
return ", ".join([f"{quote(c.name)} {c.data_type}" for c in columns])
return "<DatabricksColumn {} ({})>".format(self.name, self.data_type)
Loading

0 comments on commit dacf1cf

Please sign in to comment.