Skip to content

Commit

Permalink
Increase service spend coverage
Browse files Browse the repository at this point in the history
  • Loading branch information
@fernandobrito
fernandobrito committed Feb 6, 2025
1 parent 81de24a commit 4e7a191
Showing 1 changed file with 29 additions and 128 deletions.
157 changes: 29 additions & 128 deletions models/hourly_spend.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -403,125 +403,36 @@ cloud_services_spend_hourly as (

),

automatic_clustering_spend_hourly as (
other_costs as (
select
hours.hour,
'Automatic Clustering' as service,
null as storage_type,
null as warehouse_name,
null as database_name,
coalesce(
sum(
stg_metering_history.credits_used * daily_rates.effective_rate
),
0
) as spend,
spend as spend_net_cloud_services,
any_value(daily_rates.currency) as currency
from hours
left join {{ ref('stg_metering_history') }} as stg_metering_history on
hours.hour = convert_timezone(
'UTC', stg_metering_history.start_time
)
and stg_metering_history.service_type = 'AUTO_CLUSTERING'
left join {{ ref('daily_rates') }} as daily_rates
on hours.hour::date = daily_rates.date
and daily_rates.service_type = 'AUTOMATIC_CLUSTERING'
and daily_rates.usage_type = 'automatic clustering'
group by 1, 2, 3, 4
),

snowpipe_spend_hourly as (
select
hours.hour,
'Snowpipe' as service,
/* Sometimes Snowflake is inconsistent and the service names in metering_history
do not match the service names in our daily_rates (coming from rate_sheet_daily),
so we rename them to make it match */
case stg_metering_history.service_type
when 'AUTO_CLUSTERING' then 'AUTOMATIC_CLUSTERING'
when 'PIPE' then 'SNOWPIPE'
else stg_metering_history.service_type
end as _service_renamed,

/* Convert it to a more human-readable format
AUTOMATIC_CLUSTERING -> Automatic Clustering
*/
case _service_renamed
when 'MATERIALIZED_VIEW' then 'Materialized Views'
else initcap(replace(_service_renamed, '_', ' '))
end as service,

/* Extract useful information from the row depending on the service type */
null as storage_type,
null as warehouse_name,
null as database_name,
coalesce(
sum(
stg_metering_history.credits_used * daily_rates.effective_rate
),
0
) as spend,
spend as spend_net_cloud_services,
any_value(daily_rates.currency) as currency
from hours
left join {{ ref('stg_metering_history') }} as stg_metering_history on
hours.hour = convert_timezone(
'UTC', stg_metering_history.start_time
)
and stg_metering_history.service_type = 'PIPE'
left join {{ ref('daily_rates') }} as daily_rates
on hours.hour::date = daily_rates.date
and daily_rates.service_type = 'SNOWPIPE'
and daily_rates.usage_type = 'snowpipe'
group by 1, 2, 3, 4
),

snowpipe_streaming_spend_hourly as (
select
hours.hour,
'Snowpipe Streaming' as service,
null as storage_type,
null as warehouse_name,
null as database_name,
coalesce(
sum(
stg_metering_history.credits_used * daily_rates.effective_rate
),
0
) as spend,
spend as spend_net_cloud_services,
any_value(daily_rates.currency) as currency
from hours
left join {{ ref('stg_metering_history') }} as stg_metering_history on
hours.hour = convert_timezone(
'UTC', stg_metering_history.start_time
)
and stg_metering_history.service_type = 'SNOWPIPE_STREAMING'
left join {{ ref('daily_rates') }} as daily_rates
on hours.hour::date = daily_rates.date
and daily_rates.service_type = 'SNOWPIPE_STREAMING'
and daily_rates.usage_type = 'snowpipe streaming'
group by 1, 2, 3, 4
),

query_acceleration_spend_hourly as (
select
hours.hour,
'Query Acceleration' as service,
null as storage_type,
stg_metering_history.name as warehouse_name,
case
when stg_metering_history.service_type = 'QUERY_ACCELERATION'
then stg_metering_history.name
else null
end as warehouse_name,
null as database_name,
coalesce(
sum(
stg_metering_history.credits_used * daily_rates.effective_rate
),
0
) as spend,
spend as spend_net_cloud_services,
any_value(daily_rates.currency) as currency
from hours
left join {{ ref('stg_metering_history') }} as stg_metering_history on
hours.hour = convert_timezone(
'UTC', stg_metering_history.start_time
)
and stg_metering_history.service_type = 'QUERY_ACCELERATION'
left join {{ ref('daily_rates') }} as daily_rates
on hours.hour::date = daily_rates.date
and daily_rates.service_type = 'QUERY_ACCELERATION'
and daily_rates.usage_type = 'query acceleration'
group by 1, 2, 3, 4
),

other_costs as (
select
hours.hour,
initcap(replace(stg_metering_history.service_type, '_', ' ')) as service,
null as storage_type,
null as warehouse_name,
null as database_name,
coalesce(
sum(
stg_metering_history.credits_used * daily_rates.effective_rate
Expand All @@ -538,16 +449,14 @@ other_costs as (

left join {{ ref('daily_rates') }} as daily_rates
on hour::date = daily_rates.date
and stg_metering_history.service_type = daily_rates.service_type
and _service_renamed = daily_rates.service_type

-- Already covered by other CTEs
-- Covered by their own CTEs due to more complex logic or better sources
where stg_metering_history.service_type not in (
'AI_SERVICES', 'AUTO_CLUSTERING',
'PIPE', 'QUERY_ACCELERATION',
'SERVERLESS_TASK', 'WAREHOUSE_METERING', 'WAREHOUSE_METERING_READER'
'AI_SERVICES', 'SERVERLESS_TASK', 'WAREHOUSE_METERING', 'WAREHOUSE_METERING_READER'
)

group by 1, 2, 3, 4
group by 1, 2, 3, 4, 5
),

unioned as (
Expand Down Expand Up @@ -575,17 +484,9 @@ unioned as (
union all
select * from cloud_services_spend_hourly
union all
select * from automatic_clustering_spend_hourly
union all
select * from snowpipe_spend_hourly
union all
select * from snowpipe_streaming_spend_hourly
union all
select * from query_acceleration_spend_hourly
union all
select * from serverless_task_spend_hourly
union all
select * from other_costs
select * exclude (_service_renamed) from other_costs
)

select
Expand Down

0 comments on commit 4e7a191

Please sign in to comment.