Skip to content

Commit

Permalink
Validate M dependencies with data sources, PBIT hardening validation
Browse files Browse the repository at this point in the history
  • Loading branch information
@christianwade
christianwade committed Dec 31, 2019
1 parent ba37cf3 commit 03a330a
Show file tree
Hide file tree
Showing 7 changed files with 116 additions and 51 deletions.
2 changes: 1 addition & 1 deletion BismNormalizer/BismNormalizer/TabularCompare/ComparisonFactory.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ private static Comparison CreateComparisonInitialized(ComparisonInfo comparisonI
Telemetry.TrackEvent("CreateComparisonInitialized", new Dictionary<string, string> { { "App", comparisonInfo.AppName.Replace(" ", "") } });

//If composite models not allowed on AS, check DQ/Import at model level matches:
if (comparisonInfo.ConnectionInfoSource.ServerName != null && !comparisonInfo.ConnectionInfoSource.ServerName.StartsWith("powerbi://") && !Settings.Default.OptionCompositeModelsOverride && comparisonInfo.SourceDirectQuery != comparisonInfo.TargetDirectQuery)
if (comparisonInfo.AppName == "BISM Normalizer" && comparisonInfo.ConnectionInfoTarget.ServerName != null && !comparisonInfo.ConnectionInfoTarget.ServerName.StartsWith("powerbi://") && !Settings.Default.OptionCompositeModelsOverride && comparisonInfo.SourceDirectQuery != comparisonInfo.TargetDirectQuery)
{
throw new ConnectionException($"Mixed DirectQuery settings are not supported for AS skus.\nSource is {(comparisonInfo.SourceDirectQuery ? "On" : "Off")} and target is {(comparisonInfo.TargetDirectQuery ? "On" : "Off")}.");
}
Expand Down
4 changes: 2 additions & 2 deletions BismNormalizer/BismNormalizer/TabularCompare/MultidimensionalMetadata/Measure.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -234,15 +234,15 @@ public List<string> FindMissingCalculationDependencies()
while (whatsLeftOfLine.Contains('[') && whatsLeftOfLine.Contains(']'))
{
int openSquareBracketPosition = whatsLeftOfLine.IndexOf('[', 0);
//brilliant person at microsoft has ]] instead of ]
//someone has ]] instead of ]
int closeSquareBracketPosition = whatsLeftOfLine.Replace("]]", " ").IndexOf(']', openSquareBracketPosition + 1);

if (openSquareBracketPosition < closeSquareBracketPosition - 1)
{
string potentialDependency = whatsLeftOfLine.Substring(openSquareBracketPosition + 1, closeSquareBracketPosition - openSquareBracketPosition - 1);
if (!potentialDependency.Contains('"') && !dependencies.Contains(potentialDependency))
{
//unbelievable: some genius at m$ did a replace on ] with ]]
//someone did a replace on ] with ]]
dependencies.Add(potentialDependency);
}
}
Expand Down
13 changes: 8 additions & 5 deletions BismNormalizer/BismNormalizer/TabularCompare/TabularMetadata/CalcDependencyCollection.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,20 +39,23 @@ private void LookUpDependenciesReferenceFrom(CalcDependencyObjectType objectType
/// <param name="objectType">Type of the object to look up dependencies.</param>
/// <param name="objectName">Name of the object to look up dependencies.</param>
/// <returns></returns>
public CalcDependencyCollection DependenciesReferenceTo(CalcDependencyObjectType referencedObjectType, string referencedObjectName)
public CalcDependencyCollection DependenciesReferenceTo(CalcDependencyObjectType referencedObjectType, string referencedObjectName, string referencedTableName)
{
CalcDependencyCollection returnVal = new CalcDependencyCollection();
LookUpDependenciesReferenceTo(referencedObjectType, referencedObjectName, returnVal);
LookUpDependenciesReferenceTo(referencedObjectType, referencedObjectName, referencedTableName, returnVal);
return returnVal;
}

private void LookUpDependenciesReferenceTo(CalcDependencyObjectType referencedObjectType, string referencedObjectName, CalcDependencyCollection returnVal)
private void LookUpDependenciesReferenceTo(CalcDependencyObjectType referencedObjectType, string referencedObjectName, string referencedTableName, CalcDependencyCollection returnVal)
{
foreach (CalcDependency calcDependency in this)
{
if (calcDependency.ReferencedObjectType == referencedObjectType && calcDependency.ReferencedObjectName == referencedObjectName)
if (
(calcDependency.ReferencedObjectType == referencedObjectType && referencedObjectType != CalcDependencyObjectType.Partition && calcDependency.ReferencedObjectName == referencedObjectName) ||
(calcDependency.ReferencedObjectType == referencedObjectType && referencedObjectType == CalcDependencyObjectType.Partition && calcDependency.ReferencedTableName == referencedTableName) //References to table-partition expressions are by table name, not partition name
)
{
LookUpDependenciesReferenceTo(calcDependency.ObjectType, calcDependency.ObjectName, returnVal);
LookUpDependenciesReferenceTo(calcDependency.ObjectType, calcDependency.ObjectName, calcDependency.TableName, returnVal);
returnVal.Add(calcDependency);
}
}
Expand Down
4 changes: 2 additions & 2 deletions BismNormalizer/BismNormalizer/TabularCompare/TabularMetadata/CalculationItem.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ public List<string> FindMissingCalculationItemDependencies()
while (whatsRemainingOfLine.Contains('[') && whatsRemainingOfLine.Contains(']'))
{
int openSquareBracketPosition = whatsRemainingOfLine.IndexOf('[', 0);
//brilliant person at microsoft has ]] instead of ]
//someone has ]] instead of ]
int closeSquareBracketPosition = whatsRemainingOfLine.Replace("]]", " ").IndexOf(']', openSquareBracketPosition + 1);

if (openSquareBracketPosition < closeSquareBracketPosition - 1)
Expand All @@ -78,7 +78,7 @@ public List<string> FindMissingCalculationItemDependencies()
!_tomCalculationItem.Expression.Contains($"\"{potentialDependency}\"") && //it's possible the calculationItem itself is deriving the column name from an ADDCOLUMNS for example
!dependencies.Contains(potentialDependency))
{
//unbelievable: some genius at m$ did a replace on ] with ]]
//someone did a replace on ] with ]]
dependencies.Add(potentialDependency);
}
}
Expand Down
74 changes: 53 additions & 21 deletions BismNormalizer/BismNormalizer/TabularCompare/TabularMetadata/Comparison.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -635,8 +635,8 @@ public override void ValidateSelection()
bool reconnect = false;
try
{
_sourceTabularModel.TomDatabase.Refresh();
_targetTabularModel.TomDatabase.Refresh();
if (!_sourceTabularModel.ConnectionInfo.UseBimFile) _sourceTabularModel.TomDatabase.Refresh();
if (!_targetTabularModel.ConnectionInfo.UseBimFile) _targetTabularModel.TomDatabase.Refresh();
}
catch (Exception)
{
Expand All @@ -646,16 +646,11 @@ public override void ValidateSelection()
if (reconnect || _uncommitedChanges)
{
// Reconnect to re-initialize
if (!_comparisonInfo.ConnectionInfoSource.UseBimFile)
{
_sourceTabularModel = new TabularModel(this, _comparisonInfo.ConnectionInfoSource, _comparisonInfo);
_sourceTabularModel.Connect();
}
if (!_comparisonInfo.ConnectionInfoTarget.UseBimFile)
{
_targetTabularModel = new TabularModel(this, _comparisonInfo.ConnectionInfoTarget, _comparisonInfo);
_targetTabularModel.Connect();
}
_sourceTabularModel = new TabularModel(this, _comparisonInfo.ConnectionInfoSource, _comparisonInfo);
_sourceTabularModel.Connect();

_targetTabularModel = new TabularModel(this, _comparisonInfo.ConnectionInfoTarget, _comparisonInfo);
_targetTabularModel.Connect();
}

if (!_sourceTabularModel.ConnectionInfo.UseProject && _sourceTabularModel.TomDatabase.LastSchemaUpdate > _lastSourceSchemaUpdate)
Expand Down Expand Up @@ -989,13 +984,13 @@ Gets pretty hairy.

#region Calc dependencies validation

private bool HasBlockingToDependenciesInTarget(string targetObjectName, CalcDependencyObjectType targetObjectType, ref List<string> warningObjectList)
private bool HasBlockingToDependenciesInTarget(string targetObjectName, string referencedTableName, CalcDependencyObjectType targetObjectType, ref List<string> warningObjectList)
{
//For deletion.
//Check any objects in target that depend on this object are also going to be deleted or updated.

bool returnVal = false;
CalcDependencyCollection targetToDepdendencies = _targetTabularModel.MDependencies.DependenciesReferenceTo(targetObjectType, targetObjectName);
CalcDependencyCollection targetToDepdendencies = _targetTabularModel.MDependencies.DependenciesReferenceTo(targetObjectType, targetObjectName, referencedTableName);
foreach (CalcDependency targetToDependency in targetToDepdendencies)
{
foreach (ComparisonObject comparisonObjectToCheck in _comparisonObjects)
Expand Down Expand Up @@ -1086,7 +1081,7 @@ private bool HasBlockingFromDependenciesInSource(string sourceTableName, string
comparisonObjectToCheck.SourceObjectName == sourceFromDependency.ReferencedObjectName &&
comparisonObjectToCheck.Status == ComparisonObjectStatus.MissingInTarget && //Creates being skipped (dependency will be missing).
comparisonObjectToCheck.MergeAction == MergeAction.Skip)
//Deletes are impossible for this object to depend on, so don't need to detect. Other Skips can assume are fine, so don't need to detect.
//Deletes are impossible for this object to depend on, so don't need to detect. Other Skips can assume are fine, so don't need to detect.
{
string warningObject = $"Expression {comparisonObjectToCheck.SourceObjectName}";
if (!warningObjectList.Contains(warningObject))
Expand All @@ -1096,6 +1091,25 @@ private bool HasBlockingFromDependenciesInSource(string sourceTableName, string
returnVal = true;
}

break;
case CalcDependencyObjectType.Partition:
//Does the object about to be created/updated (sourceObjectName) have a source dependency on this table (comparisonObjectToCheck)?

if (!_targetTabularModel.Tables.ContainsName(sourceFromDependency.ReferencedTableName) &&
comparisonObjectToCheck.ComparisonObjectType == ComparisonObjectType.Table &&
comparisonObjectToCheck.SourceObjectName == sourceFromDependency.ReferencedTableName &&
comparisonObjectToCheck.Status == ComparisonObjectStatus.MissingInTarget && //Creates being skipped (dependency will be missing).
comparisonObjectToCheck.MergeAction == MergeAction.Skip)
//Deletes are impossible for this object to depend on, so don't need to detect. Other Skips can assume are fine, so don't need to detect.
{
string warningObject = $"Table {comparisonObjectToCheck.SourceObjectName}";
if (!warningObjectList.Contains(warningObject))
{
warningObjectList.Add(warningObject);
}
returnVal = true;
}

break;
case CalcDependencyObjectType.DataSource:
//Does the object about to be created/updated (sourceObjectName) have a source dependency on this data source (comparisonObjectToCheck)?
Expand Down Expand Up @@ -1257,7 +1271,7 @@ private void DeleteDataSource(ComparisonObject comparisonObject)

//Check any objects in target that depend on the DataSource are also going to be deleted
List<string> warningObjectList = new List<string>();
bool toDependencies = HasBlockingToDependenciesInTarget(comparisonObject.TargetObjectName, CalcDependencyObjectType.DataSource, ref warningObjectList);
bool toDependencies = HasBlockingToDependenciesInTarget(comparisonObject.TargetObjectName, "", CalcDependencyObjectType.DataSource, ref warningObjectList);

//For old non-M partitions, check if any such tables have reference to this DataSource, and will not be deleted
foreach (Table table in _targetTabularModel.Tables)
Expand Down Expand Up @@ -1393,7 +1407,7 @@ private void DeleteExpression(ComparisonObject comparisonObject)

//Check any objects in target that depend on the expression are also going to be deleted
List<string> warningObjectList = new List<string>();
if (!HasBlockingToDependenciesInTarget(comparisonObject.TargetObjectName, CalcDependencyObjectType.Expression, ref warningObjectList))
if (!HasBlockingToDependenciesInTarget(comparisonObject.TargetObjectName, "", CalcDependencyObjectType.Expression, ref warningObjectList))
{
_targetTabularModel.DeleteExpression(comparisonObject.TargetObjectName);
OnValidationMessage(new ValidationMessageEventArgs($"Delete expression [{comparisonObject.TargetObjectName}].", ValidationMessageType.Expression, ValidationMessageStatus.Informational));
Expand Down Expand Up @@ -1495,8 +1509,23 @@ private void DeleteTable(ComparisonObject comparisonObject)
{
return;
};
_targetTabularModel.DeleteTable(comparisonObject.TargetObjectName);
OnValidationMessage(new ValidationMessageEventArgs($"Delete {(isCalculationGroup ? "calculation group" : "table")} '{comparisonObject.TargetObjectName}'.", ValidationMessageType.Table, ValidationMessageStatus.Informational));

//Check any objects in target that depend on the table expression are also going to be deleted
List<string> warningObjectList = new List<string>();
if (!HasBlockingToDependenciesInTarget("", comparisonObject.TargetObjectName, CalcDependencyObjectType.Partition, ref warningObjectList))
{
_targetTabularModel.DeleteTable(comparisonObject.TargetObjectName);
OnValidationMessage(new ValidationMessageEventArgs($"Delete {(isCalculationGroup ? "calculation group" : "table")} '{comparisonObject.TargetObjectName}'.", ValidationMessageType.Table, ValidationMessageStatus.Informational));
}
else
{
string message = $"Unable to delete table {comparisonObject.TargetObjectName} because the following objects depend on it: {String.Join(", ", warningObjectList)}.";
if (_comparisonInfo.OptionsInfo.OptionRetainPartitions && !_comparisonInfo.OptionsInfo.OptionRetainPolicyPartitions)
{
message += " Note: the option to retain partitions is on, which may be affecting this.";
}
OnValidationMessage(new ValidationMessageEventArgs(message, ValidationMessageType.Table, ValidationMessageStatus.Warning));
}
}
}

Expand Down Expand Up @@ -1855,10 +1884,13 @@ private void UpdateCalculationItem(ComparisonObject comparisonObject, string tab

private bool DesktopHardened(ComparisonObject comparisonObject, ValidationMessageType validationMessageType)
{
if (_targetTabularModel.ConnectionInfo.UseDesktop && _targetTabularModel.ConnectionInfo.ServerMode == Microsoft.AnalysisServices.ServerMode.SharePoint)
if (
(_targetTabularModel.ConnectionInfo.UseDesktop && _targetTabularModel.ConnectionInfo.ServerMode == Microsoft.AnalysisServices.ServerMode.SharePoint) ||
(_targetTabularModel.ConnectionInfo.UseBimFile && _targetTabularModel.ConnectionInfo.BimFile != null && _targetTabularModel.ConnectionInfo.BimFile.ToUpper().EndsWith(".PBIT"))
)
{
//V3 hardening
OnValidationMessage(new ValidationMessageEventArgs($"Unable to {comparisonObject.MergeAction.ToString().ToLower()} {comparisonObject.ComparisonObjectType.ToString()} {comparisonObject.TargetObjectName} because target is Power BI Desktop, which does not yet support modifications for this object type.", validationMessageType, ValidationMessageStatus.Warning));
OnValidationMessage(new ValidationMessageEventArgs($"Unable to {comparisonObject.MergeAction.ToString().ToLower()} {comparisonObject.ComparisonObjectType.ToString()} {comparisonObject.TargetObjectName} because target is Power BI Desktop or .PBIT, which does not yet support modifications for this object type.", validationMessageType, ValidationMessageStatus.Warning));
return false;
}
else
Expand Down
4 changes: 2 additions & 2 deletions BismNormalizer/BismNormalizer/TabularCompare/TabularMetadata/Measure.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ public List<string> FindMissingMeasureDependencies()
while (whatsRemainingOfLine.Contains('[') && whatsRemainingOfLine.Contains(']'))
{
int openSquareBracketPosition = whatsRemainingOfLine.IndexOf('[', 0);
//brilliant person at microsoft has ]] instead of ]
//someone has ]] instead of ]
int closeSquareBracketPosition = whatsRemainingOfLine.Replace("]]", " ").IndexOf(']', openSquareBracketPosition + 1);

if (openSquareBracketPosition < closeSquareBracketPosition - 1)
Expand All @@ -89,7 +89,7 @@ public List<string> FindMissingMeasureDependencies()
!_tomMeasure.Expression.Contains($"\"{potentialDependency}\"") && //it's possible the measure itself is deriving the column name from an ADDCOLUMNS for example
!dependencies.Contains(potentialDependency))
{
//unbelievable: some genius at m$ did a replace on ] with ]]
//someone did a replace on ] with ]]
dependencies.Add(potentialDependency);
}
}
Expand Down
Loading

0 comments on commit 03a330a

Please sign in to comment.