Skip to content

Commit

Permalink
[Internal] Query: Adds parsing resilience to IndexMetrics (#3418)
Browse files Browse the repository at this point in the history 
* making necessary ownership change

* made change to ownerships

* header test

* Call to TryCreate instead of Create in Responsemessage

* Add baseline test infra for index metric parser

* update baseline files

* Add parse retry logic

* Update headers test

* address code review

* address code review

* fix tests

* Update csproj file

Co-authored-by: Minh Le <leminh@microsoft.com>
  • Loading branch information
@leminh98
leminh98 and Minh Le authored Sep 29, 2022
1 parent 9f46192 commit 01d56ab
Show file tree
Hide file tree
Showing 12 changed files with 531 additions and 113 deletions.
9 changes: 6 additions & 3 deletions Microsoft.Azure.Cosmos/src/Handler/ResponseMessage.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ namespace Microsoft.Azure.Cosmos
using System.Net;
using System.Text;
using Microsoft.Azure.Cosmos.Diagnostics;
using Microsoft.Azure.Cosmos.Linq;
using Microsoft.Azure.Cosmos.Query.Core.Metrics;
using Microsoft.Azure.Cosmos.Resource.CosmosExceptions;
using Microsoft.Azure.Cosmos.Tracing;
Expand Down Expand Up @@ -82,7 +83,7 @@ internal ResponseMessage(
this.CosmosException = cosmosException;
this.Headers = headers ?? new Headers();

this.IndexUtilizationText = ResponseMessage.DecodeIndexMetrics(this.Headers);
this.IndexUtilizationText = ResponseMessage.DecodeIndexMetrics(this.Headers, true);

if (requestMessage != null && requestMessage.Trace != null)
{
Expand Down Expand Up @@ -250,14 +251,16 @@ private void CheckDisposed()
/// Decode the Index Metrics from the response headers, if exists.
/// </summary>
/// <param name="responseMessageHeaders">The response headers</param>
/// <param name="isBse64Encoded">The encoding of the IndexMetrics response</param>
/// <returns>Lazy implementation of the pretty-printed IndexMetrics</returns>
static internal Lazy<string> DecodeIndexMetrics(Headers responseMessageHeaders)
static internal Lazy<string> DecodeIndexMetrics(Headers responseMessageHeaders, bool isBse64Encoded)
{
if (responseMessageHeaders?.IndexUtilizationText != null)
{
return new Lazy<string>(() =>
{
IndexUtilizationInfo parsedIndexUtilizationInfo = IndexUtilizationInfo.CreateFromString(responseMessageHeaders.IndexUtilizationText);
IndexUtilizationInfo parsedIndexUtilizationInfo = IndexUtilizationInfo.CreateFromString(responseMessageHeaders.IndexUtilizationText, isBse64Encoded);
StringBuilder stringBuilder = new StringBuilder();
IndexMetricWriter indexMetricWriter = new IndexMetricWriter(stringBuilder);
indexMetricWriter.WriteIndexMetrics(parsedIndexUtilizationInfo);
Expand Down
4 changes: 2 additions & 2 deletions Microsoft.Azure.Cosmos/src/Query/Core/Metrics/CompositeIndexUtilizationEntity.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,9 @@ public CompositeIndexUtilizationEntity(
bool indexPlanFullFidelity,
string indexImpactScore)
{
this.IndexDocumentExpressions = indexDocumentExpressions ?? throw new ArgumentNullException(nameof(indexDocumentExpressions));
this.IndexDocumentExpressions = indexDocumentExpressions;
this.IndexPlanFullFidelity = indexPlanFullFidelity;
this.IndexImpactScore = indexImpactScore ?? throw new ArgumentNullException(nameof(indexImpactScore));
this.IndexImpactScore = indexImpactScore;
}

[JsonProperty(PropertyName = "IndexSpecs")]
Expand Down
138 changes: 52 additions & 86 deletions Microsoft.Azure.Cosmos/src/Query/Core/Metrics/IndexUtilizationInfo.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ namespace Microsoft.Azure.Cosmos.Query.Core.Metrics
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using Microsoft.Azure.Cosmos.Core;
using Newtonsoft.Json;

/// <summary>
Expand Down Expand Up @@ -40,85 +42,34 @@ public IndexUtilizationInfo(
IReadOnlyList<CompositeIndexUtilizationEntity> utilizedCompositeIndexes,
IReadOnlyList<CompositeIndexUtilizationEntity> potentialCompositeIndexes)
{
if (utilizedSingleIndexes == null)
{
throw new ArgumentNullException(nameof(utilizedSingleIndexes));
}

if (potentialSingleIndexes == null)
{
throw new ArgumentNullException(nameof(potentialSingleIndexes));
}

if (utilizedCompositeIndexes == null)
{
throw new ArgumentNullException(nameof(utilizedCompositeIndexes));
}

if (potentialCompositeIndexes == null)
{
throw new ArgumentNullException(nameof(potentialCompositeIndexes));
}

List<SingleIndexUtilizationEntity> utilizedSingleIndexesCopy = new List<SingleIndexUtilizationEntity>();

foreach (SingleIndexUtilizationEntity indexUtilizationEntity in utilizedSingleIndexes)
{
if (indexUtilizationEntity == null)
{
throw new ArgumentNullException(nameof(indexUtilizationEntity));
}

utilizedSingleIndexesCopy.Add(indexUtilizationEntity);
}

List<SingleIndexUtilizationEntity> potentialSingleIndexesCopy = new List<SingleIndexUtilizationEntity>();

foreach (SingleIndexUtilizationEntity indexUtilizationEntity in potentialSingleIndexes)
{
if (indexUtilizationEntity == null)
{
throw new ArgumentNullException(nameof(indexUtilizationEntity));
}

potentialSingleIndexesCopy.Add(indexUtilizationEntity);
}

List<CompositeIndexUtilizationEntity> utilizedCompositeIndexesCopy = new List<CompositeIndexUtilizationEntity>();

foreach (CompositeIndexUtilizationEntity indexUtilizationEntity in utilizedCompositeIndexes)
{
if (indexUtilizationEntity == null)
{
throw new ArgumentNullException(nameof(indexUtilizationEntity));
}

utilizedCompositeIndexesCopy.Add(indexUtilizationEntity);
}

List<CompositeIndexUtilizationEntity> potentialCompositeIndexesCopy = new List<CompositeIndexUtilizationEntity>();

foreach (CompositeIndexUtilizationEntity indexUtilizationEntity in potentialCompositeIndexes)
{
if (indexUtilizationEntity == null)
{
throw new ArgumentNullException(nameof(indexUtilizationEntity));
}

potentialCompositeIndexesCopy.Add(indexUtilizationEntity);
}

this.UtilizedSingleIndexes = utilizedSingleIndexesCopy;
this.PotentialSingleIndexes = potentialSingleIndexesCopy;
this.UtilizedCompositeIndexes = utilizedCompositeIndexesCopy;
this.PotentialCompositeIndexes = potentialCompositeIndexesCopy;
this.UtilizedSingleIndexes = (utilizedSingleIndexes ?? Enumerable.Empty<SingleIndexUtilizationEntity>()).Where(item => item != null).ToList();
this.PotentialSingleIndexes = (potentialSingleIndexes ?? Enumerable.Empty<SingleIndexUtilizationEntity>()).Where(item => item != null).ToList();
this.UtilizedCompositeIndexes = (utilizedCompositeIndexes ?? Enumerable.Empty<CompositeIndexUtilizationEntity>()).Where(item => item != null).ToList();
this.PotentialCompositeIndexes = (potentialCompositeIndexes ?? Enumerable.Empty<CompositeIndexUtilizationEntity>()).Where(item => item != null).ToList();
}

public IReadOnlyList<SingleIndexUtilizationEntity> UtilizedSingleIndexes { get; }
public IReadOnlyList<SingleIndexUtilizationEntity> PotentialSingleIndexes { get; }
public IReadOnlyList<CompositeIndexUtilizationEntity> UtilizedCompositeIndexes { get; }
public IReadOnlyList<CompositeIndexUtilizationEntity> PotentialCompositeIndexes { get; }

/// <summary>
/// Creates a new IndexUtilizationInfo from the backend delimited base64 encoded string.
/// </summary>
/// <param name="delimitedString">The backend delimited string to deserialize from.</param>
/// <param name="result">The parsed index utilization info</param>
/// <returns>A new IndexUtilizationInfo from the backend delimited string.</returns>
internal static bool TryCreateFromDelimitedBase64String(string delimitedString, out IndexUtilizationInfo result)
{
if (delimitedString == null)
{
result = IndexUtilizationInfo.Empty;
return true;
}

return TryCreateFromDelimitedString(System.Text.Encoding.UTF8.GetString(Convert.FromBase64String(delimitedString)), out result);
}

/// <summary>
/// Creates a new IndexUtilizationInfo from the backend delimited string.
/// </summary>
Expand All @@ -132,29 +83,44 @@ internal static bool TryCreateFromDelimitedString(string delimitedString, out In
result = IndexUtilizationInfo.Empty;
return true;
}

try
{
string indexString = System.Text.Encoding.UTF8.GetString(Convert.FromBase64String(delimitedString));

result = JsonConvert.DeserializeObject<IndexUtilizationInfo>(indexString);
if (result == null)
result = JsonConvert.DeserializeObject<IndexUtilizationInfo>(delimitedString, new JsonSerializerSettings()
{
result = IndexUtilizationInfo.Empty;
}
// Allowing null values to be resilient to Json structure change
MissingMemberHandling = MissingMemberHandling.Ignore,
NullValueHandling = NullValueHandling.Ignore,
// Ignore parsing error encountered in desrialization
Error = (sender, parsingErrorEvent) => parsingErrorEvent.ErrorContext.Handled = true
}) ?? IndexUtilizationInfo.Empty;

return true;
}
catch
catch (JsonException)
{
result = IndexUtilizationInfo.Empty;
return false;
}
}

public static IndexUtilizationInfo CreateFromString(string delimitedString)
/// <summary>
/// Materialize the Index Utilization String into Concrete objects.
/// </summary>
/// <param name="delimitedString">The index utilization response string as sent by the back end.</param>
/// <param name="isBse64Encoded">The encoding of the string.</param>
/// <returns>Cpncrete Index utilization object.</returns>
public static IndexUtilizationInfo CreateFromString(string delimitedString, bool isBse64Encoded)
{
if (!TryCreateFromDelimitedString(delimitedString, out IndexUtilizationInfo indexUtilizationInfo))
IndexUtilizationInfo indexUtilizationInfo;

if (isBse64Encoded)
{
TryCreateFromDelimitedBase64String(delimitedString, out indexUtilizationInfo);
}
else
{
throw new FormatException($"Failed to parse {nameof(IndexUtilizationInfo)} : '{delimitedString}'");
TryCreateFromDelimitedString(delimitedString, out indexUtilizationInfo);
}

return indexUtilizationInfo;
Expand All @@ -168,10 +134,10 @@ public Accumulator(
IEnumerable<CompositeIndexUtilizationEntity> utilizedCompositeIndexes,
IEnumerable<CompositeIndexUtilizationEntity> potentialCompositeIndexes)
{
this.UtilizedSingleIndexes = utilizedSingleIndexes; // ?? throw new ArgumentNullException(nameof(utilizedSingleIndexes));
this.PotentialSingleIndexes = potentialSingleIndexes; // ?? throw new ArgumentNullException(nameof(potentialSingleIndexes));
this.UtilizedCompositeIndexes = utilizedCompositeIndexes; // ?? throw new ArgumentNullException(nameof(utilizedCompositeIndexes));
this.PotentialCompositeIndexes = potentialCompositeIndexes; // ?? throw new ArgumentNullException(nameof(potentialCompositeIndexes));
this.UtilizedSingleIndexes = utilizedSingleIndexes;
this.PotentialSingleIndexes = potentialSingleIndexes;
this.UtilizedCompositeIndexes = utilizedCompositeIndexes;
this.PotentialCompositeIndexes = potentialCompositeIndexes;
}

public IEnumerable<SingleIndexUtilizationEntity> UtilizedSingleIndexes { get; }
Expand Down
6 changes: 3 additions & 3 deletions Microsoft.Azure.Cosmos/src/Query/Core/Metrics/SingleIndexUtilizationEntity.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,11 +34,11 @@ public SingleIndexUtilizationEntity(
bool indexPlanFullFidelity,
string indexImpactScore)
{
this.FilterExpression = filterExpression ?? throw new ArgumentNullException(nameof(filterExpression));
this.IndexDocumentExpression = indexDocumentExpression ?? throw new ArgumentNullException(nameof(indexDocumentExpression));
this.FilterExpression = filterExpression;
this.IndexDocumentExpression = indexDocumentExpression;
this.FilterExpressionPrecision = filterExpressionPrecision;
this.IndexPlanFullFidelity = indexPlanFullFidelity;
this.IndexImpactScore = indexImpactScore ?? throw new ArgumentNullException(nameof(indexImpactScore));
this.IndexImpactScore = indexImpactScore;
}

[JsonProperty(PropertyName = "FilterExpression")]
Expand Down
2 changes: 1 addition & 1 deletion Microsoft.Azure.Cosmos/src/Query/v2Query/DefaultDocumentQueryExecutionContext.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ protected override async Task<DocumentFeedResponse<CosmosElement>> ExecuteIntern
partitionIdentifier,
new QueryMetrics(
response.ResponseHeaders[HttpConstants.HttpHeaders.QueryMetrics],
IndexUtilizationInfo.CreateFromString(response.ResponseHeaders[HttpConstants.HttpHeaders.IndexUtilization]),
IndexUtilizationInfo.CreateFromString(response.ResponseHeaders[HttpConstants.HttpHeaders.IndexUtilization], true),
new ClientSideMetrics(
this.retries,
response.RequestCharge,
Expand Down
2 changes: 1 addition & 1 deletion Microsoft.Azure.Cosmos/src/Query/v3Query/QueryResponse.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,7 @@ private QueryResponse(
cosmosArray: cosmosElements,
serializerCore: serializerCore);

this.IndexUtilizationText = ResponseMessage.DecodeIndexMetrics(responseMessageHeaders);
this.IndexUtilizationText = ResponseMessage.DecodeIndexMetrics(responseMessageHeaders, true);
this.RequestMessage = requestMessage;
}

Expand Down
Loading

0 comments on commit 01d56ab

Please sign in to comment.