Skip to content

Commit

Permalink
Merge pull request #44 from bmrvilela/master
Browse files Browse the repository at this point in the history 
Fix to RetryableDataServiceQuery
  • Loading branch information
@portilha
portilha authored Feb 28, 2025
2 parents 3f450ee + bc1d908 commit 30d1bc4
Show file tree
Hide file tree
Showing 6 changed files with 200 additions and 32 deletions.
25 changes: 25 additions & 0 deletions Checkmarx.API.Tests/CxClientUnitTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
using Checkmarx.API.Models;
using Checkmarx.API.Tests.Utils;
using Microsoft.Extensions.Configuration;
using Microsoft.OData.Client;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Moq;
using Newtonsoft.Json;
Expand Down Expand Up @@ -113,6 +114,30 @@ public void RetryPolicyTest()
);
}

[TestMethod]
public void ODataRetryableTest()
{
try
{
var projects1 = clientV9.ODataV95.Projects.Expand(x => x.Preset)
.Expand(x => x.CustomFields)
.Where(y => y.IsPublic && y.OwningTeamId != -1)
.ToList();

DateTime date = new DateTime(2024, 2, 2);

Checkmarx.API.SAST.Scan scanBeforeChange = clientV9.GetScans(4, true, scanKind: CxClient.ScanRetrieveKind.Last, maxScanDate: date, includeGhostScans: false).SingleOrDefault();
}
catch (DataServiceQueryException ex)
{
Trace.WriteLine($"Status Code: {ex.Response.StatusCode}");
}
catch (Exception ex)
{
Trace.WriteLine($"Status Code: {ex.Message}");
}
}

[TestMethod]
public void ProjectExclusionsTest()
{
Expand Down
6 changes: 3 additions & 3 deletions Checkmarx.API/Connected Services/CxSAST OData Service V94/ODataClient95.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ protected string ResolveNameFromType(global::System.Type clientType)
/// </summary>
[global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.OData.Client.Design.T4", "#VersionNumber#")]
[global::Microsoft.OData.Client.OriginalNameAttribute("Projects")]
public virtual global::Microsoft.OData.Client.DataServiceQuery<Checkmarx.API.SAST.OData.Project> Projects
public virtual RetryableDataServiceQuery<Checkmarx.API.SAST.OData.Project> Projects
{
get
{
Expand All @@ -118,7 +118,7 @@ protected string ResolveNameFromType(global::System.Type clientType)
/// </summary>
[global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.OData.Client.Design.T4", "#VersionNumber#")]
[global::Microsoft.OData.Client.OriginalNameAttribute("Scans")]
public virtual global::Microsoft.OData.Client.DataServiceQuery<Checkmarx.API.SAST.OData.Scan> Scans
public virtual RetryableDataServiceQuery<Checkmarx.API.SAST.OData.Scan> Scans
{
get
{
Expand All @@ -136,7 +136,7 @@ protected string ResolveNameFromType(global::System.Type clientType)
/// </summary>
[global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.OData.Client.Design.T4", "#VersionNumber#")]
[global::Microsoft.OData.Client.OriginalNameAttribute("Results")]
public virtual global::Microsoft.OData.Client.DataServiceQuery<Checkmarx.API.SAST.OData.Result> Results
public virtual RetryableDataServiceQuery<Checkmarx.API.SAST.OData.Result> Results
{
get
{
Expand Down
6 changes: 3 additions & 3 deletions Checkmarx.API/Connected Services/CxSAST OData Service/ODataClient8.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ public ODataClient8(global::System.Uri serviceRoot, int defaultRetries = 10) :
/// There are no comments for Projects in the schema.
/// </summary>
[global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.OData.Client.Design.T4", "#VersionNumber#")]
public virtual global::Microsoft.OData.Client.DataServiceQuery<global::CxDataRepository.Project> Projects
public virtual RetryableDataServiceQuery<global::CxDataRepository.Project> Projects
{
get
{
Expand All @@ -57,7 +57,7 @@ public ODataClient8(global::System.Uri serviceRoot, int defaultRetries = 10) :
/// There are no comments for Scans in the schema.
/// </summary>
[global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.OData.Client.Design.T4", "#VersionNumber#")]
public virtual global::Microsoft.OData.Client.DataServiceQuery<global::CxDataRepository.Scan> Scans
public virtual RetryableDataServiceQuery<global::CxDataRepository.Scan> Scans
{
get
{
Expand All @@ -74,7 +74,7 @@ public ODataClient8(global::System.Uri serviceRoot, int defaultRetries = 10) :
/// There are no comments for Results in the schema.
/// </summary>
[global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.OData.Client.Design.T4", "#VersionNumber#")]
public virtual global::Microsoft.OData.Client.DataServiceQuery<global::CxDataRepository.Result> Results
public virtual RetryableDataServiceQuery<global::CxDataRepository.Result> Results
{
get
{
Expand Down
6 changes: 3 additions & 3 deletions Checkmarx.API/Connected Services/CxSAST V9 OData Service/CxSASTODataV9.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6619,7 +6619,7 @@ public Container(global::System.Uri serviceRoot, int defaultRetries = 10) :
/// There are no comments for Projects in the schema.
/// </summary>
[global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.OData.Client.Design.T4", "#VersionNumber#")]
public virtual global::Microsoft.OData.Client.DataServiceQuery<global::CxDataRepository.Project> Projects
public virtual RetryableDataServiceQuery<global::CxDataRepository.Project> Projects
{
get
{
Expand All @@ -6636,7 +6636,7 @@ public Container(global::System.Uri serviceRoot, int defaultRetries = 10) :
/// There are no comments for Scans in the schema.
/// </summary>
[global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.OData.Client.Design.T4", "#VersionNumber#")]
public virtual global::Microsoft.OData.Client.DataServiceQuery<global::CxDataRepository.Scan> Scans
public virtual RetryableDataServiceQuery<global::CxDataRepository.Scan> Scans
{
get
{
Expand All @@ -6653,7 +6653,7 @@ public Container(global::System.Uri serviceRoot, int defaultRetries = 10) :
/// There are no comments for Results in the schema.
/// </summary>
[global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.OData.Client.Design.T4", "#VersionNumber#")]
public virtual global::Microsoft.OData.Client.DataServiceQuery<global::CxDataRepository.Result> Results
public virtual RetryableDataServiceQuery<global::CxDataRepository.Result> Results
{
get
{
Expand Down
8 changes: 4 additions & 4 deletions Checkmarx.API/CxClient.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,13 +112,13 @@ public TimeSpan ServerDateTimeOffSet
}
}

private global::Microsoft.OData.Client.DataServiceQuery<global::CxDataRepository.Scan> _oDataScans => _isV9 ? _oDataV9.Scans.Expand(x => x.ScannedLanguages) : _oData.Scans.Expand(x => x.ScannedLanguages);
private RetryableDataServiceQuery<global::CxDataRepository.Scan> _oDataScans => _isV9 ? _oDataV9.Scans.Expand(x => x.ScannedLanguages) : _oData.Scans.Expand(x => x.ScannedLanguages);

private global::Microsoft.OData.Client.DataServiceQuery<global::CxDataRepository.Project> _oDataProjects => _isV9 ? _oDataV9.Projects : _oData.Projects;
private RetryableDataServiceQuery<global::CxDataRepository.Project> _oDataProjects => _isV9 ? _oDataV9.Projects : _oData.Projects;

private global::Microsoft.OData.Client.DataServiceQuery<global::CxDataRepository.Result> _oDataResults => _isV9 ? _oDataV9.Results : _oData.Results;
private RetryableDataServiceQuery<global::CxDataRepository.Result> _oDataResults => _isV9 ? _oDataV9.Results : _oData.Results;

private global::Microsoft.OData.Client.DataServiceQuery<Checkmarx.API.SAST.OData.Result> _oDataV95Results => _isV95 ? _oDataV95.Results : null;
private RetryableDataServiceQuery<Checkmarx.API.SAST.OData.Result> _oDataV95Results => _isV95 ? _oDataV95.Results : null;

public IQueryable<Result> GetODataResults(long scanId)
{
Expand Down
181 changes: 162 additions & 19 deletions Checkmarx.API/Models/RetryableDataServiceQuery.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,64 +3,100 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Linq.Expressions;
using System.Net;
using System.Net.Http;
using System.Reflection;

namespace Checkmarx.API.Models
{
public class RetryableDataServiceQuery<T> : IQueryable<T>
{
private readonly DataServiceQuery<T> _innerQuery;
private readonly IQueryable<T> _innerQuery;
private readonly int _retryCount;
private readonly Policy<IEnumerable<T>> _retryPolicy;
private readonly Policy<object> _genericRetryPolicy;
private readonly RetryableQueryProvider<T> _retryableProvider;

public RetryableDataServiceQuery(DataServiceQuery<T> innerQuery, int retryCount = 10)
public RetryableDataServiceQuery(IQueryable<T> innerQuery, int retryCount = 10)
{
_innerQuery = innerQuery;
_retryCount = retryCount;
_retryPolicy = BuildRetryPolicy(_retryCount);
_retryPolicy = buildRetryPolicy(_retryCount);
_genericRetryPolicy = buildGenericRetryPolicy(_retryCount);

_retryableProvider = new RetryableQueryProvider<T>(_innerQuery.Provider, _retryPolicy, _genericRetryPolicy, retryCount);
}

public IEnumerator<T> GetEnumerator()
{
return _innerQuery.GetEnumerator();
var result = (IEnumerable<T>)_retryableProvider.Execute(Expression);
return result.GetEnumerator();
}

IEnumerator IEnumerable.GetEnumerator()
{
return _innerQuery.GetEnumerator();
return GetEnumerator();
}

public Type ElementType => _innerQuery.ElementType;
public Expression Expression => _innerQuery.Expression;
public IQueryProvider Provider => _innerQuery.Provider;
public IQueryProvider Provider => _retryableProvider;

// Execute query with retry logic
public IEnumerable<T> Execute()
{
return _retryPolicy.Execute(() => _innerQuery.Execute());
return _retryPolicy.Execute(() => _innerQuery.ToList());
}

public DataServiceQuery<T> Expand(Expression<Func<T, object>> expandExpression)
public RetryableDataServiceQuery<T> Expand(Expression<Func<T, object>> expandExpression)
{
var expandedQuery = _innerQuery.Expand(expandExpression);
var expandedQuery = (_innerQuery as DataServiceQuery<T>)?.Expand(expandExpression);

if (expandedQuery == null)
throw new InvalidCastException("The inner query is not of type DataServiceQuery<T>");

return new RetryableDataServiceQuery<T>(expandedQuery, _retryCount);
}

public static implicit operator DataServiceQuery<T>(RetryableDataServiceQuery<T> retryableQuery)
public RetryableDataServiceQuery<T> Expand(string expandQuery)
{
return retryableQuery._innerQuery;
var expandedQuery = (_innerQuery as DataServiceQuery<T>)?.Expand(expandQuery);

if (expandedQuery == null)
throw new InvalidCastException("The inner query is not of type DataServiceQuery<T>");

return new RetryableDataServiceQuery<T>(expandedQuery, _retryCount);
}

public RetryableDataServiceQuery<T> AddQueryOption(string name, string value)
{
if (_innerQuery is DataServiceQuery<T> dataServiceQuery)
{
var updatedQuery = dataServiceQuery.AddQueryOption(name, value);

return new RetryableDataServiceQuery<T>(updatedQuery, _retryCount);
}
else
{
throw new InvalidOperationException("AddQueryOption is not supported for the current query type.");
}
}

public static explicit operator DataServiceQuery<T>(RetryableDataServiceQuery<T> retryableQuery)
{
return retryableQuery._innerQuery as DataServiceQuery<T> ?? throw new InvalidCastException("Cannot cast to DataServiceQuery<T>");
}

#region Policy Builders

private static Policy<IEnumerable<T>> BuildRetryPolicy(int retries)
private static Policy<IEnumerable<T>> buildRetryPolicy(int retries)
{
return Policy<IEnumerable<T>>
.Handle<HttpRequestException>(ex => IsTransientError(ex))
.Or<WebException>(ex => IsTransientWebError(ex))
.Handle<DataServiceQueryException>(isTransientError)
.Or<HttpRequestException>(isTransientError)
.Or<WebException>(isTransientWebError)
.WaitAndRetry(
retries,
retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)),
Expand All @@ -78,27 +114,134 @@ private static Policy<IEnumerable<T>> BuildRetryPolicy(int retries)
});
}

private static bool IsTransientError(HttpRequestException ex)
private static Policy<object> buildGenericRetryPolicy(int retries)
{
return Policy<object>
.Handle<DataServiceQueryException>(isTransientError)
.Or<HttpRequestException>(isTransientError)
.Or<WebException>(isTransientWebError)
.WaitAndRetry(
retries,
retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)),
(outcome, timespan, retryAttempt, context) =>
{
Exception ex = outcome.Exception;
string message = ex?.Message ?? "Unknown error";

if (ex is HttpRequestException httpEx && httpEx.InnerException != null)
message = httpEx.InnerException.Message;
else if (ex is WebException webEx)
message = webEx.Message;

Console.WriteLine($"Retry {retryAttempt} after {timespan.TotalSeconds} seconds due to: {message}");
});
}

private static bool isTransientError(DataServiceQueryException ex)
{
return isTransientCode(ex.Response?.StatusCode);
}

private static bool isTransientError(HttpRequestException ex)
{
if (ex.InnerException is WebException webEx && webEx.Response is HttpWebResponse response)
{
int statusCode = (int)response.StatusCode;
return statusCode >= 500 || statusCode == 408;
return isTransientCode(statusCode);
}
return false;
}

private static bool IsTransientWebError(WebException ex)
private static bool isTransientWebError(WebException ex)
{
if (ex.Response is HttpWebResponse response)
{
int statusCode = (int)response.StatusCode;
return statusCode >= 500 || statusCode == 408;
return isTransientCode(statusCode);
}
return ex.Status == WebExceptionStatus.Timeout;
}

private static bool isTransientCode(int? statusCode)
{
if (statusCode != null)
return statusCode >= 500 || statusCode == 408;

return false;
}

#endregion
}


public class RetryableQueryProvider<T> : IQueryProvider
{
private readonly IQueryProvider _innerProvider;
private readonly Policy<IEnumerable<T>> _retryPolicy;
private readonly Policy<object> _genericRetryPolicy;
private readonly int _retryCount;

public RetryableQueryProvider(IQueryProvider innerProvider, Policy<IEnumerable<T>> retryPolicy, Policy<object> genericRetryPolicy, int retryCount = 10)
{
_innerProvider = innerProvider;
_retryPolicy = retryPolicy;
_genericRetryPolicy = genericRetryPolicy;
_retryCount = retryCount;
}

public IQueryable CreateQuery(Expression expression)
{
var query = _innerProvider.CreateQuery<T>(expression);
return new RetryableDataServiceQuery<T>(query, _retryCount);
}

public IQueryable<TElement> CreateQuery<TElement>(Expression expression)
{
var query = _innerProvider.CreateQuery<TElement>(expression);
return new RetryableDataServiceQuery<TElement>(query, _retryCount);
}

public object Execute(Expression expression)
{
var queryable = _innerProvider.CreateQuery<T>(expression);
return executeWithExceptionHandling(() => _retryPolicy.Execute(() => queryable.ToList()));
}

public TResult Execute<TResult>(Expression expression)
{
var queryable = _innerProvider.CreateQuery<T>(expression);

return executeWithExceptionHandling(() =>
{
if (typeof(TResult).IsGenericType && typeof(TResult).GetGenericTypeDefinition() == typeof(IEnumerable<>))
return (TResult)(object)_retryPolicy.Execute(() => queryable.ToList());
else
return (TResult)_genericRetryPolicy.Execute(() => queryable.Provider.Execute(expression));
});
}

private static TResult executeWithExceptionHandling<TResult>(Func<TResult> action)
{
try
{
return action();
}
catch (TargetInvocationException ex) when (ex.InnerException is DataServiceQueryException queryEx)
{
throw queryEx;
}
catch (TargetInvocationException ex) when (ex.InnerException is DataServiceClientException clientEx)
{
throw clientEx;
}
catch (DataServiceQueryException)
{
throw;
}
catch (Exception)
{
throw;
}
}
}
}

0 comments on commit 30d1bc4

Please sign in to comment.