我正在Service Fabric应用程序上执行一系列异步调用,并且有一个长时间运行的调用将在5-10分钟后抛出TimeoutException。我的代码与此类似:
public class Listener {
private async Task HandleRequestAsync(RestoreRequest request, RestoreWorker worker) {
Response response = await worker.ExecuteAsync(request).ConfigureAwait(false);
}
}
public class RestoreWorker {
public async Task<Response> ExecuteAsync(RestoreRequest request) {
RestoreService restoreService = new restoreService(request);
restoreService.Progress.ProgressChanged += async (sender, info) => await request.UpdateStatusAsync(new State(StateEnum.Running) { ProgressCurrent = info.Current, ProgressTotal = info.Total }).ConfigureAwait(false);
await restoreService.RestoreAsync(request.Id, request.Name).ConfigureAwait(false);
return new Response();
}
public Progress<ProgressInfo> Progress { get; } = new Progress<ProgressInfo>();
}
public class RestoreRequest {
public async Task UpdateStatusAsync(Status status) {
Message message = new Message { Status = status };
await sender.SendAsync(message).ConfigureAwait(false);
}
}
public class RestoreService {
private static readonly IRestoreClient restoreClient = ServiceProxyFactory.CreateServiceProxy<IRestoreClient>(new Uri($"{FabricConfig.ApplicationName}/RestoreClient"));
private async Task <Project> GetProjectByNameAsync(string name){
//return the project
}
private async Task RestoreAsync(string id, string name) {
await restoreClient.RestoreAsync(id, name).ConfigureAwait(false);
}
}
public class RestoreClient : IRestoreClient {
private async Task RestoreAsync(string id, string name) {
Project project = await GetProjectByNameAsync(name).ConfigureAwait(false);
project = await UpdateDbAsync(project.Id).ConfigureAwait(false);
if (project == null) {
throw new Exception("Could not find project.");
}
}
private async Task UpdateDbAsync(string id) {
try {
List<string> input = CreateScripts();
await ExecuteScriptsOnDbAsync(input).ConfigureAwait(false);
} catch (SqlException) {
throw new Exception($"Project with id: '{id}' could not be created.");
}
}
private async Task ExecuteScriptsOnDbAsync(List<string> scripts) {
using (var conn = new SqlConnection(connectionString)) {
try {
await conn.OpenAsync().ConfigureAwait(false);
using (var sqlCommand = new SqlCommand { Connection = conn }) {
sqlCommand.CommandTimeout = SqlCommandCommandTimeout;
foreach (string script in scripts) {
sqlCommand.CommandText = script;
await sqlCommand.ExecuteNonQueryAsync().ConfigureAwait(false);
}
}
} catch (SqlException ex) {
Log.Fatal(ex, $"Cannot execute script on {Name}");
throw;
}
}
}
}
如果UpdateTheDBAsync方法需要很长时间才能执行,我将收到TimeoutException
System.AggregateException: One or more errors occurred. ---> System.TimeoutException: This can happen if message is dropped when service is busy or its long running operation and taking more time than configured Operation Timeout.
at Microsoft.ServiceFabric.Services.Communication.Client.ServicePartitionClient`1.<InvokeWithRetryAsync>d__24`1.MoveNext()
--- End of stack trace from previous location where exception was thrown ---
at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()
at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)
at Microsoft.ServiceFabric.Services.Remoting.V1.Client.ServiceRemotingPartitionClient.<InvokeAsync>d__2.MoveNext()
--- End of stack trace from previous location where exception was thrown ---
at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()
at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)
at Microsoft.ServiceFabric.Services.Remoting.Builder.ProxyBase.<InvokeAsync>d__15.MoveNext()
--- End of stack trace from previous location where exception was thrown ---
at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()
at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)
at Microsoft.ServiceFabric.Services.Remoting.Builder.ProxyBase.<ContinueWithResult>d__16`1.MoveNext()
--- End of stack trace from previous location where exception was thrown ---
at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()
at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)
at System.Runtime.CompilerServices.ConfiguredTaskAwaitable`1.ConfiguredTaskAwaiter.GetResult()
at RestoreService.<RestoreAsync>d__14.MoveNext()
--- End of stack trace from previous location where exception was thrown ---
at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()
at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)
at System.Runtime.CompilerServices.ConfiguredTaskAwaitable.ConfiguredTaskAwaiter.GetResult()
at RestoreWorker.<ExecuteAsync>d__0.MoveNext()
--- End of stack trace from previous location where exception was thrown ---
at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()
at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)
at System.Runtime.CompilerServices.ConfiguredTaskAwaitable`1.ConfiguredTaskAwaiter.GetResult()
at Listener.<HandleRequestAsync>d__15.MoveNext()
为什么即使没有配置超时,我也会收到超时?我究竟做错了什么?任何帮助表示赞赏。
PS:这个代码非常相似
该问题与服务之间的远程处理(ServiceFabric.Services.Remoting)的默认超时时间有关。
可以使用远程处理的第2版,并根据Microsoft documentation“远程处理V2堆栈表现更好”。
升级到V2后,解决问题的一种可能方法是增加超时
new ServiceProxyFactory((c) => new FabricTransportServiceRemotingClientFactory(
new FabricTransportRemotingSettings() {
OperationTimeout = TimeSpan.FromMinutes(30)
})))
但这只会增加超时而不是完全删除它。
解决这个问题的另一种方法是启动一个直接在与远程处理一起使用的服务中处理的工作程序并等待其完成。通过这种方式,解决方案不受远程超时的约束。
例如:
替换这个:
await restoreClient.RestoreAsync(id, name).ConfigureAwait(false);
同
var workerId = StartANewWorker()
JobState jobState;
do {
//poll for the status of the new worker
var workerStatus = GetStatusOfTheWorker(workerId);
await Task.Delay(1000).ConfigureAwait(false);
if (workerStatus == Failed) {
throw new Exception("Something went wrong");
}
} while (workerStatus != Finished);