optimize for readability
This commit is contained in:
@@ -24,78 +24,60 @@ public class CachedFileProvider
|
|||||||
_basePath = configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
|
_basePath = configuration.GetValue<string>(nameof(StaticFilesServerConfiguration.CacheDirectory));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async Task DownloadTask(string hash, string auth)
|
||||||
|
{
|
||||||
|
// download file from remote
|
||||||
|
var downloadUrl = new Uri(_remoteCacheSourceUri, hash);
|
||||||
|
_logger.LogInformation("Did not find {hash}, downloading from {server}", hash, downloadUrl);
|
||||||
|
using var client = new HttpClient();
|
||||||
|
client.DefaultRequestHeaders.Add("Authorization", auth);
|
||||||
|
var response = await client.GetAsync(downloadUrl, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false);
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
response.EnsureSuccessStatusCode();
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogWarning(ex, "Failed to download {url}", downloadUrl);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
var fileName = FilePathUtil.GetFilePath(_basePath, hash);
|
||||||
|
using var fileStream = new FileStream(fileName, FileMode.OpenOrCreate, FileAccess.ReadWrite);
|
||||||
|
var bufferSize = response.Content.Headers.ContentLength > 1024 * 1024 ? 4096 : 1024;
|
||||||
|
var buffer = new byte[bufferSize];
|
||||||
|
|
||||||
|
var bytesRead = 0;
|
||||||
|
while ((bytesRead = await (await response.Content.ReadAsStreamAsync().ConfigureAwait(false)).ReadAsync(buffer).ConfigureAwait(false)) > 0)
|
||||||
|
{
|
||||||
|
await fileStream.WriteAsync(buffer.AsMemory(0, bytesRead)).ConfigureAwait(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
_metrics.IncGauge(MetricsAPI.GaugeFilesTotal);
|
||||||
|
_metrics.IncGauge(MetricsAPI.GaugeFilesTotalSize, FilePathUtil.GetFileInfoForHash(_basePath, hash).Length);
|
||||||
|
}
|
||||||
|
|
||||||
public async Task<FileStream?> GetFileStream(string hash, string auth)
|
public async Task<FileStream?> GetFileStream(string hash, string auth)
|
||||||
{
|
{
|
||||||
var fi = FilePathUtil.GetFileInfoForHash(_basePath, hash);
|
var fi = FilePathUtil.GetFileInfoForHash(_basePath, hash);
|
||||||
bool hasTransferTask = _currentTransfers.TryGetValue(hash, out var transferTask);
|
if (fi == null && IsMainServer) return null;
|
||||||
if (fi == null && !hasTransferTask)
|
|
||||||
|
if (fi == null && !_currentTransfers.ContainsKey(hash))
|
||||||
{
|
{
|
||||||
if (IsMainServer) return null;
|
_currentTransfers[hash] = DownloadTask(hash, auth).ContinueWith(r => _currentTransfers.Remove(hash, out _));
|
||||||
if (transferTask == null)
|
|
||||||
{
|
|
||||||
_currentTransfers[hash] = Task.Run(async () =>
|
|
||||||
{
|
|
||||||
// download file from remote
|
|
||||||
var downloadUrl = new Uri(_remoteCacheSourceUri, hash);
|
|
||||||
_logger.LogInformation("Did not find {hash}, downloading from {server}", hash, downloadUrl);
|
|
||||||
using var client = new HttpClient();
|
|
||||||
client.DefaultRequestHeaders.Add("Authorization", auth);
|
|
||||||
var response = await client.GetAsync(downloadUrl, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false);
|
|
||||||
|
|
||||||
try
|
|
||||||
{
|
|
||||||
response.EnsureSuccessStatusCode();
|
|
||||||
}
|
|
||||||
catch (Exception ex)
|
|
||||||
{
|
|
||||||
_logger.LogWarning(ex, "Failed to download {url}", downloadUrl);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
var fileName = FilePathUtil.GetFilePath(_basePath, hash);
|
|
||||||
using var fileStream = new FileStream(fileName, FileMode.OpenOrCreate, FileAccess.ReadWrite);
|
|
||||||
var bufferSize = response.Content.Headers.ContentLength > 1024 * 1024 ? 4096 : 1024;
|
|
||||||
var buffer = new byte[bufferSize];
|
|
||||||
|
|
||||||
var bytesRead = 0;
|
|
||||||
while ((bytesRead = await (await response.Content.ReadAsStreamAsync().ConfigureAwait(false)).ReadAsync(buffer).ConfigureAwait(false)) > 0)
|
|
||||||
{
|
|
||||||
await fileStream.WriteAsync(buffer.AsMemory(0, bytesRead)).ConfigureAwait(false);
|
|
||||||
}
|
|
||||||
|
|
||||||
_metrics.IncGauge(MetricsAPI.GaugeFilesTotal);
|
|
||||||
_metrics.IncGauge(MetricsAPI.GaugeFilesTotalSize, FilePathUtil.GetFileInfoForHash(_basePath, hash).Length);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
transferTask = _currentTransfers[hash];
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (transferTask != null)
|
if (fi == null && _currentTransfers.TryGetValue(hash, out var downloadTask))
|
||||||
{
|
{
|
||||||
await transferTask.ConfigureAwait(false);
|
await downloadTask.ConfigureAwait(false);
|
||||||
_currentTransfers.Remove(hash, out _);
|
|
||||||
fi = FilePathUtil.GetFileInfoForHash(_basePath, hash);
|
|
||||||
if (fi == null) return null;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fi = FilePathUtil.GetFileInfoForHash(_basePath, hash);
|
||||||
|
if (fi == null) return null;
|
||||||
|
|
||||||
_fileStatisticsService.LogFile(hash, fi.Length);
|
_fileStatisticsService.LogFile(hash, fi.Length);
|
||||||
|
|
||||||
int attempts = 0;
|
return new FileStream(fi.FullName, FileMode.Open, FileAccess.Read, FileShare.Inheritable | FileShare.Read);
|
||||||
while (attempts < 5)
|
|
||||||
{
|
|
||||||
try
|
|
||||||
{
|
|
||||||
return new FileStream(fi.FullName, FileMode.Open, FileAccess.Read, FileShare.Inheritable | FileShare.Read);
|
|
||||||
}
|
|
||||||
catch (Exception ex)
|
|
||||||
{
|
|
||||||
attempts++;
|
|
||||||
_logger.LogWarning(ex, "Error opening file, retrying");
|
|
||||||
await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
throw new IOException("Could not open file " + fi.FullName);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Reference in New Issue
Block a user