Lightless-Sync/LightlessClient
8
1
Fork 2
Code Issues 2 Pull Requests 1 Actions Packages Projects Releases 53 Wiki Activity

Merge branch '1.11.10' into pause-fix-and-performance

Browse Source
This commit is contained in:
defnotken
2025-09-16 02:42:13 +02:00
parent 0c3af33d5d b2bd1a5fb7
commit 0730822305
9 changed files with 159 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
LightlessSync/FileCache/CacheMonitor.cs
View File

@@ -383,7 +383,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
scanThread.Start();
while (scanThread.IsAlive)
{
await Task.Delay(250).ConfigureAwait(false);
await Task.Delay(250, token).ConfigureAwait(false);
}
TotalFiles = 0;
_currentFileProgress = 0;
@@ -619,7 +619,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase
return;
}
if (entitiesToUpdate.Any() || entitiesToRemove.Any())
if (entitiesToUpdate.Count != 0 || entitiesToRemove.Count != 0)
{
foreach (var entity in entitiesToUpdate)
{

82
LightlessSync/FileCache/FileCacheManager.cs
View File

@@ -21,7 +21,7 @@ public sealed class FileCacheManager : IHostedService
private readonly string _csvPath;
private readonly ConcurrentDictionary<string, List<FileCacheEntity>> _fileCaches = new(StringComparer.Ordinal);
private readonly SemaphoreSlim _getCachesByPathsSemaphore = new(1, 1);
private readonly object _fileWriteLock = new();
private readonly Lock _fileWriteLock = new();
private readonly IpcManager _ipcManager;
private readonly ILogger<FileCacheManager> _logger;
public string CacheFolder => _configService.Current.CacheFolder;
@@ -42,10 +42,7 @@ public sealed class FileCacheManager : IHostedService
FileInfo fi = new(path);
if (!fi.Exists) return null;
_logger.LogTrace("Creating cache entry for {path}", path);
var fullName = fi.FullName.ToLowerInvariant();
if (!fullName.Contains(_configService.Current.CacheFolder.ToLowerInvariant(), StringComparison.Ordinal)) return null;
string prefixedPath = fullName.Replace(_configService.Current.CacheFolder.ToLowerInvariant(), CachePrefix + "\\", StringComparison.Ordinal).Replace("\\\\", "\\", StringComparison.Ordinal);
return CreateFileCacheEntity(fi, prefixedPath);
return CreateFileEntity(_configService.Current.CacheFolder.ToLowerInvariant(), CachePrefix, fi);
}
public FileCacheEntity? CreateFileEntry(string path)
@@ -53,9 +50,14 @@ public sealed class FileCacheManager : IHostedService
FileInfo fi = new(path);
if (!fi.Exists) return null;
_logger.LogTrace("Creating file entry for {path}", path);
return CreateFileEntity(_ipcManager.Penumbra.ModDirectory!.ToLowerInvariant(), PenumbraPrefix, fi);
}
private FileCacheEntity? CreateFileEntity(string directory, string prefix, FileInfo fi)
{
var fullName = fi.FullName.ToLowerInvariant();
if (!fullName.Contains(_ipcManager.Penumbra.ModDirectory!.ToLowerInvariant(), StringComparison.Ordinal)) return null;
string prefixedPath = fullName.Replace(_ipcManager.Penumbra.ModDirectory!.ToLowerInvariant(), PenumbraPrefix + "\\", StringComparison.Ordinal).Replace("\\\\", "\\", StringComparison.Ordinal);
if (!fullName.Contains(_configService.Current.CacheFolder.ToLowerInvariant(), StringComparison.Ordinal)) return null;
string prefixedPath = fullName.Replace(directory, prefix + "\\", StringComparison.Ordinal).Replace("\\\\", "\\", StringComparison.Ordinal);
return CreateFileCacheEntity(fi, prefixedPath);
}
@@ -66,7 +68,7 @@ public sealed class FileCacheManager : IHostedService
List<FileCacheEntity> output = [];
if (_fileCaches.TryGetValue(hash, out var fileCacheEntities))
{
foreach (var fileCache in fileCacheEntities.Where(c => ignoreCacheEntries ? !c.IsCacheEntry : true).ToList())
foreach (var fileCache in fileCacheEntities.Where(c => !ignoreCacheEntries || !c.IsCacheEntry).ToList())
{
if (!validate) output.Add(fileCache);
else
@@ -106,7 +108,7 @@ public sealed class FileCacheManager : IHostedService
var computedHash = Crypto.GetFileHash(fileCache.ResolvedFilepath);
if (!string.Equals(computedHash, fileCache.Hash, StringComparison.Ordinal))
{
_logger.LogInformation("Failed to validate {file}, got hash {hash}, expected hash {hash}", fileCache.ResolvedFilepath, computedHash, fileCache.Hash);
_logger.LogInformation("Failed to validate {file}, got hash {computedHash}, expected hash {hash}", fileCache.ResolvedFilepath, computedHash, fileCache.Hash);
brokenEntities.Add(fileCache);
}
}
@@ -151,7 +153,7 @@ public sealed class FileCacheManager : IHostedService
{
if (_fileCaches.TryGetValue(hash, out var hashes))
{
var item = hashes.OrderBy(p => p.PrefixedFilePath.Contains(PenumbraPrefix) ? 0 : 1).FirstOrDefault();
var item = hashes.OrderBy(p => p.PrefixedFilePath.Contains(PenumbraPrefix, StringComparison.Ordinal) ? 0 : 1).FirstOrDefault();
if (item != null) return GetValidatedFileCache(item);
}
return null;
@@ -180,50 +182,66 @@ public sealed class FileCacheManager : IHostedService
try
{
var cleanedPaths = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
var seenCleaned = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
var allEntities = _fileCaches.SelectMany(f => f.Value).ToArray();
foreach (var p in paths)
var cacheDict = new ConcurrentDictionary<string, FileCacheEntity>(
StringComparer.OrdinalIgnoreCase);
Parallel.ForEach(allEntities, entity =>
{
cacheDict[entity.PrefixedFilePath] = entity;
});
var cleanedPaths = new ConcurrentDictionary<string, string>(StringComparer.OrdinalIgnoreCase);
var seenCleaned = new ConcurrentDictionary<string, byte>(StringComparer.OrdinalIgnoreCase);
Parallel.ForEach(paths, p =>
{
var cleaned = p.Replace("/", "\\", StringComparison.OrdinalIgnoreCase)
.Replace(_ipcManager.Penumbra.ModDirectory!, _ipcManager.Penumbra.ModDirectory!.EndsWith('\\') ? PenumbraPrefix + '\\' : PenumbraPrefix, StringComparison.OrdinalIgnoreCase)
.Replace(_configService.Current.CacheFolder, _configService.Current.CacheFolder.EndsWith('\\') ? CachePrefix + '\\' : CachePrefix, StringComparison.OrdinalIgnoreCase)
.Replace(
_ipcManager.Penumbra.ModDirectory!,
_ipcManager.Penumbra.ModDirectory!.EndsWith('\\')
? PenumbraPrefix + '\\' : PenumbraPrefix,
StringComparison.OrdinalIgnoreCase)
.Replace(
_configService.Current.CacheFolder,
_configService.Current.CacheFolder.EndsWith('\\')
? CachePrefix + '\\' : CachePrefix,
StringComparison.OrdinalIgnoreCase)
.Replace("\\\\", "\\", StringComparison.Ordinal);
if (seenCleaned.Add(cleaned))
if (seenCleaned.TryAdd(cleaned, 0))
{
_logger.LogDebug("Adding to cleanedPaths: {cleaned}", cleaned);
cleanedPaths[p] = cleaned;
} else
{
_logger.LogWarning("Duplicated found: {cleaned}", cleaned);
}
}
else
{
_logger.LogWarning("Duplicate found: {cleaned}", cleaned);
}
});
Dictionary<string, FileCacheEntity?> result = new(StringComparer.OrdinalIgnoreCase);
var result = new ConcurrentDictionary<string, FileCacheEntity?>(StringComparer.OrdinalIgnoreCase);
var dict = _fileCaches.SelectMany(f => f.Value).Distinct()
.ToDictionary(d => d.PrefixedFilePath, d => d, StringComparer.OrdinalIgnoreCase);
foreach (var entry in cleanedPaths)
Parallel.ForEach(cleanedPaths, entry =>
{
_logger.LogDebug("Checking if in cache: {path}", entry.Value);
if (dict.TryGetValue(entry.Value, out var entity))
if (cacheDict.TryGetValue(entry.Value, out var entity))
{
var validatedCache = GetValidatedFileCache(entity);
result.Add(entry.Key, validatedCache);
result[entry.Key] = validatedCache;
}
else
{
if (!entry.Value.Contains(CachePrefix, StringComparison.Ordinal))
result.Add(entry.Key, CreateFileEntry(entry.Key));
result[entry.Key] = CreateFileEntry(entry.Key);
else
result.Add(entry.Key, CreateCacheEntry(entry.Key));
result[entry.Key] = CreateCacheEntry(entry.Key);
}
}
});
return result;
return new Dictionary<string, FileCacheEntity?>(result, StringComparer.OrdinalIgnoreCase);
}
finally
{
@@ -452,7 +470,7 @@ public sealed class FileCacheManager : IHostedService
{
attempts++;
_logger.LogWarning(ex, "Could not open {file}, trying again", _csvPath);
Thread.Sleep(100);
Task.Delay(100, cancellationToken);
}
}

1
LightlessSync/LightlessConfiguration/Configurations/LightlessConfig.cs
View File

@@ -43,6 +43,7 @@ public class LightlessConfig : ILightlessConfiguration
public bool ShowCharacterNameInsteadOfNotesForVisible { get; set; } = false;
public bool ShowOfflineUsersSeparately { get; set; } = true;
public bool ShowSyncshellOfflineUsersSeparately { get; set; } = true;
public bool ShowGroupedSyncshellsInAll { get; set; } = true;
public bool GroupUpSyncshells { get; set; } = true;
public bool ShowOnlineNotifications { get; set; } = false;
public bool ShowOnlineNotificationsOnlyForIndividualPairs { get; set; } = true;

6
LightlessSync/Services/CharaData/CharaDataManager.cs
View File

@@ -944,9 +944,7 @@ public sealed partial class CharaDataManager : DisposableMediatorSubscriberBase
Logger.LogTrace("[{appId}] Computing local missing files", applicationId);
Dictionary<string, string> modPaths;
List<FileReplacementData> missingFiles;
_fileHandler.ComputeMissingFiles(charaDataDownloadDto, out modPaths, out missingFiles);
_fileHandler.ComputeMissingFiles(charaDataDownloadDto, out Dictionary<string, string> modPaths, out List<FileReplacementData> missingFiles);
Logger.LogTrace("[{appId}] Computing local missing files", applicationId);
@@ -990,7 +988,7 @@ public sealed partial class CharaDataManager : DisposableMediatorSubscriberBase
{
_uploadCts = _uploadCts.CancelRecreate();
var missingFiles = await _fileHandler.UploadFiles([.. missingFileList.Select(k => k.HashOrFileSwap)], UploadProgress, _uploadCts.Token).ConfigureAwait(false);
if (missingFiles.Any())
if (missingFiles.Count != 0)
{
Logger.LogInformation("Failed to upload {files}", string.Join(", ", missingFiles));
return ($"Upload failed: {missingFiles.Count} missing or forbidden to upload local files.", false);

24
LightlessSync/UI/CompactUI.cs
View File

@@ -8,7 +8,6 @@ using LightlessSync.API.Data.Extensions;
using LightlessSync.API.Dto.Group;
using LightlessSync.Interop.Ipc;
using LightlessSync.LightlessConfiguration;
using LightlessSync.LightlessConfiguration.Configurations;
using LightlessSync.PlayerData.Handlers;
using LightlessSync.PlayerData.Pairs;
using LightlessSync.Services;
@@ -35,6 +34,7 @@ public class CompactUi : WindowMediatorSubscriberBase
private readonly CharacterAnalyzer _characterAnalyzer;
private readonly ApiController _apiController;
private readonly LightlessConfigService _configService;
private readonly LightlessMediator _lightlessMediator;
private readonly ConcurrentDictionary<GameObjectHandler, Dictionary<string, FileDownloadStatus>> _currentDownloads = new();
private readonly DrawEntityFactory _drawEntityFactory;
private readonly FileUploadManager _fileTransferManager;
@@ -57,7 +57,6 @@ public class CompactUi : WindowMediatorSubscriberBase
private string _lastAddedUserComment = string.Empty;
private Vector2 _lastPosition = Vector2.One;
private Vector2 _lastSize = Vector2.One;
private int _secretKeyIdx = -1;
private bool _showModalForUserAddition;
private float _transferPartHeight;
private bool _wasOpen;
@@ -68,7 +67,7 @@ public class CompactUi : WindowMediatorSubscriberBase
TagHandler tagHandler, DrawEntityFactory drawEntityFactory,
SelectTagForPairUi selectTagForPairUi, SelectPairForTagUi selectPairForTagUi, RenamePairTagUi renameTagUi,
SelectTagForSyncshellUi selectTagForSyncshellUi, SelectSyncshellForTagUi selectSyncshellForTagUi, RenameSyncshellTagUi renameSyncshellTagUi,
PerformanceCollectorService performanceCollectorService, IpcManager ipcManager, CharacterAnalyzer characterAnalyzer, PlayerPerformanceConfigService playerPerformanceConfig)
PerformanceCollectorService performanceCollectorService, IpcManager ipcManager, CharacterAnalyzer characterAnalyzer, PlayerPerformanceConfigService playerPerformanceConfig, LightlessMediator lightlessMediator)
: base(logger, mediator, "###LightlessSyncMainUI", performanceCollectorService)
{
_uiSharedService = uiShared;
@@ -124,7 +123,7 @@ public class CompactUi : WindowMediatorSubscriberBase
}
};
_drawFolders = GetDrawFolders().ToList();
_drawFolders = [.. GetDrawFolders()];
#if DEBUG
string dev = "Dev Build";
@@ -152,6 +151,7 @@ public class CompactUi : WindowMediatorSubscriberBase
};
_characterAnalyzer = characterAnalyzer;
_playerPerformanceConfig = playerPerformanceConfig;
_lightlessMediator = lightlessMediator;
}
protected override void DrawInternal()
@@ -430,7 +430,7 @@ public class CompactUi : WindowMediatorSubscriberBase
ImGui.SetClipboardText(uidText);
}
if (_cachedAnalysis != null)
if (_cachedAnalysis != null && _apiController.ServerState is ServerState.Connected)
{
var firstEntry = _cachedAnalysis.FirstOrDefault();
var valueDict = firstEntry.Value;
@@ -460,6 +460,7 @@ public class CompactUi : WindowMediatorSubscriberBase
{
ImGui.SameLine();
_uiSharedService.IconText(FontAwesomeIcon.ExclamationTriangle, UIColors.Get("LightlessYellow"));
string warningMessage = "";
if (isOverTriHold)
{
@@ -474,6 +475,10 @@ public class CompactUi : WindowMediatorSubscriberBase
$"{UiSharedService.ByteToString(actualVramUsage - (_playerPerformanceConfig.Current.VRAMSizeWarningThresholdMiB * 1024 * 1024))}.";
}
UiSharedService.AttachToolTip(warningMessage);
if (ImGui.IsItemClicked())
{
_lightlessMediator.Publish(new UiToggleMessage(typeof(DataAnalysisUi)));
}
}
}
}
@@ -494,7 +499,7 @@ public class CompactUi : WindowMediatorSubscriberBase
UiSharedService.AttachToolTip("Click to copy");
if (ImGui.IsItemClicked())
{
ImGui.SetClipboardText(_apiController.UID);
_lightlessMediator.Publish(new UiToggleMessage(typeof(DataAnalysisUi)));
}
}
}
@@ -543,6 +548,8 @@ public class CompactUi : WindowMediatorSubscriberBase
=> u.Value.Exists(g => string.Equals(g.GID, group.GID, StringComparison.Ordinal));
bool FilterNotTaggedUsers(KeyValuePair<Pair, List<GroupFullInfoDto>> u)
=> u.Key.IsDirectlyPaired && !u.Key.IsOneSidedPair && !_tagHandler.HasAnyPairTag(u.Key.UserData.UID);
bool FilterNotTaggedSyncshells(GroupFullInfoDto group)
=> (!_tagHandler.HasAnySyncshellTag(group.GID) && !_configService.Current.ShowGroupedSyncshellsInAll) || _configService.Current.ShowGroupedSyncshellsInAll;
bool FilterOfflineUsers(KeyValuePair<Pair, List<GroupFullInfoDto>> u)
=> ((u.Key.IsDirectlyPaired && _configService.Current.ShowSyncshellOfflineUsersSeparately)
|| !_configService.Current.ShowSyncshellOfflineUsersSeparately)
@@ -566,7 +573,10 @@ public class CompactUi : WindowMediatorSubscriberBase
foreach (var group in _pairManager.GroupPairs.Select(g => g.Key).OrderBy(g => g.GroupAliasOrGID, StringComparer.OrdinalIgnoreCase))
{
GetGroups(allPairs, filteredPairs, group, out ImmutableList<Pair> allGroupPairs, out Dictionary<Pair, List<GroupFullInfoDto>> filteredGroupPairs);
groupFolders.Add(_drawEntityFactory.CreateDrawGroupFolder(group, filteredGroupPairs, allGroupPairs));
if (FilterNotTaggedSyncshells(group))
{
groupFolders.Add(_drawEntityFactory.CreateDrawGroupFolder(group, filteredGroupPairs, allGroupPairs));
}
}
if (_configService.Current.GroupUpSyncshells)

28
LightlessSync/UI/SettingsUi.cs
View File

@@ -633,25 +633,6 @@ public class SettingsUi : WindowMediatorSubscriberBase
{
_lastTab = "FileCache";
_uiShared.UnderlinedBigText("Export MCDF", UIColors.Get("LightlessBlue"));
ImGuiHelpers.ScaledDummy(10);
UiSharedService.ColorTextWrapped("Exporting MCDF has moved.", UIColors.Get("LightlessYellow"));
ImGuiHelpers.ScaledDummy(5);
UiSharedService.TextWrapped("It is now found in the Main UI under \"Your User Menu\" (");
ImGui.SameLine();
_uiShared.IconText(FontAwesomeIcon.UserCog);
ImGui.SameLine();
UiSharedService.TextWrapped(") -> \"Character Data Hub\".");
if (_uiShared.IconTextButton(FontAwesomeIcon.Running, "Open Lightless Character Data Hub"))
{
Mediator.Publish(new UiToggleMessage(typeof(CharaDataHubUi)));
}
UiSharedService.TextWrapped("Note: this entry will be removed in the near future. Please use the Main UI to open the Character Data Hub.");
ImGuiHelpers.ScaledDummy(5);
ImGui.Separator();
_uiShared.UnderlinedBigText("Storage", UIColors.Get("LightlessBlue"));
ImGuiHelpers.ScaledDummy(5);
@@ -946,6 +927,7 @@ public class SettingsUi : WindowMediatorSubscriberBase
var preferNotesInsteadOfName = _configService.Current.PreferNotesOverNamesForVisible;
var useFocusTarget = _configService.Current.UseFocusTarget;
var groupUpSyncshells = _configService.Current.GroupUpSyncshells;
var groupedSyncshells = _configService.Current.ShowGroupedSyncshellsInAll;
var groupInVisible = _configService.Current.ShowSyncshellUsersInVisible;
var syncshellOfflineSeparate = _configService.Current.ShowSyncshellOfflineUsersSeparately;
@@ -1163,6 +1145,14 @@ public class SettingsUi : WindowMediatorSubscriberBase
}
_uiShared.DrawHelpText("This will group up all Syncshells in a special 'All Syncshells' folder in the main UI.");
if (ImGui.Checkbox("Show grouped syncshells in main screen/all syncshells", ref groupedSyncshells))
{
_configService.Current.ShowGroupedSyncshellsInAll = groupedSyncshells;
_configService.Save();
Mediator.Publish(new RefreshUiMessage());
}
_uiShared.DrawHelpText("This will show grouped syncshells in main screen or group 'All Syncshells'.");
if (ImGui.Checkbox("Show player name for visible players", ref showNameInsteadOfNotes))
{
_configService.Current.ShowCharacterNameInsteadOfNotesForVisible = showNameInsteadOfNotes;

31
LightlessSync/UI/SyncshellAdminUI.cs
View File

@@ -195,9 +195,9 @@ public class SyncshellAdminUI : WindowMediatorSubscriberBase
using var table = ImRaii.Table("userList#" + GroupFullInfo.Group.GID, 3, tableFlags);
if (table)
{
ImGui.TableSetupColumn("Alias/UID/Note", ImGuiTableColumnFlags.None, 5);
ImGui.TableSetupColumn("Alias/UID/Note", ImGuiTableColumnFlags.None, 4);
ImGui.TableSetupColumn("Flags", ImGuiTableColumnFlags.None, 1);
ImGui.TableSetupColumn("Actions", ImGuiTableColumnFlags.None, 2);
ImGui.TableSetupColumn("Actions", ImGuiTableColumnFlags.None, 3);
ImGui.TableHeadersRow();
var groupedPairs = new Dictionary<Pair, GroupPairUserInfo?>(pairs.Select(p => new KeyValuePair<Pair, GroupPairUserInfo?>(p,
@@ -254,13 +254,32 @@ public class SyncshellAdminUI : WindowMediatorSubscriberBase
ImGui.TableNextColumn(); // actions
if (_isOwner)
{
if (_uiSharedService.IconButton(FontAwesomeIcon.UserShield))
using (ImRaii.PushColor(ImGuiCol.Text, UIColors.Get("LightlessYellow")))
{
GroupPairUserInfo userInfo = pair.Value ?? GroupPairUserInfo.None;
using (ImRaii.Disabled(!UiSharedService.ShiftPressed()))
{
if (_uiSharedService.IconButton(FontAwesomeIcon.Crown))
{
_ = _apiController.GroupChangeOwnership(new(GroupFullInfo.Group, pair.Key.UserData));
IsOpen = false;
}
}
userInfo.SetModerator(!userInfo.IsModerator());
}
UiSharedService.AttachToolTip("Hold SHIFT and click to transfer ownership of this Syncshell to "
+ (pair.Key.UserData.AliasOrUID) + Environment.NewLine + "WARNING: This action is irreversible and will close screen.");
ImGui.SameLine();
_ = _apiController.GroupSetUserInfo(new GroupPairUserInfoDto(GroupFullInfo.Group, pair.Key.UserData, userInfo));
using (ImRaii.PushColor(ImGuiCol.Text, pair.Value != null && pair.Value.Value.IsModerator() ? UIColors.Get("DimRed") : UIColors.Get("PairBlue")))
{
if (_uiSharedService.IconButton(FontAwesomeIcon.UserShield))
{
GroupPairUserInfo userInfo = pair.Value ?? GroupPairUserInfo.None;
userInfo.SetModerator(!userInfo.IsModerator());
_ = _apiController.GroupSetUserInfo(new GroupPairUserInfoDto(GroupFullInfo.Group, pair.Key.UserData, userInfo));
}
}
UiSharedService.AttachToolTip(pair.Value != null && pair.Value.Value.IsModerator() ? "Demod user" : "Mod user");
ImGui.SameLine();

68
LightlessSync/WebAPI/Files/FileDownloadManager.cs
View File

@@ -144,28 +144,55 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
_downloadStatus[downloadGroup].DownloadStatus = DownloadStatus.Downloading;
const int maxRetries = 3;
int retryCount = 0;
TimeSpan retryDelay = TimeSpan.FromSeconds(2);
HttpResponseMessage response = null!;
var requestUrl = LightlessFiles.CacheGetFullPath(fileTransfer[0].DownloadUri, requestId);
Logger.LogDebug("Downloading {requestUrl} for request {id}", requestUrl, requestId);
try
while (true)
{
response = await _orchestrator.SendRequestAsync(HttpMethod.Get, requestUrl, ct, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
}
catch (HttpRequestException ex)
{
Logger.LogWarning(ex, "Error during download of {requestUrl}, HttpStatusCode: {code}", requestUrl, ex.StatusCode);
if (ex.StatusCode is HttpStatusCode.NotFound or HttpStatusCode.Unauthorized)
try
{
throw new InvalidDataException($"Http error {ex.StatusCode} (cancelled: {ct.IsCancellationRequested}): {requestUrl}", ex);
Logger.LogDebug("Attempt {attempt} - Downloading {requestUrl} for request {id}", retryCount + 1, requestUrl, requestId);
response = await _orchestrator.SendRequestAsync(HttpMethod.Get, requestUrl, ct, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
break;
}
catch (HttpRequestException ex) when (ex.InnerException is TimeoutException || ex.StatusCode == null)
{
retryCount++;
Logger.LogWarning(ex, "Timeout during download of {requestUrl}. Attempt {attempt} of {maxRetries}", requestUrl, retryCount, maxRetries);
if (retryCount >= maxRetries || ct.IsCancellationRequested)
{
Logger.LogError($"Max retries reached or cancelled. Failing download for {requestUrl}");
throw;
}
await Task.Delay(retryDelay, ct).ConfigureAwait(false); // Wait before retrying
}
catch (HttpRequestException ex)
{
Logger.LogWarning(ex, "Error during download of {requestUrl}, HttpStatusCode: {code}", requestUrl, ex.StatusCode);
if (ex.StatusCode is HttpStatusCode.NotFound or HttpStatusCode.Unauthorized)
{
throw new InvalidDataException($"Http error {ex.StatusCode} (cancelled: {ct.IsCancellationRequested}): {requestUrl}", ex);
}
throw;
}
}
ThrottledStream? stream = null;
FileStream? fileStream = null;
try
{
var fileStream = File.Create(tempPath);
fileStream = File.Create(tempPath);
await using (fileStream.ConfigureAwait(false))
{
var bufferSize = response.Content.Headers.ContentLength > 1024 * 1024 ? 65536 : 8196;
@@ -174,8 +201,11 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
var bytesRead = 0;
var limit = _orchestrator.DownloadLimitPerSlot();
Logger.LogTrace("Starting Download of {id} with a speed limit of {limit} to {tempPath}", requestId, limit, tempPath);
stream = new ThrottledStream(await response.Content.ReadAsStreamAsync(ct).ConfigureAwait(false), limit);
stream = new(await response.Content.ReadAsStreamAsync(ct).ConfigureAwait(false), limit);
_activeDownloadStreams.Add(stream);
while ((bytesRead = await stream.ReadAsync(buffer, ct).ConfigureAwait(false)) > 0)
{
ct.ThrowIfCancellationRequested();
@@ -194,18 +224,22 @@ public partial class FileDownloadManager : DisposableMediatorSubscriberBase
{
throw;
}
catch (Exception ex)
catch (Exception)
{
try
{
if (!tempPath.IsNullOrEmpty())
fileStream?.Close();
if (!string.IsNullOrEmpty(tempPath) && File.Exists(tempPath))
{
File.Delete(tempPath);
}
}
catch
{
// ignore if file deletion fails
// Ignore errors during cleanup
}
throw;
throw;
}
finally
{

4
LightlessSync/WebAPI/Files/FileUploadManager.cs
View File

@@ -69,7 +69,7 @@ public sealed class FileUploadManager : DisposableMediatorSubscriberBase
Logger.LogDebug("Trying to upload files");
var filesPresentLocally = hashesToUpload.Where(h => _fileDbManager.GetFileCacheByHash(h) != null).ToHashSet(StringComparer.Ordinal);
var locallyMissingFiles = hashesToUpload.Except(filesPresentLocally, StringComparer.Ordinal).ToList();
if (locallyMissingFiles.Any())
if (locallyMissingFiles.Count != 0)
{
return locallyMissingFiles;
}
@@ -92,7 +92,7 @@ public sealed class FileUploadManager : DisposableMediatorSubscriberBase
var data = await _fileDbManager.GetCompressedFileData(file.Hash, ct ?? CancellationToken.None).ConfigureAwait(false);
Logger.LogDebug("[{hash}] Starting upload for {filePath}", data.Item1, _fileDbManager.GetFileCacheByHash(data.Item1)!.ResolvedFilepath);
await uploadTask.ConfigureAwait(false);
uploadTask = UploadFile(data.Item2, file.Hash, false, ct ?? CancellationToken.None);
uploadTask = UploadFile(data.Item2, file.Hash, postProgress: false, ct ?? CancellationToken.None);
(ct ?? CancellationToken.None).ThrowIfCancellationRequested();
}