Skip to content

Fingerprint in bulk #56

Open
wants to merge 7 commits into
base: master
Choose a base branch
from
Open

Fingerprint in bulk #56

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
e086e62
paralellize pre-processing
Dec 3, 2019
6942781
restructure manager
Dec 4, 2019
13a46b4
fix duplicate check logic
Dec 4, 2019
f95fbb1
run background pool without semaphore protection
Dec 4, 2019
75c9e2f
fix mocks to set success status
Dec 8, 2019
9ab3b36
uploads end in-order
Dec 9, 2019
5a05b47
ensure MaxUploads is respected
Jan 3, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
125 changes: 59 additions & 66 deletions Hotsapi.Uploader.Common.Test/ManagerTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,81 +1,42 @@
using Microsoft.VisualStudio.TestTools.UnitTesting;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;

namespace Hotsapi.Uploader.Common.Test
{
[TestClass]
public partial class ManagerTests
{
private Task ShortRandomDelay()
public static Task ShortRandomDelay()
{
var r = new Random();
var delay = r.Next(100, 200);
return Task.Delay(delay);
}
private static IEnumerable<ReplayFile> ThreeInOrder
private static IEnumerable<ReplayFile> FilesInOrder
{
get {
var one = new ReplayFile("one") {
Created = new DateTime(2020, 1, 1, 0, 0, 1)
};
var two = new ReplayFile("two") {
Created = new DateTime(2020, 1, 1, 0, 0, 10)
};
var three = new ReplayFile("three") {
Created = new DateTime(2020, 1, 1, 0, 0, 20)
};
var initialFiles = new List<ReplayFile>() { one, two, three };
return initialFiles;
var next = new DateTime(2020, 1, 1, 0, 0, 0);
var increment = new TimeSpan(0, 0, 1);
var rand = new Random();
var nums = Enumerable.Range(1, 24).OrderBy(rf => rand.NextDouble());

return nums.Select(i => {
next += increment;
return new ReplayFile($"upload_{i}") {
Created = next
};
});

}
}

[TestMethod]
[Ignore("Known intermittant failure: multiple uploads are started in parallel and don't always start in order")]
public async Task InitialFilesStartInOrder()
{
var initialFiles = ThreeInOrder;

var manager = new Manager(new MockStorage(initialFiles));
var uploadTester = new MockUploader();

var promise = new TaskCompletionSource<int>();
Task done = promise.Task;

var uploadsSeen = 0;
var l = new object();
ReplayFile lastUploadStarted = null;
uploadTester.SetUploadCallback(async rf => {
if (lastUploadStarted != null) {
try {
Assert.IsTrue(rf.Created >= lastUploadStarted.Created, $"upload started out of order, {lastUploadStarted} started after {rf}");
} catch (Exception e) {
promise.TrySetException(e);
}
}
lastUploadStarted = rf;
await ShortRandomDelay();
var isDone = false;
lock (l) {
uploadsSeen++;
isDone = uploadsSeen >= 3;
}
if (isDone) {
promise.TrySetResult(uploadsSeen);
}
});

manager.Start(new NoNewFilesMonitor(), new MockAnalizer(), uploadTester);
await done;
}



[TestMethod]
[Ignore("Known intermittant failure: multiple uploads are started in parallel and don't always end in order")]
public async Task InitialFilesEndInorder() {
var initialFiles = ThreeInOrder;
var initialFiles = FilesInOrder;

var manager = new Manager(new MockStorage(initialFiles));
var uploadTester = new MockUploader();
Expand All @@ -85,11 +46,11 @@ public async Task InitialFilesEndInorder() {
var uploadsSeen = 0;
var l = new object();
ReplayFile lastUploadFinished = null;
uploadTester.SetUploadCallback(async rf => {
await ShortRandomDelay();
uploadTester.UploadFinished = async rf => {
if (lastUploadFinished != null) {
try {
Assert.IsTrue(rf.Created >= lastUploadFinished.Created, $"upload completed out of order, {lastUploadFinished} completed after {rf}");
var isInOrder = rf.Created >= lastUploadFinished.Created;
Assert.IsTrue(isInOrder, $"upload completed out of order, {rf} completed after {lastUploadFinished}");
}
catch (Exception e) {
promise.TrySetException(e);
Expand All @@ -104,7 +65,7 @@ public async Task InitialFilesEndInorder() {
if (isDone) {
promise.TrySetResult(uploadsSeen);
}
});
};

manager.Start(new NoNewFilesMonitor(), new MockAnalizer(), uploadTester);
await done;
Expand All @@ -113,28 +74,60 @@ public async Task InitialFilesEndInorder() {
[TestMethod]
public async Task AllInitialFilesProcessed()
{
var initialFiles = ThreeInOrder;
var initialFiles = FilesInOrder;

var manager = new Manager(new MockStorage(initialFiles));
var uploadTester = new MockUploader();
var done = new TaskCompletionSource<int>();

var uploadsSeen = 0;
object l = new object();
uploadTester.SetUploadCallback(async rf => {
uploadTester.UploadFinished = async rf => {
await ShortRandomDelay();
lock (l) {
uploadsSeen++;
if (uploadsSeen >= 3) {
done.SetResult(uploadsSeen);
}
}
});
};

manager.Start(new NoNewFilesMonitor(), new MockAnalizer(), uploadTester);
var finished = await Task.WhenAny(Task.Delay(4000), done.Task);
await finished;
Assert.AreEqual(3, uploadsSeen);
var num = await done.Task;
Assert.AreEqual(3, num);
}

[TestMethod]
public async Task UploadIsRateLimited()
{
var initialFiles = FilesInOrder;

var manager = new Manager(new MockStorage(initialFiles));
var uploadTester = new MockUploader();
var simulaneousUploads = 0;
var processedUploads = 0;
var totalFiles = initialFiles.Count();
var isDone = new TaskCompletionSource<int>();

uploadTester.UploadStarted = rf => {
var inFlight = Interlocked.Increment(ref simulaneousUploads);
try {
Assert.IsTrue(inFlight <= Manager.MaxUploads, "may not have more uploads in flight than Manager.MaxUploads");
} catch (Exception e) {
isDone.TrySetException(e);
}
return Task.CompletedTask;
};
uploadTester.UploadFinished = rf => {
Interlocked.Decrement(ref simulaneousUploads);
if(Interlocked.Increment(ref processedUploads) >= totalFiles) {
isDone.SetResult(processedUploads);
}
return Task.CompletedTask;
};

manager.Start(new NoNewFilesMonitor(), new MockAnalizer(), uploadTester);
await isDone.Task;
}
}
}
7 changes: 6 additions & 1 deletion Hotsapi.Uploader.Common.Test/MockAnalizer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,12 @@ public partial class ManagerTests
private class MockAnalizer : IAnalyzer
{
public int MinimumBuild { get; set; }
public Replay Analyze(ReplayFile file) => new Replay();
public Replay Analyze(ReplayFile file) {
file.UploadStatus = UploadStatus.Preprocessed;
return new Replay() {
Timestamp = file.Created
};
}
public string GetFingerprint(Replay replay) => "dummy fingerprint";
}
}
Expand Down
32 changes: 16 additions & 16 deletions Hotsapi.Uploader.Common.Test/MockUploader.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,27 +10,27 @@ private class MockUploader : IUploader
{
public bool UploadToHotslogs { get; set; }

private Func<ReplayFile, Task> UploadCallback = _ => Task.CompletedTask;
public void SetUploadCallback(Func<ReplayFile, Task> onUpload)
{
var old = UploadCallback;

UploadCallback = async (ReplayFile file) => {
await old(file);
await onUpload(file);
};
}
public Func<ReplayFile, Task> UploadStarted { get; set; } = _ => Task.CompletedTask;
public Func<ReplayFile, Task> UploadFinished { get; set; } = _ => Task.CompletedTask;

public Task CheckDuplicate(IEnumerable<ReplayFile> replays) => Task.CompletedTask;
public async Task CheckDuplicate(IEnumerable<ReplayFile> replays)
{
foreach (var replay in replays) {
replay.UploadStatus = UploadStatus.ReadyForUpload;
}
await ShortRandomDelay();
}
public Task<int> GetMinimumBuild() => Task.FromResult(1);
public Task Upload(ReplayFile file)
public async Task Upload(ReplayFile file, Task mayComplete)
{
UploadCallback(file);
return Task.CompletedTask;
await UploadStarted(file);
await Upload(file.Filename, mayComplete);
await UploadFinished(file);
}
public async Task<UploadStatus> Upload(string file)
public async Task<UploadStatus> Upload(string file, Task mayComplete)
{
await Task.Delay(100);
await ShortRandomDelay();
await mayComplete;
return UploadStatus.Success;
}
}
Expand Down
38 changes: 14 additions & 24 deletions Hotsapi.Uploader.Common/Analyzer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,21 +16,17 @@ public class Analyzer : IAnalyzer
private static Logger _log = LogManager.GetCurrentClassLogger();

/// <summary>
/// Analyze replay locally before uploading
/// Analyze replay locally before uploading.
///
/// Sets file status as a side-effect.
/// </summary>
/// <param name="file">Replay file</param>
public Replay Analyze(ReplayFile file)
{
try {
var result = DataParser.ParseReplay(file.Filename, false, false, false, true);
var replay = result.Item2;
var parseResult = result.Item1;
var status = GetPreStatus(replay, parseResult);

if (status != null) {
file.UploadStatus = status.Value;
}

file.UploadStatus = UploadStatus.Preprocessing;
var (parseResult, replay) = DataParser.ParseReplay(file.Filename, false, false, false, true);
file.UploadStatus = GetPreStatus(replay, parseResult) ?? file.UploadStatus;
if (parseResult != DataParser.ReplayParseResult.Success) {
return null;
}
Expand All @@ -44,7 +40,7 @@ public Replay Analyze(ReplayFile file)
}
}

public UploadStatus? GetPreStatus(Replay replay, DataParser.ReplayParseResult parseResult)
private UploadStatus? GetPreStatus(Replay replay, DataParser.ReplayParseResult parseResult)
{
switch (parseResult) {
case DataParser.ReplayParseResult.ComputerPlayerFound:
Expand All @@ -56,21 +52,15 @@ public Replay Analyze(ReplayFile file)

case DataParser.ReplayParseResult.PreAlphaWipe:
return UploadStatus.TooOld;
case DataParser.ReplayParseResult.Incomplete:
case DataParser.ReplayParseResult.UnexpectedResult:
return UploadStatus.Incomplete;
}

if (parseResult != DataParser.ReplayParseResult.Success) {
return null;
}

if (replay.GameMode == GameMode.Custom) {
return UploadStatus.CustomGame;
}

if (replay.ReplayBuild < MinimumBuild) {
return UploadStatus.TooOld;
}

return null;
return parseResult != DataParser.ReplayParseResult.Success ? null
: replay.GameMode == GameMode.Custom ? (UploadStatus?)UploadStatus.CustomGame
: replay.ReplayBuild < MinimumBuild ? (UploadStatus?)UploadStatus.TooOld
: (UploadStatus?)UploadStatus.Preprocessed;
}

/// <summary>
Expand Down
Loading