rob/website
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add caching to API calls.

Browse source
This commit is contained in:
Robert Marshall 2020-04-18 13:08:41 +01:00
parent fcdec66861
commit ed8468105d
14 changed files with 228 additions and 56 deletions
Download patch file Download diff file Expand all files Collapse all files

2
.gitattributes vendored
View file

@ -1 +1 @@
/Website/appsettings.Development.json filter=clean-config
src/Website/appsettings.Development.json filter=clean-config

56
src/Website.Tests/Data/ApiClientTests.cs
View file

@ -2,13 +2,15 @@
using System.Net.Http;
using System.Threading.Tasks;
using FluentAssertions;
using Microsoft.Extensions.Caching.Memory;
using NSubstitute;
using Website.Data;
using Xunit;
namespace Website.Tests.Data {
public class ApiClientTests {
private class TestApiClient : ApiClient {
public TestApiClient(HttpClient client) : base(client) {
public TestApiClient(HttpClient client, IMemoryCache cache) : base(client, cache, new CacheDurations()) {
}
public async Task<T> Post<T>(string url, object value, object query = null) => await base.Post<T>(url, value, query);
@ -21,21 +23,25 @@ namespace Website.Tests.Data {
}
[Fact]
public async Task Get_WithUrl_ReturnsResult() {
public async Task Get_WithUrl_CachesAndReturnsResult() {
var httpClient = new HttpClientBuilder()
.WithMethod(HttpMethod.Get)
.WithUrl("/test")
.WithResponse(@"{""TestProperty"":1}")
.Build();
var cache = Substitute.For<IMemoryCache>();
cache.TryGetValue("TestApiClient:Get", out _).Returns(false);
var expected = new TestObject {TestProperty = 1};
var client = new TestApiClient(httpClient);
(await client.Get<TestObject>("/test")).Should().BeEquivalentTo(expected);
var client = new TestApiClient(httpClient, cache);
(await client.Get<TestObject>("test")).Should().BeEquivalentTo(expected);
cache.Received(1).CreateEntry("TestApiClient:Get");
}
[Fact]
public async Task Get_WithUrlAndQuery_ReturnsResult() {
public async Task Get_WithUrlAndQuery_CachesAndReturnsResult() {
var httpClient = new HttpClientBuilder()
.WithMethod(HttpMethod.Get)
.WithUrl("/test")
@ -44,10 +50,14 @@ namespace Website.Tests.Data {
.WithResponse(@"{""TestProperty"":1}")
.Build();
var cache = Substitute.For<IMemoryCache>();
cache.TryGetValue("TestApiClient:Get?query1=1&query2=2", out _).Returns(false);
var expected = new TestObject {TestProperty = 1};
var client = new TestApiClient(httpClient);
(await client.Get<TestObject>("/test", new {query1 = 1, query2 = 2})).Should().BeEquivalentTo(expected);
var client = new TestApiClient(httpClient, cache);
(await client.Get<TestObject>("test", new {query1 = 1, query2 = 2})).Should().BeEquivalentTo(expected);
cache.Received(1).CreateEntry("TestApiClient:Get?query1=1&query2=2");
}
[Fact]
@ -59,14 +69,16 @@ namespace Website.Tests.Data {
.WithErrorStatus(HttpStatusCode.NotFound)
.Build();
var client = new TestApiClient(httpClient);
var cache = Substitute.For<IMemoryCache>();
client.Invoking(apiClient => apiClient.Get<TestObject>("/404")).Should().Throw<ApiCallException>()
var client = new TestApiClient(httpClient, cache);
client.Invoking(apiClient => apiClient.Get<TestObject>("404")).Should().Throw<ApiCallException>()
.WithMessage("Error calling API http://example.com/404: 404, Not Found");
}
[Fact]
public async Task Post_WithUrlAndValue_ReturnsResult() {
public async Task Post_WithUrlAndValue_DoesNotCacheAndReturnsResult() {
var httpClient = new HttpClientBuilder()
.WithMethod(HttpMethod.Post)
.WithUrl("/test")
@ -74,14 +86,17 @@ namespace Website.Tests.Data {
.WithPostBody("\"value\"")
.Build();
var cache = Substitute.For<IMemoryCache>();
var expected = new TestObject {TestProperty = 1};
var client = new TestApiClient(httpClient);
(await client.Post<TestObject>("/test", "value")).Should().BeEquivalentTo(expected);
var client = new TestApiClient(httpClient, cache);
(await client.Post<TestObject>("test", "value")).Should().BeEquivalentTo(expected);
cache.Received(0).CreateEntry(Arg.Any<string>());
}
[Fact]
public async Task Post_WithUrlAndValueAndQuery_ReturnsResult() {
public async Task Post_WithUrlAndValueAndQuery_CachesAndReturnsResult() {
var httpClient = new HttpClientBuilder()
.WithMethod(HttpMethod.Post)
.WithUrl("/test")
@ -91,14 +106,17 @@ namespace Website.Tests.Data {
.WithResponse(@"{""TestProperty"":1}")
.Build();
var cache = Substitute.For<IMemoryCache>();
var expected = new TestObject {TestProperty = 1};
var client = new TestApiClient(httpClient);
(await client.Post<TestObject>("/test", "value", new {query1 = 1, query2 = 2})).Should().BeEquivalentTo(expected);
var client = new TestApiClient(httpClient, cache);
(await client.Post<TestObject>("test", "value", new {query1 = 1, query2 = 2})).Should().BeEquivalentTo(expected);
cache.Received(0).CreateEntry(Arg.Any<string>());
}
[Fact]
public async Task Post_WithUrlAndValueAndQuery_ReturnsNoResult() {
public async Task Post_WithUrlAndValueAndQuery_CachesAndReturnsNoResult() {
var httpClient = new HttpClientBuilder()
.WithMethod(HttpMethod.Post)
.WithUrl("/test")
@ -107,10 +125,12 @@ namespace Website.Tests.Data {
.WithPostBody("\"value\"")
.Build(out var mockHttpMessageHandler);
var cache = Substitute.For<IMemoryCache>();
var client = new TestApiClient(httpClient);
await client.Post<object>("/test", "value", new {query1 = 1, query2 = 2});
var client = new TestApiClient(httpClient, cache);
await client.Post<object>("test", "value", new {query1 = 1, query2 = 2});
mockHttpMessageHandler.VerifyNoOutstandingExpectation();
cache.Received(0).CreateEntry(Arg.Any<string>());
}
}
}

10
src/Website.Tests/Data/AuthenticationProviderTests.cs
View file

@ -2,6 +2,8 @@
using System.Net.Http;
using System.Threading.Tasks;
using FluentAssertions;
using Microsoft.Extensions.Caching.Memory;
using NSubstitute;
using Website.Data;
using Website.Models.Auth;
using Xunit;
@ -18,6 +20,8 @@ namespace Website.Tests.Data {
.WithResponse(json)
.Build();
var cache = Substitute.For<IMemoryCache>();
var request = new LoginRequest {
Username = "username",
Password = "password"
@ -28,7 +32,7 @@ namespace Website.Tests.Data {
Password = "password"
};
var provider = new AuthenticationProvider(httpClient);
var provider = new AuthenticationProvider(httpClient, cache, new CacheDurations());
(await provider.Authenticate(request)).Should().BeEquivalentTo(expectedUser);
}
@ -43,12 +47,14 @@ namespace Website.Tests.Data {
.WithResponse(json)
.Build();
var cache = Substitute.For<IMemoryCache>();
var request = new LoginRequest {
Username = "username",
Password = "wrong"
};
var provider = new AuthenticationProvider(httpClient);
var provider = new AuthenticationProvider(httpClient, cache, new CacheDurations());
(await provider.Authenticate(request)).Should().BeNull();
}
}

46
src/Website.Tests/Data/BlogApiTests.cs
View file

@ -2,6 +2,8 @@
using System.Net.Http;
using System.Threading.Tasks;
using FluentAssertions;
using Microsoft.Extensions.Caching.Memory;
using NSubstitute;
using Website.Data;
using Website.Models.Blog;
using Xunit;
@ -17,6 +19,8 @@ namespace Website.Tests.Data {
.WithResponse(json)
.Build();
var cache = Substitute.For<IMemoryCache>();
var expectation = new BlogPost {
Id = 1,
Title = "title",
@ -27,7 +31,7 @@ namespace Website.Tests.Data {
UserId = 0
};
var api = new BlogApi(httpClient);
var api = new BlogApi(httpClient, cache, new CacheDurations());
(await api.GetPostByUrlAsync("test")).Should().BeEquivalentTo(expectation);
}
@ -40,6 +44,8 @@ namespace Website.Tests.Data {
.WithResponse(json)
.Build();
var cache = Substitute.For<IMemoryCache>();
var expectation = new[] {
new BlogPost {
Id = 1,
@ -52,7 +58,7 @@ namespace Website.Tests.Data {
}
};
var api = new BlogApi(httpClient);
var api = new BlogApi(httpClient, cache, new CacheDurations());
(await api.GetLatestPostsAsync()).Should().BeEquivalentTo(expectation);
}
@ -65,6 +71,8 @@ namespace Website.Tests.Data {
.WithResponse(json)
.Build();
var cache = Substitute.For<IMemoryCache>();
var expectation = new[] {
new BlogPost {
Id = 1,
@ -77,7 +85,7 @@ namespace Website.Tests.Data {
}
};
var api = new BlogApi(httpClient);
var api = new BlogApi(httpClient, cache, new CacheDurations());
(await api.GetLatestPostsAsync(1)).Should().BeEquivalentTo(expectation);
}
@ -92,6 +100,8 @@ namespace Website.Tests.Data {
.WithResponse(json)
.Build();
var cache = Substitute.For<IMemoryCache>();
var expectation = new[] {
new BlogPost {
Id = 1,
@ -104,7 +114,7 @@ namespace Website.Tests.Data {
}
};
var api = new BlogApi(httpClient);
var api = new BlogApi(httpClient, cache, new CacheDurations());
(await api.GetLatestPostsAsync(1, 1)).Should().BeEquivalentTo(expectation);
}
@ -117,6 +127,8 @@ namespace Website.Tests.Data {
.WithResponse(json)
.Build();
var cache = Substitute.For<IMemoryCache>();
var expectation = new BlogPost {
Id = 1,
Title = "title",
@ -127,7 +139,7 @@ namespace Website.Tests.Data {
UserId = 0
};
var api = new BlogApi(httpClient);
var api = new BlogApi(httpClient, cache, new CacheDurations());
(await api.GetLatestPostAsync()).Should().BeEquivalentTo(expectation);
}
@ -140,7 +152,9 @@ namespace Website.Tests.Data {
.WithResponse(json)
.Build();
var api = new BlogApi(httpClient);
var cache = Substitute.For<IMemoryCache>();
var api = new BlogApi(httpClient, cache, new CacheDurations());
(await api.GetCountAsync()).Should().Be(23);
}
@ -153,6 +167,8 @@ namespace Website.Tests.Data {
.WithResponse(json)
.Build();
var cache = Substitute.For<IMemoryCache>();
var expectation = new BlogPost {
Id = 1,
Title = "title",
@ -163,7 +179,7 @@ namespace Website.Tests.Data {
UserId = 0
};
var api = new BlogApi(httpClient);
var api = new BlogApi(httpClient, cache, new CacheDurations());
(await api.GetPostByIdAsync(1)).Should().BeEquivalentTo(expectation);
}
@ -178,6 +194,8 @@ namespace Website.Tests.Data {
.WithResponse(responseJson)
.Build();
var cache = Substitute.For<IMemoryCache>();
var post = new BlogPostSubmission {Id = 1, Title = "title", Content = "content"};
var expectation = new BlogPost {
@ -190,7 +208,7 @@ namespace Website.Tests.Data {
UserId = 0
};
var api = new BlogApi(httpClient);
var api = new BlogApi(httpClient, cache, new CacheDurations());
(await api.SavePost(post)).Should().BeEquivalentTo(expectation);
}
@ -204,6 +222,8 @@ namespace Website.Tests.Data {
.WithResponse(json)
.Build();
var cache = Substitute.For<IMemoryCache>();
var expectation = new[] {
new BlogPost {
Id = 1,
@ -216,7 +236,7 @@ namespace Website.Tests.Data {
}
};
var api = new BlogApi(httpClient);
var api = new BlogApi(httpClient, cache, new CacheDurations());
(await api.GetAllPostsAsync()).Should().BeEquivalentTo(expectation);
}
@ -228,7 +248,9 @@ namespace Website.Tests.Data {
.WithPostBody("1")
.Build(out var mockHttpMessageHandler);
var api = new BlogApi(httpClient);
var cache = Substitute.For<IMemoryCache>();
var api = new BlogApi(httpClient, cache, new CacheDurations());
await api.DeletePostAsync(1);
mockHttpMessageHandler.VerifyNoOutstandingExpectation();
}
@ -241,7 +263,9 @@ namespace Website.Tests.Data {
.WithPostBody("1")
.Build(out var mockHttpMessageHandler);
var api = new BlogApi(httpClient);
var cache = Substitute.For<IMemoryCache>();
var api = new BlogApi(httpClient, cache, new CacheDurations());
await api.PublishPostAsync(1);
mockHttpMessageHandler.VerifyNoOutstandingExpectation();
}

32
src/Website.Tests/Data/CacheDurationsTests.cs Normal file
View file

@ -0,0 +1,32 @@
using System.Collections.Generic;
using FluentAssertions;
using Website.Data;
using Xunit;
namespace Website.Tests.Data {
public class CacheDurationsTests {
[Fact]
public void Index_WithExistingKey_ReturnsDuration() {
var duration = new CacheDurations {{"class", new Dictionary<string, int> {{"method", 1}}}};
duration["class:method"].Should().Be(1);
}
[Fact]
public void Index_WithNonExistentKey_ReturnsDefault() {
var duration = new CacheDurations {{"default", new Dictionary<string, int> {{"default", 1}}}};
duration["key"].Should().Be(1);
}
[Fact]
public void Index_WithNonExistentMethodKey_ReturnsDefaultForClass() {
var duration = new CacheDurations {{"class", new Dictionary<string, int> {{"default", 1}}}};
duration["class:method"].Should().Be(1);
}
[Fact]
public void Index_WithNonexistentKey_WithoutDefaultDefined_ReturnsZero() {
var duration = new CacheDurations();
duration["key"].Should().Be(0);
}
}
}

14
src/Website.Tests/Data/GitApiTests.cs
View file

@ -2,6 +2,8 @@
using System.Net.Http;
using System.Threading.Tasks;
using FluentAssertions;
using Microsoft.Extensions.Caching.Memory;
using NSubstitute;
using Website.Data;
using Website.Models.Git;
using Xunit;
@ -18,9 +20,11 @@ namespace Website.Tests.Data {
.WithResponse(json)
.Build();
var cache = Substitute.For<IMemoryCache>();
var expectation = new[] {new Repository {Url = "url", Name = "name"}};
var api = new GitApi(httpClient);
var api = new GitApi(httpClient, cache, new CacheDurations());
(await api.GetRepositories("test")).Should().BeEquivalentTo(expectation);
}
@ -35,6 +39,8 @@ namespace Website.Tests.Data {
.WithResponse(json)
.Build();
var cache = Substitute.For<IMemoryCache>();
var expectation = new[] {
new Branch {
Name = "master",
@ -46,7 +52,7 @@ namespace Website.Tests.Data {
}
};
var api = new GitApi(httpClient);
var api = new GitApi(httpClient, cache, new CacheDurations());
(await api.GetBranches("test", "repo")).Should().BeEquivalentTo(expectation);
}
@ -62,6 +68,8 @@ namespace Website.Tests.Data {
.WithResponse(json)
.Build();
var cache = Substitute.For<IMemoryCache>();
var expectation = new Commit {
Id = "0923b554309ef562fca978c7e981b3812bc4af40",
Message = "message",
@ -69,7 +77,7 @@ namespace Website.Tests.Data {
Url = "https://test.com/test/repo/commit/0923b554309ef562fca978c7e981b3812bc4af40"
};
var api = new GitApi(httpClient);
var api = new GitApi(httpClient, cache, new CacheDurations());
(await api.GetCommit("test", "repo", "hash")).Should().BeEquivalentTo(expectation);
}
}

59
src/Website/Data/ApiClient.cs
View file

@ -1,17 +1,24 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
using Microsoft.AspNetCore.WebUtilities;
using Microsoft.Extensions.Caching.Memory;
using Newtonsoft.Json;
namespace Website.Data {
public abstract class ApiClient {
private readonly HttpClient _client;
private readonly IMemoryCache _cache;
private readonly CacheDurations _cacheDurations;
protected ApiClient(HttpClient client) {
protected ApiClient(HttpClient client, IMemoryCache cache, CacheDurations cacheDurations) {
_client = client;
_cache = cache;
_cacheDurations = cacheDurations;
}
private IDictionary<string, string> ParseQueryParameters(object query) {
@ -20,28 +27,48 @@ namespace Website.Data {
return props.ToDictionary(info => info.Name, info => info.GetValue(query, null).ToString());
}
private async Task<T> Send<T>(HttpMethod method, string url, object query, HttpContent content = null) {
if (query != null)
url = QueryHelpers.AddQueryString(url, ParseQueryParameters(query));
private async Task<TResult> DoCachedCall<TResult>(HttpMethod method, string callerName, IDictionary<string, string> query, Func<Task<TResult>> call) {
if (method == HttpMethod.Post)
return await call();
using var httpRequest = new HttpRequestMessage(method, url) { Content = content };
var response = await _client.SendAsync(httpRequest);
var baseKey = GetType().Name + ":" + callerName;
var queryString = query != null ? "?" + string.Join('&', query.Select(pair => pair.Key + "=" + pair.Value)) : string.Empty;
var key = baseKey + queryString;
if (!response.IsSuccessStatusCode)
throw new ApiCallException(response);
var json = await response.Content.ReadAsStringAsync();
return JsonConvert.DeserializeObject<T>(json);
return await _cache.GetOrCreate(key, async entry => {
entry.AbsoluteExpirationRelativeToNow = TimeSpan.FromSeconds(_cacheDurations[baseKey]);
return await call();
});
}
protected async Task<T> Post<T>(string url, object value, object query = null) {
private async Task<T> Send<T>(HttpMethod method, string url, object query, string callerName, HttpContent content = null) {
IDictionary<string, string> queryParameters = null;
if (query != null) {
queryParameters = ParseQueryParameters(query);
url = QueryHelpers.AddQueryString(url, queryParameters);
}
return await DoCachedCall(method, callerName, queryParameters, async () => {
using var httpRequest = new HttpRequestMessage(method, url) { Content = content };
var response = await _client.SendAsync(httpRequest);
if (!response.IsSuccessStatusCode)
throw new ApiCallException(response);
var json = await response.Content.ReadAsStringAsync();
return JsonConvert.DeserializeObject<T>(json);
});
}
protected async Task<T> Post<T>(string url, object value, object query = null, [CallerMemberName] string callerName = "") {
var json = JsonConvert.SerializeObject(value);
using var requestBody = new StringContent(json, Encoding.UTF8, "application/json");
return await Send<T>(HttpMethod.Post, url, query, requestBody);
return await Send<T>(HttpMethod.Post, url, query, callerName, requestBody);
}
protected async Task<T> Get<T>(string url, object query = null) {
return await Send<T>(HttpMethod.Get, url, query);
protected async Task<T> Get<T>(string url, object query = null, [CallerMemberName] string callerName = "") {
return await Send<T>(HttpMethod.Get, url, query, callerName);
}
}

5
src/Website/Data/AuthenticationProvider.cs
View file

@ -1,10 +1,11 @@
using System.Net.Http;
using System.Threading.Tasks;
using Microsoft.Extensions.Caching.Memory;
using Website.Models.Auth;
namespace Website.Data {
public class AuthenticationProvider:ApiClient, IAuthenticationProvider {
public AuthenticationProvider(HttpClient client) : base(client) {
public class AuthenticationProvider : ApiClient, IAuthenticationProvider {
public AuthenticationProvider(HttpClient client, IMemoryCache cache, CacheDurations cacheDurations) : base(client, cache, cacheDurations) {
}
public async Task<User> Authenticate(LoginRequest request) {

3
src/Website/Data/BlogApi.cs
View file

@ -1,12 +1,13 @@
using System.Threading.Tasks;
using System.Collections.Generic;
using System.Net.Http;
using Microsoft.Extensions.Caching.Memory;
using Website.Models.Blog;
namespace Website.Data
{
public class BlogApi : ApiClient, IBlogApi {
public BlogApi(HttpClient client) : base(client) {
public BlogApi(HttpClient client, IMemoryCache cache, CacheDurations cacheDurations) : base(client, cache, cacheDurations) {
}
public async Task<BlogPost> GetPostByUrlAsync(string url) => await Get<BlogPost>("get/" + url);

21
src/Website/Data/CacheDurations.cs Normal file
View file

@ -0,0 +1,21 @@
using System.Collections.Generic;
namespace Website.Data {
public class CacheDurations : Dictionary<string, Dictionary<string, int>> {
const string DefaultKey = "default";
public new int this[string key] {
get {
var keyParts = key.Split(':', 2);
if (ContainsKey(keyParts[0]))
if (base[keyParts[0]].ContainsKey(keyParts[1]))
return base[keyParts[0]][keyParts[1]];
else if (base[keyParts[0]].ContainsKey(DefaultKey))
return base[keyParts[0]][DefaultKey];
return ContainsKey(DefaultKey) ? base[DefaultKey][DefaultKey] : 0;
}
}
}
}

3
src/Website/Data/GitApi.cs
View file

@ -1,12 +1,13 @@
using System.Collections.Generic;
using System.Net.Http;
using System.Threading.Tasks;
using Microsoft.Extensions.Caching.Memory;
using Website.Models.Git;
namespace Website.Data {
public class GitApi : ApiClient, IGitApi {
public GitApi(HttpClient client) : base(client) {
public GitApi(HttpClient client, IMemoryCache cache, CacheDurations cacheDurations) : base(client, cache, cacheDurations) {
}
public async Task<IEnumerable<Repository>> GetRepositories(string user) => await Get<IEnumerable<Repository>>("repositories", new { user });

3
src/Website/Startup.cs
View file

@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.Net.Http;
using Microsoft.AspNetCore.Authentication.Cookies;
using Microsoft.AspNetCore.Builder;
@ -32,6 +33,8 @@ namespace Website
services.AddSingleton(Configuration);
services.AddSingleton(Configuration.GetSection("cacheDurations").Get<CacheDurations>());
services.AddHttpClient<IGitApi, GitApi>(client => client.BaseAddress = new Uri(Configuration["gitApiEndpoint"]))
.ConfigurePrimaryHttpMessageHandler(() => new HttpClientHandler {ServerCertificateCustomValidationCallback = (message, cert, chain, errors) => true});

16
src/Website/appsettings.Development.json
View file

@ -8,5 +8,19 @@
},
"blogApiEndpoint": "",
"gitApiEndpoint": "",
"authApiEndpoint": ""
"authApiEndpoint": "",
"cacheDurations": {
"default": {
"default": 30
},
"BlogApi": {
"default": 21600, // 6 hours
"GetPostByUrlAsync": 86400, // 1 day
"GetPostByIdAsync": 86400 // 1 day
},
"GitApi": {
"default": 3600, //1 hour
"GetCommit": 86400 // 1 day
}
}
}

14
src/Website/appsettings.json
View file

@ -14,5 +14,19 @@
"Url": "http://0.0.0.0:5000"
}
}
},
"cacheDurations": {
"default": {
"default": 30
},
"BlogApi": {
"default": 21600, // 6 hours
"GetPostByUrlAsync": 86400, // 1 day
"GetPostByIdAsync": 86400 // 1 day
},
"GitApi": {
"default": 3600, //1 hour
"GetCommit": 86400 // 1 day
}
}
}