Skip to content

Commit 6b1c70c

Browse files
mbhavembhave
committed
Pre-fetch project information on startup
This avoid hitting Github's rate limit as data is not fetched on every request. A Github webhook triggers the cache update when content changes in the spring-io/spring-website-content
1 parent 8f57f4f commit 6b1c70c

20 files changed

+606
-77
lines changed

‎build.gradle

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ dependencies {
4343
implementation 'org.springframework.boot:spring-boot-starter-graphql'
4444
implementation 'com.azure.spring:spring-cloud-azure-starter-keyvault-secrets'
4545
implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310'
46-
implementation 'com.github.ben-manes.caffeine:caffeine'
46+
implementation 'jakarta.xml.bind:jakarta.xml.bind-api:4.0.2'
4747
implementation 'com.vladsch.flexmark:flexmark-all:0.64.8'
4848
implementation 'org.apache.maven:maven-artifact:3.6.3'
4949
testImplementation 'com.squareup.okhttp3:mockwebserver'

‎src/main/java/io/spring/projectapi/Application.java

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,11 +24,9 @@
2424
import org.springframework.boot.autoconfigure.SpringBootApplication;
2525
import org.springframework.boot.context.properties.EnableConfigurationProperties;
2626
import org.springframework.boot.web.client.RestTemplateBuilder;
27-
import org.springframework.cache.annotation.EnableCaching;
2827
import org.springframework.context.annotation.Bean;
2928

3029
@SpringBootApplication
31-
@EnableCaching
3230
@EnableConfigurationProperties(ApplicationProperties.class)
3331
public class Application {
3432

‎src/main/java/io/spring/projectapi/ApplicationProperties.java

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,12 +68,18 @@ public static class Github {
6868
*/
6969
private String branch;
7070

71+
/**
72+
* Secret for triggering the webhook that refreshes the cache.
73+
*/
74+
private String webhookSecret;
75+
7176
@ConstructorBinding
72-
Github(String org, String team, String accesstoken, @DefaultValue("main") String branch) {
77+
Github(String org, String team, String accesstoken, @DefaultValue("main") String branch, String webhookSecret) {
7378
this.org = org;
7479
this.team = team;
7580
this.accesstoken = accesstoken;
7681
this.branch = branch;
82+
this.webhookSecret = webhookSecret;
7783
}
7884

7985
public String getOrg() {
@@ -92,6 +98,10 @@ public String getAccesstoken() {
9298
return this.accesstoken;
9399
}
94100

101+
public String getWebhookSecret() {
102+
return this.webhookSecret;
103+
}
104+
95105
}
96106

97107
}

‎src/main/java/io/spring/projectapi/ProjectRepository.java

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
/*
2+
* Copyright 2022-2023 the original author or authors.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package io.spring.projectapi;
18+
19+
import java.util.Collection;
20+
import java.util.LinkedHashMap;
21+
import java.util.List;
22+
import java.util.Map;
23+
24+
import io.spring.projectapi.github.GithubOperations;
25+
import io.spring.projectapi.github.Project;
26+
import io.spring.projectapi.github.ProjectDocumentation;
27+
import io.spring.projectapi.github.ProjectSupport;
28+
import io.spring.projectapi.web.webhook.CacheController;
29+
30+
import org.springframework.stereotype.Component;
31+
32+
/**
33+
* Caches Github project information. Populated on start up and updates triggered via
34+
* {@link CacheController}.
35+
*
36+
* @author Madhura Bhave
37+
* @author Phillip Webb
38+
*/
39+
@Component
40+
public class ProjectRepository {
41+
42+
private final GithubOperations githubOperations;
43+
44+
private transient Data data;
45+
46+
public ProjectRepository(GithubOperations githubOperations) {
47+
this.githubOperations = githubOperations;
48+
this.data = Data.load(githubOperations);
49+
}
50+
51+
public void update() {
52+
this.data = Data.load(this.githubOperations);
53+
}
54+
55+
public Collection<Project> getProjects() {
56+
return this.data.project().values();
57+
}
58+
59+
public Project getProject(String projectSlug) {
60+
return this.data.project().get(projectSlug);
61+
}
62+
63+
public List<ProjectDocumentation> getProjectDocumentations(String projectSlug) {
64+
return this.data.documentation().get(projectSlug);
65+
}
66+
67+
public List<ProjectSupport> getProjectSupports(String projectSlug) {
68+
return this.data.support().get(projectSlug);
69+
}
70+
71+
public String getProjectSupportPolicy(String projectSlug) {
72+
return this.data.supportPolicy().get(projectSlug);
73+
}
74+
75+
record Data(Map<String, Project> project, Map<String, List<ProjectDocumentation>> documentation,
76+
Map<String, List<ProjectSupport>> support, Map<String, String> supportPolicy) {
77+
78+
public static Data load(GithubOperations githubOperations) {
79+
Map<String, Project> projects = new LinkedHashMap<>();
80+
Map<String, List<ProjectDocumentation>> documentation = new LinkedHashMap<>();
81+
Map<String, List<ProjectSupport>> support = new LinkedHashMap<>();
82+
Map<String, String> supportPolicy = new LinkedHashMap<>();
83+
githubOperations.getProjects().forEach((project) -> {
84+
String slug = project.getSlug();
85+
projects.put(slug, project);
86+
documentation.put(slug, githubOperations.getProjectDocumentations(slug));
87+
support.put(slug, githubOperations.getProjectSupports(slug));
88+
supportPolicy.put(slug, githubOperations.getProjectSupportPolicy(slug));
89+
});
90+
return new Data(Map.copyOf(projects), Map.copyOf(documentation), Map.copyOf(support),
91+
Map.copyOf(supportPolicy));
92+
}
93+
94+
}
95+
96+
}

‎src/main/java/io/spring/projectapi/github/GithubOperations.java

Lines changed: 15 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,6 @@
3535
import io.spring.projectapi.github.ProjectDocumentation.Status;
3636
import org.apache.maven.artifact.versioning.ComparableVersion;
3737
import org.jetbrains.annotations.NotNull;
38-
import org.slf4j.Logger;
39-
import org.slf4j.LoggerFactory;
4038

4139
import org.springframework.boot.web.client.RestTemplateBuilder;
4240
import org.springframework.cache.annotation.Cacheable;
@@ -56,6 +54,12 @@
5654
*/
5755
public class GithubOperations {
5856

57+
private static final TypeReference<@NotNull List<ProjectDocumentation>> DOCUMENTATION_LIST = new TypeReference<>() {
58+
};
59+
60+
private static final TypeReference<List<ProjectSupport>> SUPPORT_LIST = new TypeReference<>() {
61+
};
62+
5963
private static final String GITHUB_URI = "https://api.github.com/repos/spring-io/spring-website-content/contents";
6064

6165
private static final Comparator<ProjectDocumentation> VERSION_COMPARATOR = GithubOperations::compare;
@@ -80,8 +84,6 @@ public class GithubOperations {
8084

8185
private final String branch;
8286

83-
private static final Logger logger = LoggerFactory.getLogger(GithubOperations.class);
84-
8587
public GithubOperations(RestTemplateBuilder restTemplateBuilder, ObjectMapper objectMapper, String token,
8688
String branch) {
8789
this.restTemplate = restTemplateBuilder.rootUri(GITHUB_URI)
@@ -115,13 +117,7 @@ public void addProjectDocumentation(String projectSlug, ProjectDocumentation doc
115117

116118
@NotNull
117119
private List<ProjectDocumentation> convertToProjectDocumentation(String content) {
118-
try {
119-
return this.objectMapper.readValue(content, new TypeReference<>() {
120-
});
121-
}
122-
catch (JsonProcessingException ex) {
123-
throw new RuntimeException(ex);
124-
}
120+
return readValue(content, DOCUMENTATION_LIST);
125121
}
126122

127123
private void updateProjectDocumentation(String projectSlug, List<ProjectDocumentation> documentations, String sha) {
@@ -180,7 +176,7 @@ private List<ProjectDocumentation> computeCurrentRelease(List<ProjectDocumentati
180176
List<ProjectDocumentation> updatedGaList = new ArrayList<>(getListWithUpdatedCurrentRelease(sortedGaList));
181177
Collections.reverse(updatedGaList);
182178
preReleaseList.addAll(updatedGaList);
183-
return preReleaseList;
179+
return List.copyOf(preReleaseList);
184180
}
185181

186182
@NotNull
@@ -209,15 +205,13 @@ public void deleteDocumentation(String projectSlug, String version) {
209205
}
210206

211207
private ResponseEntity<Map<String, Object>> getFile(String projectSlug, String fileName) {
212-
logger.info("****In private getFile for project " + projectSlug);
213208
RequestEntity<Void> request = RequestEntity
214209
.get("/project/{projectSlug}/{fileName}?ref=" + this.branch, projectSlug, fileName)
215210
.build();
216211
try {
217212
return this.restTemplate.exchange(request, STRING_OBJECT_MAP);
218213
}
219214
catch (HttpClientErrorException ex) {
220-
logger.info("****In private getFile for project with exception " + projectSlug);
221215
HttpStatusCode statusCode = ex.getStatusCode();
222216
if (statusCode.value() == 404) {
223217
throwIfProjectDoesNotExist(projectSlug);
@@ -255,7 +249,6 @@ private String getFileSha(ResponseEntity<Map<String, Object>> exchange) {
255249
public List<Project> getProjects() {
256250
List<Project> projects = new ArrayList<>();
257251
try {
258-
logger.info("****In getProjects");
259252
RequestEntity<Void> request = RequestEntity.get("/project?ref=" + this.branch).build();
260253
ResponseEntity<List<Map<String, Object>>> exchange = this.restTemplate.exchange(request,
261254
STRING_OBJECT_MAP_LIST);
@@ -274,12 +267,10 @@ public List<Project> getProjects() {
274267
catch (HttpClientErrorException ex) {
275268
// Return empty list
276269
}
277-
return projects;
270+
return List.copyOf(projects);
278271
}
279272

280-
@Cacheable(value = "project", key = "#projectSlug")
281273
public Project getProject(String projectSlug) {
282-
logger.info("****In getProject with project " + projectSlug);
283274
ResponseEntity<Map<String, Object>> response = getFile(projectSlug, "index.md");
284275
String contents = getFileContents(response);
285276
Map<String, String> frontMatter = MarkdownUtils.getFrontMatter(contents);
@@ -288,26 +279,25 @@ public Project getProject(String projectSlug) {
288279
return this.objectMapper.convertValue(frontMatter, Project.class);
289280
}
290281

291-
@Cacheable(value = "documentation", key = "#projectSlug")
292282
public List<ProjectDocumentation> getProjectDocumentations(String projectSlug) {
293-
logger.info("****In getProjectDocumentations with project " + projectSlug);
294283
ResponseEntity<Map<String, Object>> response = getFile(projectSlug, "documentation.json");
295284
String content = getFileContents(response);
296-
return convertToProjectDocumentation(content);
285+
return List.copyOf(convertToProjectDocumentation(content));
297286
}
298287

299-
@Cacheable(value = "support", key = "#projectSlug")
300288
public List<ProjectSupport> getProjectSupports(String projectSlug) {
301-
logger.info("****In getProjectSupports with project " + projectSlug);
302289
ResponseEntity<Map<String, Object>> response = getFile(projectSlug, "support.json");
303290
if (response == null) {
304291
return Collections.emptyList();
305292
}
306293
String contents = getFileContents(response);
307294
getProjectSupportPolicy(projectSlug);
295+
return List.copyOf(readValue(contents, SUPPORT_LIST));
296+
}
297+
298+
private <T> T readValue(String contents, TypeReference<T> type) {
308299
try {
309-
return this.objectMapper.readValue(contents, new TypeReference<>() {
310-
});
300+
return this.objectMapper.readValue(contents, type);
311301
}
312302
catch (JsonProcessingException ex) {
313303
throw new RuntimeException(ex);
@@ -316,7 +306,6 @@ public List<ProjectSupport> getProjectSupports(String projectSlug) {
316306

317307
@Cacheable(value = "support_policy", key = "#projectSlug")
318308
public String getProjectSupportPolicy(String projectSlug) {
319-
logger.info("****In getProjectSupportPolicy with project " + projectSlug);
320309
ResponseEntity<Map<String, Object>> indexResponse = getFile(projectSlug, "index.md");
321310
String indexContents = getFileContents(indexResponse);
322311
Map<String, String> frontMatter = MarkdownUtils.getFrontMatter(indexContents);

‎src/main/java/io/spring/projectapi/security/SecurityConfiguration.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ public SecurityFilterChain configure(HttpSecurity http, RestTemplateBuilder rest
4949
http.requiresChannel((channel) -> channel.requestMatchers(this::hasXForwardedPortHeader).requiresSecure());
5050
http.authorizeHttpRequests((requests) -> {
5151
requests.requestMatchers(HttpMethod.GET, "/**").permitAll();
52+
requests.requestMatchers("/refresh_cache").permitAll();
5253
requests.anyRequest().hasRole("ADMIN");
5354
});
5455
Github github = properties.getGithub();

‎src/main/java/io/spring/projectapi/web/generation/GenerationsController.java

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
import java.time.LocalDate;
2020
import java.util.List;
2121

22-
import io.spring.projectapi.github.GithubOperations;
22+
import io.spring.projectapi.ProjectRepository;
2323
import io.spring.projectapi.github.ProjectSupport;
2424
import io.spring.projectapi.web.error.ResourceNotFoundException;
2525
import io.spring.projectapi.web.project.ProjectsController;
@@ -48,16 +48,16 @@
4848
@ExposesResourceFor(Generation.class)
4949
public class GenerationsController {
5050

51-
private final GithubOperations githubOperations;
51+
private final ProjectRepository projectRepository;
5252

53-
public GenerationsController(GithubOperations githubOperations) {
54-
this.githubOperations = githubOperations;
53+
public GenerationsController(ProjectRepository projectRepository) {
54+
this.projectRepository = projectRepository;
5555
}
5656

5757
@GetMapping
5858
public CollectionModel<EntityModel<Generation>> generations(@PathVariable String id) {
59-
List<ProjectSupport> supports = this.githubOperations.getProjectSupports(id);
60-
String supportPolicy = this.githubOperations.getProjectSupportPolicy(id);
59+
List<ProjectSupport> supports = this.projectRepository.getProjectSupports(id);
60+
String supportPolicy = this.projectRepository.getProjectSupportPolicy(id);
6161
List<Generation> generations = supports.stream()
6262
.map((support) -> asGeneration(support, supportPolicy))
6363
.toList();
@@ -69,8 +69,8 @@ public CollectionModel<EntityModel<Generation>> generations(@PathVariable String
6969

7070
@GetMapping("/{name}")
7171
public EntityModel<Generation> generation(@PathVariable String id, @PathVariable String name) {
72-
List<ProjectSupport> supports = this.githubOperations.getProjectSupports(id);
73-
String supportPolicy = this.githubOperations.getProjectSupportPolicy(id);
72+
List<ProjectSupport> supports = this.projectRepository.getProjectSupports(id);
73+
String supportPolicy = this.projectRepository.getProjectSupportPolicy(id);
7474
List<Generation> generations = supports.stream()
7575
.map((support) -> asGeneration(support, supportPolicy))
7676
.toList();

‎src/main/java/io/spring/projectapi/web/project/ProjectsController.java

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818

1919
import java.util.List;
2020

21-
import io.spring.projectapi.github.GithubOperations;
21+
import io.spring.projectapi.ProjectRepository;
2222
import io.spring.projectapi.web.generation.GenerationsController;
2323
import io.spring.projectapi.web.project.Project.Status;
2424
import io.spring.projectapi.web.release.ReleasesController;
@@ -48,18 +48,18 @@
4848
@ExposesResourceFor(Project.class)
4949
public class ProjectsController {
5050

51-
private final GithubOperations githubOperations;
51+
private final ProjectRepository projectRepository;
5252

5353
private final EntityLinks entityLinks;
5454

55-
public ProjectsController(GithubOperations githubOperations, EntityLinks entityLinks) {
56-
this.githubOperations = githubOperations;
55+
public ProjectsController(ProjectRepository projectRepository, EntityLinks entityLinks) {
56+
this.projectRepository = projectRepository;
5757
this.entityLinks = entityLinks;
5858
}
5959

6060
@GetMapping
61-
public CollectionModel<EntityModel<Project>> projects() throws Exception {
62-
List<Project> projects = this.githubOperations.getProjects().stream().map(this::asProject).toList();
61+
public CollectionModel<EntityModel<Project>> projects() {
62+
List<Project> projects = this.projectRepository.getProjects().stream().map(this::asProject).toList();
6363
CollectionModel<EntityModel<Project>> collection = CollectionModel.of(projects.stream().map((project) -> {
6464
try {
6565
return asModel(project);
@@ -74,7 +74,7 @@ public CollectionModel<EntityModel<Project>> projects() throws Exception {
7474

7575
@GetMapping("/{id}")
7676
public EntityModel<Project> project(@PathVariable String id) {
77-
Project project = asProject(this.githubOperations.getProject(id));
77+
Project project = asProject(this.projectRepository.getProject(id));
7878
return asModel(project);
7979
}
8080

0 commit comments

Comments
 (0)