diff --git a/build.gradle b/build.gradle index 4ea11ee18..aa1d137e0 100644 --- a/build.gradle +++ b/build.gradle @@ -24,9 +24,14 @@ task wrapper(type: Wrapper) { } subprojects { + + + apply plugin:'java' // apply plugin: 'com.github.johnrengelman.shadow' - apply plugin: 'com.bmuschko.nexus' + if(project.name != "micro-tutorial") { + apply plugin: 'com.bmuschko.nexus' + } sourceCompatibility = 1.8 targetCompatibility = 1.8 diff --git a/gradle.properties b/gradle.properties index 2a2714098..3f863afb5 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -version=0.91.5 +version=0.91.8 springVersion=4.3.3.RELEASE springBootVersion=1.4.1.RELEASE jerseyVersion=2.24 diff --git a/micro-application-register/src/main/java/com/aol/micro/server/application/registry/Finder.java b/micro-application-register/src/main/java/com/aol/micro/server/application/registry/Finder.java index 57e3d13eb..1153989b6 100644 --- a/micro-application-register/src/main/java/com/aol/micro/server/application/registry/Finder.java +++ b/micro-application-register/src/main/java/com/aol/micro/server/application/registry/Finder.java @@ -3,6 +3,8 @@ import java.io.File; import java.util.ArrayList; import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.commons.io.FileUtils; @@ -24,8 +26,13 @@ public Finder(RegisterConfig config) { this.config = config; } - public List find() { - return findDir(new File(config.getOutputDir())); + public List find(final Optional re) { + + List entries = findDir(new File(config.getOutputDir())); + if (re.isPresent()) { + entries = entries.stream().filter( e -> e.matches(re.get())).collect(Collectors.toList()); + } + return entries; } private List findDir(File dir) { diff --git a/micro-application-register/src/main/java/com/aol/micro/server/application/registry/RegisterEntry.java b/micro-application-register/src/main/java/com/aol/micro/server/application/registry/RegisterEntry.java index 18d9d1406..c30232bd1 100644 --- a/micro-application-register/src/main/java/com/aol/micro/server/application/registry/RegisterEntry.java +++ b/micro-application-register/src/main/java/com/aol/micro/server/application/registry/RegisterEntry.java @@ -1,11 +1,9 @@ package com.aol.micro.server.application.registry; import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.UUID; +import java.util.*; +import javax.ws.rs.QueryParam; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; @@ -14,6 +12,7 @@ import lombok.AccessLevel; import lombok.Builder; import lombok.Getter; +import lombok.ToString; import lombok.experimental.FieldDefaults; import lombok.experimental.Wither; @@ -24,43 +23,44 @@ @Getter @Wither @Builder +@ToString public class RegisterEntry { - private static SimpleDateFormat f = new SimpleDateFormat( - "EEE, d MMM yyyy HH:mm:ss Z"); + private static SimpleDateFormat f = new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss Z"); @Wither - int port; + private final int port; @Wither - String hostname; + private final String hostname; @Wither - String module; + private final String module; @Wither - String context; - Date time; + private final String context; + private final Date time; @Wither - String uuid; + private final String uuid; @Wither - String target; - String formattedDate; - Map manifest = ManifestLoader.instance.getManifest(); + private final String target; + private final String formattedDate; + private final Map manifest = new HashMap<>(); @Wither - Health health; + private final Health health; @Wither - List>> stats; + private final List>> stats; @Wither - int externalPort; + private final int externalPort; public RegisterEntry() { - this( - -1, null, null, null, null, null, null, -1); + this(-1, null, null, null, null, null, null, -1); } public RegisterEntry(int port, String hostname, String module, String context, Date time, String uuid, - String target, int externalPort) { - this( - port, hostname, module, context, time, UUID.randomUUID() - .toString(), - target, null, Health.OK, null, externalPort); + String target, int externalPort) { + this(port, hostname, module, context, time, uuid, target, null, Health.OK, null, externalPort); + } + + public RegisterEntry(int port, String hostname, String module, String context, Date time, String target, + int externalPort) { + this(port, hostname, module, context, time, UUID.randomUUID().toString(), target, externalPort); } private RegisterEntry(int port, String hostname, String module, String context, Date time, String uuid, @@ -82,14 +82,28 @@ private RegisterEntry(int port, String hostname, String module, String context, else this.formattedDate = null; + this.manifest.putAll(ManifestLoader.instance.getManifest()); + } - public RegisterEntry(int port, String hostname, String module, String context, Date time, String target, - int externalPort) { - this( - port, hostname, module, context, time, UUID.randomUUID() - .toString(), - target, externalPort); + public boolean matches(RegisterEntry re) { + //Only the fields which make sense to query is added for now. + return (re.port == -1 || re.port == port) && + (Objects.isNull(re.hostname) || Objects.nonNull(hostname) && hostname.startsWith(re.hostname)) && + (Objects.isNull(re.module) || Objects.nonNull(module) && module.startsWith(re.module)) && + (Objects.isNull(re.context) || Objects.nonNull(context) && context.startsWith(re.context)) && + (Objects.isNull(re.health) || re.health.equals(health)) && + (re.externalPort == -1 || re.externalPort == externalPort) && + (Objects.isNull(re.manifest) || re.manifest.isEmpty() || matchManifest(re.manifest)); + } + + private boolean matchManifest(Map manifest) { + return match(manifest, this.manifest, "Implementation-revision") && + match(manifest, this.manifest, "Implementation-Timestamp") && + match(manifest, this.manifest, "Implementation-Version"); } -} + private boolean match(Map map1, Map map2, String key) { + return !map1.containsKey(key) || (map2.containsKey(key) && map2.get(key).startsWith(map1.get(key))); + } +} \ No newline at end of file diff --git a/micro-application-register/src/main/java/com/aol/micro/server/application/registry/ServiceRegistryResource.java b/micro-application-register/src/main/java/com/aol/micro/server/application/registry/ServiceRegistryResource.java index 26e0b8d9d..7f67db7c8 100644 --- a/micro-application-register/src/main/java/com/aol/micro/server/application/registry/ServiceRegistryResource.java +++ b/micro-application-register/src/main/java/com/aol/micro/server/application/registry/ServiceRegistryResource.java @@ -1,14 +1,15 @@ package com.aol.micro.server.application.registry; import java.util.Arrays; +import java.util.Optional; -import javax.ws.rs.Consumes; -import javax.ws.rs.GET; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; +import javax.ws.rs.*; import javax.ws.rs.container.AsyncResponse; import javax.ws.rs.container.Suspended; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.Response.Status; +import javax.ws.rs.core.UriInfo; import cyclops.stream.ReactiveSeq; import org.slf4j.Logger; @@ -20,6 +21,8 @@ import com.aol.micro.server.auto.discovery.Rest; import com.aol.micro.server.utility.HashMapBuilder; +import static javax.ws.rs.core.Response.Status.*; + @Rest @Path("/service-registry") @@ -42,18 +45,17 @@ public ServiceRegistryResource(Cleaner cleaner, Finder finder, Register register @GET @Path("/list") @Produces("application/json") - public void list(@Suspended AsyncResponse response) { + public void list(@Context UriInfo uriInfo, @Suspended AsyncResponse response) { ReactiveSeq.of(this).foldFuture(WorkerThreads.ioExecutor.get(), s->s.forEach(Long.MAX_VALUE,next -> { try{ cleaner.clean(); - response.resume(finder.find()); + response.resume(finder.find(UriInfoParser.toRegisterEntry(uriInfo))); }catch(Exception e){ logger.error(e.getMessage(),e); - response.resume(Arrays.asList("failed due to error")); + response.resume(Arrays.asList("Bad Request: " + e.getMessage())); } })); - } @POST diff --git a/micro-application-register/src/main/java/com/aol/micro/server/application/registry/UriInfoParser.java b/micro-application-register/src/main/java/com/aol/micro/server/application/registry/UriInfoParser.java new file mode 100644 index 000000000..03fe82d34 --- /dev/null +++ b/micro-application-register/src/main/java/com/aol/micro/server/application/registry/UriInfoParser.java @@ -0,0 +1,63 @@ +package com.aol.micro.server.application.registry; + +import cyclops.stream.ReactiveSeq; + +import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.core.UriInfo; +import java.util.Arrays; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +public class UriInfoParser { + + public static Optional toRegisterEntry(UriInfo uriInfo) { + if (uriInfo.getQueryParameters().isEmpty()) { + return Optional.empty(); + } else { + MultivaluedMap parameters = uriInfo.getQueryParameters(); + RegisterEntry re = RegisterEntry.builder() + .context(parameters.getFirst("context")) + .hostname(parameters.getFirst("hostname")) + .port(toInt(parameters.getFirst("port"))) + .target(parameters.getFirst("target")) + .externalPort(toInt(parameters.getFirst("externalPort"))) + .module(parameters.getFirst("module")) + .health(toHealth(parameters.getFirst("health"))) + .build(); + + Map manifest = ReactiveSeq.fromIterable(parameters.entrySet()) + .filter(e -> e.getKey().startsWith("manifest.")) + .toMap(e -> e.getKey().replace("manifest.", ""), e -> parameters.getFirst(e.getKey())); + + re.getManifest().clear(); + re.getManifest().putAll(manifest); + + return Optional.of(re); + } + } + + private static Health toHealth(String health) { + if (Objects.nonNull(health)) { + try { + return Health.valueOf(health); + } catch (Exception e) { + throw new IllegalArgumentException("'" + health + "' is not valid, valid values are " + + Arrays.asList(Health.values())); + } + } + return null; + } + + private static int toInt(String port) { + if (Objects.isNull(port)) + return -1; + + try { + return Integer.valueOf(port); + } catch (Exception e) { + throw new IllegalArgumentException("'" + port + "' is not a valid number."); + } + } + +} diff --git a/micro-application-register/src/test/java/app/registry/com/aol/micro/server/RegistryAppRunner.java b/micro-application-register/src/test/java/app/registry/com/aol/micro/server/RegistryAppRunner.java index 229b7e0d9..f998808f0 100644 --- a/micro-application-register/src/test/java/app/registry/com/aol/micro/server/RegistryAppRunner.java +++ b/micro-application-register/src/test/java/app/registry/com/aol/micro/server/RegistryAppRunner.java @@ -1,5 +1,6 @@ package app.registry.com.aol.micro.server; +import static java.util.stream.Collectors.joining; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.not; @@ -7,8 +8,12 @@ import java.text.SimpleDateFormat; import java.util.Date; +import java.util.List; import java.util.concurrent.ExecutionException; +import java.util.stream.Stream; +import com.aol.micro.server.rest.client.RestClient; +import com.fasterxml.jackson.core.type.TypeReference; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -25,17 +30,15 @@ public class RegistryAppRunner { RestAgent rest = new RestAgent(); - private final AsyncRestClient restAsync = new AsyncRestClient( - 100, 2000); + private final AsyncRestClient restAsync = new AsyncRestClient(100, 2000); MicroserverApp server; + String baseUrl = "http://localhost:8080/registry-app/service-registry"; + @Before public void startServer() { - - server = new MicroserverApp( - () -> "registry-app"); + server = new MicroserverApp(() -> "registry-app"); server.start(); - } @After @@ -46,53 +49,115 @@ public void stopServer() { @Test public void runAppAndBasicTest() throws InterruptedException, ExecutionException { - SimpleDateFormat f = new SimpleDateFormat( - "EEE"); + SimpleDateFormat f = new SimpleDateFormat("EEE"); String date = f.format(new Date()); Thread.sleep(1000); - assertThat(rest.post("http://localhost:8080/registry-app/service-registry/schedule"), - is("{\"status\":\"success\"}")); + assertThat(rest.post(baseUrl + "/schedule"), is("{\"status\":\"success\"}")); Thread.sleep(1000); - assertThat(rest.getJson("http://localhost:8080/registry-app/service-registry/list"), - containsString("[{\"port\":8080,")); - assertThat(rest.getJson("http://localhost:8080/registry-app/service-registry/list"), - containsString("externalPort\":8080")); - sendPing(new RegisterEntry( - 8081, "use-ip", "hello", "world", new Date(), "my-target", 8082)); + String listResponse = rest.getJson(baseUrl + "/list"); + + assertThat(listResponse, containsString("[{\"port\":8080,")); + assertThat(listResponse, containsString("externalPort\":8080")); + + sendPing("1", 8081, "use-ip", "hello", "world", "my-target", 8082); Thread.sleep(1000); - System.out.println(rest.getJson("http://localhost:8080/registry-app/service-registry/list")); - assertThat(rest.getJson("http://localhost:8080/registry-app/service-registry/list"), - containsString("{\"port\":8081,")); - - assertThat(rest.getJson("http://localhost:8080/registry-app/service-registry/list"), - containsString("\"target\":\"my-target\"")); - assertThat(rest.getJson("http://localhost:8080/registry-app/service-registry/list"), - containsString("\"target\":\"configured-target\"")); - assertThat(rest.getJson("http://localhost:8080/registry-app/service-registry/list"), - not(containsString("\"hostname\":\"test-host\""))); - assertThat(rest.getJson("http://localhost:8080/registry-app/service-registry/list"), - containsString("\"formattedDate\"")); - assertThat(rest.getJson("http://localhost:8080/registry-app/service-registry/list"), - containsString("\"manifest\"")); - assertThat(rest.getJson("http://localhost:8080/registry-app/service-registry/list"), - containsString("Manifest-Version")); - - assertThat(rest.getJson("http://localhost:8080/registry-app/service-registry/list"), containsString(date)); + + listResponse = rest.getJson(baseUrl + "/list");; + + assertThat(listResponse, containsString("{\"port\":8081,")); + assertThat(listResponse, containsString("\"target\":\"my-target\"")); + assertThat(listResponse, containsString("\"target\":\"configured-target\"")); + assertThat(listResponse, not(containsString("\"hostname\":\"test-host\""))); + assertThat(listResponse, containsString("\"formattedDate\"")); + assertThat(listResponse, containsString("\"manifest\"")); + assertThat(listResponse, containsString("Manifest-Version")); + assertThat(listResponse, containsString(date)); } - private void sendPing(RegisterEntry entry) { + @Test + public void filterTest() throws Exception { + Thread.sleep(1000); - try { + List entries = list(); + assertThat(entries.size(), is(1)); - restAsync.post("http://localhost:8080/registry-app/service-registry/register", - JacksonUtil.serializeToJson(entry)) - .join(); - } catch (Exception e) { + sendPing("121", 8080, "host1", "module1", "context1", "target1", 9080); + sendPing("122", 8080, "host2", "module1", "context1", "target1", 9080); + sendPing("131", 6080, "host3", "module2", "context2", "target2", 7080); + sendPing("132", 6080, "host4", "module2", "context2", "target2", 7080); + + entries = list(); + assertThat(entries.size(), is(5)); + + entries = list("port=8080"); + assertThat(entries.size(), is(3)); + + entries = list("port=8080", "externalPort=9080"); + System.out.println(entries); + assertThat(entries.size(), is(2)); + + entries = list("port=8080", "externalPort=9080", "module=module", "context=context1"); + assertThat(entries.size(), is(2)); - } + entries = list("port=8080", "externalPort=9080", "module=module", "context=context1", "hostname=host1"); + assertThat(entries.size(), is(1)); + + entries = list("port=8080", "externalPort=9080", "module=module1", "context=context2"); + assertThat(entries.size(), is(0)); + + entries = list("manifest.Implementation-Version=version"); + assertThat(entries.size(), is(4)); + + entries = list("manifest.Implementation-Version=version1"); + assertThat(entries.size(), is(4)); + + entries = list("manifest.Implementation-Version=version121"); + assertThat(entries.size(), is(1)); + + entries = list("manifest.Implementation-revision=rev12"); + assertThat(entries.size(), is(2)); + + entries = list("manifest.Implementation-Timestamp=2017_13"); + assertThat(entries.size(), is(2)); + + entries = list("health=OK"); + assertThat(entries.size(), is(5)); + + List list = JacksonUtil.convertFromJson(rest.getJson(baseUrl + "/list?port=OK"), new TypeReference>() {}); + assertThat(list.size(), is(1)); + assertThat(list.get(0), is("Bad Request: 'OK' is not a valid number.")); + + list = JacksonUtil.convertFromJson(rest.getJson(baseUrl + "/list?health=Suspended"), new TypeReference>() {}); + assertThat(list.size(), is(1)); + assertThat(list.get(0), is("Bad Request: 'Suspended' is not valid, valid values are [OK, ERROR]")); + } + + private List list(String... parameters) { + String url = baseUrl + "/list?" + Stream.of(parameters).collect(joining("&")); + return JacksonUtil.convertFromJson(rest.getJson(url), new TypeReference>() {}); } + private void sendPing(String uuid, int port, String hostName, String module, String context, String target, int externalPort) { + try { + RegisterEntry re = RegisterEntry.builder() + .port(port) + .hostname(hostName) + .module(module) + .context(context) + .time(new Date()) + .uuid(uuid) + .target(target) + .externalPort(externalPort) + .build(); + re.getManifest().put("Implementation-revision", "rev" + uuid); + re.getManifest().put("Implementation-Version", "version" + uuid); + re.getManifest().put("Implementation-Timestamp", "2017_" + uuid); + restAsync.post("http://localhost:8080/registry-app/service-registry/register", + JacksonUtil.serializeToJson(re)) + .get(); + } catch (Exception e) {} + } } diff --git a/micro-application-register/src/test/java/com/aol/micro/server/application/registry/CleanerTest.java b/micro-application-register/src/test/java/com/aol/micro/server/application/registry/CleanerTest.java index c23bcc5ff..98f1c3400 100644 --- a/micro-application-register/src/test/java/com/aol/micro/server/application/registry/CleanerTest.java +++ b/micro-application-register/src/test/java/com/aol/micro/server/application/registry/CleanerTest.java @@ -6,6 +6,7 @@ import java.io.File; import java.util.Date; import java.util.List; +import java.util.Optional; import org.junit.Before; import org.junit.Test; @@ -50,7 +51,7 @@ public void testClean() { System.currentTimeMillis() - 2000))); cleaner.clean(); - List list = finder.find(); + List list = finder.find(Optional.empty()); assertThat(list.size(), equalTo(0)); } } diff --git a/micro-application-register/src/test/java/com/aol/micro/server/application/registry/FinderTest.java b/micro-application-register/src/test/java/com/aol/micro/server/application/registry/FinderTest.java index 98ab4e878..064c2eb28 100644 --- a/micro-application-register/src/test/java/com/aol/micro/server/application/registry/FinderTest.java +++ b/micro-application-register/src/test/java/com/aol/micro/server/application/registry/FinderTest.java @@ -7,6 +7,7 @@ import java.io.File; import java.util.Date; import java.util.List; +import java.util.Optional; import org.junit.Before; import org.junit.Test; @@ -44,7 +45,7 @@ public void setUp() throws Exception { @Test public void testFind() { writer.register(entry); - List list = finder.find(); + List list = finder.find(Optional.empty()); assertThat(list.size(), greaterThan(0)); assertThat(list.get(0) .getContext(), diff --git a/micro-application-register/src/test/java/com/aol/micro/server/application/registry/RegisterEntryTest.java b/micro-application-register/src/test/java/com/aol/micro/server/application/registry/RegisterEntryTest.java index 5c3e5d237..de27b3007 100644 --- a/micro-application-register/src/test/java/com/aol/micro/server/application/registry/RegisterEntryTest.java +++ b/micro-application-register/src/test/java/com/aol/micro/server/application/registry/RegisterEntryTest.java @@ -1,8 +1,10 @@ package com.aol.micro.server.application.registry; +import static junit.framework.TestCase.assertFalse; import static org.junit.Assert.assertTrue; import java.util.Date; +import java.util.Map; import org.junit.Before; import org.junit.Test; @@ -15,14 +17,57 @@ public class RegisterEntryTest { @Before public void setUp() throws Exception { - entry = new RegisterEntry( - 8080, "hostname", "name", "context", new Date(), null, 8080); + entry = RegisterEntry.builder() + .port(8080) + .hostname("host") + .module("module") + .context("context") + .time(new Date()) + .uuid("1") + .target("target") + .externalPort(9090) + .build(); + Map manifest = entry.getManifest(); + manifest.put("Implementation-revision", "a2edfe4bc"); + manifest.put("Implementation-Version", "version"); + manifest.put("Implementation-Timestamp", "2017_1201"); } @Test public void test() { + assertTrue(JacksonUtil.serializeToJson(entry).contains("\"context\":\"context")); + } + + @Test + public void matches() throws Exception { + RegisterEntry re = new RegisterEntry(); + re.getManifest().clear(); + assertFalse(entry.matches(re)); + + re = RegisterEntry.builder().port(8080).externalPort(-1).build(); + re.getManifest().clear(); + assertTrue(entry.matches(re)); + + re = RegisterEntry.builder().port(8080).externalPort(9090).build(); + re.getManifest().clear(); + assertTrue(entry.matches(re)); + + re = RegisterEntry.builder().port(8080).hostname("host").externalPort(9090).build(); + re.getManifest().clear(); + assertTrue(entry.matches(re)); + + re = RegisterEntry.builder().port(8080).hostname("host1").externalPort(9090).build(); + re.getManifest().clear(); + assertFalse(entry.matches(re)); + + re = RegisterEntry.builder().port(8080).hostname("host").externalPort(9090).build(); + re.getManifest().clear(); + re.getManifest().put("Implementation-revision", "a2edfe4bc"); + assertTrue(entry.matches(re)); - assertTrue(JacksonUtil.serializeToJson(entry) - .contains("\"context\":\"context")); + re = RegisterEntry.builder().port(8080).hostname("host").externalPort(9090).build(); + re.getManifest().clear(); + re.getManifest().put("Implementation-Version", "version1"); + assertFalse(entry.matches(re)); } } diff --git a/micro-application-register/src/test/java/com/aol/micro/server/application/registry/UriInfoParserTest.java b/micro-application-register/src/test/java/com/aol/micro/server/application/registry/UriInfoParserTest.java new file mode 100644 index 000000000..218bc15c3 --- /dev/null +++ b/micro-application-register/src/test/java/com/aol/micro/server/application/registry/UriInfoParserTest.java @@ -0,0 +1,61 @@ +package com.aol.micro.server.application.registry; + +import org.junit.Test; +import org.mockito.Mockito; + +import javax.ws.rs.core.*; + +import java.net.URI; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.*; +import static org.mockito.Mockito.when; + +public class UriInfoParserTest { + + @Test + public void toRegisterEntryFromQueryParameters() throws Exception { + UriInfo uriInfo = Mockito.mock(UriInfo.class); + MultivaluedMap data = new MultivaluedHashMap<>(); + data.put("port", Arrays.asList("8080")); + data.put("externalPort", Arrays.asList("9090")); + data.put("hostname", Arrays.asList("host1")); + data.put("module", Arrays.asList("module1")); + data.put("context", Arrays.asList("context1")); + data.put("target", Arrays.asList("target1")); + data.put("health", Arrays.asList("OK")); + data.put("manifest.Implementation-revision", Arrays.asList("revision1")); + data.put("manifest.Implementation-Timestamp", Arrays.asList("2017_001")); + data.put("manifest.Implementation-Version", Arrays.asList("v1")); + + when(uriInfo.getQueryParameters()).thenReturn(data); + + Optional reOptional = UriInfoParser.toRegisterEntry(uriInfo); + assertTrue(reOptional.isPresent()); + + RegisterEntry re = reOptional.get(); + assertThat(re.getPort(), is(8080)); + assertThat(re.getExternalPort(), is(9090)); + assertThat(re.getHostname(), is("host1")); + assertThat(re.getModule(), is("module1")); + assertThat(re.getContext(), is("context1")); + assertThat(re.getTarget(), is("target1")); + assertThat(re.getHealth(), is(Health.OK)); + assertThat(re.getManifest().get("Implementation-revision"), is("revision1")); + assertThat(re.getManifest().get("Implementation-Timestamp"), is("2017_001")); + assertThat(re.getManifest().get("Implementation-Version"), is("v1")); + } + + @Test + public void toRegisterEntryFromEmptyQueryParameter() throws Exception { + UriInfo uriInfo = Mockito.mock(UriInfo.class); + when(uriInfo.getQueryParameters()).thenReturn(new MultivaluedHashMap<>()); + + Optional re = UriInfoParser.toRegisterEntry(uriInfo); + assertFalse(re.isPresent()); + + } +} \ No newline at end of file diff --git a/micro-curator/src/integration/java/com/aol/micro/server/curator/lock/IntegrationTest.java b/micro-curator/src/integration/java/com/aol/micro/server/curator/lock/IntegrationTest.java index e6974c67e..dd5be2033 100644 --- a/micro-curator/src/integration/java/com/aol/micro/server/curator/lock/IntegrationTest.java +++ b/micro-curator/src/integration/java/com/aol/micro/server/curator/lock/IntegrationTest.java @@ -48,6 +48,12 @@ public void initialize() { } }).start(); + try { + // allow zooKeeperServer enough time to initialize + Thread.sleep(1000); + } catch (InterruptedException e) { + } + provider = new CuratorDistributedLockServiceProvider("localhost:12181", "1000", "1", "/test"); } diff --git a/micro-event-metrics/readme.md b/micro-event-metrics/readme.md index c25d9a5f0..41b53693c 100644 --- a/micro-event-metrics/readme.md +++ b/micro-event-metrics/readme.md @@ -108,9 +108,14 @@ Number of active jobs to cache in memory ```text com.aol.micro.server.event.metrics.MetricsCatcher.requests-started +com.aol.micro.server.event.metrics.MetricsCatcher.requests-started-interval-count com.aol.micro.server.event.metrics.MetricsCatcher.request-start- +com.aol.micro.server.event.metrics.MetricsCatcher.request-start--interval-count com.aol.micro.server.event.metrics.MetricsCatcher.requests-completed +com.aol.micro.server.event.metrics.MetricsCatcher.requests-completed-interval-type com.aol.micro.server.event.metrics.MetricsCatcher.request-completed- +com.aol.micro.server.event.metrics.MetricsCatcher.request-completed--interval-count + ``` #### Timers : @@ -172,6 +177,8 @@ event.metrics.capture.jobs.by.type=true # jobsByType, event.metrics.capture.number.of.queries=10000 # numQueries, event.metrics.capture.queries.minutes=180 # holdQueriesForMinutes, event.metrics.capture.number.of.jobs=10000 # numJobs, -event.metrics.capture.jobs.minutes=180 +event.metrics.capture.jobs.minutes=180, +event.metrics.capture.timer.interval.seconds=10 +event.metrics.capture.jobs.prefix=null ``` diff --git a/micro-event-metrics/src/main/java/com/aol/micro/server/event/metrics/Configuration.java b/micro-event-metrics/src/main/java/com/aol/micro/server/event/metrics/Configuration.java index 0d88df66d..581300776 100644 --- a/micro-event-metrics/src/main/java/com/aol/micro/server/event/metrics/Configuration.java +++ b/micro-event-metrics/src/main/java/com/aol/micro/server/event/metrics/Configuration.java @@ -24,6 +24,7 @@ class Configuration { private final int numJobs; private final int holdJobsForMinutes; + private final int timerIntervalSeconds; private final String prefix; @Autowired @@ -35,6 +36,7 @@ public Configuration(@Value("${event.metrics.capture.errors.by.type:true}") bool @Value("${event.metrics.capture.queries.minutes:180}") int holdQueriesForMinutes, @Value("${event.metrics.capture.number.of.jobs:10000}") int numJobs, @Value("${event.metrics.capture.jobs.minutes:180}") int holdJobsForMinutes, + @Value("${event.metrics.capture.timer.interval.seconds:10}") int timerIntervalSeconds, @Value("${event.metrics.capture.jobs.prefix:#{null}}") String prefix) { super(); this.errorsByType = errorsByType; @@ -45,6 +47,7 @@ public Configuration(@Value("${event.metrics.capture.errors.by.type:true}") bool this.holdQueriesForMinutes = holdQueriesForMinutes; this.numJobs = numJobs; this.holdJobsForMinutes = holdJobsForMinutes; + this.timerIntervalSeconds = timerIntervalSeconds; this.prefix = Optional.ofNullable(prefix) .orElseGet(() -> MetricsCatcher.class.getTypeName()); } diff --git a/micro-event-metrics/src/main/java/com/aol/micro/server/event/metrics/MetricsCatcher.java b/micro-event-metrics/src/main/java/com/aol/micro/server/event/metrics/MetricsCatcher.java index 6583ac37e..bcc5429ce 100644 --- a/micro-event-metrics/src/main/java/com/aol/micro/server/event/metrics/MetricsCatcher.java +++ b/micro-event-metrics/src/main/java/com/aol/micro/server/event/metrics/MetricsCatcher.java @@ -1,6 +1,9 @@ package com.aol.micro.server.event.metrics; import com.aol.micro.server.events.GenericEvent; +import com.aol.micro.server.spring.metrics.InstantGauge; +import com.codahale.metrics.SlidingTimeWindowArrayReservoir; +import com.codahale.metrics.Timer; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -9,6 +12,8 @@ import com.aol.micro.server.events.JobStartEvent; import com.aol.micro.server.events.RequestTypes.AddQuery; import com.aol.micro.server.events.RequestTypes.RemoveQuery; +import com.aol.micro.server.events.RequestTypes.AddLabelledQuery; +import com.aol.micro.server.events.RequestTypes.RemoveLabelledQuery; import com.aol.micro.server.events.RequestTypes.RequestData; import com.aol.micro.server.events.SystemData; import com.aol.micro.server.health.ErrorEvent; @@ -17,6 +22,7 @@ import com.google.common.eventbus.Subscribe; import java.util.Objects; +import java.util.concurrent.TimeUnit; @Component public class MetricsCatcher { @@ -39,6 +45,7 @@ public MetricsCatcher(MetricRegistry registry, EventBus bus, Configuration confi jobs = new TimerManager( configuration.getNumJobs(), configuration.getHoldJobsForMinutes()); this.configuration = configuration; + } @Subscribe @@ -47,16 +54,20 @@ public void requestStart(AddQuery data) { .mark(); registry.counter(prefix + ".requests-started-count") .inc(); + ((InstantGauge) registry.gauge(prefix + ".requests-started-interval-count", () -> new InstantGauge())).increment(); + if (this.configuration.isQueriesByType()) { RequestData rd = data.getData(); registry.meter(queryStartName(rd) + "-meter") .mark(); - queries.start(rd.getCorrelationId(), registry.timer(queryEndName(rd) + "-timer") - .time()); + queries.start(rd.getCorrelationId(), timer(queryEndName(rd) + "-timer").time()); registry.counter(prefix + ".requests-active-" + rd.getType() + "-count") .inc(); + ((InstantGauge) registry.gauge(prefix + ".requests-started-" + rd.getType() + "-interval-count", + () -> new InstantGauge())).increment(); + } } @@ -74,6 +85,9 @@ public void requestComplete(RemoveQuery data) { .mark(); registry.counter(prefix + ".requests-completed-count") .inc(); + ((InstantGauge) registry.gauge(prefix + ".requests-completed-interval-count", () -> new InstantGauge())) + .increment(); + if (this.configuration.isQueriesByType()) { RequestData rd = data.getData(); registry.meter(queryEndName(rd)) @@ -83,8 +97,29 @@ public void requestComplete(RemoveQuery data) { registry.counter(prefix + ".requests-active-" + rd.getType() + "-count") .dec(); + ((InstantGauge) registry.gauge(prefix + ".requests-completed-" + rd.getType() + "-interval-count", + () -> new InstantGauge())).increment(); + } + } + + @Subscribe + public void requestStart(AddLabelledQuery data) { + if (this.configuration.isQueriesByType()) { + RequestData rd = data.getData(); + ((InstantGauge) registry.gauge(prefix + ".requests-started-" + rd.getType() + "-interval-count", () -> new InstantGauge())) + .increment(); + } + } + + @Subscribe + public void requestComplete(RemoveLabelledQuery data) { + if (this.configuration.isQueriesByType()) { + RequestData rd = data.getData(); + ((InstantGauge) registry.gauge(prefix + ".requests-completed-" + rd.getType() + "-interval-count", () -> new InstantGauge())) + .increment(); + } } @Subscribe @@ -102,8 +137,7 @@ public void jobStarted(JobStartEvent data) { registry.meter(prefix + ".job-meter-" + data.getType()) .mark(); - jobs.start(data.getCorrelationId(), registry.timer(prefix + ".job-timer-" + data.getType()) - .time()); + jobs.start(data.getCorrelationId(), timer(prefix + ".job-timer-" + data.getType()).time()); registry.counter(prefix + ".jobs-active-" + data.getType() + "-count") .inc(); } @@ -190,4 +224,8 @@ public void genericEvent(GenericEvent event) { private String name(ErrorCode c) { return prefix + ".error-" + c.getSeverity() + "-" + c.getErrorId(); } + + private Timer timer (String name) { + return registry.timer(name, () -> new Timer(new SlidingTimeWindowArrayReservoir(configuration.getTimerIntervalSeconds(), TimeUnit.SECONDS))); + } } diff --git a/micro-event-metrics/src/test/java/com/aol/micro/server/event/metrics/MetricsCatcherConfigOffTest.java b/micro-event-metrics/src/test/java/com/aol/micro/server/event/metrics/MetricsCatcherConfigOffTest.java index 04bf7dd01..c5d0832c1 100644 --- a/micro-event-metrics/src/test/java/com/aol/micro/server/event/metrics/MetricsCatcherConfigOffTest.java +++ b/micro-event-metrics/src/test/java/com/aol/micro/server/event/metrics/MetricsCatcherConfigOffTest.java @@ -27,7 +27,7 @@ public void setup() { registry = new MetricRegistry(); bus = new EventBus(); config = new Configuration( - false, false, false, false, 5, 6, 7, 8, "bob"); + false, false, false, false, 5, 6, 7, 8, 10, "bob"); catcher = new MetricsCatcher<>( registry, bus, config); } diff --git a/micro-event-metrics/src/test/java/com/aol/micro/server/event/metrics/MetricsCatcherTest.java b/micro-event-metrics/src/test/java/com/aol/micro/server/event/metrics/MetricsCatcherTest.java index d1d243022..dc8ce6d19 100644 --- a/micro-event-metrics/src/test/java/com/aol/micro/server/event/metrics/MetricsCatcherTest.java +++ b/micro-event-metrics/src/test/java/com/aol/micro/server/event/metrics/MetricsCatcherTest.java @@ -29,7 +29,7 @@ public void setup() { registry = new MetricRegistry(); bus = new EventBus(); config = new Configuration( - true, true, true, true, 5, 6, 7, 8, "bob"); + true, true, true, true, 5, 6, 7, 8, 10,"bob"); catcher = new MetricsCatcher<>( registry, bus, config); } @@ -86,6 +86,35 @@ public void queriesCounterDec() { equalTo(-1l)); } + @Test + public void queriesIntervalCounterInc() { + + catcher.requestStart(new AddQuery( + RequestData.builder() + .correlationId(10l) + .type("test") + .build())); + assertThat(registry.getGauges().size(), equalTo(2)); + assertThat(registry.getGauges().get(this.config.getPrefix() + ".requests-started-interval-count").getValue(), equalTo(1l)); + assertThat(registry.getGauges().get(this.config.getPrefix() + ".requests-started-test-interval-count").getValue(), equalTo(1l)); + } + + @Test + public void queriesIntervalCounterDec() { + + catcher.requestComplete(new RemoveQuery( + RequestData.builder() + .correlationId(10l) + .type("test") + .build())); + assertThat(registry.getGauges().size(), equalTo(2)); + assertThat(registry.getGauges().get(this.config.getPrefix() + ".requests-completed-interval-count").getValue(), + equalTo(1l)); + assertThat(registry.getGauges().get(this.config.getPrefix() + ".requests-completed-test-interval-count").getValue(), + equalTo(1l)); + + } + @Test public void jobsCounterDec() { diff --git a/micro-events/src/main/java/com/aol/micro/server/events/LabelledEvents.java b/micro-events/src/main/java/com/aol/micro/server/events/LabelledEvents.java new file mode 100644 index 000000000..e77d6f512 --- /dev/null +++ b/micro-events/src/main/java/com/aol/micro/server/events/LabelledEvents.java @@ -0,0 +1,119 @@ +package com.aol.micro.server.events; + +import com.aol.micro.server.events.RequestTypes.AddLabelledQuery; +import com.aol.micro.server.events.RequestTypes.RemoveLabelledQuery; +import com.aol.micro.server.events.RequestTypes.RequestData; + +import com.google.common.eventbus.EventBus; + +/** + * Factory class for creating Start and End events which are identified by a custom label + * + */ +public class LabelledEvents { + + /** + * Publish start events for each of the specified query types + * + * 
+     * {@code
+        LabelledEvents.start("get", 1l, bus, "typeA", "custom");
+        try {
+            return "ok";
+        } finally {
+            RequestEvents.finish("get", 1l, bus, "typeA", "custom");
+        }
+     * }
+     *
+ * + * @param query Completed query + * @param correlationId Identifier + * @param bus EventBus to post events to + * @param labels Query labels to post to event bus + */ + public static void start(T query, long correlationId, EventBus bus, String... labels) { + + for (String label : labels) { + AddLabelledQuery next = start(query, correlationId, label, null); + bus.post(next); + } + + } + + /** + * Marks the start of a query identified by the provided correlationId, with additional query type and data parameters + * + * @param query - Query data + * @param correlationId - Identifier + * @param label - allows queries to be grouped by label + * @return Start event to pass to the Events systems EventBus + */ + public static AddLabelledQuery start(T query, long correlationId, String label) { + return start(query, correlationId, label, null); + } + + /** + * Marks the start of a query identified by the provided correlationId, with additional query type and data parameters + * + * @param query - Query data + * @param correlationId - Identifier + * @param label - allows queries to be grouped by label + * @param additionalData - Any additional info about the request to be rendered in the JSON view / rest endpoint + * @return Start event to pass to the Events systems EventBus + */ + public static AddLabelledQuery start(T query, long correlationId, String label, Object additionalData) { + + return new AddLabelledQuery( + RequestData.builder() + .query(query) + .correlationId(correlationId) + .type(label) + .additionalData(additionalData) + .build()); + } + + /** + * Publish finish events for each of the specified query labels + * + * 
+     * {@code
+     * LabelledEvents.start("get", 1l, bus, "typeA", "custom");
+       try {
+            return "ok";
+        } finally {
+            RequestEvents.finish("get", 1l, bus, "typeA", "custom");
+        }
+     *
+     * }
+     *
+ * + * + * @param query Completed query + * @param correlationId Identifier + * @param bus EventBus to post events to + * @param labels Query types to post to event bus + */ + public static void finish(T query, long correlationId, EventBus bus, String... labels) { + for (String type : labels) { + RemoveLabelledQuery next = finish(query, correlationId, type); + bus.post(next); + } + } + /** + * Marks the end of a query identified by the provided correlationId + * + * @param query - Query data + * @param correlationId - Identifier + * @param label - allows queries to be grouped by type + * @return RemoveLabelledQuery event to pass to the Events systems EventBus + */ + public static RemoveLabelledQuery finish(T query, long correlationId, String label) { + + return new RemoveLabelledQuery<>( + RequestData.builder() + .query(query) + .correlationId(correlationId) + .type(label) + .build()); + } +} diff --git a/micro-events/src/main/java/com/aol/micro/server/events/RequestTypes.java b/micro-events/src/main/java/com/aol/micro/server/events/RequestTypes.java index e17be567c..bc1f22086 100644 --- a/micro-events/src/main/java/com/aol/micro/server/events/RequestTypes.java +++ b/micro-events/src/main/java/com/aol/micro/server/events/RequestTypes.java @@ -79,6 +79,24 @@ public RemoveQuery(RequestData data) { } + public static class AddLabelledQuery extends AddEvent> { + + public AddLabelledQuery(RequestData data) { + super( + data); + } + + } + + public static class RemoveLabelledQuery extends RemoveEvent> { + + public RemoveLabelledQuery(RequestData data) { + super( + data); + } + + } + @AllArgsConstructor @Builder @XmlAccessorType(XmlAccessType.FIELD) diff --git a/micro-events/src/test/java/com/aol/micro/server/events/LabelledEventsTest.java b/micro-events/src/test/java/com/aol/micro/server/events/LabelledEventsTest.java new file mode 100644 index 000000000..421940ff9 --- /dev/null +++ b/micro-events/src/test/java/com/aol/micro/server/events/LabelledEventsTest.java @@ -0,0 +1,62 @@ +package com.aol.micro.server.events; + +import org.junit.Before; +import org.junit.Test; + +import com.aol.micro.server.events.RequestTypes.AddLabelledQuery; +import com.aol.micro.server.events.RequestTypes.RemoveLabelledQuery; +import com.aol.micro.server.events.RequestTypes.RequestData; + +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; + + +public class LabelledEventsTest { + + String query; + long corrId; + String label; + String addData; + + @Before + public void setUp() { + query = "query as string"; + corrId = 1234; + label = "label"; + addData = "additional data"; + } + + @Test + public void createAddLabelledQuery() { + + AddLabelledQuery userQuery = LabelledEvents.start(query, corrId, label); + RequestData rd = userQuery.getData(); + + assertThat(rd.getQuery(), is(query)); + assertThat(rd.getType(), is(label)); + assertThat(rd.getCorrelationId(), is(corrId)); + } + + @Test + public void createAddLabelledQueryWithAdditionalData() { + + AddLabelledQuery userQuery = LabelledEvents.start(query, corrId, label, addData); + RequestData rd = userQuery.getData(); + + assertThat(rd.getQuery(), is(query)); + assertThat(rd.getType(), is(label)); + assertThat(rd.getCorrelationId(), is(corrId)); + assertThat(rd.getAdditionalData(), is(addData)); + } + + @Test + public void createRemoveLabelledQuery() { + + RemoveLabelledQuery userQuery = LabelledEvents.finish(query, corrId, label); + RequestData rd = userQuery.getData(); + + assertThat(rd.getQuery(), is(query)); + assertThat(rd.getType(), is(label)); + assertThat(rd.getCorrelationId(), is(corrId)); + } +} diff --git a/micro-jackson-configuration/src/main/java/com/aol/micro/server/rest/jackson/JacksonUtil.java b/micro-jackson-configuration/src/main/java/com/aol/micro/server/rest/jackson/JacksonUtil.java index c3459c4be..d305da6c3 100644 --- a/micro-jackson-configuration/src/main/java/com/aol/micro/server/rest/jackson/JacksonUtil.java +++ b/micro-jackson-configuration/src/main/java/com/aol/micro/server/rest/jackson/JacksonUtil.java @@ -15,6 +15,7 @@ import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.databind.JavaType; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.core.type.TypeReference; public final class JacksonUtil { @@ -94,6 +95,15 @@ public static T convertFromJson(final String jsonString, final JavaType type return null; } + + public static T convertFromJson(String json, final TypeReference type) { + try { + return JacksonUtil.getMapper().readValue(json, type); + } catch (final Exception ex) { + ExceptionSoftener.throwSoftenedException(ex); + } + return null; + } public static Object serializeToJsonLogFailure(Object value) { try { diff --git a/micro-jmx-metrics/build.gradle b/micro-jmx-metrics/build.gradle index 423b7ac0a..a2ad41057 100644 --- a/micro-jmx-metrics/build.gradle +++ b/micro-jmx-metrics/build.gradle @@ -3,7 +3,12 @@ description = 'micro-jmx-metrics' dependencies { - compile ('com.ryantenney.metrics:metrics-spring:'+springMetricsVersion){ exclude(module: 'org.springframework') } + compile ('com.ryantenney.metrics:metrics-spring:'+springMetricsVersion){ + exclude(module: 'org.springframework') + exclude(group: 'io.dropwizard.metrics', module: 'metrics-core') + } + + compile("io.dropwizard.metrics:metrics-core:3.2.2") compile ("com.aol.simplereact:cyclops-react:$cyclopsReactVersion") compile project(':micro-core') diff --git a/micro-metrics/build.gradle b/micro-metrics/build.gradle index 693dc6b32..f02225eb8 100644 --- a/micro-metrics/build.gradle +++ b/micro-metrics/build.gradle @@ -1,8 +1,12 @@ description = 'micro-metrics' dependencies { - compile ('com.ryantenney.metrics:metrics-spring:'+springMetricsVersion){ + compile('com.ryantenney.metrics:metrics-spring:' + springMetricsVersion) { exclude(module: 'org.springframework') + exclude(group: 'io.dropwizard.metrics', module: 'metrics-core') } + + compile("io.dropwizard.metrics:metrics-core:3.2.3") + compile project(':micro-core') testCompile project(':micro-jackson-configuration') testCompile project(':micro-grizzly')