Skip to content

Commit

Permalink
Faster JSON parsing (up to 10x faster!)
Browse files Browse the repository at this point in the history
  • Loading branch information
@denysvitali
denysvitali committed Jun 11, 2018
1 parent f5ae0cd commit 1a0d9b1
Show file tree
Hide file tree
Showing 44 changed files with 1,364 additions and 43 deletions.
24 changes: 13 additions & 11 deletions build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ apply plugin: 'idea'
apply plugin: 'java'
apply plugin: 'application'
apply plugin: 'com.github.johnrengelman.shadow'
apply plugin: 'javafx-gradle-plugin'
//apply plugin: 'javafx-gradle-plugin'

mainClassName = "ch.supsi.dti.i2b.shrug.optitravel.Main";

Expand All @@ -35,9 +35,11 @@ dependencies {
testRuntime("org.junit.jupiter:junit-jupiter-engine:${junitJupiterVersion}")
implementation 'com.squareup.okhttp3:okhttp:3.10.0'
compile 'com.lynden:GMapsFX:2.12.0'
compile 'ca.fuzzlesoft:json-parse:1.3.4'
compile 'com.jsoniter:jsoniter:0.9.19'
// compile 'com.jfoenix:jfoenix:8.0.3' // Java 8
//compile 'com.jfoenix:jfoenix:8.0.3' // Java 8
compile 'com.jfoenix:jfoenix:9.0.3' // Java 9
compile group: 'org.javassist', name: 'javassist', version: '3.22.0-GA'
compile group: 'de.jensd', name: 'fontawesomefx-materialicons', version: '2.2.0-9.1.2'
}

Expand Down Expand Up @@ -78,23 +80,23 @@ jar {
}


jfx {
// minimal requirement for jfxJar-task
mainClass = mainClassName
// minimal requirement for jfxNative-task
vendor = '¯\\_(ツ)_/¯'
}
//jfx {
// // minimal requirement for jfxJar-task
// mainClass = mainClassName
// // minimal requirement for jfxNative-task
// vendor = '¯\\_(ツ)_/¯'
//}

run{

}

/*task jsoniterStaticCodgen(type:JavaExec) {
task jsoniterStaticCodgen(type:JavaExec) {
classpath configurations.getByName(sourceSets.main.compileConfigurationName)
classpath = sourceSets.main.runtimeClasspath + sourceSets.main.compileClasspath
workingDir = new File(project.rootDir, '/src/main')
main = 'com.jsoniter.static_codegen.StaticCodegen'
args 'ch.supsi.dti.i2b.shrug.optitravel.staticgen.StaticCodeGen'
args 'ch.supsi.dti.i2b.shrug.optitravel.codegen.CGConfig'
standardOutput = System.out
errorOutput = System.err
}*/
}
Binary file modified gradle/wrapper/gradle-wrapper.jar
View file
Binary file not shown.
3 changes: 1 addition & 2 deletions gradle/wrapper/gradle-wrapper.properties
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
#Wed Jun 06 15:46:49 CEST 2018
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-4.8-bin.zip
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-4.8-all.zip
122 changes: 103 additions & 19 deletions src/main/ch/supsi/dti/i2b/shrug/optitravel/api/GTFS_rs/GTFSrsWrapper.java
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,17 @@
package ch.supsi.dti.i2b.shrug.optitravel.api.GTFS_rs;

import ca.fuzzlesoft.JsonParse;
import ch.supsi.dti.i2b.shrug.optitravel.api.GTFS_rs.api.Error;
import ch.supsi.dti.i2b.shrug.optitravel.api.GTFS_rs.api.Meta;
import ch.supsi.dti.i2b.shrug.optitravel.api.GTFS_rs.api.Result;
import ch.supsi.dti.i2b.shrug.optitravel.api.GTFS_rs.api.ResultArray;
import ch.supsi.dti.i2b.shrug.optitravel.api.GTFS_rs.api.models.Pagination;
import ch.supsi.dti.i2b.shrug.optitravel.api.GTFS_rs.models.*;
import ch.supsi.dti.i2b.shrug.optitravel.api.GTFS_rs.search.TripSearch;
import ch.supsi.dti.i2b.shrug.optitravel.config.BuildConfig;
import ch.supsi.dti.i2b.shrug.optitravel.geography.BoundingBox;
import ch.supsi.dti.i2b.shrug.optitravel.geography.Coordinate;
import ch.supsi.dti.i2b.shrug.optitravel.models.Date;
import ch.supsi.dti.i2b.shrug.optitravel.models.StopTime;
import ch.supsi.dti.i2b.shrug.optitravel.models.Time;
import ch.supsi.dti.i2b.shrug.optitravel.utilities.HttpClient;
import com.jsoniter.JsonIterator;
Expand All @@ -21,6 +24,7 @@
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

public class GTFSrsWrapper {
Expand All @@ -31,6 +35,8 @@ public class GTFSrsWrapper {

public GTFSrsWrapper(){
this.client = new HttpClient();
restoreJsonIteratorMode();

/*JsonIterator.setMode(DecodingMode.STATIC_MODE);
JsonStream.setMode(EncodingMode.STATIC_MODE);*/
}
Expand Down Expand Up @@ -67,13 +73,18 @@ public List<Stop> getStopsByTrip(String tid) throws GTFSrsError {
private List<Stop> parseStopResponse(Response response) throws GTFSrsError {
if(response != null && response.isSuccessful() && response.body() != null){
try {
JsonIterator.setMode(DecodingMode.REFLECTION_MODE);
JsonStream.setMode(EncodingMode.REFLECTION_MODE);
ResultArray a = JsonIterator.deserialize(response.body().string(), ResultArray.class);
response.close();
return a.getResult()
List<Stop> stopList = a.getResult()
.asList()
.stream()
.map(e-> e.as(Stop.class))
.collect(Collectors.toList());

restoreJsonIteratorMode();
return stopList;
} catch(IOException ex){
return null;
}
Expand All @@ -82,6 +93,11 @@ private List<Stop> parseStopResponse(Response response) throws GTFSrsError {
}
}

private static void restoreJsonIteratorMode() {
JsonIterator.setMode(DecodingMode.DYNAMIC_MODE_AND_MATCH_FIELD_WITH_HASH);
JsonStream.setMode(EncodingMode.DYNAMIC_MODE);
}

public List<Route> getRoutes() throws GTFSrsError {
// /api/routes
HttpUrl url = new HttpUrl.Builder()
Expand Down Expand Up @@ -115,9 +131,13 @@ public Agency getAgency(String uid) throws GTFSrsError {
Response response = client.get(url);
if(response != null && response.isSuccessful() && response.body() != null){
try {
JsonIterator.setMode(DecodingMode.REFLECTION_MODE);
JsonStream.setMode(EncodingMode.STATIC_MODE);
ResultArray a = JsonIterator.deserialize(response.body().string(), ResultArray.class);
response.close();
return a.getResult().as(Agency.class);
Agency ag = a.getResult().as(Agency.class);
restoreJsonIteratorMode();
return ag;
} catch(IOException ex){
return null;
}
Expand Down Expand Up @@ -180,9 +200,13 @@ public Trip getTrip(String uid) throws GTFSrsError {
Response response = client.get(url);
if(response != null && response.isSuccessful() && response.body() != null){
try {
JsonIterator.setMode(DecodingMode.REFLECTION_MODE);
JsonStream.setMode(EncodingMode.REFLECTION_MODE);
ResultArray a = JsonIterator.deserialize(response.body().string(), ResultArray.class);
response.close();
return a.getResult().as(Trip.class);
Trip t = a.getResult().as(Trip.class);
restoreJsonIteratorMode();
return t;
} catch(IOException ex){
return null;
}
Expand Down Expand Up @@ -303,8 +327,12 @@ private StopTimes parseSingleStopTimes(Response response) throws GTFSrsError {
Result a = JsonIterator.deserialize(
response.body().string(),
Result.class);
return a.getResult()
JsonIterator.setMode(DecodingMode.REFLECTION_MODE);
JsonStream.setMode(EncodingMode.DYNAMIC_MODE);
StopTimes st = a.getResult()
.as(StopTimes.class);
restoreJsonIteratorMode();
return st;
} catch(IOException ex){
return null;
}
Expand All @@ -320,14 +348,18 @@ private PaginatedList<StopTimes> getPaginatedStopTimes(Response response) throws
response.body().string(),
ResultArray.class);
response.close();
return new PaginatedList<>(
JsonIterator.setMode(DecodingMode.REFLECTION_MODE);
JsonStream.setMode(EncodingMode.STATIC_MODE);
PaginatedList<StopTimes> pl = new PaginatedList<>(
a.getResult()
.asList()
.stream()
.map((e) -> e.as(StopTimes.class))
.collect(Collectors.toList()),
a.getMeta()
);
restoreJsonIteratorMode();
return pl;
} catch(IOException ex){
return null;
}
Expand Down Expand Up @@ -426,17 +458,65 @@ public static PaginatedList<Trip> parsePaginatedTrips(byte[] json){
}

public static PaginatedList<Trip> parsePaginatedTrips(String json){
ResultArray a = JsonIterator.deserialize(
json,
ResultArray.class);
return new PaginatedList<>(
a.getResult()
.asList()
.stream()
.map((e) -> e.as(Trip.class))
.collect(Collectors.toList()),
a.getMeta()
);

Map<String, Object> resultArray = JsonParse.map(json);
List<Map<String, Object>> trips = (List<Map<String, Object>>) resultArray.get("result");
Map<String, Object> meta = (Map<String, Object>) resultArray.get("meta");
Meta metaobj = new Meta();

Map<String, Object> pag = (Map<String, Object>) meta.get("pagination");
Map<String, Object> err = (Map<String, Object>) meta.get("error");

if(pag != null){
Pagination p = new Pagination();
p.limit = ((Long) pag.get("limit")).intValue();
p.offset = ((Long) pag.get("offset")).intValue();
metaobj.pagination = p;
}

if(err != null){
Error error = new Error();
error.message = (String) err.get("message");
error.code = ((Long) err.get("code")).intValue();
metaobj.error = error;
}

metaobj.success = (boolean) meta.get("success");

List<Trip> trips_objs = trips.stream().map(e->{
Trip t = new Trip();
t.uid = (String) e.get("uid");
t.route_id = (String) e.get("route_id");
t.service_id = (String) e.get("service_id");
t.headsign = (String) e.get("headsign");
t.short_name = (String) e.get("short_name");
t.direction_id = ((int) (long) e.get("direction_id"));
t.stop_sequence = ((List<Map<String, Object>>) e.get("stop_sequence")).stream().map(ss->{
StopTrip st = new StopTrip();
Map<String, Object> stopMap = (Map<String, Object>) ss.get("stop");
Stop s = new Stop();
s.setUid((String) stopMap.get("uid"));
s.setName((String) stopMap.get("name"));
s.setLat((Double) stopMap.get("lat"));
s.setLng((Double) stopMap.get("lng"));
s.setLocation_type(((Long) stopMap.get("location_type")).intValue());
s.setParent_station((String) stopMap.get("parent_station"));

st.setStop(s);
st.setArrival_time((String) ss.get("arrival_time"));
st.setDeparture_time((String) ss.get("departure_time"));
st.setStop_sequence(((Long) ss.get("stop_sequence")).intValue());
st.setDrop_off(ch.supsi.dti.i2b.shrug.optitravel.models.DropOff.valueOf((String) ss.get("drop_off")));
st.setPickup(ch.supsi.dti.i2b.shrug.optitravel.models.PickUp.valueOf((String) ss.get("pickup")));
return st;
}).collect(Collectors.toList());
return t;
}).collect(Collectors.toList());

PaginatedList<Trip> pl = new PaginatedList<>();
pl.setResult(trips_objs);
pl.setMeta(metaobj);
return pl;
}

private PaginatedList<Trip> getPaginatedTrips(Response response) throws GTFSrsError {
Expand Down Expand Up @@ -505,7 +585,7 @@ public PaginatedList<StopDistance> getStopsNear(Coordinate coordinate) throws GT
.addPathSegment(String.valueOf(coordinate.getLng()));

HttpUrl url = builder.build();
Response response = client.get(url);
Response response = client.get(url, 20 * 1000);
return getPaginatedStopDistances(response);
}

Expand All @@ -531,14 +611,18 @@ private PaginatedList<StopDistance> getPaginatedStopDistances(Response response)
response.body().string(),
ResultArray.class);
response.close();
return new PaginatedList<>(
JsonIterator.setMode(DecodingMode.REFLECTION_MODE);
JsonStream.setMode(EncodingMode.STATIC_MODE);
PaginatedList<StopDistance> pl = new PaginatedList<>(
a.getResult()
.asList()
.stream()
.map((e) -> e.as(StopDistance.class))
.collect(Collectors.toList()),
a.getMeta()
);
restoreJsonIteratorMode();
return pl;
} catch(IOException ex){
return null;
}
Expand Down
12 changes: 12 additions & 0 deletions src/main/ch/supsi/dti/i2b/shrug/optitravel/api/GTFS_rs/models/PaginatedList.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,23 @@ public PaginatedList(List<T> result, Meta meta) {
this.meta = meta;
}

public PaginatedList() {

}

public Meta getMeta() {
return meta;
}

public List<T> getResult() {
return result;
}

public void setResult(List<T> result) {
this.result = result;
}

public void setMeta(Meta metaobj) {
this.meta = metaobj;
}
}
28 changes: 28 additions & 0 deletions src/main/ch/supsi/dti/i2b/shrug/optitravel/api/GTFS_rs/models/Stop.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,4 +88,32 @@ public boolean equals(Object o) {
public int hashCode() {
return Objects.hash(uid, name, lat, lng, location_type, parent_station);
}

public void setUid(String uid) {
this.uid = uid;
}

public void setName(String name) {
this.name = name;
}

public void setLat(double lat) {
this.lat = lat;
}

public void setLng(double lng) {
this.lng = lng;
}

public void setLocation_type(int location_type) {
this.location_type = location_type;
}

public void setParent_station(String parent_station) {
this.parent_station = parent_station;
}

public void setGtfSrsWrapper(GTFSrsWrapper gtfSrsWrapper) {
this.gtfSrsWrapper = gtfSrsWrapper;
}
}
3 changes: 3 additions & 0 deletions src/main/ch/supsi/dti/i2b/shrug/optitravel/api/GTFS_rs/models/StopTimes.java
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
package ch.supsi.dti.i2b.shrug.optitravel.api.GTFS_rs.models;

import com.jsoniter.annotation.JsonObject;

import java.util.List;

@JsonObject
public class StopTimes {
private String stop;
private List<TripTimeStop> time;
Expand Down
Loading

0 comments on commit 1a0d9b1

Please sign in to comment.