LEAF version: 3.12.7
LEAF component(s): framework-services-mongo
Issue/Question: Loading data objects from mongo through the data object service adds significant overhead compared to loading Mongo Documents. We have followed the performance guide on the knowledge base.
We are using queries that can have more than 60K results. Comparing it to a direct Mongo query, the data object service adds overhead that gets worse as more results are returned.
Are there other recommended strategies/ways to improve load times? One way I thought of was to split the query into smaller queries and then combine the results, but I am unsure if that would help.
This is the test I used with MongoDB 4.4 deployed on a local minikube environment.
package com.leidos.soma.services.leaf;
import static org.junit.jupiter.api.Assertions.assertEquals;
import java.util.List;
import java.util.stream.IntStream;
import java.util.stream.StreamSupport;
import javax.cache.configuration.MutableConfiguration;
import javax.cache.expiry.EternalExpiryPolicy;
import org.apache.commons.lang3.tuple.Pair;
import org.bson.Document;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInstance;
import org.junit.jupiter.api.TestInstance.Lifecycle;
import com.codahale.metrics.ConsoleReporter;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Timer;
import com.leidos.leaf.beans.Context;
import com.leidos.leaf.beans.IContext;
import com.leidos.leaf.beans.dataobject.DataObject;
import com.leidos.leaf.beans.metadata.BeanMetadata;
import com.leidos.leaf.beans.metadata.MetadataTemplate;
import com.leidos.leaf.beans.metadata.TemplateNode;
import com.leidos.leaf.beans.query._static.api.DataObjectQueries;
import com.leidos.leaf.data.service.DataServiceTransient;
import com.leidos.leaf.data.service.IDataService;
import com.leidos.leaf.data.service.mongo.DataServiceMongo;
import com.leidos.leaf.dataobject.service.IDataObjectService;
import com.leidos.leaf.event.service.EventServiceLocal;
import com.leidos.leaf.event.service.IEventService;
import com.leidos.leaf.hooked.Args;
import com.leidos.leaf.hooked.default_.hooks.CacheHook;
import com.leidos.leaf.metadata.service.IMetadataService;
import com.leidos.leaf.mongo.util.MongoClientUtils;
import com.leidos.leaf.util.serialization.JsonObjectMapper;
import com.mongodb.client.MongoClient;
import com.mongodb.client.MongoIterable;
import com.mongodb.client.model.Aggregates;
@TestInstance(Lifecycle.PER_CLASS)
class DataObjectLoadingTest {
private static final String TEST_TYPE = "test";
private static final int TOTAL_DATA_OBJECTS = 60_000;
IDataService dataService;
ConsoleReporter reporter;
MetricRegistry metrics = new MetricRegistry();
@BeforeAll
void setup() {
System.setProperty("MONGO_USERNAME", "root");
System.setProperty("MONGO_PASSWORD", "l3afpl4tf0rm");
System.setProperty("MONGO_HOSTNAME", "localhost:27017");
IEventService eventService = EventServiceLocal.newBuilder()
.build();
CacheHook cacheHook = CacheHook.newBuilder()
.withConfigurationStrategy(method -> new MutableConfiguration<Pair<Args, Object>, Object>()
.setExpiryPolicyFactory(EternalExpiryPolicy::new))
.withNoCacheKeyStrategy()
.build();
IDataService transientDataService = DataServiceTransient.newBuilder(eventService)
.build();
IMetadataService metadataService = transientDataService.getMetadataService();
this.dataService = DataServiceMongo.newBuilder(eventService)
.withCacheHook(cacheHook)
.addSharedService(IMetadataService.BUILDER_KEY, metadataService)
.build();
BeanMetadata metadata = BeanMetadata.newBuilder()
.withType(TEST_TYPE)
.build();
MetadataTemplate template = MetadataTemplate.newBuilder(TEST_TYPE)
.addRootNode(TemplateNode.newBuilder(metadata.getType())
.build())
.build();
IContext ctx = Context.makeSystemContext();
metadataService.upsertBeanMetadata(List.of(metadata), ctx);
metadataService.upsertTemplates(List.of(template), ctx);
List<DataObject> dataObjects = IntStream.range(0, TOTAL_DATA_OBJECTS)
.mapToObj(i -> DataObject.newBuilder(metadata)
.build())
.toList();
this.dataService.getDataObjectService()
.addRoots(dataObjects, ctx);
this.reporter = ConsoleReporter.forRegistry(metrics)
.build();
}
@AfterAll
void teardown() {
IContext ctx = Context.makeSystemContext();
this.dataService.getDataObjectService()
.deleteByQuery(DataObjectQueries.dataObjectQuery(DataObjectQueries.types(TEST_TYPE)), ctx);
}
@Test
void test() throws Exception {
// mongo documents
MongoClient client = MongoClientUtils.createMongoClient();
Timer loadTimer = metrics.timer("load documents");
List<Document> documents;
try (Timer.Context ctx = loadTimer.time()) {
MongoIterable<Document> iterable =
// @formatter:off
client.getDatabase("leafDB")
.getCollection("dataObjects").aggregate(
List.of(
Aggregates.match(com.mongodb.client.model.Filters.eq("type", TEST_TYPE)),
Aggregates.limit(TOTAL_DATA_OBJECTS)
)
);
// @formatter:on
documents = StreamSupport.stream(iterable.spliterator(), false)
.toList();
}
assertEquals(TOTAL_DATA_OBJECTS, documents.size(), "Incorrect documents count");
System.out.println(documents.get(0)
.toJson());
// data objects
Timer dataObjectServiceTimer = metrics.timer("load data objects");
IDataObjectService dataObjectService = this.dataService.getDataObjectService();
List<DataObject> result;
try (Timer.Context tCtx = dataObjectServiceTimer.time()) {
result = dataObjectService.load(DataObjectQueries.dataObjectQuery(DataObjectQueries.types(TEST_TYPE),
DataObjectQueries.pagingSpec(TOTAL_DATA_OBJECTS, 0)), Context.makeSystemContext());
}
assertEquals(TOTAL_DATA_OBJECTS, result.size(), "Incorrect data objects count");
System.out.println(JsonObjectMapper.MAPPER_INSTANCE.writeValueAsString(result.get(0)));
reporter.report();
}
}
The following is the console reporter for the test.
-- Timers ----------------------------------------------------------------------
load data objects
count = 1
mean rate = 0.31 calls/second
1-minute rate = 0.00 calls/second
5-minute rate = 0.00 calls/second
15-minute rate = 0.00 calls/second
min = 3181.59 milliseconds
max = 3181.59 milliseconds
mean = 3181.59 milliseconds
stddev = 0.00 milliseconds
median = 3181.59 milliseconds
75% <= 3181.59 milliseconds
95% <= 3181.59 milliseconds
98% <= 3181.59 milliseconds
99% <= 3181.59 milliseconds
99.9% <= 3181.59 milliseconds
load documents
count = 1
mean rate = 0.20 calls/second
1-minute rate = 0.00 calls/second
5-minute rate = 0.00 calls/second
15-minute rate = 0.00 calls/second
min = 1729.34 milliseconds
max = 1729.34 milliseconds
mean = 1729.34 milliseconds
stddev = 0.00 milliseconds
median = 1729.34 milliseconds
75% <= 1729.34 milliseconds
95% <= 1729.34 milliseconds
98% <= 1729.34 milliseconds
99% <= 1729.34 milliseconds
99.9% <= 1729.34 milliseconds