Skip to content

Commit

Permalink
fix tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@scivey
scivey committed Nov 16, 2015
1 parent aa76980 commit b26f7e2
Show file tree
Hide file tree
Showing 7 changed files with 106 additions and 65 deletions.
4 changes: 2 additions & 2 deletions src/centroid_update_worker/test_functional/test_CentroidUpdateWorker.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,17 +57,17 @@ struct UpdateWorkerTestCtx {
UpdateWorkerTestCtx() {
threadPool1.reset(new wangle::FutureExecutor<wangle::CPUThreadPoolExecutor>(2));
threadPool2.reset(new wangle::FutureExecutor<wangle::CPUThreadPoolExecutor>(2));
sysClock.reset(new Clock);
UniquePointer<RockHandleIf> rockHandle(new InMemoryRockHandle("foo"));
UniquePointer<SyncPersistenceIf> syncPersistence(
new SyncPersistence(std::move(rockHandle))
new SyncPersistence(sysClock, std::move(rockHandle))
);
shared_ptr<PersistenceIf> result(
new Persistence(std::move(syncPersistence), threadPool1)
);
persistence = result;
metadb.reset(new CentroidMetadataDb(persistence));
accumulatorFactory.reset(new DocumentAccumulatorFactory);
sysClock.reset(new Clock);
updaterFactory.reset(new CentroidUpdaterFactory(
persistence, metadb, accumulatorFactory, sysClock
));
Expand Down
13 changes: 6 additions & 7 deletions src/document_processing_worker/test_functional/test_DocumentProcessingWorker.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
#include "document_processing_worker/DocumentProcessor.h"
#include "document_processing_worker/DocumentProcessingWorker.h"
#include "stopwords/StopwordFilter.h"
#include "stemmer/ThreadSafeUtf8Stemmer.h"
#include "stemmer/ThreadSafeStemmerManager.h"
#include "models/ProcessedDocument.h"
#include "models/Centroid.h"
#include "models/Document.h"
Expand All @@ -44,7 +44,6 @@ using namespace relevanced::centroid_update_worker;
using namespace relevanced::document_processing_worker;
using namespace relevanced::stemmer;
using namespace relevanced::stopwords;
using namespace relevanced::tokenizer;
using relevanced::thrift_protocol::Language;

using ::testing::Return;
Expand All @@ -55,7 +54,7 @@ struct ProcessingWorkerTestCtx {
shared_ptr<PersistenceIf> persistence;
shared_ptr<Sha1HasherIf> hasher;
shared_ptr<ClockIf> sysClock;
shared_ptr<StemmerIf> stemmer;
shared_ptr<StemmerManagerIf> stemmerManager;
shared_ptr<StopwordFilterIf> stopwordFilter;
shared_ptr<FutureExecutor<CPUThreadPoolExecutor>> threadPool1;
shared_ptr<FutureExecutor<CPUThreadPoolExecutor>> threadPool2;
Expand All @@ -66,18 +65,18 @@ struct ProcessingWorkerTestCtx {
threadPool1.reset(new wangle::FutureExecutor<wangle::CPUThreadPoolExecutor>(2));
threadPool2.reset(new wangle::FutureExecutor<wangle::CPUThreadPoolExecutor>(2));
UniquePointer<RockHandleIf> rockHandle(new InMemoryRockHandle("foo"));
sysClock.reset(new Clock);
UniquePointer<SyncPersistenceIf> syncPersistence(
new SyncPersistence(std::move(rockHandle))
new SyncPersistence(sysClock, std::move(rockHandle))
);
persistence.reset(
new Persistence(std::move(syncPersistence), threadPool1)
);
hasher.reset(new Sha1Hasher);
sysClock.reset(new Clock);
stemmer.reset(new ThreadSafeUtf8Stemmer);
stemmerManager.reset(new ThreadSafeStemmerManager);
stopwordFilter.reset(new StopwordFilter);
processor.reset(
new DocumentProcessor(stemmer, stopwordFilter, sysClock)
new DocumentProcessor(stemmerManager, stopwordFilter, sysClock)
);
worker.reset(new DocumentProcessingWorker(
processor, hasher, threadPool2
Expand Down
9 changes: 9 additions & 0 deletions src/server/RelevanceServer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -400,6 +400,15 @@ Future<unique_ptr<vector<string>>> RelevanceServer::listAllDocuments() {
});
}

Future<unique_ptr<vector<string>>> RelevanceServer::listUnusedDocuments(
size_t count) {
return persistence_->listUnusedDocuments(count)
.then([](vector<string> docIds) {
return std::move(
folly::make_unique<vector<string>>(docIds)
);
});
}

Future<unique_ptr<vector<string>>> RelevanceServer::listDocumentRange(
size_t offset, size_t count) {
Expand Down
6 changes: 6 additions & 0 deletions src/server/RelevanceServer.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,9 @@ class RelevanceServerIf {
virtual folly::Future<std::unique_ptr<std::vector<std::string>>>
listAllDocuments() = 0;

virtual folly::Future<std::unique_ptr<std::vector<std::string>>>
listUnusedDocuments(size_t count) = 0;

virtual folly::Future<std::unique_ptr<std::vector<std::string>>>
listCentroidRange(size_t offset, size_t count) = 0;

Expand Down Expand Up @@ -305,6 +308,9 @@ class RelevanceServer : public RelevanceServerIf {
folly::Future<std::unique_ptr<std::vector<std::string>>>
listAllDocuments() override;

folly::Future<std::unique_ptr<std::vector<std::string>>>
listUnusedDocuments(size_t count) override;

folly::Future<std::unique_ptr<std::vector<std::string>>>
listCentroidRange(size_t offset, size_t count) override;

Expand Down
111 changes: 81 additions & 30 deletions src/server/test_functional/test_RelevanceServer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,8 @@
#include "document_processing_worker/DocumentProcessingWorker.h"
#include "similarity_score_worker/SimilarityScoreWorker.h"
#include "stopwords/StopwordFilter.h"
#include "stemmer/ThreadSafeUtf8Stemmer.h"
#include "stemmer/Utf8Stemmer.h"
#include "stemmer/ThreadSafeStemmerManager.h"
#include "server/RelevanceServer.h"
#include "models/ProcessedDocument.h"
#include "server/RelevanceServer.h"
Expand All @@ -58,11 +59,8 @@ using namespace relevanced::similarity_score_worker;
using namespace relevanced::stemmer;
using namespace relevanced::stopwords;
using namespace relevanced::server;
using namespace relevanced::tokenizer;
using namespace relevanced::thrift_protocol;



using ::testing::Return;
using ::testing::_;

Expand All @@ -72,7 +70,7 @@ struct RelevanceServerTestCtx {
shared_ptr<CentroidMetadataDbIf> metadb;
shared_ptr<Sha1HasherIf> hasher;
shared_ptr<ClockIf> sysClock;
shared_ptr<StemmerIf> stemmer;
shared_ptr<StemmerManagerIf> stemmerManager;
shared_ptr<StopwordFilterIf> stopwordFilter;
shared_ptr<DocumentProcessorIf> processor;
shared_ptr<CentroidUpdaterFactoryIf> updaterFactory;
Expand All @@ -92,17 +90,17 @@ struct RelevanceServerTestCtx {
scoringThreads.reset(new wangle::FutureExecutor<wangle::CPUThreadPoolExecutor>(2));
updatingThreads.reset(new wangle::FutureExecutor<wangle::CPUThreadPoolExecutor>(2));
UniquePointer<RockHandleIf> rockHandle(new InMemoryRockHandle("foo"));
sysClock.reset(new Clock);
UniquePointer<SyncPersistenceIf> syncPersistence(
new SyncPersistence(std::move(rockHandle))
new SyncPersistence(sysClock, std::move(rockHandle))
);
persistence.reset(new Persistence(std::move(syncPersistence), persistenceThreads));
hasher.reset(new Sha1Hasher);
metadb.reset(new CentroidMetadataDb(persistence));
stemmer.reset(new ThreadSafeUtf8Stemmer);
stemmerManager.reset(new ThreadSafeStemmerManager);
stopwordFilter.reset(new StopwordFilter);
sysClock.reset(new Clock);
processor.reset(
new DocumentProcessor(stemmer, stopwordFilter, sysClock)
new DocumentProcessor(stemmerManager, stopwordFilter, sysClock)
);
accumulatorFactory.reset(new DocumentAccumulatorFactory);
updaterFactory.reset(new CentroidUpdaterFactory(
Expand Down Expand Up @@ -152,7 +150,8 @@ TEST(RelevanceServer, TestAddDocumentToCentroidHappy) {
ctx.persistence->saveCentroid("centroid-id", centroid).get();
ctx.server->createDocumentWithID(
folly::make_unique<string>("doc-id"),
folly::make_unique<string>("some text about dogs")
folly::make_unique<string>("some text about dogs"),
Language::EN
).get();
auto response = ctx.server->addDocumentToCentroid(
folly::make_unique<string>("centroid-id"),
Expand All @@ -172,7 +171,8 @@ TEST(RelevanceServer, TestAddDocumentToCentroidAlreadyInCentroid) {
ctx.persistence->saveCentroid("centroid-id", centroid).get();
ctx.server->createDocumentWithID(
folly::make_unique<string>("doc-id"),
folly::make_unique<string>("some text about dogs")
folly::make_unique<string>("some text about dogs"),
Language::EN
).get();
auto response1 = ctx.server->addDocumentToCentroid(
folly::make_unique<string>("centroid-id"),
Expand All @@ -194,7 +194,8 @@ TEST(RelevanceServer, TestAddDocumentToCentroidMissingDocument) {
ctx.persistence->saveCentroid("centroid-id", centroid).get();
ctx.server->createDocumentWithID(
folly::make_unique<string>("unrelated-doc-id"),
folly::make_unique<string>("some text about dogs")
folly::make_unique<string>("some text about dogs"),
Language::EN
).get();
auto response = ctx.server->addDocumentToCentroid(
folly::make_unique<string>("centroid-id"),
Expand All @@ -211,7 +212,8 @@ TEST(RelevanceServer, TestAddDocumentToCentroidMissingCentroid) {
ctx.persistence->saveCentroid("centroid-id", centroid).get();
ctx.server->createDocumentWithID(
folly::make_unique<string>("doc-id"),
folly::make_unique<string>("some text about dogs")
folly::make_unique<string>("some text about dogs"),
Language::EN
).get();
auto response = ctx.server->addDocumentToCentroid(
folly::make_unique<string>("missing-centroid-id"),
Expand All @@ -228,7 +230,8 @@ TEST(RelevanceServer, TestAddDocumentToCentroidMissingBoth) {
ctx.persistence->saveCentroid("centroid-id", centroid).get();
ctx.server->createDocumentWithID(
folly::make_unique<string>("doc-id"),
folly::make_unique<string>("some text about dogs")
folly::make_unique<string>("some text about dogs"),
Language::EN
).get();
auto response = ctx.server->addDocumentToCentroid(
folly::make_unique<string>("missing-centroid-id"),
Expand All @@ -249,7 +252,8 @@ TEST(RelevanceServer, TestRemoveDocumentFromCentroidHappy) {
saves.push_back(
ctx.server->createDocumentWithID(
folly::make_unique<string>("doc-id"),
folly::make_unique<string>("some text about dogs")
folly::make_unique<string>("some text about dogs"),
Language::EN
).then([](Try<unique_ptr<string>> result) {
EXPECT_FALSE(result.hasException());
return Try<bool>(true);
Expand Down Expand Up @@ -280,7 +284,8 @@ TEST(RelevanceServer, TestRemoveDocumentFromCentroidDocumentNotInCentroid) {
saves.push_back(
ctx.server->createDocumentWithID(
folly::make_unique<string>("doc-id"),
folly::make_unique<string>("some text about dogs")
folly::make_unique<string>("some text about dogs"),
Language::EN
).then([](Try<unique_ptr<string>> result) {
EXPECT_FALSE(result.hasException());
return Try<bool>(true);
Expand Down Expand Up @@ -315,7 +320,8 @@ TEST(RelevanceServer, TestRemoveDocumentFromCentroidMissingCentroid) {
ctx.persistence->saveCentroid("centroid-id", centroid).get();
ctx.server->createDocumentWithID(
folly::make_unique<string>("doc-id"),
folly::make_unique<string>("some text about dogs")
folly::make_unique<string>("some text about dogs"),
Language::EN
).get();
auto response = ctx.server->removeDocumentFromCentroid(
folly::make_unique<string>("missing-centroid-id"),
Expand All @@ -332,7 +338,8 @@ TEST(RelevanceServer, TestRemoveDocumentFromCentroidMissingBoth) {
ctx.persistence->saveCentroid("centroid-id", centroid).get();
ctx.server->createDocumentWithID(
folly::make_unique<string>("doc-id"),
folly::make_unique<string>("some text about dogs")
folly::make_unique<string>("some text about dogs"),
Language::EN
).get();
auto response = ctx.server->removeDocumentFromCentroid(
folly::make_unique<string>("missing-centroid-id"),
Expand All @@ -350,7 +357,8 @@ TEST(RelevanceServer, TestGetTextSimilarityHappy) {
ctx.scoreWorker->reloadCentroid("centroid-id").get();
auto scoreResponse = ctx.server->getTextSimilarity(
folly::make_unique<string>("centroid-id"),
folly::make_unique<string>("This is some dog related text which is also about a cat.")
folly::make_unique<string>("This is some dog related text which is also about a cat."),
Language::EN
).get();
EXPECT_FALSE(scoreResponse.hasException());
auto similarity = scoreResponse.value();
Expand All @@ -367,7 +375,8 @@ TEST(RelevanceServer, TestGetTextSimilarityMissingCentroid) {
ctx.scoreWorker->reloadCentroid("centroid-id").get();
auto scoreResponse = ctx.server->getTextSimilarity(
folly::make_unique<string>("unrelated-centroid-id"),
folly::make_unique<string>("This is some dog related text which is also about a cat.")
folly::make_unique<string>("This is some dog related text which is also about a cat."),
Language::EN
).get();
EXPECT_TRUE(scoreResponse.hasException<ECentroidDoesNotExist>());
}
Expand All @@ -391,7 +400,8 @@ TEST(RelevanceServer, TestMultiGetTextSimilarityHappy) {
vector<string> centroidIds {"centroid-1-id", "centroid-2-id"};
auto scoreResponse = ctx.server->multiGetTextSimilarity(
folly::make_unique<vector<string>>(centroidIds),
folly::make_unique<string>("This is some dog related text which is also about a cat.")
folly::make_unique<string>("This is some dog related text which is also about a cat."),
Language::EN
).get();
EXPECT_FALSE(scoreResponse.hasException());
}
Expand All @@ -401,7 +411,9 @@ TEST(RelevanceServer, TestCreateDocument) {
RelevanceServerTestCtx ctx;
auto text = folly::make_unique<string>("some text about cats and dogs and fish and so forth");

auto response = ctx.server->createDocument(std::move(text)).get();
auto response = ctx.server->createDocument(
std::move(text), Language::EN
).get();
EXPECT_TRUE(response.hasValue());
string docId = *response.value();
auto persisted = ctx.persistence->loadDocumentOption(docId).get();
Expand All @@ -413,7 +425,9 @@ TEST(RelevanceServer, TestCreateDocumentWithID) {
RelevanceServerTestCtx ctx;
auto text = folly::make_unique<string>("some text about cats and dogs and fish and so forth");
auto id = folly::make_unique<string>("doc-id");
auto response = ctx.server->createDocumentWithID(std::move(id), std::move(text)).get();
auto response = ctx.server->createDocumentWithID(
std::move(id), std::move(text), Language::EN
).get();
EXPECT_TRUE(response.hasValue());
EXPECT_EQ("doc-id", *response.value());
auto persisted = ctx.persistence->loadDocumentOption("doc-id").get();
Expand All @@ -425,11 +439,14 @@ TEST(RelevanceServer, TestCreateDocumentWithIDAlreadyExists) {
RelevanceServerTestCtx ctx;
auto text = folly::make_unique<string>("some text about cats and dogs and fish and so forth");
auto id = folly::make_unique<string>("doc-id");
auto response1 = ctx.server->createDocumentWithID(std::move(id), std::move(text)).get();
auto response1 = ctx.server->createDocumentWithID(
std::move(id), std::move(text), Language::EN
).get();
EXPECT_FALSE(response1.hasException());
auto response2 = ctx.server->createDocumentWithID(
folly::make_unique<string>("doc-id"),
folly::make_unique<string>("some text")
folly::make_unique<string>("some text"),
Language::EN
).get();
EXPECT_TRUE(response2.hasException<EDocumentAlreadyExists>());
}
Expand Down Expand Up @@ -527,7 +544,8 @@ TEST(RelevanceServer, TestListAllDocuments) {
expectedIds.insert(id);
creations.push_back(ctx.server->createDocumentWithID(
folly::make_unique<string>(id),
folly::make_unique<string>("this is some text about things")
folly::make_unique<string>("this is some text about things"),
Language::EN
));
}
set<string> createdIds;
Expand All @@ -553,7 +571,8 @@ TEST(RelevanceServer, TestListDocumentRange) {
auto id = sformat("some-doc-{}", i);
creations.push_back(ctx.server->createDocumentWithID(
folly::make_unique<string>(id),
folly::make_unique<string>("this is some text about things")
folly::make_unique<string>("this is some text about things"),
Language::EN
));
}
collect(creations).get();
Expand All @@ -575,7 +594,8 @@ TEST(RelevanceServer, TestListDocumentRangeFromID) {
auto id = sformat("some-doc-{}", i);
creations.push_back(ctx.server->createDocumentWithID(
folly::make_unique<string>(id),
folly::make_unique<string>("this is some text about things")
folly::make_unique<string>("this is some text about things"),
Language::EN
));
}
collect(creations).get();
Expand All @@ -599,7 +619,8 @@ TEST(RelevanceServer, TestDeleteDocument) {
auto id = sformat("some-doc-{}", i);
creations.push_back(ctx.server->createDocumentWithID(
folly::make_unique<string>(id),
folly::make_unique<string>("this is some text about things")
folly::make_unique<string>("this is some text about things"),
Language::EN
));
}
collect(creations).get();
Expand All @@ -623,10 +644,40 @@ TEST(RelevanceServer, TestDeleteDocumentMissing) {
auto id = sformat("some-doc-{}", i);
creations.push_back(ctx.server->createDocumentWithID(
folly::make_unique<string>(id),
folly::make_unique<string>("this is some text about things")
folly::make_unique<string>("this is some text about things"),
Language::EN
));
}
collect(creations).get();
auto result = ctx.server->deleteDocument(folly::make_unique<string>("some-doc-8")).get();
EXPECT_TRUE(result.hasException<EDocumentDoesNotExist>());
}

// TEST(RelevanceServer, TestListUnusedDocuments) {
// RelevanceServerTestCtx ctx;
// vector<Future<Try<unique_ptr<string>>>> documentCreations;
// for (size_t i = 0; i < 6; i++) {
// auto id = sformat("some-doc-{}", i);
// documentCreations.push_back(ctx.server->createDocumentWithID(
// folly::make_unique<string>(id),
// folly::make_unique<string>("this is some text about things"),
// Language::EN
// ));
// }
// collect(documentCreations).get();
// ctx.server->createCentroid("c1").get();
// vector<Future<Try<bool>>> additions;
// vector<string> toAdd {"some-doc-1", "some-doc-3", "some-doc-4"};
// for (string id: toAdd) {
// additions.push_back(ctx.server->addDocumentToCentroid(
// folly::make_unique<string>("c1"),
// folly::make_unique<string>(id)
// ));
// }
// collect(additions).get();
// auto unused = ctx.server->listUnusedDocuments(10).get();
// vector<string> expected {
// "some-doc-0", "some-doc-2", "some-doc-5"
// };
// EXPECT_EQ(expected, unused);
// }
Loading

0 comments on commit b26f7e2

Please sign in to comment.