From dca423e1ace3814b0691fa9fcb80729d521f1fc9 Mon Sep 17 00:00:00 2001 From: Scott Ivey Date: Mon, 23 Nov 2015 20:06:19 -0500 Subject: [PATCH] tests --- clients/nodejs/client/index.js | 118 +++++++++++- clients/nodejs/client/package.json | 4 + clients/nodejs/client/test/test.js | 300 +++++++++++++++++++++++++++++ src/RelevancedProtocol.thrift | 6 - src/server/RelevanceServer.cpp | 53 ----- src/server/RelevanceServer.h | 5 - 6 files changed, 418 insertions(+), 68 deletions(-) create mode 100644 clients/nodejs/client/test/test.js diff --git a/clients/nodejs/client/index.js b/clients/nodejs/client/index.js index 4bf13af..6eb8dd1 100644 --- a/clients/nodejs/client/index.js +++ b/clients/nodejs/client/index.js @@ -55,13 +55,17 @@ var CLIENT_METHODS = [ 'createDocument', 'createDocumentWithID', 'deleteDocument', + 'multiDeleteDocuments', 'getDocumentMetadata', + 'deleteCentroid', + 'multiDeleteCentroids', 'createCentroid', 'multiCreateCentroids', 'listAllDocumentsForCentroid', - 'addDocumentToCentroid', - 'removeDocumentFromCentroid', + 'addDocumentsToCentroid', + 'removeDocumentsFromCentroid', 'joinCentroid', + 'multiJoinCentroids', 'listAllCentroids', 'listAllDocuments', 'listUnusedDocuments' @@ -100,7 +104,7 @@ _.each(CLIENT_METHODS, function(methodName) { (function() { var original = RelevancedClient.prototype.createDocumentWithID; - RelevancedClient.prototype.createDocumentWithId = function(id, text, lang) { + RelevancedClient.prototype.createDocumentWithID = function(id, text, lang) { if (!_.isNumber(lang)) { lang = genTypes.Language.EN; } @@ -157,13 +161,119 @@ _.each(CLIENT_METHODS, function(methodName) { if (!_.isBoolean(ignoreExisting)) { ignoreExisting = false; } - var request = new genTypes.CreateCentroidRequest; + var request = new genTypes.MultiCreateCentroidsRequest; request.ids = ids; request.ignoreExisting = ignoreExisting; return original.apply(this, [request]); }; })(); +(function() { + var original = RelevancedClient.prototype.deleteCentroid; + RelevancedClient.prototype.deleteCentroid = function(id, ignoreMissing) { + if (!_.isBoolean(ignoreMissing)) { + ignoreMissing = false; + } + var request = new genTypes.DeleteCentroidRequest; + request.id = id; + request.ignoreMissing = ignoreMissing; + return original.apply(this, [request]); + }; +})(); + +(function() { + var original = RelevancedClient.prototype.multiDeleteCentroids; + RelevancedClient.prototype.multiDeleteCentroids = function(ids, ignoreMissing) { + if (!_.isBoolean(ignoreMissing)) { + ignoreMissing = false; + } + var request = new genTypes.MultiDeleteCentroidsRequest; + request.ids = ids; + request.ignoreMissing = ignoreMissing; + return original.apply(this, [request]); + }; +})(); + +(function() { + var original = RelevancedClient.prototype.addDocumentsToCentroid; + RelevancedClient.prototype.addDocumentsToCentroid = function(centroidId, documentIds, ignoreAlreadyInCentroid) { + if (!_.isBoolean(ignoreAlreadyInCentroid)) { + ignoreAlreadyInCentroid = false; + } + var request = new genTypes.AddDocumentsToCentroidRequest; + request.centroidId = centroidId; + request.documentIds = documentIds; + request.ignoreAlreadyInCentroid = ignoreAlreadyInCentroid; + return original.apply(this, [request]); + }; +})(); + +(function() { + var original = RelevancedClient.prototype.removeDocumentsFromCentroid; + RelevancedClient.prototype.removeDocumentsFromCentroid = function(centroidId, documentIds, ignoreNotInCentroid) { + if (!_.isBoolean(ignoreNotInCentroid)) { + ignoreNotInCentroid = false; + } + var request = new genTypes.RemoveDocumentsFromCentroidRequest; + request.centroidId = centroidId; + request.documentIds = documentIds; + request.ignoreNotInCentroid = ignoreNotInCentroid; + return original.apply(this, [request]); + }; +})(); + +(function() { + var original = RelevancedClient.prototype.deleteDocument; + RelevancedClient.prototype.deleteDocument = function(documentId, ignoreMissing) { + if (!_.isBoolean(ignoreMissing)) { + ignoreMissing = false; + } + var request = new genTypes.DeleteDocumentRequest; + request.id = documentId; + request.ignoreMissing = ignoreMissing; + return original.apply(this, [request]); + }; +})(); + +(function() { + var original = RelevancedClient.prototype.multiDeleteDocuments; + RelevancedClient.prototype.multiDeleteDocuments = function(documentIds, ignoreMissing) { + if (!_.isBoolean(ignoreMissing)) { + ignoreMissing = false; + } + var request = new genTypes.MultiDeleteDocumentsRequest; + request.ids = documentIds; + request.ignoreMissing = ignoreMissing; + return original.apply(this, [request]); + }; +})(); + +(function() { + var original = RelevancedClient.prototype.joinCentroid; + RelevancedClient.prototype.joinCentroid = function(centroidId, ignoreMissing) { + if (!_.isBoolean(ignoreMissing)) { + ignoreMissing = false; + } + var request = new genTypes.JoinCentroidRequest; + request.id = centroidId; + request.ignoreMissing = ignoreMissing; + return original.apply(this, [request]); + }; +})(); + +(function() { + var original = RelevancedClient.prototype.multiJoinCentroids; + RelevancedClient.prototype.multiJoinCentroids = function(centroidIds, ignoreMissing) { + if (!_.isBoolean(ignoreMissing)) { + ignoreMissing = false; + } + var request = new genTypes.MultiJoinCentroidsRequest; + request.ids = centroidIds; + request.ignoreMissing = ignoreMissing; + return original.apply(this, [request]); + }; +})(); + module.exports = { RelevancedClient: RelevancedClient, diff --git a/clients/nodejs/client/package.json b/clients/nodejs/client/package.json index 82431c2..9a53201 100644 --- a/clients/nodejs/client/package.json +++ b/clients/nodejs/client/package.json @@ -17,5 +17,9 @@ "bluebird": "^2.10.1", "lodash": "^3.10.1", "thrift": "^0.9.2" + }, + "devDependencies": { + "chai": "^3.4.1", + "mocha": "^2.3.4" } } diff --git a/clients/nodejs/client/test/test.js b/clients/nodejs/client/test/test.js new file mode 100644 index 0000000..88f9ee1 --- /dev/null +++ b/clients/nodejs/client/test/test.js @@ -0,0 +1,300 @@ +var RelevancedClient = require('../').RelevancedClient; +var _ = require('lodash'); +var log = _.bind(console.log, console); +var assert = require('chai').assert; +var Promise = require('bluebird'); + +var eraseAllCentroids = function(client) { + return client + .listAllCentroids() + .then(function(response) { + var deletions = _.map(response.centroids, function(id) { + var ignoreMissing = true; + return client.deleteCentroid(id, ignoreMissing); + }); + return Promise.all(deletions); + }); +} + +var eraseAllDocuments = function(client) { + return client + .listAllDocuments() + .then(function(response) { + var deletions = _.map(response.documents, function(id) { + var ignoreMissing = true; + return client.deleteDocument(id, ignoreMissing); + }); + return Promise.all(deletions); + }); +} + +var eraseEverything = function(client) { + return Promise.all([ + eraseAllCentroids(client), eraseAllDocuments(client) + ]); +} + +describe('RelevancedClient', function() { + before(function(done) { + var self = this; + RelevancedClient + .connect('localhost', 8097) + .then(function(client) { + self.client = client; + done(); + }).catch(done); + }); + beforeEach(function(done) { + eraseEverything(this.client) + .then(function() { + done(); + }).catch(done); + }); + afterEach(function(done) { + eraseEverything(this.client) + .then(function() { + done(); + }).catch(done); + }) + describe('centroid CRUD', function() { + it('creation and deletion work', function(done) { + var client = this.client; + Promise.all([ + client.createCentroid('c-1'), + client.createCentroid('c-2'), + client.multiCreateCentroids(['c-3', 'c-4']) + ]).then(function() { + return client.listAllCentroids(); + }) + .then(function(listCentroids) { + assert.deepEqual( + ['c-1', 'c-2', 'c-3', 'c-4'], + listCentroids.centroids + ); + return Promise.all([ + client.deleteCentroid('c-1'), + client.multiDeleteCentroids([ + 'c-2', 'c-4' + ]) + ]); + }).then(function() { + return client.listAllCentroids(); + }).then(function(listCentroids) { + assert.deepEqual( + ['c-3'], listCentroids.centroids + ); + done(); + }).catch(done); + }); + }); + describe('document CRUD', function() { + it('creation and deletion work', function(done) { + var client = this.client; + var creations = _.map(_.range(0, 6), function(id) { + id = 'doc-id-' + id; + return client.createDocumentWithID(id, 'stub text'); + }); + Promise.all(creations) + .then(function() { + return client.listAllDocuments(); + }) + .then(function(res) { + assert.deepEqual( + ['doc-id-0', 'doc-id-1', 'doc-id-2', 'doc-id-3', 'doc-id-4', 'doc-id-5'], + res.documents + ); + return Promise.all([ + client.createDocumentWithID('other-doc-1', 'some text'), + client.createDocumentWithID('other-doc-2', 'some text'), + client.deleteDocument('doc-id-2'), + client.multiDeleteDocuments(['doc-id-3', 'doc-id-5']) + ]); + }) + .then(function() { + return client.listAllDocuments(); + }) + .then(function(res) { + assert.deepEqual( + ['doc-id-0', 'doc-id-1', 'doc-id-4', 'other-doc-1', 'other-doc-2'], + res.documents + ); + done(); + }).catch(done); + }); + }); + describe('centroid document operations', function() { + beforeEach(function(done) { + var client = this.client; + var docIds = ['doc-1', 'doc-2', 'doc-3', 'doc-4', 'doc-5', 'doc-6', 'doc-7']; + var centroidIds = ['cx-1', 'cx-2', 'cx-3']; + var docCreations = Promise.all(_.map(docIds, function(id) { + return client.createDocumentWithID( + id, 'some text' + ); + })); + Promise.all([ + client.multiCreateCentroids(centroidIds), + docCreations + ]).then(function() { + done(); + }).catch(done); + }); + it('works', function(done) { + var client = this.client; + client + .listAllDocumentsForCentroid('cx-1') + .then(function(response) { + assert.deepEqual([], response.documents); + return Promise.all([ + client.addDocumentsToCentroid( + 'cx-1', ['doc-2', 'doc-4', 'doc-5'] + ), + client.addDocumentsToCentroid( + 'cx-3', ['doc-5', 'doc-6', 'doc-7'] + ) + ]); + }) + .then(function() { + return Promise.all([ + client.listAllDocumentsForCentroid('cx-1'), + client.listAllDocumentsForCentroid('cx-2'), + client.listAllDocumentsForCentroid('cx-3') + ]); + }) + .then(function(lists) { + assert.deepEqual( + ['doc-2', 'doc-4', 'doc-5'], lists[0].documents + ); + assert.deepEqual([], lists[1].documents); + assert.deepEqual( + ['doc-5', 'doc-6', 'doc-7'], lists[2].documents + ); + return Promise.all([ + client.removeDocumentsFromCentroid( + 'cx-1', ['doc-4', 'doc-5'] + ), + client.addDocumentsToCentroid( + 'cx-2', ['doc-1', 'doc-3'] + ), + client.removeDocumentsFromCentroid( + 'cx-3', ['doc-6'] + ) + ]); + }).then(function() { + return Promise.all([ + client.listAllDocumentsForCentroid('cx-1'), + client.listAllDocumentsForCentroid('cx-2'), + client.listAllDocumentsForCentroid('cx-3') + ]); + }).then(function(lists) { + assert.deepEqual( + ['doc-2'], lists[0].documents + ); + assert.deepEqual( + ['doc-1', 'doc-3'], lists[1].documents + ); + assert.deepEqual( + ['doc-5', 'doc-7'], lists[2].documents + ); + done(); + }).catch(function(err) { + console.log('err', err); + done(err); + }); + }); + }); + describe('similarity scoring', function() { + var data; + beforeEach(function(done) { + data = { + 'monkeys': { + 'doc-monkeys-1': 'monkey gorilla ape banana cat', + 'doc-monkeys-2': 'gorilla gorilla gorilla fish banana', + 'doc-monkeys-3': 'ape lemur gorilla monkey grill' + }, + 'cars': { + 'doc-cars-1': 'engine motor wheel gear clutch', + 'doc-cars-2': 'motor motor nascar left wheel', + 'doc-cars-3': 'mudflap column engine engine engine' + } + }; + var client = this.client; + var creations = []; + _.each(data, function(docSet, centroidId) { + creations.push(client.createCentroid(centroidId)); + _.each(docSet, function(docText, docId) { + creations.push(client.createDocumentWithID(docId, docText)); + }); + }); + Promise.all(creations) + .then(function() { + return Promise.all(_.map(data, function(docSet, centroid) { + return client.addDocumentsToCentroid( + centroid, _.keys(docSet) + ); + })); + }).then(function() { + return Promise.all([ + client.joinCentroid('monkeys'), + client.joinCentroid('cars') + ]); + }).then(function() { + done(); + }).catch(done); + }); + it('joinCentroid works', function(done) { + var client = this.client; + client.createDocumentWithID( + 'another-monkey-doc', 'monkey monkey ape banana banana' + ).then(function() { + return client.addDocumentsToCentroid( + 'monkeys', ['another-monkey-doc'] + ); + }).then(function() { + return client.joinCentroid('monkeys'); + }).then(function(res) { + assert.equal('monkeys', res.id); + done(); + }).catch(done); + }); + it('multiJoinCentroids works', function(done) { + var client = this.client; + Promise.all([ + client.createDocumentWithID('another-monkey-doc', 'monkey banana'), + client.createDocumentWithID('another-car-doc', 'engine engine') + ]).then(function() { + return Promise.all([ + client.addDocumentsToCentroid('monkeys', ['another-monkey-doc']), + client.addDocumentsToCentroid('cars', ['another-car-doc']) + ]); + }).then(function() { + return client.multiJoinCentroids(['monkeys', 'cars']); + }).then(function(res) { + assert.deepEqual( + ['monkeys', 'cars'], res.ids + ); + done(); + }).catch(done); + }); + it('getTextSimilarity works', function(done) { + var client = this.client; + var testText = 'ape monkey monkey gorilla engine'; + Promise.all([ + client.getTextSimilarity('monkeys', testText), + client.getTextSimilarity('cars', testText) + ]).then(function(results) { + assert.isNumber(results[0]); + assert.isNumber(results[1]); + assert.isAbove(results[0], results[1]); + assert.isAbove(results[0], 0); + assert.isAbove(results[1], 0); + assert.isBelow(results[0], 1.0001); + assert.isBelow(results[1], 1.0001); + done(); + }).catch(function(err) { + console.log(err); + done(err); + }); + }); + }); +}); \ No newline at end of file diff --git a/src/RelevancedProtocol.thrift b/src/RelevancedProtocol.thrift index 92631c2..a0134cb 100644 --- a/src/RelevancedProtocol.thrift +++ b/src/RelevancedProtocol.thrift @@ -68,11 +68,6 @@ struct GetCentroidMetadataResponse { 1: required CentroidMetadataDTO metadata; } -struct CentroidSimilarityMatrix { - 1: required list centroidIds; - 2: required list> scores; -} - struct MultiSimilarityResponse { 1: required map scores; } @@ -237,7 +232,6 @@ service Relevanced { double getTextSimilarity(1: string centroidId, 2: string text, 3: Language lang) throws (1: ECentroidDoesNotExist err), MultiSimilarityResponse multiGetTextSimilarity(1: list centroidIds, 2: string text, 3: Language lang) throws (1: ECentroidDoesNotExist err), double getCentroidSimilarity(1: string centroid1Id, 2: string centroid2Id) throws (1: ECentroidDoesNotExist err), - CentroidSimilarityMatrix getCentroidSimilarityMatrix(1: list centroidIds) throws (1: ECentroidDoesNotExist err), CreateDocumentResponse createDocument(1: string text, 2: Language language), CreateDocumentResponse createDocumentWithID(1: string id, 2: string text, 3: Language language) throws (1: EDocumentAlreadyExists err), DeleteDocumentResponse deleteDocument(1: DeleteDocumentRequest request) throws (1: EDocumentDoesNotExist err), diff --git a/src/server/RelevanceServer.cpp b/src/server/RelevanceServer.cpp index 0e8c023..348328d 100644 --- a/src/server/RelevanceServer.cpp +++ b/src/server/RelevanceServer.cpp @@ -86,59 +86,6 @@ Future> RelevanceServer::getDocumentSimilarity( }); } -Future>> RelevanceServer::getCentroidSimilarityMatrix( - unique_ptr> centroidIds) { - auto cIds = std::make_shared>(*centroidIds); - auto scoreMap = std::make_shared, size_t>>; - vector>> tasks; - for (size_t i = 0; i < centroidIds->size(); i++) { - for (size_t j = 0; j < centroidIds->size(); j++) { - if (i == j) { - continue; - } - auto asPair = make_pair(i, j); - auto existing = scoreMap->find(asPair); - if (existing == scoreMap->end()) { - auto index = tasks.size(); - tasks.push_back(getCentroidSimilarity( - folly::make_unique(centroidIds->at(i)), - folly::make_unique(centroidIds->at(j)) - )); - scoreMap->insert(make_pair(asPair, index)); - auto altPair = make_pair(j, i); - scoreMap->insert(make_pair(altPair, index)); - } - } - } - collect(tasks) - .then([cIds, scoreMap](vector> results) { - auto response = folly::make_unique(); - response->scores.reserve(cIds->size()); - for (size_t i = 0; i < cIds->size(); i++) { - vector row; - row.reserve(cIds->size()); - for (size_t j = 0; j < cIds->size(); j++) { - if (i == j) { - row.push_back(0.0); - } else { - auto idx = scoreMap->find(make_pair(i, j))->second; - auto score = results.at(idx); - if (score.hasException()) { - return Try>( - score.exception() - ); - } - row.push_back(score.value()); - } - } - response->scores.push_back(row); - } - return Try>( - std::move(response) - ); - }); -} - Future> RelevanceServer::getCentroidSimilarity( unique_ptr centroid1Id, unique_ptr centroid2Id) { return scoreWorker_->getCentroidSimilarity( diff --git a/src/server/RelevanceServer.h b/src/server/RelevanceServer.h index c07868b..2f8ceb3 100644 --- a/src/server/RelevanceServer.h +++ b/src/server/RelevanceServer.h @@ -271,11 +271,6 @@ class RelevanceServer : public RelevanceServerIf { std::unique_ptr centroid2Id ) override; - folly::Future>> - getCentroidSimilarityMatrix( - std::unique_ptr> centroidIds - ); - folly::Future>> createDocument( std::unique_ptr text,