From 3840068cad8915fd0c11913e6b48b71c4aa4b233 Mon Sep 17 00:00:00 2001 From: Cristian Balas Date: Tue, 11 May 2021 20:23:42 +0300 Subject: [PATCH] diff endpoints + added version in ServerInfo (#235) --- .circleci/build.sh | 2 +- packages/frontend/nginx/nginx.conf | 2 +- packages/server/Dockerfile | 3 + .../modules/core/graph/schemas/server.graphql | 1 + packages/server/modules/core/index.js | 4 + .../server/modules/core/rest/authUtils.js | 55 +++++++++++ .../server/modules/core/rest/diffDownload.js | 96 +++++++++++++++++++ .../server/modules/core/rest/diffUpload.js | 31 ++++++ packages/server/modules/core/rest/download.js | 44 +++------ packages/server/modules/core/rest/upload.js | 24 ++--- .../server/modules/core/services/generic.js | 6 +- .../server/modules/core/services/objects.js | 25 +++++ .../server/modules/core/tests/graph.spec.js | 1 + .../server/modules/core/tests/rest.spec.js | 79 ++++++++++++++- .../template-nginx-site.conf | 2 +- 15 files changed, 320 insertions(+), 55 deletions(-) create mode 100644 packages/server/modules/core/rest/authUtils.js create mode 100644 packages/server/modules/core/rest/diffDownload.js create mode 100644 packages/server/modules/core/rest/diffUpload.js diff --git a/.circleci/build.sh b/.circleci/build.sh index 39414696a0..9ebe82eeb2 100755 --- a/.circleci/build.sh +++ b/.circleci/build.sh @@ -10,7 +10,7 @@ if [[ "$CIRCLE_TAG" =~ ^v.* ]]; then IMAGE_VERSION_TAG=$CIRCLE_TAG fi -docker build -t $DOCKER_IMAGE_TAG:latest . -f packages/$SPECKLE_SERVER_PACKAGE/Dockerfile +docker build --build-arg SPECKLE_SERVER_VERSION=$IMAGE_VERSION_TAG -t $DOCKER_IMAGE_TAG:latest . -f packages/$SPECKLE_SERVER_PACKAGE/Dockerfile docker tag $DOCKER_IMAGE_TAG:latest $DOCKER_IMAGE_TAG:$IMAGE_VERSION_TAG echo "$DOCKER_REG_PASS" | docker login -u "$DOCKER_REG_USER" --password-stdin $DOCKER_REG_URL diff --git a/packages/frontend/nginx/nginx.conf b/packages/frontend/nginx/nginx.conf index 11acdac79f..ecf5c03736 100644 --- a/packages/frontend/nginx/nginx.conf +++ b/packages/frontend/nginx/nginx.conf @@ -7,7 +7,7 @@ server { try_files $uri $uri/ /app.html; } - location ~* ^/(graphql|explorer|(auth/.*)|(objects/.*)|(preview/.*)) { + location ~* ^/(graphql|explorer|(auth/.*)|(objects/.*)|(preview/.*)|(api/.*)) { resolver 127.0.0.11 valid=30s; set $upstream_speckle_server speckle-server; client_max_body_size 100m; diff --git a/packages/server/Dockerfile b/packages/server/Dockerfile index dec114a28b..6638def468 100644 --- a/packages/server/Dockerfile +++ b/packages/server/Dockerfile @@ -9,6 +9,9 @@ RUN chmod +x /wait ARG NODE_ENV=production ENV NODE_ENV=${NODE_ENV} + +ARG SPECKLE_SERVER_VERSION=custom +ENV SPECKLE_SERVER_VERSION=${SPECKLE_SERVER_VERSION} WORKDIR /app COPY packages/server/package*.json ./ diff --git a/packages/server/modules/core/graph/schemas/server.graphql b/packages/server/modules/core/graph/schemas/server.graphql index 00dcc8f74a..71f73a7a9f 100644 --- a/packages/server/modules/core/graph/schemas/server.graphql +++ b/packages/server/modules/core/graph/schemas/server.graphql @@ -15,6 +15,7 @@ type ServerInfo { roles: [Role]! scopes: [Scope]! inviteOnly: Boolean + version: String } """ diff --git a/packages/server/modules/core/index.js b/packages/server/modules/core/index.js index 78babab80f..523738c54d 100644 --- a/packages/server/modules/core/index.js +++ b/packages/server/modules/core/index.js @@ -11,6 +11,10 @@ exports.init = async ( app, options ) => { require( './rest/upload' )( app ) require( './rest/download' )( app ) + // Initialises the two diff-based upload/download endpoints + require( './rest/diffUpload' )( app ) + require( './rest/diffDownload' )( app ) + // Register core-based scoeps const scopes = require( './scopes.js' ) for ( let scope of scopes ) { diff --git a/packages/server/modules/core/rest/authUtils.js b/packages/server/modules/core/rest/authUtils.js new file mode 100644 index 0000000000..e1c9bc593c --- /dev/null +++ b/packages/server/modules/core/rest/authUtils.js @@ -0,0 +1,55 @@ +'use strict' +const appRoot = require( 'app-root-path' ) +const { contextMiddleware, validateScopes, authorizeResolver } = require( `${appRoot}/modules/shared` ) + +const { getStream } = require( '../services/streams' ) + +module.exports = { + async validatePermissionsReadStream( streamId, req ) { + const stream = await getStream( { streamId: streamId, userId: req.context.userId } ) + + if ( !stream ) { + return { result: false, status: 404 } + } + + if ( !stream.isPublic && req.context.auth === false ) { + return { result: false, status: 401 } + } + + if ( !stream.isPublic ) { + try { + await validateScopes( req.context.scopes, 'streams:read' ) + } catch ( err ) { + return { result: false, status: 401 } + } + + try { + await authorizeResolver( req.context.userId, streamId, 'stream:reviewer' ) + } catch ( err ) { + return { result: false, status: 401 } + } + } + + return { result: true, status: 200 } + }, + + async validatePermissionsWriteStream( streamId, req ) { + if ( !req.context || !req.context.auth ) { + return { result: false, status: 401 } + } + + try { + await validateScopes( req.context.scopes, 'streams:write' ) + } catch ( err ) { + return { result: false, status: 401 } + } + + try { + await authorizeResolver( req.context.userId, streamId, 'stream:contributor' ) + } catch ( err ) { + return { result: false, status: 401 } + } + + return { result: true, status: 200 } + } +} diff --git a/packages/server/modules/core/rest/diffDownload.js b/packages/server/modules/core/rest/diffDownload.js new file mode 100644 index 0000000000..821e2b9ebe --- /dev/null +++ b/packages/server/modules/core/rest/diffDownload.js @@ -0,0 +1,96 @@ +'use strict' +const zlib = require( 'zlib' ) +const Busboy = require( 'busboy' ) +const debug = require( 'debug' ) +const appRoot = require( 'app-root-path' ) +const cors = require( 'cors' ) + +const { matomoMiddleware } = require( `${appRoot}/logging/matomoHelper` ) +const { contextMiddleware, validateScopes, authorizeResolver } = require( `${appRoot}/modules/shared` ) +const { validatePermissionsReadStream } = require( './authUtils' ) + +const { getObjectsStream } = require( '../services/objects' ) + +module.exports = ( app ) => { + + app.options( '/api/getobjects/:streamId', cors() ) + app.post( '/api/getobjects/:streamId', cors(), contextMiddleware, matomoMiddleware, async ( req, res ) => { + let hasStreamAccess = await validatePermissionsReadStream( req.params.streamId, req ) + if ( !hasStreamAccess.result ) { + return res.status( hasStreamAccess.status ).end() + } + + let childrenList = JSON.parse( req.body.objects ) + + let simpleText = req.headers.accept === 'text/plain' + + let dbStream = await getObjectsStream( { streamId: req.params.streamId, objectIds: childrenList } ) + + let currentChunkSize = 0 + let maxChunkSize = 50000 + let chunk = simpleText ? '' : [ ] + let isFirst = true + + + res.writeHead( 200, { 'Content-Encoding': 'gzip', 'Content-Type': simpleText ? 'text/plain' : 'application/json' } ) + + const gzip = zlib.createGzip( ) + + if ( !simpleText ) gzip.write( '[' ) + + // helper func to flush the gzip buffer + const writeBuffer = ( addTrailingComma ) => { + // console.log( `writing buff ${currentChunkSize}` ) + if ( simpleText ) { + gzip.write( chunk ) + } else { + gzip.write( chunk.join( ',' ) ) + if ( addTrailingComma ) { + gzip.write( ',' ) + } + } + gzip.flush( ) + chunk = simpleText ? '' : [ ] + } + + let k = 0 + let requestDropped = false + dbStream.on( 'data', row => { + try { + let data = JSON.stringify( row.data ) + currentChunkSize += Buffer.byteLength( data, 'utf8' ) + if ( simpleText ) { + chunk += `${row.data.id}\t${data}\n` + } else { + chunk.push( data ) + } + if ( currentChunkSize >= maxChunkSize ) { + currentChunkSize = 0 + writeBuffer( true ) + } + k++ + } catch ( e ) { + requestDropped = true + debug( 'speckle:error' )( `'Failed to find object, or object is corrupted.' ${req.params.objectId}` ) + return + } + } ) + + dbStream.on( 'error', err => { + debug( 'speckle:error' )( `Error in streaming object children for ${req.params.objectId}: ${err}` ) + requestDropped = true + return + } ) + + dbStream.on( 'end', ( ) => { + if ( currentChunkSize !== 0 ) { + writeBuffer( false ) + if ( !simpleText ) gzip.write( ']' ) + } + gzip.end( ) + } ) + + // 🚬 + gzip.pipe( res ) + } ) +} diff --git a/packages/server/modules/core/rest/diffUpload.js b/packages/server/modules/core/rest/diffUpload.js new file mode 100644 index 0000000000..00b72e0fab --- /dev/null +++ b/packages/server/modules/core/rest/diffUpload.js @@ -0,0 +1,31 @@ +'use strict' +const zlib = require( 'zlib' ) +const Busboy = require( 'busboy' ) +const debug = require( 'debug' ) +const appRoot = require( 'app-root-path' ) + +const { matomoMiddleware } = require( `${appRoot}/logging/matomoHelper` ) +const { contextMiddleware } = require( `${appRoot}/modules/shared` ) +const { validatePermissionsWriteStream } = require( './authUtils' ) + +const { hasObjects } = require( '../services/objects' ) + +module.exports = ( app ) => { + app.post( '/api/diff/:streamId', contextMiddleware, matomoMiddleware, async ( req, res ) => { + let hasStreamAccess = await validatePermissionsWriteStream( req.params.streamId, req ) + if ( !hasStreamAccess.result ) { + return res.status( hasStreamAccess.status ).end() + } + + let objectList = JSON.parse( req.body.objects ) + + let response = await hasObjects( { streamId: req.params.streamId, objectIds: objectList } ) + // console.log(response) + res.writeHead( 200, { 'Content-Encoding': 'gzip', 'Content-Type': 'application/json' } ) + const gzip = zlib.createGzip( ) + gzip.write( JSON.stringify( response ) ) + gzip.flush( ) + gzip.end( ) + gzip.pipe( res ) + } ) +} diff --git a/packages/server/modules/core/rest/download.js b/packages/server/modules/core/rest/download.js index ca9432c2fb..4bc7737ff8 100644 --- a/packages/server/modules/core/rest/download.js +++ b/packages/server/modules/core/rest/download.js @@ -6,38 +6,19 @@ const appRoot = require( 'app-root-path' ) const cors = require( 'cors' ) const { matomoMiddleware } = require( `${appRoot}/logging/matomoHelper` ) -const { contextMiddleware, validateScopes, authorizeResolver } = require( `${appRoot}/modules/shared` ) +const { contextMiddleware } = require( `${appRoot}/modules/shared` ) +const { validatePermissionsReadStream } = require( './authUtils' ) + const { getObject, getObjectChildrenStream } = require( '../services/objects' ) -const { getStream } = require( '../services/streams' ) module.exports = ( app ) => { app.options( '/objects/:streamId/:objectId', cors() ) app.get( '/objects/:streamId/:objectId', cors(), contextMiddleware, matomoMiddleware, async ( req, res ) => { - - const stream = await getStream( { streamId: req.params.streamId, userId: req.context.userId } ) - - if ( !stream ) { - return res.status( 404 ).end() - } - - if ( !stream.isPublic && req.context.auth === false ) { - return res.status( 401 ).end( ) - } - - if ( !stream.isPublic ) { - try { - await validateScopes( req.context.scopes, 'streams:read' ) - } catch ( err ) { - return res.status( 401 ).end( ) - } - - try { - await authorizeResolver( req.context.userId, req.params.streamId, 'stream:reviewer' ) - } catch ( err ) { - return res.status( 401 ).end( ) - } + let hasStreamAccess = await validatePermissionsReadStream( req.params.streamId, req ) + if ( !hasStreamAccess.result ) { + return res.status( hasStreamAccess.status ).end() } // Populate first object (the "commit") @@ -129,12 +110,15 @@ module.exports = ( app ) => { gzip.pipe( res ) } ) - // TODO: is this needed/used? - app.get( '/objects/:streamId/:objectId/single', async ( req, res ) => { - // TODO: authN & authZ checks + app.options( '/objects/:streamId/:objectId/single', cors() ) + app.get( '/objects/:streamId/:objectId/single', cors(), contextMiddleware, matomoMiddleware, async ( req, res ) => { + let hasStreamAccess = await validatePermissionsReadStream( req.params.streamId, req ) + if ( !hasStreamAccess.result ) { + return res.status( hasStreamAccess.status ).end() + } - let obj = await getObject( req.params.streamId, req.params.objectId ) + let obj = await getObject( { streamId: req.params.streamId, objectId: req.params.objectId } ) - res.send( obj ) + res.send( obj.data ) } ) } diff --git a/packages/server/modules/core/rest/upload.js b/packages/server/modules/core/rest/upload.js index 512f9c99c4..994658b1af 100644 --- a/packages/server/modules/core/rest/upload.js +++ b/packages/server/modules/core/rest/upload.js @@ -5,29 +5,17 @@ const debug = require( 'debug' ) const appRoot = require( 'app-root-path' ) const { matomoMiddleware } = require( `${appRoot}/logging/matomoHelper` ) -const { contextMiddleware, validateScopes, authorizeResolver } = require( `${appRoot}/modules/shared` ) +const { contextMiddleware } = require( `${appRoot}/modules/shared` ) +const { validatePermissionsWriteStream } = require( './authUtils' ) const { createObjects, createObjectsBatched } = require( '../services/objects' ) + module.exports = ( app ) => { app.post( '/objects/:streamId', contextMiddleware, matomoMiddleware, async ( req, res ) => { - - debug( 'speckle:upload-endpoint' )( 'booom upload endpoint' ) - - if ( !req.context || !req.context.auth ) { - return res.status( 401 ).end( ) - } - - try { - await validateScopes( req.context.scopes, 'streams:write' ) - } catch ( err ) { - return res.status( 401 ).end( ) - } - - try { - await authorizeResolver( req.context.userId, req.params.streamId, 'stream:contributor' ) - } catch ( err ) { - return res.status( 401 ).end( ) + let hasStreamAccess = await validatePermissionsWriteStream( req.params.streamId, req ) + if ( !hasStreamAccess.result ) { + return res.status( hasStreamAccess.status ).end() } debug( 'speckle:upload-endpoint' )( 'Upload started' ) diff --git a/packages/server/modules/core/services/generic.js b/packages/server/modules/core/services/generic.js index 14f3e1ccbd..df6971b5bf 100644 --- a/packages/server/modules/core/services/generic.js +++ b/packages/server/modules/core/services/generic.js @@ -9,9 +9,9 @@ const Info = ( ) => knex( 'server_config' ) module.exports = { async getServerInfo( ) { - - return await Info( ).select( '*' ).first( ) - + let serverInfo = await Info( ).select( '*' ).first( ) + serverInfo.version = process.env.SPECKLE_SERVER_VERSION || 'dev' + return serverInfo }, async getAllScopes( ) { diff --git a/packages/server/modules/core/services/objects.js b/packages/server/modules/core/services/objects.js index 15e32f25bb..9c7bcbdcef 100644 --- a/packages/server/modules/core/services/objects.js +++ b/packages/server/modules/core/services/objects.js @@ -436,6 +436,31 @@ module.exports = { return res }, + async getObjectsStream( { streamId, objectIds } ) { + let res = Objects( ) + .whereIn( 'id', objectIds ) + .andWhere( 'streamId', streamId ) + .orderBy( 'id' ) + .select( 'id', 'speckleType', 'totalChildrenCount', 'totalChildrenCountByDepth', 'createdAt', 'data' ) + return res.stream( ) + }, + + async hasObjects( { streamId, objectIds } ) { + let dbRes = await Objects( ) + .whereIn( 'id', objectIds ) + .andWhere( 'streamId', streamId ) + .select( 'id' ) + + let res = {} + for ( let i in objectIds ) { + res[ objectIds[ i ] ] = false + } + for ( let i in dbRes ) { + res [ dbRes[ i ].id ] = true + } + return res + }, + // NOTE: Derive Object async updateObject( ) { throw new Error( 'not implemeneted' ) diff --git a/packages/server/modules/core/tests/graph.spec.js b/packages/server/modules/core/tests/graph.spec.js index ee1a91d32f..87e9078011 100644 --- a/packages/server/modules/core/tests/graph.spec.js +++ b/packages/server/modules/core/tests/graph.spec.js @@ -966,6 +966,7 @@ describe( 'GraphQL API Core @core-api', ( ) => { adminContact termsOfService description + version roles{ name description diff --git a/packages/server/modules/core/tests/rest.spec.js b/packages/server/modules/core/tests/rest.spec.js index 9fe476e6de..1d6dd7d8ee 100644 --- a/packages/server/modules/core/tests/rest.spec.js +++ b/packages/server/modules/core/tests/rest.spec.js @@ -126,9 +126,9 @@ describe( 'Upload/Download Routes @api-rest', ( ) => { let parentId let numObjs = 5000 + let objBatches = [ createManyObjects( numObjs ), createManyObjects( numObjs ), createManyObjects( numObjs ) ] it( 'Should properly upload a bunch of objects', async ( ) => { - let objBatches = [ createManyObjects( numObjs ), createManyObjects( numObjs ), createManyObjects( numObjs ) ] parentId = objBatches[ 0 ][ 0 ].id let res = @@ -208,6 +208,83 @@ describe( 'Upload/Download Routes @api-rest', ( ) => { } ) + it( 'Should properly download a list of objects', ( done ) => { + let objectIds = [] + for ( let i = 0; i < objBatches[0].length; i++ ) { + objectIds.push( objBatches[0][i].id ) + } + let res = request( expressApp ) + .post( `/api/getobjects/${testStream.id}` ) + .set( 'Authorization', userA.token ) + .set( 'Accept', 'text/plain' ) + .send( { objects: JSON.stringify( objectIds ) } ) + .buffer( ) + .parse( ( res, cb ) => { + res.data = '' + res.on( 'data', chunk => { + res.data += chunk.toString( ) + } ) + res.on( 'end', ( ) => { + cb( null, res.data ) + } ) + } ) + .end( ( err, res ) => { + if ( err ) done( err ) + try { + let o = res.body.split( '\n' ).filter( l => l !== '' ) + expect( o.length ).to.equal( objectIds.length ) + expect( res ).to.be.text + done( ) + } catch ( err ) { + done( err ) + } + } ) + } ) + + it( 'Should properly check if the server has a list of objects', ( done ) => { + let objectIds = [] + for ( let i = 0; i < objBatches[0].length; i++ ) { + objectIds.push( objBatches[0][i].id ) + } + let fakeIds = [] + for ( let i = 0; i < 100; i++ ) { + let fakeId = crypto.createHash( 'md5' ).update( 'fakefake' + i ).digest( 'hex' ) + fakeIds.push( fakeId ) + objectIds.push( fakeId ) + } + + let res = request( expressApp ) + .post( `/api/diff/${testStream.id}` ) + .set( 'Authorization', userA.token ) + .send( { objects: JSON.stringify( objectIds ) } ) + .buffer( ) + .parse( ( res, cb ) => { + res.data = '' + res.on( 'data', chunk => { + res.data += chunk.toString( ) + } ) + res.on( 'end', ( ) => { + cb( null, res.data ) + } ) + } ) + .end( ( err, res ) => { + if ( err ) done( err ) + try { + let o = JSON.parse( res.body ) + expect( Object.keys(o).length ).to.equal( objectIds.length ) + for ( let i = 0; i < objBatches[0].length; i++ ) { + assert( o[objBatches[0][i].id] === true, 'Server is missing an object' ) + } + for ( let i = 0; i < fakeIds.length; i++ ) { + assert( o[fakeIds[i]] === false, 'Server wrongly reports it has an extra object' ) + } + done( ) + } catch ( err ) { + done( err ) + } + } ) + } ) + } ) function createManyObjects( amount, noise ) { diff --git a/utils/1click_image_scripts/template-nginx-site.conf b/utils/1click_image_scripts/template-nginx-site.conf index d46fc94d77..d039c6f11b 100644 --- a/utils/1click_image_scripts/template-nginx-site.conf +++ b/utils/1click_image_scripts/template-nginx-site.conf @@ -16,7 +16,7 @@ server { proxy_set_header Connection "upgrade"; } - location ~* ^/(graphql|explorer|(auth/.*)|(objects/.*)|(preview/.*)) { + location ~* ^/(graphql|explorer|(auth/.*)|(objects/.*)|(preview/.*)|(api/.*)) { client_max_body_size 100m; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr;