From c0aa11a6406fb9f3ca4e04a75c05a5cfad75d45c Mon Sep 17 00:00:00 2001 From: Majid Date: Mon, 8 Jul 2024 20:35:14 -0400 Subject: [PATCH] feat(): added-cache-id Signed-off-by: Majid --- package-lock.json | 9 ++++++++- package.json | 3 ++- src/serve_data.js | 8 ++++++-- src/serve_rendered.js | 2 ++ src/server.js | 6 +++++- src/utils.js | 18 +++++++++++++++++- 6 files changed, 40 insertions(+), 6 deletions(-) diff --git a/package-lock.json b/package-lock.json index 70c47c7bc..e4f8758de 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25,6 +25,7 @@ "color": "4.2.3", "commander": "12.1.0", "cors": "2.8.5", + "crypto": "^1.0.1", "express": "4.19.2", "handlebars": "4.7.8", "http-shutdown": "1.2.2", @@ -33,7 +34,7 @@ "pmtiles": "3.0.6", "proj4": "2.11.0", "sanitize-filename": "1.6.3", - "sharp": "0.33.4", + "sharp": "^0.33.4", "tileserver-gl-styles": "2.0.0" }, "bin": { @@ -2810,6 +2811,12 @@ "node": ">= 8" } }, + "node_modules/crypto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/crypto/-/crypto-1.0.1.tgz", + "integrity": "sha512-VxBKmeNcqQdiUQUW2Tzq0t377b54N2bMtXO/qiLa+6eRRmmC4qT3D4OnTGoT/U6O9aklQ/jTwbOtRMTTY8G0Ig==", + "deprecated": "This package is no longer supported. It's now a built-in Node module. If you've depended on crypto, you should switch to the one that's built-in." + }, "node_modules/d3-queue": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/d3-queue/-/d3-queue-3.0.7.tgz", diff --git a/package.json b/package.json index 862b95cd7..be340c9a4 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "color": "4.2.3", "commander": "12.1.0", "cors": "2.8.5", + "crypto": "^1.0.1", "express": "4.19.2", "handlebars": "4.7.8", "http-shutdown": "1.2.2", @@ -42,7 +43,7 @@ "pmtiles": "3.0.6", "proj4": "2.11.0", "sanitize-filename": "1.6.3", - "sharp": "0.33.4", + "sharp": "^0.33.4", "tileserver-gl-styles": "2.0.0" }, "devDependencies": { diff --git a/src/serve_data.js b/src/serve_data.js index 7f1284db1..6fc6bd84c 100644 --- a/src/serve_data.js +++ b/src/serve_data.js @@ -10,7 +10,7 @@ import MBTiles from '@mapbox/mbtiles'; import Pbf from 'pbf'; import { VectorTile } from '@mapbox/vector-tile'; -import { getTileUrls, isValidHttpUrl, fixTileJSONCenter } from './utils.js'; +import { getTileUrls, isValidHttpUrl, fixTileJSONCenter, getFileHash } from './utils.js'; import { openPMtiles, getPMtilesInfo, @@ -179,6 +179,7 @@ export const serve_data = { { pbf: options.pbfAlias, }, + info.cacheId ); return res.send(info); }); @@ -230,10 +231,11 @@ export const serve_data = { Object.assign(tileJSON, metadata); tileJSON['tilejson'] = '2.0.0'; + tileJSON['cacheId']= btoa(tileJSON['mtime']); delete tileJSON['filesize']; delete tileJSON['mtime']; delete tileJSON['scheme']; - + Object.assign(tileJSON, params.tilejson || {}); fixTileJSONCenter(tileJSON); @@ -242,6 +244,8 @@ export const serve_data = { } } else if (inputType === 'mbtiles') { sourceType = 'mbtiles'; + tileJSON['cacheId']= await getFileHash(inputFile) + const sourceInfoPromise = new Promise((resolve, reject) => { source = new MBTiles(inputFile + '?mode=ro', (err) => { if (err) { diff --git a/src/serve_rendered.js b/src/serve_rendered.js index c6953e7ff..811a93b73 100644 --- a/src/serve_rendered.js +++ b/src/serve_rendered.js @@ -847,6 +847,8 @@ export const serve_rendered = { tileSize, info.format, item.publicUrl, + null, + info.cacheId ); return res.send(info); }); diff --git a/src/server.js b/src/server.js index 55e3f2e23..13356cf62 100644 --- a/src/server.js +++ b/src/server.js @@ -373,6 +373,7 @@ function start(opts) { { pbf: options.pbfAlias, }, + info.cacheId ); arr.push(info); } @@ -479,6 +480,8 @@ function start(opts) { tileSize, style.serving_rendered.tileJSON.format, opts.publicUrl, + null, + style.serving_rendered.tileJSON.cacheId )[0]; } @@ -517,6 +520,7 @@ function start(opts) { { pbf: options.pbfAlias, }, + tileJSON.cacheId )[0]; if (data.filesize) { @@ -598,7 +602,7 @@ function start(opts) { serveTemplate('/data/:id/$', 'data', (req) => { const { id } = req.params; const data = serving.data[id]; - + debugger if (!data) { return null; } diff --git a/src/utils.js b/src/utils.js index 550a0da3e..92fce71ed 100644 --- a/src/utils.js +++ b/src/utils.js @@ -5,7 +5,7 @@ import fsPromises from 'fs/promises'; import fs, { existsSync } from 'node:fs'; import clone from 'clone'; import { combine } from '@jsse/pbfont'; - +import * as crypto from 'crypto'; /** * Restrict user input to an allowed set of options. * @param opts @@ -72,6 +72,7 @@ export const getTileUrls = ( format, publicUrl, aliases, + cacheId ) => { const urlObject = getUrlObject(req); if (domains) { @@ -107,6 +108,11 @@ export const getTileUrls = ( if (req.query.style) { queryParams.push(`style=${encodeURIComponent(req.query.style)}`); } + + if(cacheId){ + queryParams.push(`cache-id=${encodeURIComponent(cacheId)}`); + } + const query = queryParams.length > 0 ? `?${queryParams.join('&')}` : ''; if (aliases && aliases[format]) { @@ -130,6 +136,7 @@ export const getTileUrls = ( uris.push(`${publicUrl}${path}/${tileParams}.${format}${query}`); } + return uris; }; @@ -245,3 +252,12 @@ export const isValidHttpUrl = (string) => { return url.protocol === 'http:' || url.protocol === 'https:'; }; + + +export const getFileHash = path => new Promise((resolve, reject) => { + const hash = crypto.createHash('sha1'); + const rs = fs.createReadStream(path); + rs.on('error', reject); + rs.on('data', chunk => hash.update(chunk)); + rs.on('end', () => resolve(hash.digest('base64'))); + }) \ No newline at end of file