sync-assets.js

var https = require('https');
var fs = require('fs');
var crypto = require('crypto');
var path = require('node:path');
const LOG_TAG = '[sync-assets]';
let verbose = false;
let MEMPOOL_CDN = false;
let DRY_RUN = false;

if (parseInt(process.env.SKIP_SYNC) === 1) {
  console.log(`${LOG_TAG} SKIP_SYNC is set, not checking any assets`);
  process.exit(0);
}

if (parseInt(process.env.VERBOSE) === 1) {
  console.log(`${LOG_TAG} VERBOSE is set, logs will be more verbose`);
  verbose = true;
}

if (parseInt(process.env.MEMPOOL_CDN) === 1) {
  console.log(`${LOG_TAG} MEMPOOL_CDN is set, assets will be downloaded from mempool.space`);
  MEMPOOL_CDN = true;
}

if (parseInt(process.env.DRY_RUN) === 1) {
  console.log(`${LOG_TAG} DRY_RUN is set, not downloading any assets`);
  DRY_RUN = true;
}

const githubSecret = process.env.GITHUB_TOKEN;

const CONFIG_FILE_NAME = 'mempool-frontend-config.json';
let configContent = {};

var PATH;
if (process.argv[2]) {
  PATH = process.argv[2];
  PATH += PATH.endsWith("/") ? "" : "/"
  PATH = path.resolve(path.normalize(PATH));
  console.log(`[sync-assets] using PATH ${PATH}`);
  if (!fs.existsSync(PATH)){
    console.log(`${LOG_TAG} ${PATH} does not exist, creating`);
    fs.mkdirSync(PATH, { recursive: true });
  }
}

if (!PATH) {
  throw new Error('Resource path argument is not set');
}

try {
  const rawConfig = fs.readFileSync(CONFIG_FILE_NAME);
  configContent = JSON.parse(rawConfig);
  console.log(`${LOG_TAG} ${CONFIG_FILE_NAME} file found, using provided config`);
} catch (e) {
  if (e.code !== 'ENOENT') {
    throw new Error(e);
  } else {
    console.log(`${LOG_TAG} ${CONFIG_FILE_NAME} file not found, using default config`);
  }
}

function download(filename, url) {
  https.get(url, (response) => {
    if (response.statusCode < 200 || response.statusCode > 299) {
      throw new Error('HTTP Error ' + response.statusCode + ' while fetching \'' + filename + '\'');
    }
    response.pipe(fs.createWriteStream(filename));
  })
  .on('error', function(e) {
    throw new Error(e);
  })
  .on('finish', () => {
    if (verbose) {
      console.log(`${LOG_TAG} \tFinished downloading ${url} to ${filename}`);
    }
  });
}

function getLocalHash(filePath) {
  const size = fs.statSync(filePath);
  const buffer = fs.readFileSync(filePath);
  const bufferWithHeader = Buffer.concat([Buffer.from('blob '), Buffer.from(`${size.size}`), Buffer.from('\0'), buffer]);
  const hash = crypto.createHash('sha1').update(bufferWithHeader).digest('hex');

  if (verbose) {
    console.log(`${LOG_TAG} \t\tgetLocalHash ${filePath} ${hash}`);
  }

  return hash;
}

function downloadMiningPoolLogos$() {
  return new Promise((resolve, reject) => {
    console.log(`${LOG_TAG} \tChecking if mining pool logos needs downloading or updating...`);
    const options = {
      host: 'api.github.com',
      path: '/repos/mempool/mining-pool-logos/contents/',
      method: 'GET',
      headers: {'user-agent': 'node.js'}
    };

    if (githubSecret) {
      console.log(`${LOG_TAG} Downloading the mining pool logos with authentication`);
      options.headers['authorization'] = `Bearer ${githubSecret}`;
      options.headers['X-GitHub-Api-Version'] = '2022-11-28';
    }

    https.get(options, (response) => {
      const chunks_of_data = [];

      response.on('data', (fragments) => {
        chunks_of_data.push(fragments);
      });

      response.on('end', () => {
        const response_body = Buffer.concat(chunks_of_data);
        try {
          const poolLogos = JSON.parse(response_body.toString());
          if (poolLogos.message) {
            reject(poolLogos.message);
          }
          let downloadedCount = 0;
          for (const poolLogo of poolLogos) {
            if (verbose) {
              console.log(`${LOG_TAG} Processing ${poolLogo.name}`);
            }
            const filePath = `${PATH}/mining-pools/${poolLogo.name}`;
            if (fs.existsSync(filePath)) {
              const localHash = getLocalHash(filePath);
              if (verbose) {
                console.log(`${LOG_TAG} \t\tremote ${poolLogo.name} logo hash ${poolLogo.sha}`);
                console.log(`${LOG_TAG} \t\t\tchecking if ${filePath} exists: ${fs.existsSync(filePath)}`);
              }
              if (localHash !== poolLogo.sha) {
                console.log(`${LOG_TAG} \t\t\t\t${poolLogo.name} is different on the remote, downloading...`);
                let download_url = poolLogo.download_url;
                if (MEMPOOL_CDN) {
                  download_url = download_url.replace("raw.githubusercontent.com/mempool/mining-pool-logos/master", "mempool.space/resources/mining-pools");
                }
                if (DRY_RUN) {
                  console.log(`${LOG_TAG} \t\tDRY_RUN is set, not downloading ${poolLogo.name} but we should`);
                } else {
                  if (verbose) {
                    console.log(`${LOG_TAG} \t\tDownloading ${download_url} to ${filePath}`);
                  }
                  download(filePath, download_url);
                  downloadedCount++;
                }
              } else {
                console.log(`${LOG_TAG} \t\t${poolLogo.name} is already up to date. Skipping.`);
              }
            } else {
              console.log(`${LOG_TAG} \t\t${poolLogo.name} is missing, downloading...`);
              const miningPoolsDir = `${PATH}/mining-pools/`;
              if (!fs.existsSync(miningPoolsDir)){
                fs.mkdirSync(miningPoolsDir, { recursive: true });
              }
              let download_url = poolLogo.download_url;
              if (MEMPOOL_CDN) {
                download_url = download_url.replace("raw.githubusercontent.com/mempool/mining-pool-logos/master", "mempool.space/resources/mining-pools");
              }
              if (DRY_RUN) {
                console.log(`${LOG_TAG} DRY_RUN is set, not downloading ${poolLogo.name} but it should`);
              } else {
                console.log(`${LOG_TAG} \tDownloading ${download_url} to ${filePath}`);
                download(filePath, download_url);
                downloadedCount++;
              }
            }
          }
          console.log(`${LOG_TAG} \t\tDownloaded ${downloadedCount} and skipped ${poolLogos.length - downloadedCount} existing mining pool logos`);
          resolve();
        } catch (e) {
          reject(`Unable to download mining pool logos. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`);
        }
      });

      response.on('error', (error) => {
        reject(error);
      });
    });
  });
}

function downloadPromoVideoSubtiles$() {
  return new Promise((resolve, reject) => {
    console.log(`${LOG_TAG} \tChecking if promo video subtitles needs downloading or updating...`);
    const options = {
      host: 'api.github.com',
      path: '/repos/mempool/mempool-promo/contents/subtitles',
      method: 'GET',
      headers: {'user-agent': 'node.js'}
    };

    if (githubSecret) {
      console.log(`${LOG_TAG} \tDownloading the promo video subtitles with authentication`);
      options.headers['authorization'] = `Bearer ${githubSecret}`;
      options.headers['X-GitHub-Api-Version'] = '2022-11-28';
    }


    https.get(options, (response) => {
      const chunks_of_data = [];

      response.on('data', (fragments) => {
        chunks_of_data.push(fragments);
      });

      response.on('end', () => {
        const response_body = Buffer.concat(chunks_of_data);
        try {
          const videoLanguages = JSON.parse(response_body.toString());
          if (videoLanguages.message) {
            reject(videoLanguages.message);
          }
          let downloadedCount = 0;
          for (const language of videoLanguages) {
            if (verbose) {
              console.log(`${LOG_TAG} Processing ${language.name}`);
            }
            const filePath = `${PATH}/promo-video/${language.name}`;
            if (fs.existsSync(filePath)) {
              if (verbose) {
                console.log(`${LOG_TAG} \t${language.name} remote promo video hash ${language.sha}`);
              }
              const localHash = getLocalHash(filePath);
              if (localHash !== language.sha) {
                console.log(`${LOG_TAG} \t\t${language.name} is different on the remote, updating`);
                let download_url = language.download_url;
                if (MEMPOOL_CDN) {
                  download_url = download_url.replace("raw.githubusercontent.com/mempool/mempool-promo/master/subtitles", "mempool.space/resources/promo-video");
                }
                if (DRY_RUN) {
                  console.log(`${LOG_TAG} \t\tDRY_RUN is set, not downloading ${language.name} but we should`);
                } else {
                  if (verbose) {
                    console.log(`${LOG_TAG} \t\tdownloading ${download_url} to ${filePath}`);
                  }
                  download(filePath, download_url);
                  downloadedCount++;
                }
              } else {
                console.log(`${LOG_TAG} \t\t${language.name} is already up to date. Skipping.`);
              }
            } else {
              console.log(`${LOG_TAG} \t\t${language.name} is missing, downloading`);
              const promoVideosDir = `${PATH}/promo-video/`;
              if (!fs.existsSync(promoVideosDir)){
                fs.mkdirSync(promoVideosDir, { recursive: true });
              }

              let download_url = language.download_url;
              if (MEMPOOL_CDN) {
                download_url = downloadownload_url = download_url.replace("raw.githubusercontent.com/mempool/mempool-promo/master/subtitles", "mempool.space/resources/promo-video");
              }
              if (DRY_RUN) {
                console.log(`${LOG_TAG} \tDRY_RUN is set, not downloading ${language.name} but we should`);
              } else {
                if (verbose) {
                  console.log(`${LOG_TAG} downloading ${download_url} to ${filePath}`);
                }
                download(filePath, download_url);
                downloadedCount++;
              }
            }
          }
          console.log(`${LOG_TAG} Downloaded ${downloadedCount} and skipped ${videoLanguages.length - downloadedCount} existing video subtitles`);
          resolve();
        } catch (e) {
          reject(`Unable to download video subtitles. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`);
        }
      });

      response.on('error', (error) => {
        reject(error);
      });
    });
  });
}

function downloadPromoVideo$() {
  return new Promise((resolve, reject) => {
    console.log(`${LOG_TAG} \tChecking if promo video needs downloading or updating...`);
    const options = {
      host: 'api.github.com',
      path: '/repos/mempool/mempool-promo/contents',
      method: 'GET',
      headers: {'user-agent': 'node.js'}
    };

    if (githubSecret) {
      console.log(`${LOG_TAG} \tDownloading the promo video with authentication`);
      options.headers['authorization'] = `Bearer ${githubSecret}`;
      options.headers['X-GitHub-Api-Version'] = '2022-11-28';
    }

    https.get(options, (response) => {
      const chunks_of_data = [];

      response.on('data', (fragments) => {
        chunks_of_data.push(fragments);
      });

      response.on('end', () => {
        const response_body = Buffer.concat(chunks_of_data);
        try {
          const contents = JSON.parse(response_body.toString());
          if (contents.message) {
            reject(contents.message);
          }
          for (const item of contents) {
            if (item.name !== 'promo.mp4') {
              continue;
            }
            const filePath = `${PATH}/promo-video/mempool-promo.mp4`;
            if (fs.existsSync(filePath)) {
              const localHash = getLocalHash(filePath);

              if (localHash !== item.sha) {
                console.log(`${LOG_TAG} \tmempool-promo.mp4 is different on the remote, updating`);
                let download_url = item.download_url;
                if (MEMPOOL_CDN) {
                  download_url = download_url.replace("raw.githubusercontent.com/mempool/mempool-promo/master/promo.mp4", "mempool.space/resources/promo-video/mempool-promo.mp4");
                }
                if (DRY_RUN) {
                  console.log(`${LOG_TAG} DRY_RUN is set, not downloading mempool-promo.mp4 but we should`);
                } else {
                  if (verbose) {
                    console.log(`${LOG_TAG} downloading ${download_url} to ${filePath}`);
                  }
                  download(filePath, download_url);
                  console.log(`${LOG_TAG} \tmempool-promo.mp4 downloaded.`);
                }
              } else {
                console.log(`${LOG_TAG} \t\tmempool-promo.mp4 is already up to date. Skipping.`);
              }
            } else {
              console.log(`${LOG_TAG} \tmempool-promo.mp4 is missing, downloading`);
              let download_url = item.download_url;
              if (MEMPOOL_CDN) {
                download_url = download_url.replace("raw.githubusercontent.com/mempool/mempool-promo/master/promo.mp4", "mempool.space/resources/promo-video/mempool-promo.mp4");
              }
              if (DRY_RUN) {
                console.log(`${LOG_TAG} DRY_RUN is set, not downloading mempool-promo.mp4 but we should`);
              } else {
                if (verbose) {
                  console.log(`${LOG_TAG} downloading ${download_url} to ${filePath}`);
                }
                download(filePath, download_url);
              }
            }
          }
          resolve();
        } catch (e) {
          reject(`Unable to download video. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`);
        }
      });

      response.on('error', (error) => {
        reject(error);
      });
    });
  });

}


if (configContent.BASE_MODULE && configContent.BASE_MODULE === 'liquid') {
  const assetsJsonUrl = 'https://raw.githubusercontent.com/Blockstream/asset_registry_db/master/index.json';
  const assetsMinimalJsonUrl = 'https://raw.githubusercontent.com/Blockstream/asset_registry_db/master/index.minimal.json';
  const testnetAssetsJsonUrl = 'https://raw.githubusercontent.com/Blockstream/asset_registry_testnet_db/master/index.json';
  const testnetAssetsMinimalJsonUrl = 'https://raw.githubusercontent.com/Blockstream/asset_registry_testnet_db/master/index.minimal.json';

  console.log(`${LOG_TAG} Downloading assets`);
  download(`${PATH}/assets.json`, assetsJsonUrl);

  console.log(`${LOG_TAG} Downloading assets minimal`);
  download(`${PATH}/assets.minimal.json`, assetsMinimalJsonUrl);

  console.log(`${LOG_TAG} Downloading testnet assets`);
  download(`${PATH}/assets-testnet.json`, testnetAssetsJsonUrl);

  console.log(`${LOG_TAG} Downloading testnet assets minimal`);
  download(`${PATH}/assets-testnet.minimal.json`, testnetAssetsMinimalJsonUrl);
} else {
  if (verbose) {
    console.log(`${LOG_TAG} BASE_MODULE is not set to Liquid (currently ${configContent.BASE_MODULE}), skipping downloading assets`);
  }
}

(() => {
  if (verbose) {
    console.log(`${LOG_TAG} Downloading mining pool logos`);
  }
  downloadMiningPoolLogos$()
  .then(() => {
    if (verbose) {
      console.log(`${LOG_TAG} Downloading promo video subtitles`);
    }
    downloadPromoVideoSubtiles$();
  })
  .then(() => {
    if (verbose) {
      console.log(`${LOG_TAG} Downloading promo video`);
    }
    downloadPromoVideo$();
  })
  .catch((error) => {
    throw new Error(error);
  });
})();