Remove HCL parsing and definition - fixes TF v0.12 issue (#5)

* Remove HCL parsing and definition - fixes TF v0.12 issue * eslint fixes * Remove invalid tests * eslint fixes * Bump ngrok to fix TravisCI issue * Fix package-lock issues
robburger · Aug 22, 2019 · 24f6fe1 · 24f6fe1
1 parent 56cc625
commit 24f6fe1
Show file tree

Hide file tree

Showing 31 changed files with 908 additions and 441 deletions.
diff --git a/app.js b/app.js
@@ -17,7 +17,9 @@ app.set('view engine', 'jten');
 // uncomment after placing your favicon in /public
 app.use(morgan('dev'));
 app.use(bodyParser.json());
-app.use(bodyParser.urlencoded({ extended: false }));
+app.use(bodyParser.urlencoded({
+  extended: false,
+}));
 
 app.use('/', require('./routes/index'));
 app.use('/', require('./routes/service-discovery'));

diff --git a/lib/citizen.js b/lib/citizen.js
@@ -1,8 +1,12 @@
 /* eslint-disable no-console */
-const { format } = require('util');
+const {
+  format,
+} = require('util');
 const commander = require('commander');
 const Listr = require('listr');
-const { red } = require('colors');
+const {
+  red,
+} = require('colors');
 const debug = require('debug');
 
 const citizen = require('../package.json');
@@ -56,33 +60,32 @@ const runPublish = (namespace, name, provider, version, cmd) => {
   const targetDir = process.cwd();
   const moduleFullPath = `${namespace}/${name}/${provider}/${moduleversion}`;
 
-  const tasks = new Listr([
-    {
-      title: 'compress the terraform module',
-      task: async (ctx) => {
-        const files = await makeFileList(targetDir);
-        verbose(`files to compress: ${files}`);
-        const tarball = await makeTarball(targetDir, files);
-        ctx.tarball = tarball;
-      },
+  const tasks = new Listr([{
+    title: 'compress the terraform module',
+    task: async (ctx) => {
+      const files = await makeFileList(targetDir);
+      verbose(`files to compress: ${files}`);
+      const tarball = await makeTarball(targetDir, files);
+      ctx.tarball = tarball;
     },
-    {
-      title: `publish ${moduleFullPath}`,
-      task: async (ctx) => {
-        try {
-          const res = await publish(registryAddr, moduleFullPath, ctx.tarball, owner);
-          verbose(`response status code form registry: ${res.statusCode}`);
-          if (res.statusCode >= 400) {
-            throw new Error(`The registry server is something wrong: ${res.statusMessage}`);
-          }
-        } catch (err) {
-          if (err.code === 'ECONNREFUSED') {
-            throw new Error('The registry server doesn\'t response. Please check the registry.');
-          }
-          throw err;
+  },
+  {
+    title: `publish ${moduleFullPath}`,
+    task: async (ctx) => {
+      try {
+        const res = await publish(registryAddr, moduleFullPath, ctx.tarball, owner);
+        verbose(`response status code form registry: ${res.statusCode}`);
+        if (res.statusCode >= 400) {
+          throw new Error(`The registry server is something wrong: ${res.statusMessage}`);
+        }
+      } catch (err) {
+        if (err.code === 'ECONNREFUSED') {
+          throw new Error('The registry server doesn\'t response. Please check the registry.');
         }
-      },
+        throw err;
+      }
     },
+  },
   ]);
 
   tasks.run().catch((err) => {
@@ -125,7 +128,9 @@ commander
   });
 
 commander
-  .command('*', { noHelp: true })
+  .command('*', {
+    noHelp: true,
+  })
   .action((env) => {
     console.error(`No command: ${env}`);
   });

diff --git a/lib/logger.js b/lib/logger.js
@@ -2,6 +2,9 @@ const pino = require('pino');
 
 const loggerName = process.env.npm_package_name || 'citizen';
 
-const logger = pino({ name: loggerName, level: 'error' });
+const logger = pino({
+  name: loggerName,
+  level: 'error',
+});
 
 module.exports = logger;
diff --git a/lib/module.js b/lib/module.js
@@ -2,7 +2,9 @@ const globby = require('globby');
 const tar = require('tar');
 const semver = require('semver');
 const request = require('request');
-const { promisify } = require('util');
+const {
+  promisify,
+} = require('util');
 const verbose = require('debug')('citizen:client');
 
 const post = promisify(request.post);
@@ -32,13 +34,11 @@ const makeFileList = async (target = __dirname) => {
 };
 
 const makeTarball = (target = __dirname, filelist = []) => new Promise((resolve, reject) => {
-  const compressed = tar.create(
-    {
-      cwd: target,
-      gzip: true,
-    },
-    filelist,
-  );
+  const compressed = tar.create({
+    cwd: target,
+    gzip: true,
+  },
+  filelist);
   const buffers = [];
 
   compressed

diff --git a/lib/module.spec.js b/lib/module.spec.js
@@ -1,9 +1,15 @@
 /* eslint-disable no-unused-expressions */
 const fs = require('fs');
 const path = require('path');
-const { expect } = require('chai');
-const { Duplex } = require('stream');
-const { promisify } = require('util');
+const {
+  expect,
+} = require('chai');
+const {
+  Duplex,
+} = require('stream');
+const {
+  promisify,
+} = require('util');
 const tar = require('tar');
 const recursive = require('recursive-readdir');
 const rimraf = require('rimraf');
@@ -113,7 +119,9 @@ describe('module\'s', () => {
           tarball.push(compressed);
           tarball.push(null);
 
-          tarball.pipe(tar.x({ cwd: UNTAR_DIR }))
+          tarball.pipe(tar.x({
+            cwd: UNTAR_DIR,
+          }))
             .on('finish', () => {
               recursive(UNTAR_DIR, (err, list) => {
                 expect(list).to.have.members([
@@ -142,7 +150,9 @@ describe('module\'s', () => {
           tarball.push(compressed);
           tarball.push(null);
 
-          tarball.pipe(tar.x({ cwd: UNTAR_DIR }))
+          tarball.pipe(tar.x({
+            cwd: UNTAR_DIR,
+          }))
             .on('finish', () => {
               recursive(UNTAR_DIR, (err, list) => {
                 expect(list).to.have.members([

diff --git a/lib/storage.js b/lib/storage.js
@@ -6,11 +6,17 @@ let getModule;
 
 if (process.env.CITIZEN_STORAGE === 's3') {
   ({
-    type, saveModule, hasModule, getModule,
+    type,
+    saveModule,
+    hasModule,
+    getModule,
   } = require('../storages/s3'));
 } else {
   ({
-    type, saveModule, hasModule, getModule,
+    type,
+    saveModule,
+    hasModule,
+    getModule,
   } = require('../storages/file'));
 }
 

diff --git a/lib/store.js b/lib/store.js
@@ -9,11 +9,23 @@ let increaseDownload;
 
 if (process.env.CITIZEN_DATABASE === 'mongodb') {
   ({
-    db, save, findOne, findAll, getVersions, getLatestVersion, increaseDownload,
+    db,
+    save,
+    findOne,
+    findAll,
+    getVersions,
+    getLatestVersion,
+    increaseDownload,
   } = require('../stores/mongodb'));
 } else {
   ({
-    db, save, findOne, findAll, getVersions, getLatestVersion, increaseDownload,
+    db,
+    save,
+    findOne,
+    findAll,
+    getVersions,
+    getLatestVersion,
+    increaseDownload,
   } = require('../stores/nedb'));
 }
 

diff --git a/lib/util.js b/lib/util.js
@@ -1,15 +1,6 @@
-const { readFile } = require('fs');
-const { basename, extname, join } = require('path');
-const { promisify } = require('util');
-const hcl = require('gopher-hcl');
-const tar = require('tar');
-const { Duplex } = require('stream');
-const { URLSearchParams } = require('url');
-const recursive = require('recursive-readdir');
-const _ = require('lodash');
-const tmp = require('tmp');
-
-const readFileProm = promisify(readFile);
+const {
+  URLSearchParams,
+} = require('url');
 
 const makeUrl = (req, search) => {
   const params = new URLSearchParams(search);
@@ -19,120 +10,6 @@ const makeUrl = (req, search) => {
   return `${req.baseUrl}${req._parsedUrl.pathname}${searchStr}`;
 };
 
-const ignore = (file, stats) => {
-  if ((stats.isDirectory() || extname(file) === '.tf') && !basename(file).startsWith('._')) {
-    return false;
-  }
-  return true;
-};
-
-const hclToJson = async (filePath) => {
-  const content = await readFileProm(filePath);
-  const json = hcl.parse(content.toString());
-
-  return json;
-};
-
-const extractDefinition = async (files, targetPath) => {
-  const tfFiles = files.filter((f) => {
-    const relativePath = f.replace(targetPath, '');
-    if (relativePath.lastIndexOf('/') > 0) {
-      return false;
-    }
-    return true;
-  });
-
-  const promises = tfFiles.map(async (r) => {
-    const j = await hclToJson(r);
-    return j;
-  });
-  const list = await Promise.all(promises);
-
-  return _.reduce(list, (l, accum) => _.merge(accum, l), {});
-};
-
-const nomarlizeModule = module => ({
-  path: '',
-  name: module.name || '',
-  readme: '',
-  empty: !module,
-  inputs: module.variable
-    ? Object.keys(module.variable).map(name => ({ name, ...module.variable[name] }))
-    : [],
-  outputs: module.output
-    ? Object.keys(module.output).map(name => ({ name, ...module.output[name] }))
-    : [],
-  dependencies: [],
-  resources: module.resource
-    ? Object.keys(module.resource).map(type => ({ name: Object.keys(module.resource)[0], type }))
-    : [],
-});
-
-const extractSubmodules = async (definition, files, targetPath) => {
-  let pathes = [];
-  if (definition.module) {
-    const submodules = Object.keys(definition.module).map(key => definition.module[key].source);
-    pathes = _.uniq(submodules);
-  }
-
-  const promises = pathes.map(async (p) => {
-    const data = await extractDefinition(files, join(targetPath, p));
-    data.name = p.substr(p.lastIndexOf('/') + 1);
-
-    let result = [data];
-    if (data.module) {
-      const m = await extractSubmodules(data, files, join(targetPath, p));
-      result = result.concat(m);
-    }
-    return result;
-  });
-
-  const submodules = _.flatten(await Promise.all(promises));
-  return submodules;
-};
-
-const parseHcl = (moduleName, compressedModule) => new Promise((resolve, reject) => {
-  const stream = new Duplex();
-  stream.push(compressedModule);
-  stream.push(null);
-
-  tmp.dir({ unsafeCleanup: true }, (err, tempDir, cleanupCallback) => {
-    if (err) { return reject(err); }
-
-    return stream.pipe(tar.x({ cwd: tempDir }))
-      .on('finish', async () => {
-        try {
-          const files = await recursive(tempDir, [ignore]);
-
-          // make a root module definition
-          const rootData = await extractDefinition(files, tempDir);
-          rootData.name = moduleName;
-          const rootDefinition = nomarlizeModule(rootData);
-
-          // make submodules definition
-          const submodulesData = await extractSubmodules(rootData, files, tempDir);
-          const submodulesDefinition = submodulesData.map(s => nomarlizeModule(s));
-          submodulesDefinition.forEach((s) => {
-            let modulePath = files.find(f => f.includes(`/${s.name}/`));
-            modulePath = modulePath.replace(`${tempDir}/`, '');
-            modulePath = modulePath.substr(0, modulePath.lastIndexOf('/'));
-            s.path = modulePath; // eslint-disable-line no-param-reassign
-          });
-
-          resolve({
-            root: rootDefinition,
-            submodules: submodulesDefinition,
-          });
-        } catch (e) {
-          reject(e);
-        } finally {
-          cleanupCallback();
-        }
-      });
-  });
-});
-
 module.exports = {
   makeUrl,
-  parseHcl,
 };