Changes for Koenig and Ghost 2.0 (#9750)

refs #9742, refs #9724

- handle König Editor format for 2.0
- adapted importer to be able to import 1.0 and 2.0 exports
- added migration scripts
  - remove labs flag for Koenig
  - migrate all old editor posts to new editor format
- ensure we protect the code against mobiledoc or html field being null
- ensure we create a blank mobiledoc structure if mobiledoc field is null (model layer)
- ensure you can fully rollback 2.0 to 1.0
- keep mobiledoc/markdown version 1 logic to be able to rollback (deprecated code)

Author: kirrg001

core/server/data/importer/importers/data/posts.js

@@ -2,6 +2,7 @@ const debug = require('ghost-ignition').debug('importer:posts'),
     _ = require('lodash'),
     uuid = require('uuid'),
     BaseImporter = require('./base'),
+    converters = require('../../../../lib/mobiledoc/converters'),
     validation = require('../../../validation');
 
 class PostsImporter extends BaseImporter {
@@ -160,6 +161,33 @@ class PostsImporter extends BaseImporter {
                     model.comment_id = model.id;
                 }
             }
+
+            // CASE 1: you are importing old editor posts
+            // CASE 2: you are importing Koenig Beta posts
+            if (model.mobiledoc || (model.mobiledoc && model.html && model.html.match(/^<div class="kg-card-markdown">/))) {
+                let mobiledoc;
+
+                try {
+                    mobiledoc = JSON.parse(model.mobiledoc);
+
+                    if (!mobiledoc.cards || !_.isArray(mobiledoc.cards)) {
+                        model.mobiledoc = converters.mobiledocConverter.blankStructure();
+                        mobiledoc = model.mobiledoc;
+                    }
+                } catch (err) {
+                    mobiledoc = converters.mobiledocConverter.blankStructure();
+                }
+
+                mobiledoc.cards.forEach((card) => {
+                    if (card[0] === 'image') {
+                        card[1].cardWidth = card[1].imageStyle;
+                        delete card[1].imageStyle;
+                    }
+                });
+
+                model.mobiledoc = JSON.stringify(mobiledoc);
+                model.html = converters.mobiledocConverter.render(JSON.parse(model.mobiledoc));
+            }
         });
 
         // NOTE: We only support removing duplicate posts within the file to import.

core/server/data/migrations/versions/2.0/2-update-posts.js

@@ -2,32 +2,66 @@ const _ = require('lodash'),
     Promise = require('bluebird'),
     common = require('../../../../lib/common'),
     models = require('../../../../models'),
-    message1 = 'Updating post data (comment_id)',
-    message2 = 'Updated post data (comment_id)',
-    message3 = 'Rollback: Keep correct comment_id values in amp column.';
+    converters = require('../../../../lib/mobiledoc/converters'),
+    message1 = 'Updating posts: apply new editor format and set comment_id field.',
+    message2 = 'Updated posts: apply new editor format and set comment_id field.',
+    message3 = 'Rollback: Updating posts: use old editor format',
+    message4 = 'Rollback: Updated posts: use old editor format';
 
 module.exports.config = {
     transaction: true
 };
 
+let mobiledocIsCompatibleWithV1 = function mobiledocIsCompatibleWithV1(doc) {
+    if (doc
+        && doc.markups.length === 0
+        && doc.cards.length === 1
+        && doc.cards[0][0].match(/(?:card-)?markdown/)
+        && doc.sections.length === 1
+        && doc.sections[0].length === 2
+        && doc.sections[0][0] === 10
+        && doc.sections[0][1] === 0
+    ) {
+        return true;
+    }
+
+    return false;
+};
+
 module.exports.up = (options) => {
-    const postAllColumns = ['id', 'comment_id'];
+    const postAllColumns = ['id', 'comment_id', 'html', 'mobiledoc'];
 
     let localOptions = _.merge({
-        context: {internal: true}
+        context: {internal: true},
+        migrating: true
     }, options);
 
     common.logging.info(message1);
 
     return models.Post.findAll(_.merge({columns: postAllColumns}, localOptions))
         .then(function (posts) {
             return Promise.map(posts.models, function (post) {
-                if (post.get('comment_id')) {
-                    return Promise.resolve();
+                let mobiledoc;
+                let html;
+
+                try {
+                    mobiledoc = JSON.parse(post.get('mobiledoc') || null);
+                } catch (err) {
+                    common.logging.warn(`Invalid mobiledoc structure for ${post.id}. Falling back to blank structure.`);
+                    mobiledoc = converters.mobiledocConverter.blankStructure();
+                }
+
+                // CASE: convert all old editor posts to the new editor format
+                // CASE: if mobiledoc field is null, we auto set a blank structure in the model layer
+                // CASE: if html field is null, we auto generate the html in the model layer
+                if (mobiledoc && post.get('html') && post.get('html').match(/^<div class="kg-card-markdown">/)) {
+                    html = converters.mobiledocConverter.render(mobiledoc);
                 }
 
                 return models.Post.edit({
-                    comment_id: post.id
+                    comment_id: post.get('comment_id') || post.id,
+                    html: html || post.get('html'),
+                    mobiledoc: JSON.stringify(mobiledoc)
                 }, _.merge({id: post.id}, localOptions));
             }, {concurrency: 100});
         })
@@ -36,7 +70,38 @@ module.exports.up = (options) => {
         });
 };
 
-module.exports.down = () => {
-    common.logging.warn(message3);
-    return Promise.resolve();
+module.exports.down = (options) => {
+    const postAllColumns = ['id', 'html', 'mobiledoc'];
+
+    let localOptions = _.merge({
+        context: {internal: true},
+        migrating: true
+    }, options);
+
+    common.logging.info(message3);
+
+    return models.Post.findAll(_.merge({columns: postAllColumns}, localOptions))
+        .then(function (posts) {
+            return Promise.map(posts.models, function (post) {
+                let version = 1;
+                let html;
+                let mobiledoc = JSON.parse(post.get('mobiledoc') || null);
+
+                if (!mobiledocIsCompatibleWithV1(mobiledoc)) {
+                    version = 2;
+                }
+
+                // CASE: revert: all new editor posts to the old editor format
+                if (mobiledoc && post.get('html')) {
+                    html = converters.mobiledocConverter.render(mobiledoc, version);
+                }
+
+                return models.Post.edit({
+                    html: html || post.get('html')
+                }, _.merge({id: post.id}, localOptions));
+            }, {concurrency: 100});
+        })
+        .then(() => {
+            common.logging.info(message4);
+        });
 };

core/server/data/migrations/versions/2.0/3-remove-koenig-labs.js

@@ -0,0 +1,42 @@
+const _ = require('lodash'),
+    Promise = require('bluebird'),
+    common = require('../../../../lib/common'),
+    models = require('../../../../models'),
+    message1 = 'Removing `koenigEditor` from labs.',
+    message2 = 'Removed `koenigEditor` from labs.',
+    message3 = 'Rollback: Please re-enable König Beta if required. We can\'t rollback this change.';
+
+module.exports.config = {
+    transaction: true
+};
+
+module.exports.up = (options) => {
+    let localOptions = _.merge({
+        context: {internal: true}
+    }, options);
+
+    return models.Settings.findOne({key: 'labs'}, localOptions)
+        .then(function (settingsModel) {
+            if (!settingsModel) {
+                common.logging.warn('Labs field does not exist.');
+                return;
+            }
+
+            const labsValue = JSON.parse(settingsModel.get('value'));
+            delete labsValue.koenigEditor;
+
+            common.logging.info(message1);
+            return models.Settings.edit({
+                key: 'labs',
+                value: JSON.stringify(labsValue)
+            }, localOptions);
+        })
+        .then(() => {
+            common.logging.info(message2);
+        });
+};
+
+module.exports.down = () => {
+    common.logging.warn(message3);
+    return Promise.resolve();
+};

core/server/lib/mobiledoc/cards/markdown.js

@@ -4,11 +4,13 @@ module.exports = {
     render: function (opts) {
         let converters = require('../converters');
         let payload = opts.payload;
-        let version = opts.options.version;
+        let version = opts.options && opts.options.version || 2;
         // convert markdown to HTML ready for insertion into dom
         let html = converters.markdownConverter.render(payload.markdown || '');
 
-        // Ghost 1.0's markdown-only renderer wrapped cards
+        /**
+         * @deprecated Ghost 1.0's markdown-only renderer wrapped cards
+         */
         if (version === 1) {
             html = `<div class="kg-card-markdown">${html}</div>`;
         }

core/server/lib/mobiledoc/converters/mobiledoc-converter.js

@@ -88,14 +88,15 @@ class DomModifier {
 }
 
 module.exports = {
-    // version 1 === Ghost 1.0 markdown-only mobiledoc
-    // version 2 === Ghost 2.0 full mobiledoc
     render(mobiledoc, version) {
-        version = version || 1;
+        /**
+         * @deprecated: version 1 === Ghost 1.0 markdown-only mobiledoc
+         *              We keep the version 1 logic till Ghost 3.0 to be able to rollback posts.
+         *
+         * version 2 (latest) === Ghost 2.0 full mobiledoc
+         */
+        version = version || 2;
 
-        // pass the version through to the card renderers.
-        // create a new object here to avoid modifying the default options
-        // object because the version can change per-render until 2.0 is released
         let versionedOptions = Object.assign({}, options, {
             cardOptions: {version}
         });
@@ -116,8 +117,20 @@ module.exports = {
         let modifier = new DomModifier();
         modifier.modifyChildren(rendered.result);
 
-        let html = serializer.serializeChildren(rendered.result);
-
-        return html;
+        return serializer.serializeChildren(rendered.result);
+    },
+
+    blankStructure() {
+        return {
+            version: '0.3.1',
+            markups: [],
+            atoms: [],
+            cards: [],
+            sections: [
+                [1, 'p', [
+                    [0, [], 0, '']
+                ]]
+            ]
+        };
     }
 };

core/server/models/post.js

@@ -8,7 +8,6 @@ var _ = require('lodash'),
     htmlToText = require('html-to-text'),
     ghostBookshelf = require('./base'),
     config = require('../config'),
-    labs = require('../services/labs'),
     converters = require('../lib/mobiledoc/converters'),
     urlService = require('../services/url'),
     relations = require('./relations'),
@@ -198,7 +197,6 @@ Post = ghostBookshelf.Model.extend({
             prevSlug = this.previous('slug'),
             publishedAt = this.get('published_at'),
             publishedAtHasChanged = this.hasDateChanged('published_at', {beforeWrite: true}),
-            mobiledoc = JSON.parse(this.get('mobiledoc') || null),
             generatedFields = ['html', 'plaintext'],
             tagsToSave,
             ops = [];
@@ -262,42 +260,21 @@ Post = ghostBookshelf.Model.extend({
         ghostBookshelf.Model.prototype.onSaving.call(this, model, attr, options);
 
         // do not allow generated fields to be overridden via the API
-        generatedFields.forEach((field) => {
-            if (this.hasChanged(field)) {
-                this.set(field, this.previous(field));
-            }
-        });
-
-        // render mobiledoc to HTML. Switch render version if Koenig is enabled
-        // or has been edited with Koenig and is no longer compatible with the
-        // Ghost 1.0 markdown-only renderer
-        // TODO: re-render all content and remove the version toggle for Ghost 2.0
-        if (mobiledoc) {
-            let version = 1;
-            let koenigEnabled = labs.isSet('koenigEditor') === true;
-
-            let mobiledocIsCompatibleWithV1 = function mobiledocIsCompatibleWithV1(doc) {
-                if (doc
-                    && doc.markups.length === 0
-                    && doc.cards.length === 1
-                    && doc.cards[0][0].match(/(?:card-)?markdown/)
-                    && doc.sections.length === 1
-                    && doc.sections[0].length === 2
-                    && doc.sections[0][0] === 10
-                    && doc.sections[0][1] === 0
-                ) {
-                    return true;
+        if (!options.migrating) {
+            generatedFields.forEach((field) => {
+                if (this.hasChanged(field)) {
+                    this.set(field, this.previous(field));
                 }
+            });
+        }
 
-                return false;
-            };
-
-            if (koenigEnabled || !mobiledocIsCompatibleWithV1(mobiledoc)) {
-                version = 2;
-            }
+        if (!this.get('mobiledoc')) {
+            this.set('mobiledoc', JSON.stringify(converters.mobiledocConverter.blankStructure()));
+        }
 
-            let html = converters.mobiledocConverter.render(mobiledoc, version);
-            this.set('html', html);
+        // render mobiledoc to HTML
+        if (this.hasChanged('mobiledoc') || !this.get('html')) {
+            this.set('html', converters.mobiledocConverter.render(JSON.parse(this.get('mobiledoc'))));
         }
 
         if (this.hasChanged('html') || !this.get('plaintext')) {

core/test/integration/data/importer/importers/data_spec.js

@@ -1402,20 +1402,3 @@ describe('LTS', function () {
             });
     });
 });
-
-describe('LTS', function () {
-    beforeEach(testUtils.teardown);
-    beforeEach(testUtils.setup('roles', 'owner', 'settings'));
-
-    it('disallows importing LTS imports', function () {
-        const exportData = exportedLegacyBody().db[0];
-
-        return dataImporter.doImport(exportData, importOptions)
-            .then(function () {
-                "0".should.eql(1, 'LTS import should fail');
-            })
-            .catch(function (err) {
-                err.message.should.eql('Importing a LTS export into Ghost 2.0 is not allowed.');
-            });
-    });
-});

core/test/unit/helpers/reading_time_spec.js

@@ -4,7 +4,7 @@ var should = require('should'),
     helpers = require('../../../server/helpers');
 
 var almostOneMinute =
-    '<div class="kg-card-markdown"><p>Ghost has a number of different user roles for your team</p>' +
+    '<p>Ghost has a number of different user roles for your team</p>' +
     '<h3 id="authors">Authors</h3><p>The base user level in Ghost is an author. Authors can write posts,' +
     ' edit their own posts, and publish their own posts. Authors are <strong>trusted</strong> users. If you ' +
     'don\'t trust users to be allowed to publish their own posts, you shouldn\'t invite them to Ghost admin.</p>' +
@@ -18,7 +18,7 @@ var almostOneMinute =
     'The Owner can never be deleted. And in some circumstances the owner will have access to additional special settings ' +
     'if applicable — for example, billing details, if using Ghost(Pro).</p><hr><p>It\'s a good idea to ask all of your' +
     ' users to fill out their user profiles, including bio and social links. These will populate rich structured data ' +
-    'for posts and generally create more opportunities for themes to fully populate their design.</p></div>';
+    'for posts and generally create more opportunities for themes to fully populate their design.</p>';
 
 var almostOneAndAHalfMinute = almostOneMinute +
     '<div>' +