Scene importer (Segs#917)

* Scene importer work.

* CMake: Add editor plugin creation helper macro
* Common/Runtime: Record the depth of scene inclusion stack, mark runtime
data state as ready on a successful load.
* Add segs_plugin_coh_scene plugin to the editor sub-project.
* Switched to c++17, same as the engine
* More importer work
* Saner qt logging in the editor.
* Make RuntimeData non-copyable
* Introduce a wrapper class that abstracts the underlying filesystem.
* Update ace to version supporting glibc >= 2.30
* Update sdk version + stop deleting 'prebuilt' directory

3rd_party/prebuilt/.gitignore

@@ -0,0 +1,4 @@
+# no files and subdirectories 
+*
+*/
+!.gitignore

CMakeLists.txt

@@ -1,7 +1,7 @@
 cmake_minimum_required(VERSION 3.16)
 project(SEGS)
 
-set(CMAKE_CXX_STANDARD 14)
+set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
 # Tell cmake to honor CXX_VISIBILITY_PRESET and ignore it
@@ -52,6 +52,16 @@ add_definitions(
 
 if(MSVC)
     add_definitions( -D_CRT_SECURE_NO_WARNINGS -D_CRT_NONSTDC_NO_WARNINGS -DNOMINMAX -D_USE_MATH_DEFINES)
+    add_compile_options (/wd4251) # disable warning related to dll interface needed ... clients of class
+    if (NOT (MSVC_VERSION LESS 1910))
+      #add_compile_options (/MP)
+      add_compile_options (/permissive-)
+      #add_compile_options (/d2cgsummary)
+      #add_compile_options (/d1reportTime)
+      add_compile_options (/diagnostics:caret)
+      #add_compile_options (/sdl)
+      #add_compile_options (/arch:AVX2)
+    endif()
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /bigobj")
 elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pedantic -Wall -Wextra -Wuseless-cast")

CMakeScripts/SDKHelper.cmake

@@ -27,7 +27,10 @@ function(update_binary_deps DEP_PREFIX DEP_NAME DEP_FOLDER DEP_VERSION)
         COMMAND ${CMAKE_COMMAND} -E tar xzf ${PROJECT_SOURCE_DIR}/Temp/Dependencies.7z
         WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/Temp/unpack
 	)
-    execute_process(COMMAND ${CMAKE_COMMAND} -E remove_directory ${DEP_FOLDER})
+    execute_process(COMMAND ${CMAKE_COMMAND} -E remove_directory ${DEP_FOLDER}/bin)
+    execute_process(COMMAND ${CMAKE_COMMAND} -E remove_directory ${DEP_FOLDER}/cmake)
+    execute_process(COMMAND ${CMAKE_COMMAND} -E remove_directory ${DEP_FOLDER}/include)
+    execute_process(COMMAND ${CMAKE_COMMAND} -E remove_directory ${DEP_FOLDER}/lib)
     execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory ${DEP_FOLDER})
     execute_process(COMMAND ${CMAKE_COMMAND} -E copy_directory ${PROJECT_SOURCE_DIR}/Temp/unpack ${DEP_FOLDER})
 #    execute_process(COMMAND ${CMAKE_COMMAND} -E remove_directory ${PROJECT_SOURCE_DIR}/Temp)

Components/CMakeLists.txt

@@ -4,6 +4,7 @@ endif()
 
 SET(target_CPP
     serialization_types.cpp
+    serialization_common.cpp
     BitStream.cpp
     Buffer.cpp
     ConfigExtension.cpp

Components/serialization_common.cpp

@@ -0,0 +1,24 @@
+#include "serialization_common.h"
+
+#include <QtCore/QDir>
+
+QIODevice *QFSWrapper::open(const QString &path, bool read_only, bool text_only)
+{
+    QFile *res = new QFile(path);
+    if (!res->open((read_only ? QFile::ReadOnly : QFile::ReadWrite) | (text_only ? QFile::Text : QFile::NotOpen)))
+    {
+        delete res;
+        return nullptr;
+    }
+    return res;
+}
+
+bool QFSWrapper::exists(const QString &path)
+{
+    return QFile::exists(path);
+}
+
+QStringList QFSWrapper::dir_entries(const QString &path)
+{
+    return QDir(path).entryList(QDir::Files | QDir::NoSymLinks);
+}

Components/serialization_common.h

@@ -15,10 +15,30 @@
 #include <cereal/cereal.hpp>
 
 #include <QtCore/QString>
-#include <QtCore/QDir>
 #include <QtCore/QFile>
 #include <QtCore/QDebug>
 
+ // A simple file access wrapper to allow re-locating/packing files
+struct FSWrapper
+{
+    virtual ~FSWrapper() = default;
+
+    virtual QIODevice* open(const QString &path, bool read_only = true, bool text_only = false)=0;
+    virtual bool exists(const QString& path)=0;
+    virtual QStringList dir_entries(const QString& path) = 0;
+};
+
+struct QFSWrapper : public FSWrapper
+{
+    ~QFSWrapper() override = default;
+
+    QIODevice *open(const QString &path, bool read_only = true, bool text_only = false) override;
+
+    bool exists(const QString &path) override;
+
+    QStringList dir_entries(const QString &path) override;
+};
+
 template<class T>
 void commonSaveTo(const T & target, const char *classname, const QString & baseName, bool text_format)
 {
@@ -60,18 +80,20 @@ void commonSaveTo(const T & target, const char *classname, const QString & baseN
 }
 
 template<class T>
-bool commonReadFrom(const QString &crl_path,const char *classname, T &target)
+bool commonReadFrom(FSWrapper &fs,const QString &crl_path,const char *classname, T &target)
 {
-    QFile ifl(crl_path);
+    QIODevice *ifl=nullptr;
     if(crl_path.endsWith("json") || crl_path.endsWith("crl_json"))
     {
-        if(!ifl.open(QFile::ReadOnly|QFile::Text))
+        ifl = fs.open(crl_path,true,true);
+        if(!ifl)
         {
             qWarning() << "Failed to open" << crl_path;
             return false;
         }
 
-        std::istringstream istr(ifl.readAll().toStdString());
+        std::istringstream istr(ifl->readAll().toStdString());
+        delete ifl;
 
         try
         {
@@ -89,13 +111,14 @@ bool commonReadFrom(const QString &crl_path,const char *classname, T &target)
     }
     else if(crl_path.endsWith(".crl.bin"))
     {
-        if(!ifl.open(QFile::ReadOnly))
+        ifl = fs.open(qPrintable(crl_path), true, false);
+        if(!ifl)
         {
             qWarning() << "Failed to open" << crl_path;
             return false;
         }
-        std::istringstream istr(ifl.readAll().toStdString());
-
+        std::istringstream istr(ifl->readAll().toStdString());
+        delete ifl;
         try
         {
             cereal::BinaryInputArchive arc(istr);

Projects/CoX/Common/GameData/CMakeLists.txt

@@ -150,6 +150,10 @@ set_target_properties(gameData PROPERTIES AUTOMOC ON)
 add_dependencies(gameData ace_IMP cereal_IMP) # used for Singleton stuff.
 target_link_libraries(gameData PUBLIC Qt5::Core)
 target_link_libraries(gameData INTERFACE SEGS_Components)
+target_include_directories(gameData INTERFACE
+  $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../>
+  #$<INSTALL_INTERFACE:gameData>
+)
 set_property(TARGET gameData APPEND PROPERTY INCLUDE_DIRECTORIES "$<TARGET_PROPERTY:cereal_IMP,INTERFACE_INCLUDE_DIRECTORIES>")
 
 # Retain directory structure in visual studio

Projects/CoX/Common/GameData/CharacterAttributes.cpp

@@ -2,6 +2,7 @@
 
 #include "serialization_common.h"
 #include "serialization_types.h"
+#include <QDebug>
 
 template<class Archive>
 void Parse_AttribDesc::serialize(Archive & archive)

Projects/CoX/Common/GameData/DataStorage.cpp

@@ -12,7 +12,7 @@
 
 #include "DataStorage.h"
 #include "Colors.h"
-#include <QtCore/QFile>
+
 #include <QtCore/QString>
 #include <QtCore/QFileInfo>
 
@@ -21,12 +21,12 @@ bool BinStore::check_bin_version_and_crc(uint32_t req_crc)
     QString tgt;
     uint32_t crc_from_file;
     char magic_contents[8];
-    m_str.read(magic_contents,8);
+    m_str->read(magic_contents,8);
     read(crc_from_file);
     tgt=read_pstr(4096);
     if( 0!=strncmp(magic_contents,"CrypticS",8) || tgt.midRef(0,6)!="Parse4" || (req_crc!=0 && crc_from_file != req_crc) ) //
     {
-        m_str.close();
+        m_str->close();
         return false;
     }
     return true;
@@ -42,7 +42,7 @@ const QByteArray &BinStore::read_pstr( size_t maxlen )
     if(len<=maxlen)
     {
         buf.resize(len);
-        m_str.read(buf.data(),len);
+        m_str->read(buf.data(),len);
         if(m_file_sizes.size()>0)
         {
             (*m_file_sizes.rbegin())-=len;
@@ -58,7 +58,7 @@ void BinStore::skip_pstr()
 {
     uint16_t len=0;
     read(len);
-    m_str.seek(len+m_str.pos());
+    m_str->seek(len+m_str->pos());
 }
 
 bool BinStore::read_data_blocks( bool file_data_blocks )
@@ -69,14 +69,14 @@ bool BinStore::read_data_blocks( bool file_data_blocks )
         uint32_t v;
         read(v);
         if(v)
-            m_str.seek(v+m_str.pos());
+            m_str->seek(v+m_str->pos());
         return true;
     }
     const QByteArray &hdr(read_pstr(20));
     uint32_t sz;
     read_internal(sz);
 
-    quint64 read_start = m_str.pos();
+    quint64 read_start = m_str->pos();
     if(!hdr.startsWith("Files1")||sz<=0)
         return false;
     int num_data_blocks;
@@ -88,17 +88,21 @@ bool BinStore::read_data_blocks( bool file_data_blocks )
         read_internal(fe.date);
         m_entries.push_back(fe);
     }
-    quint64 read_end = m_str.pos();
-    m_file_sizes.push_back(m_str.size()-read_end);
+    quint64 read_end = m_str->pos();
+    m_file_sizes.push_back(m_str->size()-read_end);
     return (sz==(read_end-read_start));
 }
 
-bool BinStore::open(const QString &name,uint32_t required_crc )
+bool BinStore::open(FSWrapper& fs, const QString &name,uint32_t required_crc )
 {
-    if(!m_str.isOpen())
+    if(m_str && m_str->isOpen())
     {
-        m_str.setFileName(name);
-        if(!m_str.open(QFile::ReadOnly))
+        m_str->close();
+        delete m_str;
+        m_str = nullptr;
+    }
+    m_str=fs.open(name,true);
+    if(!m_str) {
             return false;
     }
     bool result = check_bin_version_and_crc(required_crc);
@@ -251,7 +255,7 @@ bool BinStore::read(std::vector<float> &res)
 
 bool BinStore::read_bytes( char *tgt,size_t sz )
 {
-    m_str.read(tgt,sz);
+    m_str->read(tgt,sz);
     bytes_read+=sz;
     return true;
 }
@@ -300,10 +304,10 @@ bool BinStore::nesting_name(QByteArray &name)
 
 void BinStore::fixup()
 {
-    qint64 nonmult4 = ((m_str.pos() + 3) & ~3) - m_str.pos();
+    qint64 nonmult4 = ((m_str->pos() + 3) & ~3) - m_str->pos();
     if(nonmult4)
     {
-        m_str.seek(nonmult4+m_str.pos());
+        m_str->seek(nonmult4+m_str->pos());
         bytes_read+=nonmult4;
         if(m_file_sizes.size()>0)
             (*m_file_sizes.rbegin())-=nonmult4;

Projects/CoX/Common/GameData/DataStorage.h

@@ -9,13 +9,11 @@
 
 #include <glm/vec3.hpp>
 #include <glm/vec2.hpp>
-#include <string>
 #include <vector>
-#include <QtCore/QString>
-#include <QtCore/QFile>
-#include <fstream>
 #include <map>
 #include <type_traits>
+
+#include "serialization_common.h"
 struct RGBA;
 
 typedef glm::vec3 Vec3;
@@ -26,7 +24,7 @@ class BinStore // binary storage
         QString name;
         uint32_t date=0;
     };
-    QFile m_str;
+    QIODevice *m_str=nullptr;
     size_t bytes_read=0;
     uint32_t bytes_to_read=0;
     std::vector<uint32_t> m_file_sizes; // implicit stack
@@ -35,10 +33,10 @@ class BinStore // binary storage
     template<class V>
     size_t read_internal(V &res)
     {
-        if(m_file_sizes.size()>0 && current_fsize()<sizeof(V))
+        if(!m_file_sizes.empty() && current_fsize()<sizeof(V))
             return 0;
-        m_str.read((char *)&res,sizeof(V));
-        if(m_file_sizes.size()>0)
+        m_str->read((char *)&res,sizeof(V));
+        if(!m_file_sizes.empty())
         {
             bytes_read+=sizeof(V);
             (*m_file_sizes.rbegin())-=sizeof(V);
@@ -105,5 +103,5 @@ class BinStore // binary storage
     void        nest_in() {  }
     void        nest_out() { m_file_sizes.pop_back(); }
     bool        end_encountered() const;
-    bool        open(const QString & name, uint32_t required_crc);
+    bool        open(FSWrapper &fs,const QString & name, uint32_t required_crc);
 };

Projects/CoX/Common/GameData/GameDataStore.cpp

@@ -225,12 +225,12 @@ template<class TARGET,unsigned int CRC>
 bool read_data_to(const QString &directory_path,const QString &storage,TARGET &target)
 {
     QElapsedTimer timer;
-
+    QFSWrapper wrap;
     QDebug deb=qDebug().noquote().nospace();
     deb << "Reading "<<directory_path<<storage<<" ... ";
     timer.start();
     BinStore bin_store;
-    if(!bin_store.open(directory_path+storage,CRC))
+    if(!bin_store.open(wrap,directory_path+storage,CRC))
     {
         deb << "failure";
         qWarning().noquote() << "Couldn't load "<<storage<<" from" << directory_path;
@@ -367,10 +367,11 @@ uint32_t GameDataStore::countForLevel(uint32_t lvl, const std::vector<uint32_t>
 
 bool GameDataStore::read_costumes(const QString &directory_path)
 {
+    QFSWrapper wrap;
     QDebug deb=qDebug().noquote().nospace();
     deb << "Reading " << directory_path << "bin/costume.bin ... ";
     BinStore costumes_store;
-    if(!costumes_store.open(directory_path + "bin/costume.bin", costumesets_i0_requiredCrc))
+    if(!costumes_store.open(wrap,directory_path + "bin/costume.bin", costumesets_i0_requiredCrc))
     {
         deb << "failure";
         qWarning().noquote() << "Couldn't load bin/costume.bin from" << directory_path;
@@ -391,11 +392,12 @@ bool GameDataStore::read_costumes(const QString &directory_path)
 
 bool GameDataStore::read_colors( const QString &directory_path )
 {
+    QFSWrapper wrap;
     QDebug deb=qDebug().noquote().nospace();
     deb << "Reading " << directory_path << "bin/supergroupColors.bin ... ";
     BinStore sg_color_store;
 
-    if(!sg_color_store.open(directory_path + "bin/supergroupColors.bin", palette_i0_requiredCrc))
+    if(!sg_color_store.open(wrap,directory_path + "bin/supergroupColors.bin", palette_i0_requiredCrc))
     {
         deb << "failure";
         qWarning().noquote() << "Couldn't load bin/supergroupColors.bin from" << directory_path;