Merge branch 'refactor/tokenizer' into autobuild/tokenizer

source/backend/control/messagefactory.cpp

@@ -63,15 +63,15 @@ MessageFactory::~MessageFactory()
 {}
 
 void MessageFactory::SendMessage(MessageClass mc, WarningLevel level, const char *text,
-                                 const UCS2 *filename, POV_LONG line, POV_LONG column, POV_OFF_T offset)
+                                 const UCS2String& filename, POV_LONG line, POV_LONG column, POV_OFF_T offset)
 {
     POVMSObject msg;
     unsigned int msgIdent;
 
     (void)POVMSObject_New(&msg, kPOVObjectClass_ControlData);
 
-    if (filename != nullptr)
-        (void)POVMSUtil_SetUCS2String(&msg, kPOVAttrib_FileName, filename);
+    if (!filename.empty())
+        (void)POVMSUtil_SetUCS2String(&msg, kPOVAttrib_FileName, filename.c_str());
     if (line != -1)
         (void)POVMSUtil_SetLong(&msg, kPOVAttrib_Line, line);
     if (column != -1)

source/backend/control/messagefactory.h

@@ -63,7 +63,7 @@ class MessageFactory : public GenericMessenger
         RenderBackend::ViewId viewId;
 
         virtual void SendMessage(MessageClass mc, WarningLevel level, const char *text,
-                                 const UCS2 *filename = nullptr, POV_LONG line = -1, POV_LONG column = -1, POV_OFF_T offset = -1);
+                                 const UCS2String& filename = u"", POV_LONG line = -1, POV_LONG column = -1, POV_OFF_T offset = -1) override;
 };
 
 }

source/base/messenger.cpp

@@ -102,7 +102,7 @@ void GenericMessenger::InfoAt(const MessageContext& context, const char *format,
     SendMessage(kMessageClass_Info, level, localvsbuffer, context);
 }
 
-void GenericMessenger::InfoAt(const UCS2 *filename, POV_LONG line, POV_LONG column, POV_OFF_T offset, const char *format, ...)
+void GenericMessenger::InfoAt(const UCS2String& filename, POV_LONG line, POV_LONG column, POV_OFF_T offset, const char *format, ...)
 {
     WarningLevel level = kWarningGeneral;
     if(warningLevel < level)
@@ -160,7 +160,7 @@ void GenericMessenger::WarningAt(WarningLevel level, const MessageContext& conte
     SendMessage(kMessageClass_Warning, level, localvsbuffer, context);
 }
 
-void GenericMessenger::WarningAt(WarningLevel level, const UCS2 *filename, POV_LONG line, POV_LONG column, POV_OFF_T offset, const char *format, ...)
+void GenericMessenger::WarningAt(WarningLevel level, const UCS2String& filename, POV_LONG line, POV_LONG column, POV_OFF_T offset, const char *format, ...)
 {
     if(warningLevel < level)
         return;
@@ -217,7 +217,7 @@ void GenericMessenger::PossibleErrorAt(const MessageContext& context, const char
     SendMessage(kMessageClass_PossibleError, kWarningNone, localvsbuffer, context);
 }
 
-void GenericMessenger::PossibleErrorAt(const UCS2 *filename, POV_LONG line, POV_LONG column, POV_OFF_T offset, const char *format, ...)
+void GenericMessenger::PossibleErrorAt(const UCS2String& filename, POV_LONG line, POV_LONG column, POV_OFF_T offset, const char *format, ...)
 {
     if(warningLevel == 0)
         return;
@@ -238,7 +238,7 @@ void GenericMessenger::PossibleErrorAt(const UCS2 *filename, POV_LONG line, POV_
 
 void GenericMessenger::SendMessage(MessageClass mc, WarningLevel level, const char *text, const MessageContext& context)
 {
-    SendMessage(mc, level, text, context.GetFileName().c_str(), context.GetLine(), context.GetColumn(), context.GetOffset());
+    SendMessage(mc, level, text, context.GetFileName(), context.GetLine(), context.GetColumn(), context.GetOffset());
 }
 
 std::string GenericMessenger::SendError(const char *format, va_list arglist, const MessageContext& context)
@@ -255,7 +255,7 @@ std::string GenericMessenger::SendError(const char *format, va_list arglist, con
 }
 
 // filename defaults to `nullptr`, and line, column, and offset default to -1
-std::string GenericMessenger::SendError(const char *format, va_list arglist, const UCS2 *filename, POV_LONG line, POV_LONG column, POV_OFF_T offset)
+std::string GenericMessenger::SendError(const char *format, va_list arglist, const UCS2String& filename, POV_LONG line, POV_LONG column, POV_OFF_T offset)
 {
     char localvsbuffer[1024];
 
@@ -365,7 +365,7 @@ void GenericMessenger::ErrorAt(Exception& ex, const MessageContext& context, con
     throw ex;
 }
 
-void GenericMessenger::ErrorAt(const UCS2 *filename, POV_LONG line, POV_LONG column, POV_OFF_T offset, const char *format, ...)
+void GenericMessenger::ErrorAt(const UCS2String& filename, POV_LONG line, POV_LONG column, POV_OFF_T offset, const char *format, ...)
 {
     va_list marker;
 
@@ -380,7 +380,7 @@ void GenericMessenger::ErrorAt(const UCS2 *filename, POV_LONG line, POV_LONG col
     throw ex;
 }
 
-void GenericMessenger::ErrorAt(const Exception& ex, const UCS2 *filename, POV_LONG line, POV_LONG column, POV_OFF_T offset, const char *format, ...)
+void GenericMessenger::ErrorAt(const Exception& ex, const UCS2String& filename, POV_LONG line, POV_LONG column, POV_OFF_T offset, const char *format, ...)
 {
     va_list marker;
 
@@ -396,7 +396,7 @@ void GenericMessenger::ErrorAt(const Exception& ex, const UCS2 *filename, POV_LO
     throw local_ex;
 }
 
-void GenericMessenger::ErrorAt(Exception& ex, const UCS2 *filename, POV_LONG line, POV_LONG column, POV_OFF_T offset, const char *format, ...)
+void GenericMessenger::ErrorAt(Exception& ex, const UCS2String& filename, POV_LONG line, POV_LONG column, POV_OFF_T offset, const char *format, ...)
 {
     va_list marker;
 

source/base/messenger.h

@@ -96,32 +96,32 @@ class GenericMessenger
 
         void Info(const char *format,...);
         void InfoAt(const MessageContext& context, const char *format, ...);
-        void InfoAt(const UCS2 *filename, POV_LONG line, POV_LONG column, POV_OFF_T offset, const char *format, ...);
+        void InfoAt(const UCS2String& filename, POV_LONG line, POV_LONG column, POV_OFF_T offset, const char *format, ...);
 
         void Warning(WarningLevel level, const char *format,...);
         void WarningAt(WarningLevel level, const MessageContext& context, const char *format, ...);
-        void WarningAt(WarningLevel level, const UCS2 *filename, POV_LONG line, POV_LONG column, POV_OFF_T offset, const char *format, ...);
+        void WarningAt(WarningLevel level, const UCS2String& filename, POV_LONG line, POV_LONG column, POV_OFF_T offset, const char *format, ...);
 
         void PossibleError(const char *format,...);
         void PossibleErrorAt(const MessageContext& context, const char *format, ...);
-        void PossibleErrorAt(const UCS2 *filename, POV_LONG line, POV_LONG column, POV_OFF_T offset, const char *format, ...);
+        void PossibleErrorAt(const UCS2String& filename, POV_LONG line, POV_LONG column, POV_OFF_T offset, const char *format, ...);
 
         void Error(const char *format,...);
         void Error(const Exception& ex, const char *format,...);
         void Error(Exception& ex, const char *format,...);
         void ErrorAt(const MessageContext& context, const char *format, ...);
         void ErrorAt(const Exception& ex, const MessageContext& context, const char *format, ...);
         void ErrorAt(Exception& ex, const MessageContext& context, const char *format, ...);
-        void ErrorAt(const UCS2 *filename, POV_LONG line, POV_LONG column, POV_OFF_T offset, const char *format, ...);
-        void ErrorAt(const Exception& ex, const UCS2 *filename, POV_LONG line, POV_LONG column, POV_OFF_T offset, const char *format, ...);
-        void ErrorAt(Exception& ex, const UCS2 *filename, POV_LONG line, POV_LONG column, POV_OFF_T offset, const char *format, ...);
+        void ErrorAt(const UCS2String& filename, POV_LONG line, POV_LONG column, POV_OFF_T offset, const char *format, ...);
+        void ErrorAt(const Exception& ex, const UCS2String& filename, POV_LONG line, POV_LONG column, POV_OFF_T offset, const char *format, ...);
+        void ErrorAt(Exception& ex, const UCS2String& filename, POV_LONG line, POV_LONG column, POV_OFF_T offset, const char *format, ...);
 
         void SetWarningLevel(unsigned int Val) { warningLevel = Val ; } // TODO FIXME - not here, not this way
 
     protected:
 
         virtual void SendMessage(MessageClass mc, WarningLevel level, const char *text,
-                                 const UCS2 *filename = nullptr, POV_LONG line = -1, POV_LONG column = -1, POV_OFF_T offset = -1) = 0;
+                                 const UCS2String& filename = u"", POV_LONG line = -1, POV_LONG column = -1, POV_OFF_T offset = -1) = 0;
 
     private:
 
@@ -132,7 +132,7 @@ class GenericMessenger
         void SendMessage(MessageClass mc, WarningLevel level, const char *text, const MessageContext& context);
         std::string SendError(const char *format, va_list arglist, const MessageContext& context);
         std::string SendError(const char *format, va_list arglist,
-                              const UCS2 *filename = nullptr, POV_LONG line = -1, POV_LONG column = -1, POV_OFF_T offset = -1);
+                              const UCS2String& filename = u"", POV_LONG line = -1, POV_LONG column = -1, POV_OFF_T offset = -1);
 };
 
 /// @}

source/base/stringutilities.cpp

@@ -267,6 +267,16 @@ const char *pov_tsprintf(const char *format,...)
 
 //******************************************************************************
 
+std::size_t UCS2_strlen(const UCS2* str)
+{
+    const UCS2* end = str;
+    while (*end != u'\0')
+        ++end;
+    return (end - str);
+}
+
+//******************************************************************************
+
 namespace UCS
 {
 

source/base/stringutilities.h

@@ -39,6 +39,14 @@
 // Module config header file must be the first file included within POV-Ray unit header files
 #include "base/configbase.h"
 
+// C++ variants of C standard header files
+#include <cstring>
+
+// C++ standard header files
+// Boost header files
+//  (none at the moment)
+
+// POV-Ray header files (base module)
 #include "base/types.h"
 
 namespace pov_base
@@ -59,6 +67,8 @@ UCS2String UTF8toUCS2String(const UTF8String& s);
 int pov_stricmp(const char *, const char *);
 const char *pov_tsprintf(const char *, ...);
 
+std::size_t UCS2_strlen(const UCS2* str);
+
 //******************************************************************************
 
 /// UCS Decoding Primitives

source/base/version.h

@@ -100,7 +100,7 @@
 /// where `N` is a serial number starting at 1 in each phase, `TIME` is the number of minutes
 /// since 2000-01-01 00:00, and `FEATURE` is an arbitrary alphanumeric moniker for a particular
 /// experimental feature.
-#define POV_RAY_PRERELEASE          "x.tokenizer.9960461"
+#define POV_RAY_PRERELEASE          "x.tokenizer.9971030"
 
 #if defined(DOXYGEN) && !defined(POV_RAY_PRERELEASE)
     // Work around doxygen being unable to document undefined macros.

source/parser/configparser.h

@@ -106,10 +106,27 @@
 ///
 /// @{
 
+/// @def POV_PARSER_ASSERT
+/// Assert a condition that should hold true by design.
+/// In debug builds, this macro evaluates the specified expression, and halts execution if the
+/// expression does not hold true.
+/// In release builds, this macro evaluates to an empty statement.
 #if POV_PARSER_DEBUG
     #define POV_PARSER_ASSERT(expr) POV_ASSERT_HARD(expr)
 #else
-    #define POV_PARSER_ASSERT(expr) POV_ASSERT_DISABLE(expr)
+    #define POV_PARSER_ASSERT(expr) POV_ASSERT_SOFT(expr) // POV_ASSERT_DISABLE(expr)
+#endif
+
+/// @def POV_PARSER_PANIC
+/// Indicates code paths that should never be reached.
+/// In debug builds, this macro halts execution.
+/// In release builds, this macro throws an exception to allow the application to fail gracefully.
+/// In static code analysis, this macro may be used to inform the analysis tool that the code branch
+/// is expected to be dead.
+#if POV_PARSER_DEBUG
+    #define POV_PARSER_PANIC()      POV_ASSERT_HARD(false)
+#else
+    #define POV_PARSER_PANIC()      POV_ASSERT_SOFT(false)
 #endif
 
 /// @}

source/parser/fncode.cpp

@@ -43,6 +43,9 @@
 #include <cstring>
 #include <algorithm>
 
+// POV-Ray header files (base module)
+#include "base/stringutilities.h"
+
 #include "core/scene/scenedata.h"
 
 #include "vm/fnintern.h"