Merge pull request #52 from OpenVicProject/add/backslash-identifier

Add backslash identifier support to v2script

include/openvic-dataloader/Error.hpp

@@ -7,6 +7,7 @@
 
 #include <dryad/abstract_node.hpp>
 #include <dryad/node.hpp>
+#include <dryad/symbol.hpp>
 
 namespace ovdl {
 	template<typename>
@@ -50,6 +51,13 @@ namespace ovdl::error {
 		LastAnnotation = SecondaryAnnotation,
 	};
 
+	struct ErrorSymbolInterner {
+		struct SymbolId;
+		using index_type = std::uint32_t;
+		using symbol_type = dryad::symbol<SymbolId, index_type>;
+		using symbol_interner_type = dryad::symbol_interner<SymbolId, char, index_type>;
+	};
+
 	static constexpr std::string_view get_kind_name(ErrorKind kind) {
 		switch (kind) {
 			using enum ErrorKind;
@@ -62,13 +70,13 @@ namespace ovdl::error {
 	}
 
 	struct Error : dryad::abstract_node_all<ErrorKind> {
-		const char* message() const { return _message; }
+		const char* message(const ErrorSymbolInterner::symbol_interner_type& symbols) const { return _message.c_str(symbols); }
 
 	protected:
 		DRYAD_ABSTRACT_NODE_CTOR(Error);
 
-		void _set_message(const char* message) { _message = message; }
-		const char* _message = "";
+		void _set_message(ErrorSymbolInterner::symbol_type message) { _message = message; }
+		ErrorSymbolInterner::symbol_type _message;
 
 		template<typename>
 		friend struct ovdl::BasicDiagnosticLogger;
@@ -94,7 +102,7 @@ namespace ovdl::error {
 	};
 
 	struct BufferError : dryad::basic_node<ErrorKind::BufferError, Error> {
-		explicit BufferError(dryad::node_ctor ctor, const char* message) : node_base(ctor) {
+		explicit BufferError(dryad::node_ctor ctor, ErrorSymbolInterner::symbol_type message) : node_base(ctor) {
 			_set_message(message);
 		}
 
@@ -103,7 +111,7 @@ namespace ovdl::error {
 
 	struct Annotation : dryad::abstract_node_range<Error, ErrorKind::FirstAnnotation, ErrorKind::LastAnnotation> {
 	protected:
-		explicit Annotation(dryad::node_ctor ctor, ErrorKind kind, const char* message) : node_base(ctor, kind) {
+		explicit Annotation(dryad::node_ctor ctor, ErrorKind kind, ErrorSymbolInterner::symbol_type message) : node_base(ctor, kind) {
 			_set_message(message);
 		}
 	};
@@ -129,7 +137,7 @@ namespace ovdl::error {
 	protected:
 		explicit ParseError(dryad::node_ctor ctor,
 			ErrorKind kind,
-			const char* message,
+			ErrorSymbolInterner::symbol_type message,
 			const char* production_name)
 			: node_base(ctor, kind),
 			  _production_name(production_name) {
@@ -143,7 +151,7 @@ namespace ovdl::error {
 	struct _ParseError_t : dryad::basic_node<NodeKind, ParseError> {
 		using base_node = dryad::basic_node<NodeKind, ParseError>;
 
-		explicit _ParseError_t(dryad::node_ctor ctor, const char* message, const char* production_name)
+		explicit _ParseError_t(dryad::node_ctor ctor, ErrorSymbolInterner::symbol_type message, const char* production_name)
 			: base_node(ctor, message, production_name) {}
 	};
 
@@ -157,12 +165,12 @@ namespace ovdl::error {
 		explicit Semantic(dryad::node_ctor ctor, ErrorKind kind)
 			: node_base(ctor, kind) {};
 
-		explicit Semantic(dryad::node_ctor ctor, ErrorKind kind, const char* message)
+		explicit Semantic(dryad::node_ctor ctor, ErrorKind kind, ErrorSymbolInterner::symbol_type message)
 			: node_base(ctor, kind) {
 			_set_message(message);
 		};
 
-		explicit Semantic(dryad::node_ctor ctor, ErrorKind kind, const char* message, AnnotationList annotations)
+		explicit Semantic(dryad::node_ctor ctor, ErrorKind kind, ErrorSymbolInterner::symbol_type message, AnnotationList annotations)
 			: node_base(ctor, kind) {
 			push_back(annotations);
 			_set_message(message);
@@ -176,10 +184,10 @@ namespace ovdl::error {
 		explicit _SemanticError_t(dryad::node_ctor ctor)
 			: base_node(ctor) {}
 
-		explicit _SemanticError_t(dryad::node_ctor ctor, const char* message)
+		explicit _SemanticError_t(dryad::node_ctor ctor, ErrorSymbolInterner::symbol_type message)
 			: base_node(ctor, message) {}
 
-		explicit _SemanticError_t(dryad::node_ctor ctor, const char* message, AnnotationList annotations)
+		explicit _SemanticError_t(dryad::node_ctor ctor, ErrorSymbolInterner::symbol_type message, AnnotationList annotations)
 			: base_node(ctor, message, annotations) {}
 	};
 
@@ -192,7 +200,7 @@ namespace ovdl::error {
 
 	template<ErrorKind NodeKind>
 	struct _Annotation_t : dryad::basic_node<NodeKind, Annotation> {
-		explicit _Annotation_t(dryad::node_ctor ctor, const char* message)
+		explicit _Annotation_t(dryad::node_ctor ctor, ErrorSymbolInterner::symbol_type message)
 			: dryad::basic_node<NodeKind, Annotation>(ctor, message) {}
 	};
 

include/openvic-dataloader/csv/Parser.hpp

@@ -42,6 +42,8 @@ namespace ovdl::csv {
 		using error_range = ovdl::detail::error_range<error::Root>;
 		Parser::error_range get_errors() const;
 
+		std::string_view error(const ovdl::error::Error* error) const;
+
 		const FilePosition get_error_position(const error::Error* error) const;
 
 		void print_errors_to(std::basic_ostream<char>& stream) const;

include/openvic-dataloader/v2script/Parser.hpp

@@ -61,6 +61,8 @@ namespace ovdl::v2script {
 		using error_range = ovdl::detail::error_range<error::Root>;
 		Parser::error_range get_errors() const;
 
+		std::string_view error(const ovdl::error::Error* error) const;
+
 		const FilePosition get_error_position(const error::Error* error) const;
 
 		void print_errors_to(std::basic_ostream<char>& stream) const;

src/openvic-dataloader/AbstractSyntaxTree.hpp

@@ -3,6 +3,7 @@
 #include <concepts>
 #include <cstdio>
 #include <string_view>
+#include <type_traits>
 #include <utility>
 
 #include <openvic-dataloader/NodeLocation.hpp>
@@ -51,12 +52,12 @@ namespace ovdl {
 		using node_type = typename file_type::node_type;
 
 		explicit BasicAbstractSyntaxTree(file_type&& file)
-			: AbstractSyntaxTree(file.size()),
+			: AbstractSyntaxTree(file.size() * file.visit_buffer([](auto&& buffer) -> size_t { return sizeof(typename std::decay_t<decltype(buffer)>::char_type); })),
 			  _file { std::move(file) } {}
 
 		template<typename Encoding, typename MemoryResource = void>
 		explicit BasicAbstractSyntaxTree(lexy::buffer<Encoding, MemoryResource>&& buffer)
-			: AbstractSyntaxTree(buffer.size()),
+			: AbstractSyntaxTree(buffer.size() * sizeof(Encoding::char_type)),
 			  _file { std::move(buffer) } {}
 
 		void set_location(const node_type* n, NodeLocation loc) {

src/openvic-dataloader/DiagnosticLogger.hpp

@@ -2,6 +2,7 @@
 
 #include <concepts> // IWYU pragma: keep
 #include <cstdio>
+#include <iostream>
 #include <ostream>
 #include <string>
 #include <type_traits>
@@ -37,12 +38,7 @@ namespace ovdl {
 	template<typename ParseState>
 	struct BasicDiagnosticLogger;
 
-	struct DiagnosticLogger {
-		struct SymbolId;
-		using index_type = std::uint32_t;
-		using symbol_type = dryad::symbol<SymbolId, index_type>;
-		using symbol_interner_type = dryad::symbol_interner<SymbolId, char, index_type>;
-
+	struct DiagnosticLogger : error::ErrorSymbolInterner {
 		using AnnotationKind = lexy_ext::annotation_kind;
 		using DiagnosticKind = lexy_ext::diagnostic_kind;
 
@@ -116,23 +112,23 @@ namespace ovdl {
 						}
 						result = writer.error();
 					} else {
-						auto production = _logger.intern_cstr(production_name);
+						auto production = production_name;
 						if constexpr (std::is_same_v<Tag, lexy::expected_literal>) {
 							auto string = lexy::_detail::make_literal_lexeme<typename Reader::encoding>(error.string(), error.length());
 							NodeLocation loc = NodeLocation::make_from(context.position(), error.position() - 1);
-							auto message = _logger.intern_cstr(fmt::format("expected '{}'", string.data()));
+							auto message = _logger.intern(fmt::format("expected '{}'", string.data()));
 							result = _logger.template create<error::ExpectedLiteral>(loc, message, production);
 						} else if constexpr (std::is_same_v<Tag, lexy::expected_keyword>) {
 							auto string = lexy::_detail::make_literal_lexeme<typename Reader::encoding>(error.string(), error.length());
 							NodeLocation loc = NodeLocation::make_from(context.position(), error.position() - 1);
-							auto message = _logger.intern_cstr(fmt::format("expected keyword '{}'", string.data()));
+							auto message = _logger.intern(fmt::format("expected keyword '{}'", string.data()));
 							result = _logger.template create<error::ExpectedKeyword>(loc, message, production);
 						} else if constexpr (std::is_same_v<Tag, lexy::expected_char_class>) {
-							auto message = _logger.intern_cstr(fmt::format("expected {}", error.name()));
+							auto message = _logger.intern(fmt::format("expected {}", error.name()));
 							result = _logger.template create<error::ExpectedCharClass>(error.position(), message, production);
 						} else {
 							NodeLocation loc = NodeLocation::make_from(error.begin(), error.end());
-							auto message = _logger.intern_cstr(error.message());
+							auto message = _logger.intern(error.message());
 							result = _logger.template create<error::GenericParseError>(loc, message, production);
 						}
 					}
@@ -361,7 +357,7 @@ namespace ovdl {
 				});
 
 				error::Annotation* annotation;
-				auto message = _logger.intern_cstr(output);
+				auto message = _logger.intern(output);
 				switch (kind) {
 					case AnnotationKind::primary:
 						annotation = _logger.create<error::PrimaryAnnotation>(loc, message);
@@ -404,7 +400,7 @@ namespace ovdl {
 				});
 			impl.write_path(iter, file().path());
 
-			auto message = intern_cstr(output);
+			auto message = intern(output);
 			error->_set_message(message);
 			if (!error->is_linked_in_tree())
 				insert(error);
@@ -422,8 +418,8 @@ namespace ovdl {
 				});
 			impl.write_path(iter, file().path());
 
-			auto production = intern_cstr(production_name);
-			auto message = intern_cstr(output);
+			auto production = production_name;
+			auto message = intern(output);
 			auto* error = [&] {
 				if constexpr (std::is_same_v<Tag, lexy::expected_literal>) {
 					return create<error::ExpectedLiteral>(loc, message, production);

src/openvic-dataloader/csv/CsvGrammar.hpp

@@ -11,10 +11,7 @@
 
 #include <lexy/_detail/config.hpp>
 #include <lexy/callback.hpp>
-#include <lexy/callback/string.hpp>
 #include <lexy/dsl.hpp>
-#include <lexy/dsl/ascii.hpp>
-#include <lexy/dsl/option.hpp>
 #include <lexy/encoding.hpp>
 
 #include "detail/Convert.hpp"
@@ -73,6 +70,9 @@ namespace ovdl::csv::grammar {
 	constexpr auto escaped_quote = lexy::symbol_table<char> //
 									   .map<'"'>('"');
 
+	constexpr auto escaped_newline = lexy::symbol_table<char> //
+										 .map<'n'>('\n');
+
 	template<ParseOptions Options>
 	struct CsvGrammar {
 		struct StringValue : lexy::scan_production<std::string>,
@@ -118,6 +118,11 @@ namespace ovdl::csv::grammar {
 			template<auto character>
 			static constexpr auto _escape_check = character - (lexy::dsl::lit_b<Options.SepChar> / lexy::dsl::ascii::newline);
 
+			struct Backslash {
+				static constexpr auto rule = LEXY_LIT("\\n");
+				static constexpr auto value = lexy::constant('\n');
+			};
+
 			template<typename Context, typename Reader>
 			static constexpr scan_result scan(lexy::rule_scanner<Context, Reader>& scanner, detail::IsFileParseState auto& state) {
 				using encoding = typename Reader::encoding;
@@ -134,13 +139,16 @@ namespace ovdl::csv::grammar {
 					if constexpr (Options.SupportStrings) {
 						return lexy::dsl::identifier(character - (lexy::dsl::lit_b<Options.SepChar> / lexy::dsl::ascii::newline));
 					} else {
-						auto escape_check_char = _escape_check<character>;
-						auto id_check_char = escape_check_char - lexy::dsl::lit_b<'\\'>;
-						auto id_segment = lexy::dsl::identifier(id_check_char);
-						auto escape_segement = lexy::dsl::token(escape_check_char);
-						auto escape_sym = lexy::dsl::symbol<escaped_symbols>(escape_segement);
-						auto escape_rule = lexy::dsl::lit_b<'\\'> >> escape_sym;
-						return lexy::dsl::list(id_segment | escape_rule);
+						constexpr auto backslash = lexy::dsl::lit_b<'\\'>;
+
+						constexpr auto escape_check_char = _escape_check<character>;
+						constexpr auto escape_rule = lexy::dsl::p<Backslash>;
+
+						return lexy::dsl::list(
+							lexy::dsl::identifier(escape_check_char - backslash) |
+							escape_rule |
+							lexy::dsl::capture(escape_check_char) //
+						);
 					}
 				}();
 

src/openvic-dataloader/csv/Parser.cpp

@@ -195,6 +195,10 @@ typename Parser::error_range Parser::get_errors() const {
 	return _parse_handler->get_errors();
 }
 
+std::string_view Parser::error(const ovdl::error::Error* error) const {
+	return error->message(_parse_handler->parse_state().logger().symbol_interner());
+}
+
 const FilePosition Parser::get_error_position(const error::Error* error) const {
 	if (!error || !error->is_linked_in_tree()) {
 		return {};
@@ -231,20 +235,20 @@ void Parser::print_errors_to(std::basic_ostream<char>& stream) const {
 		dryad::visit_tree(
 			error,
 			[&](const error::BufferError* buffer_error) {
-				stream << "buffer error: " << buffer_error->message() << '\n';
+				stream << "buffer error: " << this->error(buffer_error) << '\n';
 			},
 			[&](dryad::child_visitor<error::ErrorKind> visitor, const error::AnnotatedError* annotated_error) {
-				stream << annotated_error->message() << '\n';
+				stream << this->error(annotated_error) << '\n';
 				auto annotations = annotated_error->annotations();
 				for (auto annotation : annotations) {
 					visitor(annotation);
 				}
 			},
 			[&](const error::PrimaryAnnotation* primary) {
-				stream << primary->message() << '\n';
+				stream << this->error(primary) << '\n';
 			},
 			[&](const error::SecondaryAnnotation* secondary) {
-				stream << secondary->message() << '\n';
+				stream << this->error(secondary) << '\n';
 			});
 	}
 }

src/openvic-dataloader/v2script/Parser.cpp

@@ -320,6 +320,10 @@ Parser::error_range Parser::get_errors() const {
 	return _parse_handler->get_errors();
 }
 
+std::string_view Parser::error(const ovdl::error::Error* error) const {
+	return error->message(_parse_handler->parse_state().logger().symbol_interner());
+}
+
 const FilePosition Parser::get_error_position(const error::Error* error) const {
 	if (!error || !error->is_linked_in_tree()) {
 		return {};
@@ -352,20 +356,20 @@ void Parser::print_errors_to(std::basic_ostream<char>& stream) const {
 		dryad::visit_tree(
 			error,
 			[&](const error::BufferError* buffer_error) {
-				stream << "buffer error: " << buffer_error->message() << '\n';
+				stream << "buffer error: " << this->error(buffer_error) << '\n';
 			},
 			[&](dryad::child_visitor<error::ErrorKind> visitor, const error::AnnotatedError* annotated_error) {
-				stream << annotated_error->message() << '\n';
+				stream << this->error(annotated_error) << '\n';
 				auto annotations = annotated_error->annotations();
 				for (auto annotation : annotations) {
 					visitor(annotation);
 				}
 			},
 			[&](const error::PrimaryAnnotation* primary) {
-				stream << primary->message() << '\n';
+				stream << this->error(primary) << '\n';
 			},
 			[&](const error::SecondaryAnnotation* secondary) {
-				stream << secondary->message() << '\n';
+				stream << this->error(secondary) << '\n';
 			});
 	}
 }