Overload StringUtilities get_token(...) function to add escape characters

[waewos]

For example the delimiter <,> is used to separate a string into tokens.

Let's say the string looks like this:
<123,String,"String1,String2">

When calling get_token with <,> as delimiter, the result would be:

• 123
• String
• "String1
• String2"

It would be useful in many cases, to be able to declare an escape character, in the case above, it would be nice to be able to tell the function to escape separating by <,> when a <"> is found and start using <,> again after the second <"> has been found, so that the result looks like this:

• 123
• String
• "String1, String2"

Also if more than one escape character has been specified, the other escape characters should be ignored until the escape sequence ended. For example:

Escape characters are: <"> and <'>

<123,String,"String'1',String'2'"> should become:

• 123
• String
• "String'1',String'2'"

The code Below is a quick and dirty solution which I'm currently using for testing, so probably not the thing anyone wants to show up in production.

  //***************************************************************************
  /// get_token
  //***************************************************************************
  template <typename TInput, typename TStringView>
  etl::optional<TStringView> get_token(const TInput& input, typename TInput::const_pointer delimiters, typename TInput::const_pointer delimiters_esc, const etl::optional<TStringView>& last_view, bool ignore_empty_tokens)
  {
    using const_pointer = typename TInput::const_pointer;

    bool token_found = false;
    typename TStringView::size_type position  = 0U;
    TStringView view = last_view.value_or(TStringView());
    const_pointer begin_ptr = input.data();

    if (begin_ptr == ETL_NULLPTR)
    {
      return etl::optional<TStringView>();
    }

    const_pointer end_ptr   = begin_ptr + input.size();

    while (!token_found)
    {
      // Does the last view have valid data?
      if (view.data() != ETL_NULLPTR)
      {
        position = etl::distance(begin_ptr, view.data() + view.size() + 1U);

        // Have we reached the end of the string?
        if (position > input.size())
        {
          return etl::optional<TStringView>();
        }
      }

      // Look for the next token.
      const_pointer first_ptr = begin_ptr + position;
      const_pointer last_ptr  = find_first_of(first_ptr, end_ptr, delimiters);

      // MODIFICATION STARTING HERE ////////////////////////////////////////////////
      const_pointer first_esc_ptr = find_first_of(first_ptr, end_ptr, delimiters_esc);

      auto first_esc_ptr_tmp = *first_esc_ptr;

      // brace types -> ({[<>]}) if escape delimiter is one of the opening braces
      // automatically wait for the closing version of it to end the escape sequence!
      if (first_esc_ptr_tmp != '\0') {

        switch (first_esc_ptr_tmp) {
        case '(': {
          first_esc_ptr_tmp = ')';
          break;
        }
        case '{': {
          first_esc_ptr_tmp = '}';
          break;
        }
        case '[': {
          first_esc_ptr_tmp = ']';
          break;
        }
        case '<': {
          first_esc_ptr_tmp = '>';
          break;
        }
        default:
          // do nothing, first_esc_ptr_tmp stays what it is because <"> closes with <">
          break;
        }

        const_pointer last_esc_ptr = find_first_of(first_esc_ptr + 1, end_ptr, &first_esc_ptr_tmp);

        if (first_esc_ptr < last_ptr) {
          last_ptr  = find_first_of(last_esc_ptr, end_ptr, delimiters);
        }
      }
      // MODIFICATION ENDING HERE ////////////////////////////////////////////////

      view = TStringView(first_ptr, etl::distance(first_ptr, last_ptr));

      token_found = ((view.size() != 0U) || !ignore_empty_tokens);
    }

    return etl::optional<TStringView>(view);
  }