D ui3 123 receiving non native attributes revit params gh attributes …

…etc (#3526)

* collect attributes as strings (Brep randomly fails)

* pass function to get object attributes

* update passed function; add traversal for Revit parameter wrapper Base

* remove stackOverflowException

* fix field path functions

* don't add already existing layers

* add Rhino userStrings; get only dynamic values from Objects.Geometry; prepare to get correct FieldType

* some restructuring

* speed up check for existing datasets

* only use "parameters" Base obj, if the elements are actually RevitParameters

* only search for existing Tables if Feature Class wasn't already deleted

* get FieldType from actual values, default to String if needed

* cast bool to string

* receive only Typed primitive properties from non-native apps

* typo

* skip the properties for now

* Revert "skip the properties for now"

This reverts commit c6a1031.

* Revert "typo"

This reverts commit 37f6b9a.

* Revert "receive only Typed primitive properties from non-native apps"

This reverts commit 9754ac5.

* cast to string

* option for dealing with nullable types (#3528)

* option for dealing with nullable types

* fmt

* remove unnecessary null check

* default to null, if cast to string fails

---------

Co-authored-by: Adam Hathcock <[email protected]>

DUI3-DX/Converters/ArcGIS/Speckle.Converters.ArcGIS3/Utils/ArcGISFieldUtils.cs

@@ -1,6 +1,8 @@
+using System.Collections;
 using ArcGIS.Core.Data;
 using ArcGIS.Core.Data.Exceptions;
 using Objects.GIS;
+using Speckle.Core.Logging;
 using Speckle.Core.Models;
 using FieldDescription = ArcGIS.Core.Data.DDL.FieldDescription;
 
@@ -16,38 +18,39 @@ public ArcGISFieldUtils(ICharacterCleaner characterCleaner)
     _characterCleaner = characterCleaner;
   }
 
-  public RowBuffer AssignFieldValuesToRow(RowBuffer rowBuffer, List<FieldDescription> fields, GisFeature feat)
+  public RowBuffer AssignFieldValuesToRow(
+    RowBuffer rowBuffer,
+    List<FieldDescription> fields,
+    Dictionary<string, object?> attributes
+  )
   {
     foreach (FieldDescription field in fields)
     {
       // try to assign values to writeable fields
-      if (feat.attributes is not null)
+      if (attributes is not null)
       {
         string key = field.AliasName; // use Alias, as Name is simplified to alphanumeric
         FieldType fieldType = field.FieldType;
-        var value = feat.attributes[key];
-        if (value is not null)
+        var value = attributes[key];
+
+        try
         {
-          // POC: get all values in a correct format
-          try
-          {
-            rowBuffer[key] = GISAttributeFieldType.SpeckleValueToNativeFieldType(fieldType, value);
-          }
-          catch (GeodatabaseFeatureException)
-          {
-            //'The value type is incompatible.'
-            // log error!
-            rowBuffer[key] = null;
-          }
-          catch (GeodatabaseFieldException)
-          {
-            // non-editable Field, do nothing
-          }
+          rowBuffer[key] = GISAttributeFieldType.SpeckleValueToNativeFieldType(fieldType, value);
         }
-        else
+        catch (GeodatabaseFeatureException)
         {
+          //'The value type is incompatible.'
+          // log error!
           rowBuffer[key] = null;
         }
+        catch (GeodatabaseFieldException)
+        {
+          // non-editable Field, do nothing
+        }
+        catch (GeodatabaseGeneralException)
+        {
+          // The index passed was not within the valid range. // unclear reason of the error
+        }
       }
     }
     return rowBuffer;
@@ -88,4 +91,186 @@ public List<FieldDescription> GetFieldsFromSpeckleLayer(VectorLayer target)
     }
     return fields;
   }
+
+  public List<(FieldDescription, Func<Base, object?>)> CreateFieldsFromListOfBase(List<Base> target)
+  {
+    List<(FieldDescription, Func<Base, object?>)> fieldsAndFunctions = new();
+    List<string> fieldAdded = new();
+
+    foreach (var baseObj in target)
+    {
+      // get all members by default, but only Dynamic ones from the basic geometry
+      Dictionary<string, object?> members = new();
+
+      // leave out until we decide which properties to support on Receive
+      /*
+      if (baseObj.speckle_type.StartsWith("Objects.Geometry"))
+      {
+        members = baseObj.GetMembers(DynamicBaseMemberType.Dynamic);
+      }
+      else
+      {
+        members = baseObj.GetMembers(DynamicBaseMemberType.All);
+      }
+      */
+
+      foreach (KeyValuePair<string, object?> field in members)
+      {
+        // POC: TODO check for the forbidden characters/combinations: https://support.esri.com/en-us/knowledge-base/what-characters-should-not-be-used-in-arcgis-for-field--000005588
+        Func<Base, object?> function = x => x[field.Key];
+        TraverseAttributes(field, function, fieldsAndFunctions, fieldAdded);
+      }
+    }
+
+    // change all FieldType.Blob to String
+    // "Blob" will never be used on receive, so it is a placeholder for non-properly identified fields
+    for (int i = 0; i < fieldsAndFunctions.Count; i++)
+    {
+      (FieldDescription description, Func<Base, object?> function) = fieldsAndFunctions[i];
+      if (description.FieldType is FieldType.Blob)
+      {
+        fieldsAndFunctions[i] = new(
+          new FieldDescription(description.Name, FieldType.String) { AliasName = description.AliasName },
+          function
+        );
+      }
+    }
+
+    return fieldsAndFunctions;
+  }
+
+  private void TraverseAttributes(
+    KeyValuePair<string, object?> field,
+    Func<Base, object?> function,
+    List<(FieldDescription, Func<Base, object?>)> fieldsAndFunctions,
+    List<string> fieldAdded
+  )
+  {
+    if (field.Value is Base attributeBase)
+    {
+      // only traverse Base if it's Rhino userStrings, or Revit parameter, or Base containing Revit parameters
+      if (field.Key == "parameters")
+      {
+        foreach (KeyValuePair<string, object?> attributField in attributeBase.GetMembers(DynamicBaseMemberType.Dynamic))
+        {
+          // only iterate through elements if they are actually Revit Parameters or parameter IDs
+          if (
+            attributField.Value is Objects.BuiltElements.Revit.Parameter
+            || attributField.Key == "applicationId"
+            || attributField.Key == "id"
+          )
+          {
+            KeyValuePair<string, object?> newAttributField =
+              new($"{field.Key}.{attributField.Key}", attributField.Value);
+            Func<Base, object?> functionAdded = x => (function(x) as Base)?[attributField.Key];
+            TraverseAttributes(newAttributField, functionAdded, fieldsAndFunctions, fieldAdded);
+          }
+        }
+      }
+      else if (field.Key == "userStrings")
+      {
+        foreach (KeyValuePair<string, object?> attributField in attributeBase.GetMembers(DynamicBaseMemberType.Dynamic))
+        {
+          KeyValuePair<string, object?> newAttributField = new($"{field.Key}.{attributField.Key}", attributField.Value);
+          Func<Base, object?> functionAdded = x => (function(x) as Base)?[attributField.Key];
+          TraverseAttributes(newAttributField, functionAdded, fieldsAndFunctions, fieldAdded);
+        }
+      }
+      else if (field.Value is Objects.BuiltElements.Revit.Parameter)
+      {
+        foreach (
+          KeyValuePair<string, object?> attributField in attributeBase.GetMembers(DynamicBaseMemberType.Instance)
+        )
+        {
+          KeyValuePair<string, object?> newAttributField = new($"{field.Key}.{attributField.Key}", attributField.Value);
+          Func<Base, object?> functionAdded = x => (function(x) as Base)?[attributField.Key];
+          TraverseAttributes(newAttributField, functionAdded, fieldsAndFunctions, fieldAdded);
+        }
+      }
+      else
+      {
+        // for now, ignore all other properties of Base type
+      }
+    }
+    else if (field.Value is IList attributeList)
+    {
+      int count = 0;
+      foreach (var attributField in attributeList)
+      {
+        KeyValuePair<string, object?> newAttributField = new($"{field.Key}[{count}]", attributField);
+        Func<Base, object?> functionAdded = x => (function(x) as List<object?>)?[count];
+        TraverseAttributes(newAttributField, functionAdded, fieldsAndFunctions, fieldAdded);
+        count += 1;
+      }
+    }
+    else
+    {
+      TryAddField(field, function, fieldsAndFunctions, fieldAdded);
+    }
+  }
+
+  private void TryAddField(
+    KeyValuePair<string, object?> field,
+    Func<Base, object?> function,
+    List<(FieldDescription, Func<Base, object?>)> fieldsAndFunctions,
+    List<string> fieldAdded
+  )
+  {
+    try
+    {
+      string key = field.Key;
+      string cleanKey = _characterCleaner.CleanCharacters(key);
+
+      if (cleanKey == FID_FIELD_NAME) // we cannot add field with reserved name
+      {
+        return;
+      }
+
+      if (!fieldAdded.Contains(cleanKey))
+      {
+        // use field.Value to define FieldType
+        FieldType fieldType = GISAttributeFieldType.GetFieldTypeFromRawValue(field.Value);
+
+        FieldDescription fieldDescription = new(cleanKey, fieldType) { AliasName = key };
+        fieldsAndFunctions.Add((fieldDescription, function));
+        fieldAdded.Add(cleanKey);
+      }
+      else
+      {
+        // if field exists, check field.Value again, and revise FieldType if needed
+        int index = fieldsAndFunctions.TakeWhile(x => x.Item1.Name != cleanKey).Count();
+
+        (FieldDescription, Func<Base, object?>) itemInList;
+        try
+        {
+          itemInList = fieldsAndFunctions[index];
+        }
+        catch (Exception ex) when (!ex.IsFatal())
+        {
+          return;
+        }
+
+        FieldType existingFieldType = itemInList.Item1.FieldType;
+        FieldType newFieldType = GISAttributeFieldType.GetFieldTypeFromRawValue(field.Value);
+
+        // adjust FieldType if needed, default everything to Strings if fields types differ:
+        // 1. change to NewType, if old type was undefined ("Blob")
+        // 2. change to NewType if it's String (and the old one is not)
+        if (
+          newFieldType != FieldType.Blob && existingFieldType == FieldType.Blob
+          || (newFieldType == FieldType.String && existingFieldType != FieldType.String)
+        )
+        {
+          fieldsAndFunctions[index] = (
+            new FieldDescription(itemInList.Item1.Name, newFieldType) { AliasName = itemInList.Item1.AliasName },
+            itemInList.Item2
+          );
+        }
+      }
+    }
+    catch (GeodatabaseFieldException)
+    {
+      // do nothing
+    }
+  }
 }

DUI3-DX/Converters/ArcGIS/Speckle.Converters.ArcGIS3/Utils/FeatureClassUtils.cs

@@ -22,7 +22,15 @@ public void AddFeaturesToTable(Table newFeatureClass, List<GisFeature> gisFeatur
     {
       using (RowBuffer rowBuffer = newFeatureClass.CreateRowBuffer())
       {
-        newFeatureClass.CreateRow(_fieldsUtils.AssignFieldValuesToRow(rowBuffer, fields, feat)).Dispose();
+        newFeatureClass
+          .CreateRow(
+            _fieldsUtils.AssignFieldValuesToRow(
+              rowBuffer,
+              fields,
+              feat.attributes.GetMembers(DynamicBaseMemberType.Dynamic)
+            )
+          )
+          .Dispose();
       }
     }
   }
@@ -50,34 +58,55 @@ public void AddFeaturesToFeatureClass(
         }
 
         // get attributes
-        newFeatureClass.CreateRow(_fieldsUtils.AssignFieldValuesToRow(rowBuffer, fields, feat)).Dispose();
+        newFeatureClass
+          .CreateRow(
+            _fieldsUtils.AssignFieldValuesToRow(
+              rowBuffer,
+              fields,
+              feat.attributes.GetMembers(DynamicBaseMemberType.Dynamic)
+            )
+          )
+          .Dispose();
       }
     }
   }
 
   public void AddNonGISFeaturesToFeatureClass(
     FeatureClass newFeatureClass,
-    List<ACG.Geometry> features,
-    List<FieldDescription> fields
+    List<(Base baseObj, ACG.Geometry convertedGeom)> featuresTuples,
+    List<(FieldDescription, Func<Base, object?>)> fieldsAndFunctions
   )
   {
-    foreach (ACG.Geometry geom in features)
+    foreach ((Base baseObj, ACG.Geometry geom) in featuresTuples)
     {
       using (RowBuffer rowBuffer = newFeatureClass.CreateRowBuffer())
       {
         ACG.Geometry newGeom = geom;
+
+        // exception for Points: turn into MultiPoint layer
         if (geom is ACG.MapPoint pointGeom)
         {
           newGeom = new ACG.MultipointBuilderEx(
             new List<ACG.MapPoint>() { pointGeom },
             ACG.AttributeFlags.HasZ
           ).ToGeometry();
         }
+
         rowBuffer[newFeatureClass.GetDefinition().GetShapeField()] = newGeom;
 
-        // TODO: get attributes
-        // newFeatureClass.CreateRow(_fieldsUtils.AssignFieldValuesToRow(rowBuffer, fields, feat)).Dispose();
-        newFeatureClass.CreateRow(rowBuffer).Dispose();
+        // set and pass attributes
+        Dictionary<string, object?> attributes = new();
+        foreach ((FieldDescription field, Func<Base, object?> function) in fieldsAndFunctions)
+        {
+          string key = field.AliasName;
+          attributes[key] = function(baseObj);
+        }
+        // newFeatureClass.CreateRow(rowBuffer).Dispose(); // without extra attributes
+        newFeatureClass
+          .CreateRow(
+            _fieldsUtils.AssignFieldValuesToRow(rowBuffer, fieldsAndFunctions.Select(x => x.Item1).ToList(), attributes)
+          )
+          .Dispose();
       }
     }
   }

DUI3-DX/Converters/ArcGIS/Speckle.Converters.ArcGIS3/Utils/GISAttributeFieldType.cs

@@ -90,30 +90,52 @@ public static FieldType FieldTypeToNative(object fieldType)
         return value;
     }
 
-    if (value is not null)
+    if (value != null)
     {
       try
       {
+        static object? GetValue(string? s, Func<string, object> func) => s is null ? null : func(s);
         return fieldType switch
         {
-          FieldType.String => (string)value,
+          FieldType.String => Convert.ToString(value),
           FieldType.Single => Convert.ToSingle(value),
           FieldType.Integer => Convert.ToInt32(value), // need this step because sent "ints" seem to be received as "longs"
           FieldType.BigInteger => Convert.ToInt64(value),
           FieldType.SmallInteger => Convert.ToInt16(value),
           FieldType.Double => Convert.ToDouble(value),
-          FieldType.Date => DateTime.Parse((string)value, null),
-          FieldType.DateOnly => DateOnly.Parse((string)value),
-          FieldType.TimeOnly => TimeOnly.Parse((string)value),
+          FieldType.Date => GetValue(value.ToString(), s => DateTime.Parse(s, null)),
+          FieldType.DateOnly => GetValue(value.ToString(), s => DateOnly.Parse(s, null)),
+          FieldType.TimeOnly => GetValue(value.ToString(), s => TimeOnly.Parse(s, null)),
           _ => value,
         };
       }
-      catch (InvalidCastException)
+      catch (Exception ex) when (ex is InvalidCastException or FormatException or ArgumentNullException)
       {
-        return value;
+        return null;
       }
     }
+    else
+    {
+      return null;
+    }
+  }
+
+  public static FieldType GetFieldTypeFromRawValue(object? value)
+  {
+    // using "Blob" as a placeholder for unrecognized values/nulls.
+    // Once all elements are iterated, FieldType.Blob will be replaced with FieldType.String if no better type found
+    if (value is not null)
+    {
+      return value switch
+      {
+        string => FieldType.String,
+        int => FieldType.Integer,
+        long => FieldType.BigInteger,
+        double => FieldType.Double,
+        _ => FieldType.Blob,
+      };
+    }
 
-    return value;
+    return FieldType.Blob;
   }
 }