From 2930bcf90c50f2c64138259e8482cc35b8888460 Mon Sep 17 00:00:00 2001 From: KatKatKateryna <89912278+KatKatKateryna@users.noreply.github.com> Date: Fri, 21 Jun 2024 18:45:54 +0800 Subject: [PATCH] D ui3 123 receiving non native attributes revit params gh attributes etc (#3526) * collect attributes as strings (Brep randomly fails) * pass function to get object attributes * update passed function; add traversal for Revit parameter wrapper Base * remove stackOverflowException * fix field path functions * don't add already existing layers * add Rhino userStrings; get only dynamic values from Objects.Geometry; prepare to get correct FieldType * some restructuring * speed up check for existing datasets * only use "parameters" Base obj, if the elements are actually RevitParameters * only search for existing Tables if Feature Class wasn't already deleted * get FieldType from actual values, default to String if needed * cast bool to string * receive only Typed primitive properties from non-native apps * typo * skip the properties for now * Revert "skip the properties for now" This reverts commit c6a103146d3200d65e09644ef074f22de8e8445e. * Revert "typo" This reverts commit 37f6b9a41b03c39bfee77d4fe3cb3a8ce932b646. * Revert "receive only Typed primitive properties from non-native apps" This reverts commit 9754ac58759eba077a12f3c77dee9b19f513f742. * cast to string * option for dealing with nullable types (#3528) * option for dealing with nullable types * fmt * remove unnecessary null check * default to null, if cast to string fails --------- Co-authored-by: Adam Hathcock --- .../Utils/ArcGISFieldUtils.cs | 225 ++++++++++++++++-- .../Utils/FeatureClassUtils.cs | 45 +++- .../Utils/GISAttributeFieldType.cs | 38 ++- .../Utils/IArcGISFieldUtils.cs | 9 +- .../Utils/IFeatureClassUtils.cs | 4 +- .../Utils/NonNativeFeaturesUtils.cs | 39 +-- 6 files changed, 306 insertions(+), 54 deletions(-) diff --git a/DUI3-DX/Converters/ArcGIS/Speckle.Converters.ArcGIS3/Utils/ArcGISFieldUtils.cs b/DUI3-DX/Converters/ArcGIS/Speckle.Converters.ArcGIS3/Utils/ArcGISFieldUtils.cs index 1f98758611..aff09a3d2c 100644 --- a/DUI3-DX/Converters/ArcGIS/Speckle.Converters.ArcGIS3/Utils/ArcGISFieldUtils.cs +++ b/DUI3-DX/Converters/ArcGIS/Speckle.Converters.ArcGIS3/Utils/ArcGISFieldUtils.cs @@ -1,6 +1,8 @@ +using System.Collections; using ArcGIS.Core.Data; using ArcGIS.Core.Data.Exceptions; using Objects.GIS; +using Speckle.Core.Logging; using Speckle.Core.Models; using FieldDescription = ArcGIS.Core.Data.DDL.FieldDescription; @@ -16,38 +18,39 @@ public ArcGISFieldUtils(ICharacterCleaner characterCleaner) _characterCleaner = characterCleaner; } - public RowBuffer AssignFieldValuesToRow(RowBuffer rowBuffer, List fields, GisFeature feat) + public RowBuffer AssignFieldValuesToRow( + RowBuffer rowBuffer, + List fields, + Dictionary attributes + ) { foreach (FieldDescription field in fields) { // try to assign values to writeable fields - if (feat.attributes is not null) + if (attributes is not null) { string key = field.AliasName; // use Alias, as Name is simplified to alphanumeric FieldType fieldType = field.FieldType; - var value = feat.attributes[key]; - if (value is not null) + var value = attributes[key]; + + try { - // POC: get all values in a correct format - try - { - rowBuffer[key] = GISAttributeFieldType.SpeckleValueToNativeFieldType(fieldType, value); - } - catch (GeodatabaseFeatureException) - { - //'The value type is incompatible.' - // log error! - rowBuffer[key] = null; - } - catch (GeodatabaseFieldException) - { - // non-editable Field, do nothing - } + rowBuffer[key] = GISAttributeFieldType.SpeckleValueToNativeFieldType(fieldType, value); } - else + catch (GeodatabaseFeatureException) { + //'The value type is incompatible.' + // log error! rowBuffer[key] = null; } + catch (GeodatabaseFieldException) + { + // non-editable Field, do nothing + } + catch (GeodatabaseGeneralException) + { + // The index passed was not within the valid range. // unclear reason of the error + } } } return rowBuffer; @@ -88,4 +91,186 @@ public List GetFieldsFromSpeckleLayer(VectorLayer target) } return fields; } + + public List<(FieldDescription, Func)> CreateFieldsFromListOfBase(List target) + { + List<(FieldDescription, Func)> fieldsAndFunctions = new(); + List fieldAdded = new(); + + foreach (var baseObj in target) + { + // get all members by default, but only Dynamic ones from the basic geometry + Dictionary members = new(); + + // leave out until we decide which properties to support on Receive + /* + if (baseObj.speckle_type.StartsWith("Objects.Geometry")) + { + members = baseObj.GetMembers(DynamicBaseMemberType.Dynamic); + } + else + { + members = baseObj.GetMembers(DynamicBaseMemberType.All); + } + */ + + foreach (KeyValuePair field in members) + { + // POC: TODO check for the forbidden characters/combinations: https://support.esri.com/en-us/knowledge-base/what-characters-should-not-be-used-in-arcgis-for-field--000005588 + Func function = x => x[field.Key]; + TraverseAttributes(field, function, fieldsAndFunctions, fieldAdded); + } + } + + // change all FieldType.Blob to String + // "Blob" will never be used on receive, so it is a placeholder for non-properly identified fields + for (int i = 0; i < fieldsAndFunctions.Count; i++) + { + (FieldDescription description, Func function) = fieldsAndFunctions[i]; + if (description.FieldType is FieldType.Blob) + { + fieldsAndFunctions[i] = new( + new FieldDescription(description.Name, FieldType.String) { AliasName = description.AliasName }, + function + ); + } + } + + return fieldsAndFunctions; + } + + private void TraverseAttributes( + KeyValuePair field, + Func function, + List<(FieldDescription, Func)> fieldsAndFunctions, + List fieldAdded + ) + { + if (field.Value is Base attributeBase) + { + // only traverse Base if it's Rhino userStrings, or Revit parameter, or Base containing Revit parameters + if (field.Key == "parameters") + { + foreach (KeyValuePair attributField in attributeBase.GetMembers(DynamicBaseMemberType.Dynamic)) + { + // only iterate through elements if they are actually Revit Parameters or parameter IDs + if ( + attributField.Value is Objects.BuiltElements.Revit.Parameter + || attributField.Key == "applicationId" + || attributField.Key == "id" + ) + { + KeyValuePair newAttributField = + new($"{field.Key}.{attributField.Key}", attributField.Value); + Func functionAdded = x => (function(x) as Base)?[attributField.Key]; + TraverseAttributes(newAttributField, functionAdded, fieldsAndFunctions, fieldAdded); + } + } + } + else if (field.Key == "userStrings") + { + foreach (KeyValuePair attributField in attributeBase.GetMembers(DynamicBaseMemberType.Dynamic)) + { + KeyValuePair newAttributField = new($"{field.Key}.{attributField.Key}", attributField.Value); + Func functionAdded = x => (function(x) as Base)?[attributField.Key]; + TraverseAttributes(newAttributField, functionAdded, fieldsAndFunctions, fieldAdded); + } + } + else if (field.Value is Objects.BuiltElements.Revit.Parameter) + { + foreach ( + KeyValuePair attributField in attributeBase.GetMembers(DynamicBaseMemberType.Instance) + ) + { + KeyValuePair newAttributField = new($"{field.Key}.{attributField.Key}", attributField.Value); + Func functionAdded = x => (function(x) as Base)?[attributField.Key]; + TraverseAttributes(newAttributField, functionAdded, fieldsAndFunctions, fieldAdded); + } + } + else + { + // for now, ignore all other properties of Base type + } + } + else if (field.Value is IList attributeList) + { + int count = 0; + foreach (var attributField in attributeList) + { + KeyValuePair newAttributField = new($"{field.Key}[{count}]", attributField); + Func functionAdded = x => (function(x) as List)?[count]; + TraverseAttributes(newAttributField, functionAdded, fieldsAndFunctions, fieldAdded); + count += 1; + } + } + else + { + TryAddField(field, function, fieldsAndFunctions, fieldAdded); + } + } + + private void TryAddField( + KeyValuePair field, + Func function, + List<(FieldDescription, Func)> fieldsAndFunctions, + List fieldAdded + ) + { + try + { + string key = field.Key; + string cleanKey = _characterCleaner.CleanCharacters(key); + + if (cleanKey == FID_FIELD_NAME) // we cannot add field with reserved name + { + return; + } + + if (!fieldAdded.Contains(cleanKey)) + { + // use field.Value to define FieldType + FieldType fieldType = GISAttributeFieldType.GetFieldTypeFromRawValue(field.Value); + + FieldDescription fieldDescription = new(cleanKey, fieldType) { AliasName = key }; + fieldsAndFunctions.Add((fieldDescription, function)); + fieldAdded.Add(cleanKey); + } + else + { + // if field exists, check field.Value again, and revise FieldType if needed + int index = fieldsAndFunctions.TakeWhile(x => x.Item1.Name != cleanKey).Count(); + + (FieldDescription, Func) itemInList; + try + { + itemInList = fieldsAndFunctions[index]; + } + catch (Exception ex) when (!ex.IsFatal()) + { + return; + } + + FieldType existingFieldType = itemInList.Item1.FieldType; + FieldType newFieldType = GISAttributeFieldType.GetFieldTypeFromRawValue(field.Value); + + // adjust FieldType if needed, default everything to Strings if fields types differ: + // 1. change to NewType, if old type was undefined ("Blob") + // 2. change to NewType if it's String (and the old one is not) + if ( + newFieldType != FieldType.Blob && existingFieldType == FieldType.Blob + || (newFieldType == FieldType.String && existingFieldType != FieldType.String) + ) + { + fieldsAndFunctions[index] = ( + new FieldDescription(itemInList.Item1.Name, newFieldType) { AliasName = itemInList.Item1.AliasName }, + itemInList.Item2 + ); + } + } + } + catch (GeodatabaseFieldException) + { + // do nothing + } + } } diff --git a/DUI3-DX/Converters/ArcGIS/Speckle.Converters.ArcGIS3/Utils/FeatureClassUtils.cs b/DUI3-DX/Converters/ArcGIS/Speckle.Converters.ArcGIS3/Utils/FeatureClassUtils.cs index dce6ce6853..c80b8473f1 100644 --- a/DUI3-DX/Converters/ArcGIS/Speckle.Converters.ArcGIS3/Utils/FeatureClassUtils.cs +++ b/DUI3-DX/Converters/ArcGIS/Speckle.Converters.ArcGIS3/Utils/FeatureClassUtils.cs @@ -22,7 +22,15 @@ public void AddFeaturesToTable(Table newFeatureClass, List gisFeatur { using (RowBuffer rowBuffer = newFeatureClass.CreateRowBuffer()) { - newFeatureClass.CreateRow(_fieldsUtils.AssignFieldValuesToRow(rowBuffer, fields, feat)).Dispose(); + newFeatureClass + .CreateRow( + _fieldsUtils.AssignFieldValuesToRow( + rowBuffer, + fields, + feat.attributes.GetMembers(DynamicBaseMemberType.Dynamic) + ) + ) + .Dispose(); } } } @@ -50,22 +58,32 @@ public void AddFeaturesToFeatureClass( } // get attributes - newFeatureClass.CreateRow(_fieldsUtils.AssignFieldValuesToRow(rowBuffer, fields, feat)).Dispose(); + newFeatureClass + .CreateRow( + _fieldsUtils.AssignFieldValuesToRow( + rowBuffer, + fields, + feat.attributes.GetMembers(DynamicBaseMemberType.Dynamic) + ) + ) + .Dispose(); } } } public void AddNonGISFeaturesToFeatureClass( FeatureClass newFeatureClass, - List features, - List fields + List<(Base baseObj, ACG.Geometry convertedGeom)> featuresTuples, + List<(FieldDescription, Func)> fieldsAndFunctions ) { - foreach (ACG.Geometry geom in features) + foreach ((Base baseObj, ACG.Geometry geom) in featuresTuples) { using (RowBuffer rowBuffer = newFeatureClass.CreateRowBuffer()) { ACG.Geometry newGeom = geom; + + // exception for Points: turn into MultiPoint layer if (geom is ACG.MapPoint pointGeom) { newGeom = new ACG.MultipointBuilderEx( @@ -73,11 +91,22 @@ List fields ACG.AttributeFlags.HasZ ).ToGeometry(); } + rowBuffer[newFeatureClass.GetDefinition().GetShapeField()] = newGeom; - // TODO: get attributes - // newFeatureClass.CreateRow(_fieldsUtils.AssignFieldValuesToRow(rowBuffer, fields, feat)).Dispose(); - newFeatureClass.CreateRow(rowBuffer).Dispose(); + // set and pass attributes + Dictionary attributes = new(); + foreach ((FieldDescription field, Func function) in fieldsAndFunctions) + { + string key = field.AliasName; + attributes[key] = function(baseObj); + } + // newFeatureClass.CreateRow(rowBuffer).Dispose(); // without extra attributes + newFeatureClass + .CreateRow( + _fieldsUtils.AssignFieldValuesToRow(rowBuffer, fieldsAndFunctions.Select(x => x.Item1).ToList(), attributes) + ) + .Dispose(); } } } diff --git a/DUI3-DX/Converters/ArcGIS/Speckle.Converters.ArcGIS3/Utils/GISAttributeFieldType.cs b/DUI3-DX/Converters/ArcGIS/Speckle.Converters.ArcGIS3/Utils/GISAttributeFieldType.cs index 332a6d077a..4da3d09004 100644 --- a/DUI3-DX/Converters/ArcGIS/Speckle.Converters.ArcGIS3/Utils/GISAttributeFieldType.cs +++ b/DUI3-DX/Converters/ArcGIS/Speckle.Converters.ArcGIS3/Utils/GISAttributeFieldType.cs @@ -90,30 +90,52 @@ public static FieldType FieldTypeToNative(object fieldType) return value; } - if (value is not null) + if (value != null) { try { + static object? GetValue(string? s, Func func) => s is null ? null : func(s); return fieldType switch { - FieldType.String => (string)value, + FieldType.String => Convert.ToString(value), FieldType.Single => Convert.ToSingle(value), FieldType.Integer => Convert.ToInt32(value), // need this step because sent "ints" seem to be received as "longs" FieldType.BigInteger => Convert.ToInt64(value), FieldType.SmallInteger => Convert.ToInt16(value), FieldType.Double => Convert.ToDouble(value), - FieldType.Date => DateTime.Parse((string)value, null), - FieldType.DateOnly => DateOnly.Parse((string)value), - FieldType.TimeOnly => TimeOnly.Parse((string)value), + FieldType.Date => GetValue(value.ToString(), s => DateTime.Parse(s, null)), + FieldType.DateOnly => GetValue(value.ToString(), s => DateOnly.Parse(s, null)), + FieldType.TimeOnly => GetValue(value.ToString(), s => TimeOnly.Parse(s, null)), _ => value, }; } - catch (InvalidCastException) + catch (Exception ex) when (ex is InvalidCastException or FormatException or ArgumentNullException) { - return value; + return null; } } + else + { + return null; + } + } + + public static FieldType GetFieldTypeFromRawValue(object? value) + { + // using "Blob" as a placeholder for unrecognized values/nulls. + // Once all elements are iterated, FieldType.Blob will be replaced with FieldType.String if no better type found + if (value is not null) + { + return value switch + { + string => FieldType.String, + int => FieldType.Integer, + long => FieldType.BigInteger, + double => FieldType.Double, + _ => FieldType.Blob, + }; + } - return value; + return FieldType.Blob; } } diff --git a/DUI3-DX/Converters/ArcGIS/Speckle.Converters.ArcGIS3/Utils/IArcGISFieldUtils.cs b/DUI3-DX/Converters/ArcGIS/Speckle.Converters.ArcGIS3/Utils/IArcGISFieldUtils.cs index 959e9793fe..6bda9e1bff 100644 --- a/DUI3-DX/Converters/ArcGIS/Speckle.Converters.ArcGIS3/Utils/IArcGISFieldUtils.cs +++ b/DUI3-DX/Converters/ArcGIS/Speckle.Converters.ArcGIS3/Utils/IArcGISFieldUtils.cs @@ -1,11 +1,18 @@ using ArcGIS.Core.Data; using Objects.GIS; +using Speckle.Core.Models; using FieldDescription = ArcGIS.Core.Data.DDL.FieldDescription; namespace Speckle.Converters.ArcGIS3.Utils; public interface IArcGISFieldUtils { - public RowBuffer AssignFieldValuesToRow(RowBuffer rowBuffer, List fields, GisFeature feat); + public RowBuffer AssignFieldValuesToRow( + RowBuffer rowBuffer, + List fields, + Dictionary attributes + ); public List GetFieldsFromSpeckleLayer(VectorLayer target); + + public List<(FieldDescription, Func)> CreateFieldsFromListOfBase(List target); } diff --git a/DUI3-DX/Converters/ArcGIS/Speckle.Converters.ArcGIS3/Utils/IFeatureClassUtils.cs b/DUI3-DX/Converters/ArcGIS/Speckle.Converters.ArcGIS3/Utils/IFeatureClassUtils.cs index 289c05eeb6..b60cf4bfea 100644 --- a/DUI3-DX/Converters/ArcGIS/Speckle.Converters.ArcGIS3/Utils/IFeatureClassUtils.cs +++ b/DUI3-DX/Converters/ArcGIS/Speckle.Converters.ArcGIS3/Utils/IFeatureClassUtils.cs @@ -16,8 +16,8 @@ void AddFeaturesToFeatureClass( ); void AddNonGISFeaturesToFeatureClass( FeatureClass newFeatureClass, - List features, - List fields + List<(Base baseObj, ACG.Geometry convertedGeom)> featuresTuples, + List<(FieldDescription, Func)> fieldsAndFunctions ); void AddFeaturesToTable(Table newFeatureClass, List gisFeatures, List fields); public ACG.GeometryType GetLayerGeometryType(VectorLayer target); diff --git a/DUI3-DX/Converters/ArcGIS/Speckle.Converters.ArcGIS3/Utils/NonNativeFeaturesUtils.cs b/DUI3-DX/Converters/ArcGIS/Speckle.Converters.ArcGIS3/Utils/NonNativeFeaturesUtils.cs index 0d458029bf..3a17ff8796 100644 --- a/DUI3-DX/Converters/ArcGIS/Speckle.Converters.ArcGIS3/Utils/NonNativeFeaturesUtils.cs +++ b/DUI3-DX/Converters/ArcGIS/Speckle.Converters.ArcGIS3/Utils/NonNativeFeaturesUtils.cs @@ -6,6 +6,7 @@ using FieldDescription = ArcGIS.Core.Data.DDL.FieldDescription; using Speckle.Core.Logging; using Speckle.Core.Models.GraphTraversal; +using Speckle.Core.Models; namespace Speckle.Converters.ArcGIS3.Utils; @@ -13,14 +14,17 @@ public class NonNativeFeaturesUtils : INonNativeFeaturesUtils { private readonly IFeatureClassUtils _featureClassUtils; private readonly IConversionContextStack _contextStack; + private readonly IArcGISFieldUtils _fieldUtils; public NonNativeFeaturesUtils( IFeatureClassUtils featureClassUtils, - IConversionContextStack contextStack + IConversionContextStack contextStack, + IArcGISFieldUtils fieldUtils ) { _featureClassUtils = featureClassUtils; _contextStack = contextStack; + _fieldUtils = fieldUtils; } public void WriteGeometriesToDatasets( @@ -29,7 +33,7 @@ Dictionary conversionTracker ) { // 1. Sort features into groups by path and geom type - Dictionary> geometryGroups = new(); + Dictionary> geometryGroups = new(); foreach (var item in conversionTracker) { try @@ -40,9 +44,7 @@ Dictionary conversionTracker string? datasetId = trackerItem.DatasetId; if (geom != null && datasetId == null) // only non-native geomerties, not written into a dataset yet { - string nestedParentPath = trackerItem.NestedLayerName; - string speckle_type = nestedParentPath.Split('\\')[^1]; - + string speckle_type = trackerItem.Base.speckle_type.Split(".")[^1]; string? parentId = context.Parent?.Current.id; // add dictionnary item if doesn't exist yet @@ -50,10 +52,10 @@ Dictionary conversionTracker string uniqueKey = $"speckleTYPE_{speckle_type}_speckleID_{parentId}"; if (!geometryGroups.TryGetValue(uniqueKey, out _)) { - geometryGroups[uniqueKey] = new List(); + geometryGroups[uniqueKey] = new(); } - geometryGroups[uniqueKey].Add(geom); + geometryGroups[uniqueKey].Add((trackerItem.Base, geom)); // record changes in conversion tracker trackerItem.AddDatasetId(uniqueKey); @@ -83,10 +85,10 @@ Dictionary conversionTracker foreach (var item in geometryGroups) { string uniqueKey = item.Key; - List geomList = item.Value; + List<(Base, ACG.Geometry)> listOfGeometryTuples = item.Value; try { - CreateDatasetInDatabase(uniqueKey, geomList); + CreateDatasetInDatabase(uniqueKey, listOfGeometryTuples); } catch (GeodatabaseGeometryException ex) { @@ -105,7 +107,10 @@ Dictionary conversionTracker } } - private void CreateDatasetInDatabase(string featureClassName, List geomList) + private void CreateDatasetInDatabase( + string featureClassName, + List<(Base baseObj, ACG.Geometry convertedGeom)> listOfGeometryTuples + ) { FileGeodatabaseConnectionPath fileGeodatabaseConnectionPath = new(_contextStack.Current.Document.SpeckleDatabasePath); @@ -115,8 +120,10 @@ private void CreateDatasetInDatabase(string featureClassName, List // get Spatial Reference from the document ACG.SpatialReference spatialRef = _contextStack.Current.Document.Map.SpatialReference; - // TODO: create Fields - List fields = new(); // _fieldsUtils.GetFieldsFromSpeckleLayer(target); + // create Fields + List<(FieldDescription, Func)> fieldsAndFunctions = _fieldUtils.CreateFieldsFromListOfBase( + listOfGeometryTuples.Select(x => x.baseObj).ToList() + ); // delete FeatureClass if already exists try @@ -147,14 +154,16 @@ private void CreateDatasetInDatabase(string featureClassName, List try { // POC: make sure class has a valid crs - ACG.GeometryType geomType = geomList[0].GeometryType; + ACG.GeometryType geomType = listOfGeometryTuples[0].convertedGeom.GeometryType; ShapeDescription shpDescription = new(geomType, spatialRef) { HasZ = true }; - FeatureClassDescription featureClassDescription = new(featureClassName, fields, shpDescription); + FeatureClassDescription featureClassDescription = + new(featureClassName, fieldsAndFunctions.Select(x => x.Item1), shpDescription); FeatureClassToken featureClassToken = schemaBuilder.Create(featureClassDescription); } catch (ArgumentException ex) { // if name has invalid characters/combinations + // or 'The table contains multiple fields with the same name.: throw new ArgumentException($"{ex.Message}: {featureClassName}", ex); } bool buildStatus = schemaBuilder.Build(); @@ -168,7 +177,7 @@ private void CreateDatasetInDatabase(string featureClassName, List // Add features to the FeatureClass geodatabase.ApplyEdits(() => { - _featureClassUtils.AddNonGISFeaturesToFeatureClass(newFeatureClass, geomList, fields); + _featureClassUtils.AddNonGISFeaturesToFeatureClass(newFeatureClass, listOfGeometryTuples, fieldsAndFunctions); }); } }