From f6a23ebd7dcc7652f4992cc5d2f7656a39b1d98d Mon Sep 17 00:00:00 2001 From: KatKatKateryna <89912278+KatKatKateryna@users.noreply.github.com> Date: Thu, 13 Jun 2024 16:48:05 +0800 Subject: [PATCH] DUI3-295 track non native received objects for the full report (#3497) * introduce ConverionTracker; simplify variables * track conversions up to writing groups into dataset * throw exceptions for failed datasets. TODO: write to results * correct results * correct result order * properly updating Trackers * more comments * remove fake Exception from GIS layers * Move BUILD function to the top --- .../Bindings/BasicConnectorBinding.cs | 8 +- .../Operations/Receive/HostObjectBuilder.cs | 142 +++++++++++------- .../Utils/ArcGISProjectUtils.cs | 0 .../Utils/ConversionTracker.cs | 88 +++++++++++ .../Utils/INonNativeFeaturesUtils.cs | 4 +- .../Utils/NonNativeFeaturesUtils.cs | 82 ++++++---- 6 files changed, 237 insertions(+), 87 deletions(-) delete mode 100644 DUI3-DX/Converters/ArcGIS/Speckle.Converters.ArcGIS3/Utils/ArcGISProjectUtils.cs create mode 100644 DUI3-DX/Converters/ArcGIS/Speckle.Converters.ArcGIS3/Utils/ConversionTracker.cs diff --git a/DUI3-DX/Connectors/ArcGIS/Speckle.Connectors.ArcGIS3/Bindings/BasicConnectorBinding.cs b/DUI3-DX/Connectors/ArcGIS/Speckle.Connectors.ArcGIS3/Bindings/BasicConnectorBinding.cs index bfa36dfc87..dd2fa19efa 100644 --- a/DUI3-DX/Connectors/ArcGIS/Speckle.Connectors.ArcGIS3/Bindings/BasicConnectorBinding.cs +++ b/DUI3-DX/Connectors/ArcGIS/Speckle.Connectors.ArcGIS3/Bindings/BasicConnectorBinding.cs @@ -171,6 +171,12 @@ private void SelectMapMembersInTOC(List mapMembers) } } MapView.Active.SelectLayers(layers); - // MapView.Active.SelectStandaloneTables(tables); // clears previous selection, not clear how to ADD selection instead + + // this step clears previous selection, not clear how to ADD selection instead + // this is why, activating it only if no layers are selected + if (layers.Count == 0) + { + MapView.Active.SelectStandaloneTables(tables); + } } } diff --git a/DUI3-DX/Connectors/ArcGIS/Speckle.Connectors.ArcGIS3/Operations/Receive/HostObjectBuilder.cs b/DUI3-DX/Connectors/ArcGIS/Speckle.Connectors.ArcGIS3/Operations/Receive/HostObjectBuilder.cs index 2b770782ef..a681f5317a 100644 --- a/DUI3-DX/Connectors/ArcGIS/Speckle.Connectors.ArcGIS3/Operations/Receive/HostObjectBuilder.cs +++ b/DUI3-DX/Connectors/ArcGIS/Speckle.Connectors.ArcGIS3/Operations/Receive/HostObjectBuilder.cs @@ -36,40 +36,6 @@ GraphTraversal traverseFunction _traverseFunction = traverseFunction; } - private (string path, Geometry converted) ConvertNonNativeGeometries(Base obj, string[] path) - { - Geometry converted = (Geometry)_converter.Convert(obj); - List objPath = path.ToList(); - objPath.Add(obj.speckle_type.Split(".")[^1]); - return (string.Join("\\", objPath), converted); - } - - private (string path, string converted) ConvertNativeLayers(Collection obj, string[] path) - { - string converted = (string)_converter.Convert(obj); - string objPath = $"{string.Join("\\", path)}\\{obj.name}"; - return (objPath, converted); - } - - private string AddDatasetsToMap((string nestedLayerName, string datasetId) databaseObj) - { - Uri uri = - new( - $"{_contextStack.Current.Document.SpeckleDatabasePath.AbsolutePath.Replace('/', '\\')}\\{databaseObj.datasetId}" - ); - Map map = _contextStack.Current.Document.Map; - try - { - return LayerFactory.Instance.CreateLayer(uri, map, layerName: databaseObj.nestedLayerName).URI; - } - catch (ArgumentException) - { - return StandaloneTableFactory.Instance - .CreateStandaloneTable(uri, map, tableName: databaseObj.nestedLayerName) - .URI; - } - } - public HostObjectBuilderResult Build( Base rootObject, string projectName, @@ -89,8 +55,7 @@ CancellationToken cancellationToken int allCount = objectsToConvert.Count; int count = 0; - Dictionary convertedGeometries = new(); - List<(string path, string converted)> convertedGISObjects = new(); + Dictionary conversionTracker = new(); // 1. convert everything List results = new(objectsToConvert.Count); @@ -105,21 +70,15 @@ CancellationToken cancellationToken { if (IsGISType(obj)) { - var result = ConvertNativeLayers((Collection)obj, path); - convertedGISObjects.Add(result); - // NOTE: Dim doesn't really know what is what - is the result.path the id of the obj? - // TODO: is the type in here basically a GIS Layer? - results.Add(new(Status.SUCCESS, obj, result.path, "GIS Layer")); + string nestedLayerPath = $"{string.Join("\\", path)}\\{((Collection)obj).name}"; + string datasetId = (string)_converter.Convert(obj); + conversionTracker[ctx] = new ObjectConversionTracker(obj, nestedLayerPath, datasetId); } else { - var result = ConvertNonNativeGeometries(obj, path); - convertedGeometries[ctx] = result; - - // NOTE: Dim doesn't really know what is what - is the result.path the id of the obj? - results.Add(new(Status.SUCCESS, obj, result.path, result.converted.GetType().ToString())); //POC: what native id?, path may not be unique - // TODO: Do we need this here? I remember oguzhan saying something that selection/object highlighting is weird in arcgis (weird is subjective) - // bakedObjectIds.Add(result.path); + string nestedLayerPath = $"{string.Join("\\", path)}\\{obj.speckle_type.Split(".")[^1]}"; + Geometry converted = (Geometry)_converter.Convert(obj); + conversionTracker[ctx] = new ObjectConversionTracker(obj, nestedLayerPath, converted); } } catch (Exception ex) when (!ex.IsFatal()) // DO NOT CATCH SPECIFIC STUFF, conversion errors should be recoverable @@ -130,26 +89,101 @@ CancellationToken cancellationToken } // 2. convert Database entries with non-GIS geometry datasets - onOperationProgressed?.Invoke("Writing to Database", null); - convertedGISObjects.AddRange(_nonGisFeaturesUtils.WriteGeometriesToDatasets(convertedGeometries)); + _nonGisFeaturesUtils.WriteGeometriesToDatasets(conversionTracker); + // 3. add layer and tables to the Table Of Content int bakeCount = 0; + Dictionary bakedMapMembers = new(); onOperationProgressed?.Invoke("Adding to Map", bakeCount); - // 3. add layer and tables to the Table Of Content - foreach (var databaseObj in convertedGISObjects) + foreach (var item in conversionTracker) { cancellationToken.ThrowIfCancellationRequested(); + var trackerItem = conversionTracker[item.Key]; // updated tracker object // BAKE OBJECTS HERE - bakedObjectIds.Add(AddDatasetsToMap(databaseObj)); - onOperationProgressed?.Invoke("Adding to Map", (double)++bakeCount / convertedGISObjects.Count); + if (trackerItem.Exception != null) + { + results.Add(new(Status.ERROR, trackerItem.Base, null, null, trackerItem.Exception)); + } + else if (trackerItem.DatasetId == null) + { + results.Add( + new(Status.ERROR, trackerItem.Base, null, null, new ArgumentException("Unknown error: Dataset not created")) + ); + } + else if (bakedMapMembers.TryGetValue(trackerItem.DatasetId, out MapMember? value)) + { + // only add a report item + AddResultsFromTracker(trackerItem, results); + } + else + { + // add layer and layer URI to tracker + MapMember mapMember = AddDatasetsToMap(trackerItem); + trackerItem.AddConvertedMapMember(mapMember); + trackerItem.AddLayerURI(mapMember.URI); + conversionTracker[item.Key] = trackerItem; + + // add layer URI to bakedIds + bakedObjectIds.Add(trackerItem.MappedLayerURI == null ? "" : trackerItem.MappedLayerURI); + + // add report item + AddResultsFromTracker(trackerItem, results); + } + onOperationProgressed?.Invoke("Adding to Map", (double)++bakeCount / conversionTracker.Count); } // TODO: validated a correct set regarding bakedobject ids return new(bakedObjectIds, results); } + private void AddResultsFromTracker(ObjectConversionTracker trackerItem, List results) + { + // prioritize individual hostAppGeometry type, if available: + if (trackerItem.HostAppGeom != null) + { + results.Add( + new(Status.SUCCESS, trackerItem.Base, trackerItem.MappedLayerURI, trackerItem.HostAppGeom.GetType().ToString()) + ); + } + else + { + results.Add( + new( + Status.SUCCESS, + trackerItem.Base, + trackerItem.MappedLayerURI, + trackerItem.HostAppMapMember?.GetType().ToString() + ) + ); + } + } + + private MapMember AddDatasetsToMap(ObjectConversionTracker trackerItem) + { + string? datasetId = trackerItem.DatasetId; // should not ne null here + string nestedLayerName = trackerItem.NestedLayerName; + + Uri uri = new($"{_contextStack.Current.Document.SpeckleDatabasePath.AbsolutePath.Replace('/', '\\')}\\{datasetId}"); + Map map = _contextStack.Current.Document.Map; + + // Most of the Speckle-written datasets will be containing geometry and added as Layers + // although, some datasets might be just tables (e.g. native GIS Tables, in the future maybe Revit schedules etc. + // We can create a connection to the dataset in advance and determine its type, but this will be more + // expensive, than assuming by default that it's a layer with geometry (which in most cases it's expected to be) + try + { + var layer = LayerFactory.Instance.CreateLayer(uri, map, layerName: nestedLayerName); + return layer; + } + catch (ArgumentException) + { + var table = StandaloneTableFactory.Instance.CreateStandaloneTable(uri, map, tableName: nestedLayerName); + return table; + } + } + [Pure] private static string[] GetLayerPath(TraversalContext context) { diff --git a/DUI3-DX/Converters/ArcGIS/Speckle.Converters.ArcGIS3/Utils/ArcGISProjectUtils.cs b/DUI3-DX/Converters/ArcGIS/Speckle.Converters.ArcGIS3/Utils/ArcGISProjectUtils.cs deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/DUI3-DX/Converters/ArcGIS/Speckle.Converters.ArcGIS3/Utils/ConversionTracker.cs b/DUI3-DX/Converters/ArcGIS/Speckle.Converters.ArcGIS3/Utils/ConversionTracker.cs new file mode 100644 index 0000000000..cb01772ed7 --- /dev/null +++ b/DUI3-DX/Converters/ArcGIS/Speckle.Converters.ArcGIS3/Utils/ConversionTracker.cs @@ -0,0 +1,88 @@ +using ArcGIS.Desktop.Mapping; +using Speckle.Core.Models; + +namespace Speckle.Converters.ArcGIS3.Utils; + +/// +/// Container connecting the received Base object, converted hostApp object, dataset it was written to, +/// URI of the layer mapped from the dataset and, if applicable, feature/row id. +/// +public struct ObjectConversionTracker +{ + public Base Base { get; set; } + public string NestedLayerName { get; set; } + public ACG.Geometry? HostAppGeom { get; set; } + public MapMember? HostAppMapMember { get; set; } + public string? DatasetId { get; set; } + public int? DatasetRow { get; set; } + public string? MappedLayerURI { get; set; } + public Exception? Exception { get; set; } + + public void AddException(Exception ex) + { + Exception = ex; + HostAppGeom = null; + DatasetId = null; + DatasetRow = null; + MappedLayerURI = null; + } + + public void AddDatasetId(string datasetId) + { + DatasetId = datasetId; + } + + public void AddDatasetRow(int datasetRow) + { + DatasetRow = datasetRow; + } + + public void AddConvertedMapMember(MapMember mapMember) + { + HostAppMapMember = mapMember; + } + + public void AddLayerURI(string layerURIstring) + { + MappedLayerURI = layerURIstring; + } + + /// + /// Initializes a new instance of . + /// + /// Original received Base object. + /// String with the full traversed path to the object. Will be used to create nested layer structure in the TOC. + public ObjectConversionTracker(Base baseObj, string nestedLayerName) + { + Base = baseObj; + NestedLayerName = nestedLayerName; + } + + /// + /// Constructor for received non-GIS geometries. + /// Initializes a new instance of , accepting converted hostApp geometry. + /// + /// Original received Base object. + /// String with the full traversed path to the object. Will be used to create nested layer structure in the TOC. + /// Converted ArcGIS.Core.Geometry. + public ObjectConversionTracker(Base baseObj, string nestedLayerName, ACG.Geometry hostAppGeom) + { + Base = baseObj; + NestedLayerName = nestedLayerName; + HostAppGeom = hostAppGeom; + } + + /// + /// Constructor for received native GIS layers. + /// Initializes a new instance of , accepting datasetID of a coverted Speckle layer. + /// + /// Original received Base object. + /// String with the full traversed path to the object. Will be used to create nested layer structure in the TOC. + /// ID of the locally written dataset, created from received Speckle layer. + public ObjectConversionTracker(Base baseObj, string nestedLayerName, string datasetId) + { + Base = baseObj; + NestedLayerName = nestedLayerName; + DatasetId = datasetId; + } +} diff --git a/DUI3-DX/Converters/ArcGIS/Speckle.Converters.ArcGIS3/Utils/INonNativeFeaturesUtils.cs b/DUI3-DX/Converters/ArcGIS/Speckle.Converters.ArcGIS3/Utils/INonNativeFeaturesUtils.cs index 1f4295b457..25cd904fca 100644 --- a/DUI3-DX/Converters/ArcGIS/Speckle.Converters.ArcGIS3/Utils/INonNativeFeaturesUtils.cs +++ b/DUI3-DX/Converters/ArcGIS/Speckle.Converters.ArcGIS3/Utils/INonNativeFeaturesUtils.cs @@ -4,7 +4,5 @@ namespace Speckle.Converters.ArcGIS3.Utils; public interface INonNativeFeaturesUtils { - public List<(string parentPath, string converted)> WriteGeometriesToDatasets( - Dictionary convertedObjs - ); + public void WriteGeometriesToDatasets(Dictionary conversionTracker); } diff --git a/DUI3-DX/Converters/ArcGIS/Speckle.Converters.ArcGIS3/Utils/NonNativeFeaturesUtils.cs b/DUI3-DX/Converters/ArcGIS/Speckle.Converters.ArcGIS3/Utils/NonNativeFeaturesUtils.cs index 621ad260ca..7f91dbbf32 100644 --- a/DUI3-DX/Converters/ArcGIS/Speckle.Converters.ArcGIS3/Utils/NonNativeFeaturesUtils.cs +++ b/DUI3-DX/Converters/ArcGIS/Speckle.Converters.ArcGIS3/Utils/NonNativeFeaturesUtils.cs @@ -23,35 +23,58 @@ public NonNativeFeaturesUtils( _contextStack = contextStack; } - public List<(string parentPath, string converted)> WriteGeometriesToDatasets( - Dictionary convertedObjs + public void WriteGeometriesToDatasets( + // Dictionary conversionTracker + Dictionary conversionTracker ) { - List<(string parentPath, string converted)> result = new(); // 1. Sort features into groups by path and geom type - Dictionary geometries, string? parentId)> geometryGroups = new(); - foreach (var item in convertedObjs) + Dictionary> geometryGroups = new(); + foreach (var item in conversionTracker) { try { TraversalContext context = item.Key; - (string parentPath, ACG.Geometry geom) = item.Value; + var trackerItem = item.Value; + ACG.Geometry? geom = trackerItem.HostAppGeom; + string? datasetId = trackerItem.DatasetId; + if (geom != null && datasetId == null) // only non-native geomerties, not written into a dataset yet + { + string nestedParentPath = trackerItem.NestedLayerName; + string speckle_type = nestedParentPath.Split('\\')[^1]; + + string? parentId = context.Parent?.Current.id; + + // add dictionnary item if doesn't exist yet + // Key must be unique per parent and speckle_type + string uniqueKey = $"speckleTYPE_{speckle_type}_speckleID_{parentId}"; + if (!geometryGroups.TryGetValue(uniqueKey, out _)) + { + geometryGroups[uniqueKey] = new List(); + } - string? parentId = context.Parent?.Current.id; - // add dictionnary item if doesn't exist yet - // Key must be unique per parent and speckle_type - // Key is composed of parentId and parentPath (that contains speckle_type) - string uniqueKey = $"{parentId}_{parentPath}"; - if (!geometryGroups.TryGetValue(uniqueKey, out _)) + geometryGroups[uniqueKey].Add(geom); + + // record changes in conversion tracker + trackerItem.AddDatasetId(uniqueKey); + trackerItem.AddDatasetRow(geometryGroups[uniqueKey].Count - 1); + conversionTracker[item.Key] = trackerItem; + } + else if (geom == null && datasetId != null) // GIS layers, already written to a dataset { - geometryGroups[uniqueKey] = (new List(), parentId); + continue; + } + else + { + throw new ArgumentException($"Unexpected geometry and datasetId values: {geom}, {datasetId}"); } - - geometryGroups[uniqueKey].geometries.Add(geom); } catch (Exception ex) when (!ex.IsFatal()) { // POC: report, etc. + var trackerItem = item.Value; + trackerItem.AddException(ex); + conversionTracker[item.Key] = trackerItem; Debug.WriteLine($"conversion error happened. {ex.Message}"); } } @@ -59,24 +82,30 @@ public NonNativeFeaturesUtils( // 2. for each group create a Dataset and add geometries there as Features foreach (var item in geometryGroups) { - string uniqueKey = item.Key; // parentId_parentPath - string parentPath = uniqueKey.Split('_', 2)[^1]; - string speckle_type = parentPath.Split('\\')[^1]; - (List geomList, string? parentId) = item.Value; + string uniqueKey = item.Key; + List geomList = item.Value; try { - string converted = CreateDatasetInDatabase(speckle_type, geomList, parentId); - result.Add((parentPath, converted)); + CreateDatasetInDatabase(uniqueKey, geomList); } - catch (GeodatabaseGeometryException) + catch (GeodatabaseGeometryException ex) { // do nothing if writing of some geometry groups fails + // only record in conversionTracker: + foreach (var conversionItem in conversionTracker) + { + if (conversionItem.Value.DatasetId == uniqueKey) + { + var trackerItem = conversionItem.Value; + trackerItem.AddException(ex); + conversionTracker[conversionItem.Key] = trackerItem; + } + } } } - return result; } - private string CreateDatasetInDatabase(string speckle_type, List geomList, string? parentId) + private void CreateDatasetInDatabase(string featureClassName, List geomList) { FileGeodatabaseConnectionPath fileGeodatabaseConnectionPath = new(_contextStack.Current.Document.SpeckleDatabasePath); @@ -89,9 +118,6 @@ private string CreateDatasetInDatabase(string speckle_type, List g // TODO: create Fields List fields = new(); // _fieldsUtils.GetFieldsFromSpeckleLayer(target); - // TODO: generate meaningful name - string featureClassName = $"speckleTYPE_{speckle_type}_speckleID_{parentId}"; - // delete FeatureClass if already exists foreach (FeatureClassDefinition fClassDefinition in geodatabase.GetDefinitions()) { @@ -131,7 +157,5 @@ private string CreateDatasetInDatabase(string speckle_type, List g { _featureClassUtils.AddNonGISFeaturesToFeatureClass(newFeatureClass, geomList, fields); }); - - return featureClassName; } }