From 4910ab1384dd21cbc0b9d417d2db7c8fca21d996 Mon Sep 17 00:00:00 2001 From: Dimitrie Stefanescu Date: Mon, 29 Oct 2018 12:51:59 +0000 Subject: [PATCH 01/27] base-not working --- SpeckleCore/Accounts/AccountsUtils.cs | 45 +++++++++++ SpeckleCore/{ => Conversion}/Converter.cs | 0 .../{ => Conversion}/JsonConverters.cs | 0 .../ClassesForAbstractTests.cs | 0 .../{ => Generic Utils}/GzipContent.cs | 0 SpeckleCore/{ => Models}/ModelBase.cs | 0 SpeckleCore/{ => Models}/ModelObjects.cs | 0 SpeckleCore/{ => Models}/ModelObjectsExt.cs | 80 ++++++++++--------- SpeckleCore/{ => Models}/ModelResponses.cs | 0 SpeckleCore/SpeckleCore.csproj | 25 +++--- 10 files changed, 104 insertions(+), 46 deletions(-) create mode 100644 SpeckleCore/Accounts/AccountsUtils.cs rename SpeckleCore/{ => Conversion}/Converter.cs (100%) rename SpeckleCore/{ => Conversion}/JsonConverters.cs (100%) rename SpeckleCore/{ => Generic Utils}/ClassesForAbstractTests.cs (100%) rename SpeckleCore/{ => Generic Utils}/GzipContent.cs (100%) rename SpeckleCore/{ => Models}/ModelBase.cs (100%) rename SpeckleCore/{ => Models}/ModelObjects.cs (100%) rename SpeckleCore/{ => Models}/ModelObjectsExt.cs (92%) rename SpeckleCore/{ => Models}/ModelResponses.cs (100%) diff --git a/SpeckleCore/Accounts/AccountsUtils.cs b/SpeckleCore/Accounts/AccountsUtils.cs new file mode 100644 index 0000000..d1fd0e4 --- /dev/null +++ b/SpeckleCore/Accounts/AccountsUtils.cs @@ -0,0 +1,45 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; + +namespace SpeckleCore +{ + + public class SpeckleLocalContext : DbContext + { + private static bool created = false; + + public DbSet Accounts { get; set; } + + public SpeckleLocalContext( ) + { + + } + + protected override void OnConfiguring( DbContextOptionsBuilder optionsBuilder ) + { + //base.OnConfiguring( optionsBuilder ); + //optionsBuilder.UseSqlite( "Data Source=/*" + System.Environment.GetFolderPath( System.Environment.SpecialFolder.LocalApplicationData ) + @"\SpeckleSettings\SpeckleLocal.db" );*/ + + optionsBuilder.UseSqlite( "Data Source=SpeckleLocal.db" ); + + } + + } + + /// + /// An account is a speckle account + /// + public class Account + { + public int AccountId { get; set; } + public string ServerName { get; set; } + public string RestApi { get; set; } + public string Email { get; set; } + public string Token { get; set; } + public bool IsDefault { get; set; } = false; + } +} diff --git a/SpeckleCore/Converter.cs b/SpeckleCore/Conversion/Converter.cs similarity index 100% rename from SpeckleCore/Converter.cs rename to SpeckleCore/Conversion/Converter.cs diff --git a/SpeckleCore/JsonConverters.cs b/SpeckleCore/Conversion/JsonConverters.cs similarity index 100% rename from SpeckleCore/JsonConverters.cs rename to SpeckleCore/Conversion/JsonConverters.cs diff --git a/SpeckleCore/ClassesForAbstractTests.cs b/SpeckleCore/Generic Utils/ClassesForAbstractTests.cs similarity index 100% rename from SpeckleCore/ClassesForAbstractTests.cs rename to SpeckleCore/Generic Utils/ClassesForAbstractTests.cs diff --git a/SpeckleCore/GzipContent.cs b/SpeckleCore/Generic Utils/GzipContent.cs similarity index 100% rename from SpeckleCore/GzipContent.cs rename to SpeckleCore/Generic Utils/GzipContent.cs diff --git a/SpeckleCore/ModelBase.cs b/SpeckleCore/Models/ModelBase.cs similarity index 100% rename from SpeckleCore/ModelBase.cs rename to SpeckleCore/Models/ModelBase.cs diff --git a/SpeckleCore/ModelObjects.cs b/SpeckleCore/Models/ModelObjects.cs similarity index 100% rename from SpeckleCore/ModelObjects.cs rename to SpeckleCore/Models/ModelObjects.cs diff --git a/SpeckleCore/ModelObjectsExt.cs b/SpeckleCore/Models/ModelObjectsExt.cs similarity index 92% rename from SpeckleCore/ModelObjectsExt.cs rename to SpeckleCore/Models/ModelObjectsExt.cs index a1f4b08..6ab7679 100644 --- a/SpeckleCore/ModelObjectsExt.cs +++ b/SpeckleCore/Models/ModelObjectsExt.cs @@ -21,7 +21,7 @@ public partial class SpeckleObject /// public string GetMd5FromObject( object fromWhat, int length = 0 ) { - if(fromWhat == null) + if ( fromWhat == null ) { return "null"; } @@ -69,12 +69,12 @@ public virtual void Scale( double factor ) /// /// /// - internal Dictionary ScaleProperties(Dictionary dict, double factor) + internal Dictionary ScaleProperties( Dictionary dict, double factor ) { if ( dict == null ) return null; - foreach(var kvp in dict) + foreach ( var kvp in dict ) { - switch(kvp.Value) + switch ( kvp.Value ) { case Dictionary d: dict[ kvp.Key ] = ScaleProperties( d, factor ); @@ -568,7 +568,7 @@ public override void GenerateHash( ) public partial class SpecklePolycurve { - public SpecklePolycurve() { } + public SpecklePolycurve( ) { } public override void Scale( double factor ) { @@ -713,7 +713,7 @@ public SpeckleExtrusion( SpeckleObject profile, double length, bool capped, stri public override void Scale( double factor ) { this.Length *= factor; - switch(this.Profile) + switch ( this.Profile ) { case SpeckleCurve c: c.Scale( factor ); @@ -734,12 +734,12 @@ public override void Scale( double factor ) e.Scale( factor ); break; case SpeckleLine l: - l.Scale( factor); + l.Scale( factor ); break; default: break; } - + this.Properties = ScaleProperties( this.Properties, factor ); GenerateHash(); } @@ -829,50 +829,56 @@ public int GetHashCode( Layer obj ) } } -public partial class SpeckleInput : SpeckleObject -{ - public SpeckleInput() { } - - [Newtonsoft.Json.JsonProperty("name", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + + // These two classes are used for the stream controller functionality. + + // Input parameter (ie, width, height) + public partial class SpeckleInput : SpeckleObject + { + public SpeckleInput( ) { } + + [Newtonsoft.Json.JsonProperty( "name", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore )] public string Name { get; set; } - [Newtonsoft.Json.JsonProperty("guid", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + [Newtonsoft.Json.JsonProperty( "guid", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore )] public string Guid { get; set; } - [Newtonsoft.Json.JsonProperty("value", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + [Newtonsoft.Json.JsonProperty( "value", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore )] public float Value { get; set; } - [Newtonsoft.Json.JsonProperty("inputType", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + [Newtonsoft.Json.JsonProperty( "inputType", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore )] public string InputType { get; set; } - [Newtonsoft.Json.JsonProperty("max", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + [Newtonsoft.Json.JsonProperty( "max", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore )] public float Max { get; set; } - [Newtonsoft.Json.JsonProperty("min", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + [Newtonsoft.Json.JsonProperty( "min", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore )] public float Min { get; set; } - public SpeckleInput(string name, float min, float max, float value, string inputType, string guid) + public SpeckleInput( string name, float min, float max, float value, string inputType, string guid ) { - this.Name = name; - this.Guid = guid; - this.Min = min; - this.Max = max; - this.Value = value; - this.InputType = inputType; + this.Name = name; + this.Guid = guid; + this.Min = min; + this.Max = max; + this.Value = value; + this.InputType = inputType; } -} + } + + // Output parameter (price, area) public partial class SpeckleOutput : SpeckleObject - { - public SpeckleOutput() { } - public string Name { get; set; } - public string Guid { get; set; } - public string Value { get; set; } + { + public SpeckleOutput( ) { } + public string Name { get; set; } + public string Guid { get; set; } + public string Value { get; set; } - public SpeckleOutput(string name, string value, string guid) - { - this.Name = name; - this.Guid = guid; - this.Value = value; - } + public SpeckleOutput( string name, string value, string guid ) + { + this.Name = name; + this.Guid = guid; + this.Value = value; } + } } diff --git a/SpeckleCore/ModelResponses.cs b/SpeckleCore/Models/ModelResponses.cs similarity index 100% rename from SpeckleCore/ModelResponses.cs rename to SpeckleCore/Models/ModelResponses.cs diff --git a/SpeckleCore/SpeckleCore.csproj b/SpeckleCore/SpeckleCore.csproj index 92f45d8..6f5df90 100644 --- a/SpeckleCore/SpeckleCore.csproj +++ b/SpeckleCore/SpeckleCore.csproj @@ -9,7 +9,7 @@ Properties SpeckleCore SpeckleCore - v4.5 + v4.6.1 512 @@ -42,20 +42,27 @@ - - - - - - - + + + + + + + + - + + + 2.1.4 + + + 2.1.4 + 11.0.1 From 76e6e39d73d6e8b60fbb4e589778a481d6cca50d Mon Sep 17 00:00:00 2001 From: Dimitrie Stefanescu Date: Mon, 29 Oct 2018 16:05:14 +0000 Subject: [PATCH 02/27] scafolding up sqlite db access --- SpeckleCore/Accounts/AccountsUtils.cs | 45 ------------- SpeckleCore/LocalData/AccountsUtils.cs | 90 ++++++++++++++++++++++++++ SpeckleCore/SpeckleCore.csproj | 11 ++-- 3 files changed, 94 insertions(+), 52 deletions(-) delete mode 100644 SpeckleCore/Accounts/AccountsUtils.cs create mode 100644 SpeckleCore/LocalData/AccountsUtils.cs diff --git a/SpeckleCore/Accounts/AccountsUtils.cs b/SpeckleCore/Accounts/AccountsUtils.cs deleted file mode 100644 index d1fd0e4..0000000 --- a/SpeckleCore/Accounts/AccountsUtils.cs +++ /dev/null @@ -1,45 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Microsoft.EntityFrameworkCore; - -namespace SpeckleCore -{ - - public class SpeckleLocalContext : DbContext - { - private static bool created = false; - - public DbSet Accounts { get; set; } - - public SpeckleLocalContext( ) - { - - } - - protected override void OnConfiguring( DbContextOptionsBuilder optionsBuilder ) - { - //base.OnConfiguring( optionsBuilder ); - //optionsBuilder.UseSqlite( "Data Source=/*" + System.Environment.GetFolderPath( System.Environment.SpecialFolder.LocalApplicationData ) + @"\SpeckleSettings\SpeckleLocal.db" );*/ - - optionsBuilder.UseSqlite( "Data Source=SpeckleLocal.db" ); - - } - - } - - /// - /// An account is a speckle account - /// - public class Account - { - public int AccountId { get; set; } - public string ServerName { get; set; } - public string RestApi { get; set; } - public string Email { get; set; } - public string Token { get; set; } - public bool IsDefault { get; set; } = false; - } -} diff --git a/SpeckleCore/LocalData/AccountsUtils.cs b/SpeckleCore/LocalData/AccountsUtils.cs new file mode 100644 index 0000000..12c9544 --- /dev/null +++ b/SpeckleCore/LocalData/AccountsUtils.cs @@ -0,0 +1,90 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using SQLite; + +namespace SpeckleCore +{ + + public static class SpeckleLocalContext + { + public static SQLiteConnection Database; + + public static string DbPath = System.Environment.GetFolderPath( System.Environment.SpecialFolder.LocalApplicationData ) + @"\SpeckleSettings\SpeckleCache.db"; + + public static string SettingsFolderPath = System.Environment.GetFolderPath( System.Environment.SpecialFolder.LocalApplicationData ) + @"\SpeckleSettings\"; + + public static void Init( ) + { + Database = new SQLiteConnection( DbPath ); + Database.CreateTable(); + Database.CreateTable(); + } + + private static void MigrateAccounts( ) + { + List accounts = new List(); + + if ( Directory.Exists( SettingsFolderPath ) && Directory.EnumerateFiles( SettingsFolderPath, "*.txt" ).Count() > 0 ) + { + foreach ( string file in Directory.EnumerateFiles( SettingsFolderPath, "*.txt" ) ) + { + string content = File.ReadAllText( file ); + string[ ] pieces = content.TrimEnd( '\r', '\n' ).Split( ',' ); + + accounts.Add( new Account() { Email = pieces[ 0 ], Token = pieces[ 1 ], ServerName = pieces[ 2 ], RestApi = pieces[ 3 ] } ); + } + } + else + Debug.WriteLine( "No existing account text files found." ); + } + + public static void AddAccount( Account a ) + { + var res = Database.Insert( a ); + } + + public static List GetAccountByRestApi( string RestApi ) + { + return Database.Query( "SELECT * from Account WHERE RestApi = ?", RestApi ); + } + + public static Account GetDefaultAccount( ) + { + var res = Database.Query( "SELECT * FROM Account WHERE IsDefault='true' LIMIT 1" ); + if ( res.Count == 1 ) return res[ 0 ]; + else throw new Exception( "No default account set." ); + } + + + + } + + /// + /// An account is a speckle account + /// + public class Account + { + [PrimaryKey, AutoIncrement] + public int AccountId { get; set; } + public string ServerName { get; set; } + [Indexed] + public string RestApi { get; set; } + [Indexed] + public string Email { get; set; } + public string Token { get; set; } + public bool IsDefault { get; set; } = false; + } + + public class CachedObject + { + [PrimaryKey, Indexed, Unique] + public string Hash { get; set; } + public string Json { get; set; } + } + +} diff --git a/SpeckleCore/SpeckleCore.csproj b/SpeckleCore/SpeckleCore.csproj index 6f5df90..22b3500 100644 --- a/SpeckleCore/SpeckleCore.csproj +++ b/SpeckleCore/SpeckleCore.csproj @@ -42,7 +42,7 @@ - + @@ -57,15 +57,12 @@ - - 2.1.4 - - - 2.1.4 - 11.0.1 + + 1.5.231 + 1.0.3-rc11 From eb77324afa21b7b131e1dbe7d1b0959f8e569de0 Mon Sep 17 00:00:00 2001 From: Dimitrie Stefanescu Date: Mon, 29 Oct 2018 17:38:53 +0000 Subject: [PATCH 03/27] migrating accounts & general scaffolding --- SpeckleCore/LocalData/AccountsUtils.cs | 90 ---------- SpeckleCore/LocalData/Models.cs | 44 +++++ SpeckleCore/LocalData/SpeckleLocalContext.cs | 164 +++++++++++++++++++ 3 files changed, 208 insertions(+), 90 deletions(-) delete mode 100644 SpeckleCore/LocalData/AccountsUtils.cs create mode 100644 SpeckleCore/LocalData/Models.cs create mode 100644 SpeckleCore/LocalData/SpeckleLocalContext.cs diff --git a/SpeckleCore/LocalData/AccountsUtils.cs b/SpeckleCore/LocalData/AccountsUtils.cs deleted file mode 100644 index 12c9544..0000000 --- a/SpeckleCore/LocalData/AccountsUtils.cs +++ /dev/null @@ -1,90 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using SQLite; - -namespace SpeckleCore -{ - - public static class SpeckleLocalContext - { - public static SQLiteConnection Database; - - public static string DbPath = System.Environment.GetFolderPath( System.Environment.SpecialFolder.LocalApplicationData ) + @"\SpeckleSettings\SpeckleCache.db"; - - public static string SettingsFolderPath = System.Environment.GetFolderPath( System.Environment.SpecialFolder.LocalApplicationData ) + @"\SpeckleSettings\"; - - public static void Init( ) - { - Database = new SQLiteConnection( DbPath ); - Database.CreateTable(); - Database.CreateTable(); - } - - private static void MigrateAccounts( ) - { - List accounts = new List(); - - if ( Directory.Exists( SettingsFolderPath ) && Directory.EnumerateFiles( SettingsFolderPath, "*.txt" ).Count() > 0 ) - { - foreach ( string file in Directory.EnumerateFiles( SettingsFolderPath, "*.txt" ) ) - { - string content = File.ReadAllText( file ); - string[ ] pieces = content.TrimEnd( '\r', '\n' ).Split( ',' ); - - accounts.Add( new Account() { Email = pieces[ 0 ], Token = pieces[ 1 ], ServerName = pieces[ 2 ], RestApi = pieces[ 3 ] } ); - } - } - else - Debug.WriteLine( "No existing account text files found." ); - } - - public static void AddAccount( Account a ) - { - var res = Database.Insert( a ); - } - - public static List GetAccountByRestApi( string RestApi ) - { - return Database.Query( "SELECT * from Account WHERE RestApi = ?", RestApi ); - } - - public static Account GetDefaultAccount( ) - { - var res = Database.Query( "SELECT * FROM Account WHERE IsDefault='true' LIMIT 1" ); - if ( res.Count == 1 ) return res[ 0 ]; - else throw new Exception( "No default account set." ); - } - - - - } - - /// - /// An account is a speckle account - /// - public class Account - { - [PrimaryKey, AutoIncrement] - public int AccountId { get; set; } - public string ServerName { get; set; } - [Indexed] - public string RestApi { get; set; } - [Indexed] - public string Email { get; set; } - public string Token { get; set; } - public bool IsDefault { get; set; } = false; - } - - public class CachedObject - { - [PrimaryKey, Indexed, Unique] - public string Hash { get; set; } - public string Json { get; set; } - } - -} diff --git a/SpeckleCore/LocalData/Models.cs b/SpeckleCore/LocalData/Models.cs new file mode 100644 index 0000000..a0aa5e2 --- /dev/null +++ b/SpeckleCore/LocalData/Models.cs @@ -0,0 +1,44 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using SQLite; + +namespace SpeckleCore +{ + /// + /// A class for a generic speckle account, composed of all the identification props for a user & server combination. + /// + public class Account + { + [PrimaryKey, AutoIncrement] + public int AccountId { get; set; } + + public string ServerName { get; set; } + + [Indexed] + public string RestApi { get; set; } + + [Indexed] + public string Email { get; set; } + + public string Token { get; set; } + + public bool IsDefault { get; set; } = false; + } + + /// + /// A class for storing cached objects (that have been retrieved from a database). + /// + public class CachedObject + { + [PrimaryKey, Indexed] + public string Hash { get; set; } + + [Indexed] + public string RestApi { get; set; } + + public byte[ ] Bytes { get; set; } + } +} diff --git a/SpeckleCore/LocalData/SpeckleLocalContext.cs b/SpeckleCore/LocalData/SpeckleLocalContext.cs new file mode 100644 index 0000000..475570f --- /dev/null +++ b/SpeckleCore/LocalData/SpeckleLocalContext.cs @@ -0,0 +1,164 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using SQLite; + +namespace SpeckleCore +{ + + public static partial class SpeckleLocalContext + { + private static bool IsInit = false; + + private static SQLiteConnection Database; + + public static string DbPath = System.Environment.GetFolderPath( System.Environment.SpecialFolder.LocalApplicationData ) + @"\SpeckleSettings\SpeckleCache.db"; + + public static string SettingsFolderPath = System.Environment.GetFolderPath( System.Environment.SpecialFolder.LocalApplicationData ) + @"\SpeckleSettings\"; + + /// + /// Initialises the database context, ensures tables are created and powers up the rocket engines. + /// + public static void Init( ) + { + if ( IsInit ) return; + + Database = new SQLiteConnection( DbPath ); + Database.CreateTable(); + Database.CreateTable(); + + MigrateAccounts(); + + IsInit = true; + } + + public static void Close( ) + { + Database?.Close(); + } + + #region Accounts + /// + /// Migrates existing accounts stored in text files to the sqlite db. + /// + private static void MigrateAccounts( ) + { + List accounts = new List(); + + if ( Directory.Exists( SettingsFolderPath ) && Directory.EnumerateFiles( SettingsFolderPath, "*.txt" ).Count() > 0 ) + { + foreach ( string file in Directory.EnumerateFiles( SettingsFolderPath, "*.txt" ) ) + { + string content = File.ReadAllText( file ); + string[ ] pieces = content.TrimEnd( '\r', '\n' ).Split( ',' ); + + accounts.Add( new Account() { Email = pieces[ 0 ], Token = pieces[ 1 ], ServerName = pieces[ 2 ], RestApi = pieces[ 3 ] } ); + } + + var res = Database.InsertAll( accounts ); + + Directory.CreateDirectory( SettingsFolderPath + @"\MigratedAccounts\" ); + int k = 0; + foreach ( string file in Directory.EnumerateFiles( SettingsFolderPath, "*.txt" ) ) + File.Move( file, SettingsFolderPath + @"\MigratedAccounts\old_account_" + k++ + ".txt" ); + } + else + Debug.WriteLine( "No existing account text files found." ); + } + + /// + /// Adds a new account. + /// + /// + public static void AddAccount( Account account ) + { + var res = Database.Insert( account ); + } + + /// + /// Gets all accounts present. + /// + /// + public static List GetAllAccounts( ) + { + return Database.Query( "SELECT * FROM Account" ); + } + + /// + /// Gets all the accounts associated with the provided rest api. + /// + /// + /// + public static List GetAccountsByRestApi( string RestApi ) + { + return Database.Query( "SELECT * from Account WHERE RestApi = ?", RestApi ); + } + + /// + /// Gets all the accounts associated with the provided email. + /// + /// + /// + public static List GetAccountsByEmail( string email ) + { + return Database.Query( "SELECT * from Account WHERE Email = ?", email ); + } + + /// + /// If more accounts present, will return the first one only. + /// + /// + /// + /// + public static Account GetAccountByEmailAndRestApi( string email, string restApi ) + { + var res = Database.Query( String.Format("SELECT * from Account WHERE RestApi = '{0}' AND Email='{1}'", restApi, email) ); + if ( res.Count >= 1 ) return res[ 0 ]; + else throw new Exception( "Could not find account." ); + } + + /// + /// Returns the default account, if any. Otherwise throws an error. + /// + /// + public static Account GetDefaultAccount( ) + { + var res = Database.Query( "SELECT * FROM Account WHERE IsDefault='true' LIMIT 1" ); + if ( res.Count == 1 ) return res[ 0 ]; + else throw new Exception( "No default account set." ); + } + + /// + /// Sets an account as being the default one, and de-sets defaultness on all others. + /// + /// + public static void SetDefaultAccount( Account account ) + { + try + { + var currentDefault = GetDefaultAccount(); currentDefault.IsDefault = false; + Database.Update( currentDefault ); + } + catch ( Exception e ) + { + + } + + account.IsDefault = true; + Database.Update( account ); + } + + #endregion + + #region Objects + // TODO + #endregion + } + + + +} From 9f0d495f650daf782b14f09f2663a7eb2bfd0b89 Mon Sep 17 00:00:00 2001 From: Dimitrie Stefanescu Date: Mon, 29 Oct 2018 17:55:20 +0000 Subject: [PATCH 04/27] csproj includes fixed --- SpeckleCore/SpeckleCore.csproj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/SpeckleCore/SpeckleCore.csproj b/SpeckleCore/SpeckleCore.csproj index 22b3500..a210be7 100644 --- a/SpeckleCore/SpeckleCore.csproj +++ b/SpeckleCore/SpeckleCore.csproj @@ -42,7 +42,8 @@ - + + From dbbfa9fb495a82590e95b0e11c72ad47a821e532 Mon Sep 17 00:00:00 2001 From: Dimitrie Stefanescu Date: Mon, 29 Oct 2018 22:17:23 +0000 Subject: [PATCH 05/27] added remove account routine --- SpeckleCore/LocalData/SpeckleLocalContext.cs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/SpeckleCore/LocalData/SpeckleLocalContext.cs b/SpeckleCore/LocalData/SpeckleLocalContext.cs index 475570f..6f121c7 100644 --- a/SpeckleCore/LocalData/SpeckleLocalContext.cs +++ b/SpeckleCore/LocalData/SpeckleLocalContext.cs @@ -40,7 +40,7 @@ public static void Close( ) { Database?.Close(); } - + #region Accounts /// /// Migrates existing accounts stored in text files to the sqlite db. @@ -143,15 +143,20 @@ public static void SetDefaultAccount( Account account ) var currentDefault = GetDefaultAccount(); currentDefault.IsDefault = false; Database.Update( currentDefault ); } - catch ( Exception e ) + catch { - + // fail silently, it's not major } account.IsDefault = true; Database.Update( account ); } + public static void RemoveAccount(Account ac) + { + Database.Delete( ac.AccountId ); + } + #endregion #region Objects From aff70d9dd9e4b7e5b3a9342e143fb2d25fd15c76 Mon Sep 17 00:00:00 2001 From: Dimitrie Stefanescu Date: Tue, 30 Oct 2018 15:20:34 +0000 Subject: [PATCH 06/27] adds get and add routines for cached objects in the local context --- SpeckleCore/Conversion/Converter.cs | 27 ++++++++ SpeckleCore/LocalData/Models.cs | 18 +++++- SpeckleCore/LocalData/SpeckleLocalContext.cs | 65 ++++++++++++++++++-- 3 files changed, 104 insertions(+), 6 deletions(-) diff --git a/SpeckleCore/Conversion/Converter.cs b/SpeckleCore/Conversion/Converter.cs index 307071b..4e76c0e 100644 --- a/SpeckleCore/Conversion/Converter.cs +++ b/SpeckleCore/Conversion/Converter.cs @@ -74,6 +74,33 @@ public static byte[ ] base64ToBytes( string str ) return Convert.FromBase64String( str ); } + /// + /// Returns a stringifed MD5 hash of a string. + /// + /// String from which to generate the hash + /// If 0, the full hasdh will be returned, otherwise it will be trimmed to the specified lenght + /// + public static string getMd5Hash( string str, int length = 0) + { + using ( System.IO.MemoryStream ms = new System.IO.MemoryStream() ) + { + new System.Runtime.Serialization.Formatters.Binary.BinaryFormatter().Serialize( ms, str ); + byte[ ] hash; + using ( System.Security.Cryptography.MD5 md5 = System.Security.Cryptography.MD5.Create() ) + { + hash = md5.ComputeHash( ms.ToArray() ); + StringBuilder sb = new StringBuilder(); + foreach ( byte bbb in hash ) + sb.Append( bbb.ToString( "X2" ) ); + + if ( length != 0 ) + return sb.ToString().ToLower().Substring( 0, length ); + else + return sb.ToString().ToLower(); + } + } + } + // https://stackoverflow.com/a/299526/3446736 private static IEnumerable GetExtensionMethods( Assembly assembly, Type extendedType, string methodName ) { diff --git a/SpeckleCore/LocalData/Models.cs b/SpeckleCore/LocalData/Models.cs index a0aa5e2..7929ab5 100644 --- a/SpeckleCore/LocalData/Models.cs +++ b/SpeckleCore/LocalData/Models.cs @@ -33,12 +33,26 @@ public class Account /// public class CachedObject { - [PrimaryKey, Indexed] - public string Hash { get; set; } + /// + /// Represents hash(databaseId + restApi) + /// + [PrimaryKey, Indexed(Unique =true)] + public string CombinedHash { get; set; } [Indexed] public string RestApi { get; set; } + [Indexed] + public string DatabaseId { get; set; } + + [Indexed] + public string Hash { get; set; } + public byte[ ] Bytes { get; set; } + + public SpeckleObject ToSpeckle( ) + { + return SpeckleCore.Converter.getObjFromBytes( this.Bytes ) as SpeckleObject; + } } } diff --git a/SpeckleCore/LocalData/SpeckleLocalContext.cs b/SpeckleCore/LocalData/SpeckleLocalContext.cs index 6f121c7..3515b53 100644 --- a/SpeckleCore/LocalData/SpeckleLocalContext.cs +++ b/SpeckleCore/LocalData/SpeckleLocalContext.cs @@ -40,7 +40,7 @@ public static void Close( ) { Database?.Close(); } - + #region Accounts /// /// Migrates existing accounts stored in text files to the sqlite db. @@ -116,7 +116,7 @@ public static List GetAccountsByEmail( string email ) /// public static Account GetAccountByEmailAndRestApi( string email, string restApi ) { - var res = Database.Query( String.Format("SELECT * from Account WHERE RestApi = '{0}' AND Email='{1}'", restApi, email) ); + var res = Database.Query( String.Format( "SELECT * from Account WHERE RestApi = '{0}' AND Email='{1}'", restApi, email ) ); if ( res.Count >= 1 ) return res[ 0 ]; else throw new Exception( "Could not find account." ); } @@ -152,7 +152,7 @@ public static void SetDefaultAccount( Account account ) Database.Update( account ); } - public static void RemoveAccount(Account ac) + public static void RemoveAccount( Account ac ) { Database.Delete( ac.AccountId ); } @@ -160,7 +160,64 @@ public static void RemoveAccount(Account ac) #endregion #region Objects - // TODO + + /// + /// Adds a speckle object to the local cache. + /// + /// The object to add. + /// The server url of where it has been persisted. + public static void AddObject( SpeckleObject obj, string restApi ) + { + var bytes = SpeckleCore.Converter.getBytes( obj ); + var combinedHash = Converter.getMd5Hash( obj._id + restApi ); + var cached = new CachedObject() + { + RestApi = restApi, + Bytes = bytes, + DatabaseId = obj._id, + CombinedHash = combinedHash, + Hash = obj.Hash + }; + + try + { + Database.Insert( cached ); + } + catch + { + // object was already there + } + } + + /// + /// + /// + /// + /// + public static List GetObjects( List objs, string restApi ) + { + var combinedHashes = objs.Select( obj => Converter.getMd5Hash( obj._id + restApi ) ).ToList(); + var res = Database.Table().Where( obj => combinedHashes.Contains( obj.CombinedHash ) ).Select( o => o.ToSpeckle() ).ToList(); + + // populate the original list with whatever objects we found in the database. + for ( int i = 0; i < objs.Count; i++ ) + { + var placeholder = objs[ i ]; + var myObject = res.Find( o => o._id == placeholder._id ); + if ( myObject != null ) objs[ i ] = myObject; + } + + return objs; + } + + public static SpeckleObject GetObject( string combinedHash ) + { + + return null; + } + + //public static + #endregion } From bc027890abe226a9244c2683a363a896d1c54609 Mon Sep 17 00:00:00 2001 From: Dimitrie Stefanescu Date: Wed, 31 Oct 2018 08:53:11 +0000 Subject: [PATCH 07/27] added CachedStreams & added on date for objects --- SpeckleCore/LocalData/Models.cs | 42 +++++++++++++++- SpeckleCore/LocalData/SpeckleLocalContext.cs | 53 +++++++++++++++++--- 2 files changed, 85 insertions(+), 10 deletions(-) diff --git a/SpeckleCore/LocalData/Models.cs b/SpeckleCore/LocalData/Models.cs index 7929ab5..9e1059b 100644 --- a/SpeckleCore/LocalData/Models.cs +++ b/SpeckleCore/LocalData/Models.cs @@ -36,9 +36,12 @@ public class CachedObject /// /// Represents hash(databaseId + restApi) /// - [PrimaryKey, Indexed(Unique =true)] + [PrimaryKey, Indexed( Unique = true )] public string CombinedHash { get; set; } - + + /// + /// Represents the api this object came from + /// [Indexed] public string RestApi { get; set; } @@ -48,11 +51,46 @@ public class CachedObject [Indexed] public string Hash { get; set; } + public DateTime AddedOn {get;set;} + public byte[ ] Bytes { get; set; } + /// + /// Returns the speckle object from cache. + /// + /// public SpeckleObject ToSpeckle( ) { return SpeckleCore.Converter.getObjFromBytes( this.Bytes ) as SpeckleObject; } } + + public class CachedStream + { + /// + /// Represents hash(streamId + restApi) + /// + [PrimaryKey, Indexed( Unique = true )] + public string CombinedHash { get; set; } + + /// + /// Represents the api this object came from + /// + [Indexed] + public string RestApi { get; set; } + + [Indexed] + public string StreamId { get; set; } + + public DateTime AddedOn { get; set; } + + public DateTime UpdatedOn { get; set; } + + public byte[ ] Bytes { get; set; } + + public SpeckleStream ToSpeckle() + { + return SpeckleCore.Converter.getObjFromBytes( this.Bytes ) as SpeckleStream; + } + } } diff --git a/SpeckleCore/LocalData/SpeckleLocalContext.cs b/SpeckleCore/LocalData/SpeckleLocalContext.cs index 3515b53..36b0440 100644 --- a/SpeckleCore/LocalData/SpeckleLocalContext.cs +++ b/SpeckleCore/LocalData/SpeckleLocalContext.cs @@ -9,8 +9,11 @@ namespace SpeckleCore { - - public static partial class SpeckleLocalContext + /// + /// This class holds the keys to the local sqlite database that acts as a local cache for various speckle things. + /// You can access accounts from here across speckle integrations, as well as the local object cache. + /// + public static partial class LocalContext { private static bool IsInit = false; @@ -30,6 +33,7 @@ public static void Init( ) Database = new SQLiteConnection( DbPath ); Database.CreateTable(); Database.CreateTable(); + Database.CreateTable(); MigrateAccounts(); @@ -176,7 +180,8 @@ public static void AddObject( SpeckleObject obj, string restApi ) Bytes = bytes, DatabaseId = obj._id, CombinedHash = combinedHash, - Hash = obj.Hash + Hash = obj.Hash, + AddedOn = DateTime.Now }; try @@ -190,9 +195,10 @@ public static void AddObject( SpeckleObject obj, string restApi ) } /// - /// + /// Does a cache check on a list of speckle object placeholders. It will populate the original list with any objects it can find in the cache. If none are found, the list is returned unmodified. /// - /// + /// Speckle object placeholders to check against the cache. + /// The rest api these objects are expected to come from. /// public static List GetObjects( List objs, string restApi ) { @@ -209,14 +215,45 @@ public static List GetObjects( List objs, string r return objs; } + #endregion + + #region Streams - public static SpeckleObject GetObject( string combinedHash ) + // TODO + + public static void AddOrUpdateStream( SpeckleStream stream, string restApi ) { + var bytes = SpeckleCore.Converter.getBytes( stream ); + var combinedHash = Converter.getMd5Hash( stream._id + restApi ); + + var cacheRes = Database.Table().Where( existing => existing.CombinedHash == combinedHash ).ToList(); + + if ( cacheRes.Count >= 1 ) + { + var toUpdate = cacheRes[ 0 ]; + toUpdate.Bytes = bytes; + toUpdate.UpdatedOn = DateTime.Now; + Database.Update( toUpdate ); + } + else + { + var toCache = new CachedStream() + { + CombinedHash = combinedHash, + Bytes = bytes, + RestApi = restApi, + StreamId = stream.StreamId, + AddedOn = DateTime.Now, + UpdatedOn = DateTime.Now + }; + Database.Insert( toCache ); + } - return null; + //throw new NotImplementedException(); } - //public static + public static void UpdateStream( SpeckleStream stream, string restApi ) { throw new NotImplementedException(); } + public static void GetStream( string streamId, string restApi ) { throw new NotImplementedException(); } #endregion } From d9463470ec528cfc8393ba11498e84ea82980305 Mon Sep 17 00:00:00 2001 From: Dimitrie Stefanescu Date: Wed, 31 Oct 2018 16:23:47 +0000 Subject: [PATCH 08/27] code doc + get stream from cache --- SpeckleCore/LocalData/Models.cs | 2 +- SpeckleCore/LocalData/SpeckleLocalContext.cs | 25 +++++++++++++++----- SpeckleCore/Models/ModelBase.cs | 6 ++--- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/SpeckleCore/LocalData/Models.cs b/SpeckleCore/LocalData/Models.cs index 9e1059b..e8872f0 100644 --- a/SpeckleCore/LocalData/Models.cs +++ b/SpeckleCore/LocalData/Models.cs @@ -90,7 +90,7 @@ public class CachedStream public SpeckleStream ToSpeckle() { - return SpeckleCore.Converter.getObjFromBytes( this.Bytes ) as SpeckleStream; + return SpeckleStream.FromJson( SpeckleCore.Converter.getObjFromBytes( this.Bytes ) as string ); // ((SpeckleCore.Converter.getObjFromBytes( this.Bytes ) as SpeckleStream; } } } diff --git a/SpeckleCore/LocalData/SpeckleLocalContext.cs b/SpeckleCore/LocalData/SpeckleLocalContext.cs index 36b0440..7b9a115 100644 --- a/SpeckleCore/LocalData/SpeckleLocalContext.cs +++ b/SpeckleCore/LocalData/SpeckleLocalContext.cs @@ -219,11 +219,14 @@ public static List GetObjects( List objs, string r #region Streams - // TODO - + /// + /// Updates or inserts a stream in the local cache. + /// + /// + /// public static void AddOrUpdateStream( SpeckleStream stream, string restApi ) { - var bytes = SpeckleCore.Converter.getBytes( stream ); + var bytes = SpeckleCore.Converter.getBytes( stream.ToJson() ); var combinedHash = Converter.getMd5Hash( stream._id + restApi ); var cacheRes = Database.Table().Where( existing => existing.CombinedHash == combinedHash ).ToList(); @@ -248,12 +251,22 @@ public static void AddOrUpdateStream( SpeckleStream stream, string restApi ) }; Database.Insert( toCache ); } - //throw new NotImplementedException(); } - public static void UpdateStream( SpeckleStream stream, string restApi ) { throw new NotImplementedException(); } - public static void GetStream( string streamId, string restApi ) { throw new NotImplementedException(); } + /// + /// Gets a stream from the local cache. + /// + /// + /// + /// Null, if nothing found, or the speckle stream. + public static SpeckleStream GetStream( string streamId, string restApi ) + { + var combinedHash = Converter.getMd5Hash( streamId + restApi ); + var res = Database.Table().Where( str => str.CombinedHash == combinedHash ).ToArray(); + if ( res.Length > 0 ) return res[ 0 ].ToSpeckle(); + return null; + } #endregion } diff --git a/SpeckleCore/Models/ModelBase.cs b/SpeckleCore/Models/ModelBase.cs index e1b7e5c..8ab348e 100644 --- a/SpeckleCore/Models/ModelBase.cs +++ b/SpeckleCore/Models/ModelBase.cs @@ -251,17 +251,17 @@ public partial class SpeckleStream : ResourceBase /// Units, tolerances, etc. [Newtonsoft.Json.JsonProperty( "baseProperties", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore )] - public object BaseProperties { get; set; } + public dynamic BaseProperties { get; set; } /// Any performance measures can go in here. [Newtonsoft.Json.JsonProperty( "globalMeasures", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore )] - public object GlobalMeasures { get; set; } + public dynamic GlobalMeasures { get; set; } [Newtonsoft.Json.JsonProperty( "isComputedResult", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore )] public bool? IsComputedResult { get; set; } [Newtonsoft.Json.JsonProperty( "viewerLayers", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore )] - public List ViewerLayers { get; set; } + public List ViewerLayers { get; set; } /// If this stream is a child, the parent's streamId. [Newtonsoft.Json.JsonProperty( "parent", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore )] From 36a1759a8b5892db51927dfcc4c4f55c4a6983b9 Mon Sep 17 00:00:00 2001 From: Dimitrie Stefanescu Date: Wed, 31 Oct 2018 19:01:46 +0000 Subject: [PATCH 09/27] added prune existing objects routine --- SpeckleCore/LocalData/SpeckleLocalContext.cs | 25 ++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/SpeckleCore/LocalData/SpeckleLocalContext.cs b/SpeckleCore/LocalData/SpeckleLocalContext.cs index 7b9a115..aeca79f 100644 --- a/SpeckleCore/LocalData/SpeckleLocalContext.cs +++ b/SpeckleCore/LocalData/SpeckleLocalContext.cs @@ -215,6 +215,31 @@ public static List GetObjects( List objs, string r return objs; } + + /// + /// Replaces any objects in the given list with placeholders if they're found in the local cache, as this means they were sent before and most probably exist on the server. + /// + /// + /// + /// (Optinoal) The modified list. + public static List PruneExistingObjects( List objs, string restApi) + { + var objHashes = objs.Select( obj => true ? obj.Hash : restApi ).ToList(); + var res = Database.Table().Where( obj => objHashes.Contains( true ? obj.Hash : restApi)/* && obj.RestApi == restApi*/ ).ToList(); + + for ( int i = 0; i < objs.Count; i++ ) + { + var placeholder = objs[ i ]; + var myObject = res.Find( o => o.Hash == objs[ i ].Hash ); + if ( myObject != null ) + { + objs[ i ] = new SpecklePlaceholder() { _id = myObject.DatabaseId }; + } + } + + return objs; + } + #endregion #region Streams From 6fce630558ef922f4f8dcfbfd7fd3eb9b406d754 Mon Sep 17 00:00:00 2001 From: Dimitrie Stefanescu Date: Thu, 1 Nov 2018 07:48:36 +0000 Subject: [PATCH 10/27] add api check on prune objects --- SpeckleCore/LocalData/SpeckleLocalContext.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SpeckleCore/LocalData/SpeckleLocalContext.cs b/SpeckleCore/LocalData/SpeckleLocalContext.cs index aeca79f..3be0f71 100644 --- a/SpeckleCore/LocalData/SpeckleLocalContext.cs +++ b/SpeckleCore/LocalData/SpeckleLocalContext.cs @@ -225,7 +225,7 @@ public static List GetObjects( List objs, string r public static List PruneExistingObjects( List objs, string restApi) { var objHashes = objs.Select( obj => true ? obj.Hash : restApi ).ToList(); - var res = Database.Table().Where( obj => objHashes.Contains( true ? obj.Hash : restApi)/* && obj.RestApi == restApi*/ ).ToList(); + var res = Database.Table().Where( obj => objHashes.Contains( obj.Hash) && obj.RestApi == restApi ).ToList(); for ( int i = 0; i < objs.Count; i++ ) { From 16f7a6909aa7ff284b88ba31490f68a09d7a67af Mon Sep 17 00:00:00 2001 From: Dimitrie Stefanescu Date: Thu, 1 Nov 2018 08:43:55 +0000 Subject: [PATCH 11/27] this should take care of setting default accounts --- SpeckleCore/LocalData/SpeckleLocalContext.cs | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/SpeckleCore/LocalData/SpeckleLocalContext.cs b/SpeckleCore/LocalData/SpeckleLocalContext.cs index 3be0f71..70fb546 100644 --- a/SpeckleCore/LocalData/SpeckleLocalContext.cs +++ b/SpeckleCore/LocalData/SpeckleLocalContext.cs @@ -142,15 +142,8 @@ public static Account GetDefaultAccount( ) /// public static void SetDefaultAccount( Account account ) { - try - { - var currentDefault = GetDefaultAccount(); currentDefault.IsDefault = false; - Database.Update( currentDefault ); - } - catch - { - // fail silently, it's not major - } + + Database.Execute( "UPDATE Account SET IsDefault='false'" ); account.IsDefault = true; Database.Update( account ); From f429069e97859f1b390e153b07f2d541fb7967b4 Mon Sep 17 00:00:00 2001 From: Dimitrie Stefanescu Date: Thu, 1 Nov 2018 08:51:55 +0000 Subject: [PATCH 12/27] this should take care of setting default accounts *properly* --- SpeckleCore/LocalData/SpeckleLocalContext.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SpeckleCore/LocalData/SpeckleLocalContext.cs b/SpeckleCore/LocalData/SpeckleLocalContext.cs index 70fb546..7aaf06d 100644 --- a/SpeckleCore/LocalData/SpeckleLocalContext.cs +++ b/SpeckleCore/LocalData/SpeckleLocalContext.cs @@ -143,7 +143,7 @@ public static Account GetDefaultAccount( ) public static void SetDefaultAccount( Account account ) { - Database.Execute( "UPDATE Account SET IsDefault='false'" ); + Database.Execute( "UPDATE Account SET IsDefault=0" ); account.IsDefault = true; Database.Update( account ); From bc12da416bc2ecfae113460148f9c2cc4a628323 Mon Sep 17 00:00:00 2001 From: Dimitrie Stefanescu Date: Fri, 2 Nov 2018 17:46:08 +0000 Subject: [PATCH 13/27] adding second try for assembly check based only on short name re abstract deserialisation --- SpeckleCore/Conversion/Converter.cs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/SpeckleCore/Conversion/Converter.cs b/SpeckleCore/Conversion/Converter.cs index 4e76c0e..5b24a19 100644 --- a/SpeckleCore/Conversion/Converter.cs +++ b/SpeckleCore/Conversion/Converter.cs @@ -184,7 +184,16 @@ public static object Deserialise( SpeckleObject obj, object root = null ) if ( absObj._type == "ref" ) return null; + //var shortName = absObj._assembly.Split( ',' )[ 0 ]; + var assembly = System.AppDomain.CurrentDomain.GetAssemblies().FirstOrDefault( a => a.FullName == absObj._assembly ); + + //try again + if(assembly == null) + { + var shortName = absObj._assembly.Split( ',' )[ 0 ]; + assembly = System.AppDomain.CurrentDomain.GetAssemblies().FirstOrDefault( a => a.FullName.Contains(shortName )); + } if ( assembly == null ) // we can't deserialise for sure return Converter.ShallowConvert( absObj ); From e63eb20f31e7d4dfd9c2588124068ae48cca6c51 Mon Sep 17 00:00:00 2001 From: Matteo Cominetti Date: Sat, 3 Nov 2018 17:19:05 +0000 Subject: [PATCH 14/27] fixed bug if no specklesettings folder exists --- SpeckleCore/LocalData/SpeckleLocalContext.cs | 141 ++++++++++--------- 1 file changed, 72 insertions(+), 69 deletions(-) mode change 100644 => 100755 SpeckleCore/LocalData/SpeckleLocalContext.cs diff --git a/SpeckleCore/LocalData/SpeckleLocalContext.cs b/SpeckleCore/LocalData/SpeckleLocalContext.cs old mode 100644 new mode 100755 index 7aaf06d..f495604 --- a/SpeckleCore/LocalData/SpeckleLocalContext.cs +++ b/SpeckleCore/LocalData/SpeckleLocalContext.cs @@ -19,18 +19,21 @@ public static partial class LocalContext private static SQLiteConnection Database; - public static string DbPath = System.Environment.GetFolderPath( System.Environment.SpecialFolder.LocalApplicationData ) + @"\SpeckleSettings\SpeckleCache.db"; + public static string DbPath = System.Environment.GetFolderPath(System.Environment.SpecialFolder.LocalApplicationData) + @"\SpeckleSettings\SpeckleCache.db"; - public static string SettingsFolderPath = System.Environment.GetFolderPath( System.Environment.SpecialFolder.LocalApplicationData ) + @"\SpeckleSettings\"; + public static string SettingsFolderPath = System.Environment.GetFolderPath(System.Environment.SpecialFolder.LocalApplicationData) + @"\SpeckleSettings\"; /// /// Initialises the database context, ensures tables are created and powers up the rocket engines. /// - public static void Init( ) + public static void Init() { - if ( IsInit ) return; + if (IsInit) return; - Database = new SQLiteConnection( DbPath ); + if (!Directory.Exists(SettingsFolderPath)) + Directory.CreateDirectory(SettingsFolderPath); + + Database = new SQLiteConnection(DbPath); Database.CreateTable(); Database.CreateTable(); Database.CreateTable(); @@ -40,7 +43,7 @@ public static void Init( ) IsInit = true; } - public static void Close( ) + public static void Close() { Database?.Close(); } @@ -49,47 +52,47 @@ public static void Close( ) /// /// Migrates existing accounts stored in text files to the sqlite db. /// - private static void MigrateAccounts( ) + private static void MigrateAccounts() { List accounts = new List(); - if ( Directory.Exists( SettingsFolderPath ) && Directory.EnumerateFiles( SettingsFolderPath, "*.txt" ).Count() > 0 ) + if (Directory.Exists(SettingsFolderPath) && Directory.EnumerateFiles(SettingsFolderPath, "*.txt").Count() > 0) { - foreach ( string file in Directory.EnumerateFiles( SettingsFolderPath, "*.txt" ) ) + foreach (string file in Directory.EnumerateFiles(SettingsFolderPath, "*.txt")) { - string content = File.ReadAllText( file ); - string[ ] pieces = content.TrimEnd( '\r', '\n' ).Split( ',' ); + string content = File.ReadAllText(file); + string[] pieces = content.TrimEnd('\r', '\n').Split(','); - accounts.Add( new Account() { Email = pieces[ 0 ], Token = pieces[ 1 ], ServerName = pieces[ 2 ], RestApi = pieces[ 3 ] } ); + accounts.Add(new Account() { Email = pieces[0], Token = pieces[1], ServerName = pieces[2], RestApi = pieces[3] }); } - var res = Database.InsertAll( accounts ); + var res = Database.InsertAll(accounts); - Directory.CreateDirectory( SettingsFolderPath + @"\MigratedAccounts\" ); + Directory.CreateDirectory(SettingsFolderPath + @"\MigratedAccounts\"); int k = 0; - foreach ( string file in Directory.EnumerateFiles( SettingsFolderPath, "*.txt" ) ) - File.Move( file, SettingsFolderPath + @"\MigratedAccounts\old_account_" + k++ + ".txt" ); + foreach (string file in Directory.EnumerateFiles(SettingsFolderPath, "*.txt")) + File.Move(file, SettingsFolderPath + @"\MigratedAccounts\old_account_" + k++ + ".txt"); } else - Debug.WriteLine( "No existing account text files found." ); + Debug.WriteLine("No existing account text files found."); } /// /// Adds a new account. /// /// - public static void AddAccount( Account account ) + public static void AddAccount(Account account) { - var res = Database.Insert( account ); + var res = Database.Insert(account); } /// /// Gets all accounts present. /// /// - public static List GetAllAccounts( ) + public static List GetAllAccounts() { - return Database.Query( "SELECT * FROM Account" ); + return Database.Query("SELECT * FROM Account"); } /// @@ -97,9 +100,9 @@ public static List GetAllAccounts( ) /// /// /// - public static List GetAccountsByRestApi( string RestApi ) + public static List GetAccountsByRestApi(string RestApi) { - return Database.Query( "SELECT * from Account WHERE RestApi = ?", RestApi ); + return Database.Query("SELECT * from Account WHERE RestApi = ?", RestApi); } /// @@ -107,9 +110,9 @@ public static List GetAccountsByRestApi( string RestApi ) /// /// /// - public static List GetAccountsByEmail( string email ) + public static List GetAccountsByEmail(string email) { - return Database.Query( "SELECT * from Account WHERE Email = ?", email ); + return Database.Query("SELECT * from Account WHERE Email = ?", email); } /// @@ -118,40 +121,40 @@ public static List GetAccountsByEmail( string email ) /// /// /// - public static Account GetAccountByEmailAndRestApi( string email, string restApi ) + public static Account GetAccountByEmailAndRestApi(string email, string restApi) { - var res = Database.Query( String.Format( "SELECT * from Account WHERE RestApi = '{0}' AND Email='{1}'", restApi, email ) ); - if ( res.Count >= 1 ) return res[ 0 ]; - else throw new Exception( "Could not find account." ); + var res = Database.Query(String.Format("SELECT * from Account WHERE RestApi = '{0}' AND Email='{1}'", restApi, email)); + if (res.Count >= 1) return res[0]; + else throw new Exception("Could not find account."); } /// /// Returns the default account, if any. Otherwise throws an error. /// /// - public static Account GetDefaultAccount( ) + public static Account GetDefaultAccount() { - var res = Database.Query( "SELECT * FROM Account WHERE IsDefault='true' LIMIT 1" ); - if ( res.Count == 1 ) return res[ 0 ]; - else throw new Exception( "No default account set." ); + var res = Database.Query("SELECT * FROM Account WHERE IsDefault='true' LIMIT 1"); + if (res.Count == 1) return res[0]; + else throw new Exception("No default account set."); } /// /// Sets an account as being the default one, and de-sets defaultness on all others. /// /// - public static void SetDefaultAccount( Account account ) + public static void SetDefaultAccount(Account account) { - Database.Execute( "UPDATE Account SET IsDefault=0" ); + Database.Execute("UPDATE Account SET IsDefault=0"); account.IsDefault = true; - Database.Update( account ); + Database.Update(account); } - public static void RemoveAccount( Account ac ) + public static void RemoveAccount(Account ac) { - Database.Delete( ac.AccountId ); + Database.Delete(ac.AccountId); } #endregion @@ -163,10 +166,10 @@ public static void RemoveAccount( Account ac ) /// /// The object to add. /// The server url of where it has been persisted. - public static void AddObject( SpeckleObject obj, string restApi ) + public static void AddObject(SpeckleObject obj, string restApi) { - var bytes = SpeckleCore.Converter.getBytes( obj ); - var combinedHash = Converter.getMd5Hash( obj._id + restApi ); + var bytes = SpeckleCore.Converter.getBytes(obj); + var combinedHash = Converter.getMd5Hash(obj._id + restApi); var cached = new CachedObject() { RestApi = restApi, @@ -179,7 +182,7 @@ public static void AddObject( SpeckleObject obj, string restApi ) try { - Database.Insert( cached ); + Database.Insert(cached); } catch { @@ -193,17 +196,17 @@ public static void AddObject( SpeckleObject obj, string restApi ) /// Speckle object placeholders to check against the cache. /// The rest api these objects are expected to come from. /// - public static List GetObjects( List objs, string restApi ) + public static List GetObjects(List objs, string restApi) { - var combinedHashes = objs.Select( obj => Converter.getMd5Hash( obj._id + restApi ) ).ToList(); - var res = Database.Table().Where( obj => combinedHashes.Contains( obj.CombinedHash ) ).Select( o => o.ToSpeckle() ).ToList(); + var combinedHashes = objs.Select(obj => Converter.getMd5Hash(obj._id + restApi)).ToList(); + var res = Database.Table().Where(obj => combinedHashes.Contains(obj.CombinedHash)).Select(o => o.ToSpeckle()).ToList(); // populate the original list with whatever objects we found in the database. - for ( int i = 0; i < objs.Count; i++ ) + for (int i = 0; i < objs.Count; i++) { - var placeholder = objs[ i ]; - var myObject = res.Find( o => o._id == placeholder._id ); - if ( myObject != null ) objs[ i ] = myObject; + var placeholder = objs[i]; + var myObject = res.Find(o => o._id == placeholder._id); + if (myObject != null) objs[i] = myObject; } return objs; @@ -215,18 +218,18 @@ public static List GetObjects( List objs, string r /// /// /// (Optinoal) The modified list. - public static List PruneExistingObjects( List objs, string restApi) + public static List PruneExistingObjects(List objs, string restApi) { - var objHashes = objs.Select( obj => true ? obj.Hash : restApi ).ToList(); - var res = Database.Table().Where( obj => objHashes.Contains( obj.Hash) && obj.RestApi == restApi ).ToList(); + var objHashes = objs.Select(obj => true ? obj.Hash : restApi).ToList(); + var res = Database.Table().Where(obj => objHashes.Contains(obj.Hash) && obj.RestApi == restApi).ToList(); - for ( int i = 0; i < objs.Count; i++ ) + for (int i = 0; i < objs.Count; i++) { - var placeholder = objs[ i ]; - var myObject = res.Find( o => o.Hash == objs[ i ].Hash ); - if ( myObject != null ) + var placeholder = objs[i]; + var myObject = res.Find(o => o.Hash == objs[i].Hash); + if (myObject != null) { - objs[ i ] = new SpecklePlaceholder() { _id = myObject.DatabaseId }; + objs[i] = new SpecklePlaceholder() { _id = myObject.DatabaseId }; } } @@ -242,19 +245,19 @@ public static List PruneExistingObjects( List objs /// /// /// - public static void AddOrUpdateStream( SpeckleStream stream, string restApi ) + public static void AddOrUpdateStream(SpeckleStream stream, string restApi) { - var bytes = SpeckleCore.Converter.getBytes( stream.ToJson() ); - var combinedHash = Converter.getMd5Hash( stream._id + restApi ); + var bytes = SpeckleCore.Converter.getBytes(stream.ToJson()); + var combinedHash = Converter.getMd5Hash(stream._id + restApi); - var cacheRes = Database.Table().Where( existing => existing.CombinedHash == combinedHash ).ToList(); + var cacheRes = Database.Table().Where(existing => existing.CombinedHash == combinedHash).ToList(); - if ( cacheRes.Count >= 1 ) + if (cacheRes.Count >= 1) { - var toUpdate = cacheRes[ 0 ]; + var toUpdate = cacheRes[0]; toUpdate.Bytes = bytes; toUpdate.UpdatedOn = DateTime.Now; - Database.Update( toUpdate ); + Database.Update(toUpdate); } else { @@ -267,7 +270,7 @@ public static void AddOrUpdateStream( SpeckleStream stream, string restApi ) AddedOn = DateTime.Now, UpdatedOn = DateTime.Now }; - Database.Insert( toCache ); + Database.Insert(toCache); } //throw new NotImplementedException(); } @@ -278,11 +281,11 @@ public static void AddOrUpdateStream( SpeckleStream stream, string restApi ) /// /// /// Null, if nothing found, or the speckle stream. - public static SpeckleStream GetStream( string streamId, string restApi ) + public static SpeckleStream GetStream(string streamId, string restApi) { - var combinedHash = Converter.getMd5Hash( streamId + restApi ); - var res = Database.Table().Where( str => str.CombinedHash == combinedHash ).ToArray(); - if ( res.Length > 0 ) return res[ 0 ].ToSpeckle(); + var combinedHash = Converter.getMd5Hash(streamId + restApi); + var res = Database.Table().Where(str => str.CombinedHash == combinedHash).ToArray(); + if (res.Length > 0) return res[0].ToSpeckle(); return null; } From 50c1599f92c6769abc0b37676f5ca6b043dfef9c Mon Sep 17 00:00:00 2001 From: Dimitrie Stefanescu Date: Sat, 3 Nov 2018 23:25:09 +0000 Subject: [PATCH 15/27] implementing speckle object equality comparer --- SpeckleCore/Models/ModelObjects.cs | 14 ++++++++++++-- SpeckleCore/Models/ModelObjectsExt.cs | 15 ++++++++++++++- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/SpeckleCore/Models/ModelObjects.cs b/SpeckleCore/Models/ModelObjects.cs index 45dfb12..0c046b2 100644 --- a/SpeckleCore/Models/ModelObjects.cs +++ b/SpeckleCore/Models/ModelObjects.cs @@ -1,5 +1,6 @@ using Newtonsoft.Json; using System; +using System.Collections; using System.Collections.Generic; using System.Linq; using System.Text; @@ -115,7 +116,7 @@ public enum SpeckleObjectType [JsonInheritanceAttribute( "SpeckleBlock", typeof( SpeckleBlock ) )] [System.CodeDom.Compiler.GeneratedCode( "NJsonSchema", "9.10.41.0 (Newtonsoft.Json v9.0.0.0)" )] [Serializable] - public partial class SpeckleObject : ResourceBase + public partial class SpeckleObject : ResourceBase, IEqualityComparer { [Newtonsoft.Json.JsonProperty( "type", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore )] [Newtonsoft.Json.JsonConverter( typeof( Newtonsoft.Json.Converters.StringEnumConverter ) )] @@ -164,7 +165,16 @@ public static SpeckleObject FromJson( string data ) return Newtonsoft.Json.JsonConvert.DeserializeObject( data ); } - } + public bool Equals( SpeckleObject x, SpeckleObject y ) + { + return x.Hash == y.Hash; + } + + public int GetHashCode( SpeckleObject obj ) + { + return obj.Hash.GetHashCode(); + } + } [System.CodeDom.Compiler.GeneratedCode( "NJsonSchema", "9.10.41.0 (Newtonsoft.Json v9.0.0.0)" )] [Serializable] diff --git a/SpeckleCore/Models/ModelObjectsExt.cs b/SpeckleCore/Models/ModelObjectsExt.cs index 6ab7679..6fd0f21 100644 --- a/SpeckleCore/Models/ModelObjectsExt.cs +++ b/SpeckleCore/Models/ModelObjectsExt.cs @@ -13,7 +13,20 @@ namespace SpeckleCore { - public partial class SpeckleObject + public class SpeckleObjectComparer : IEqualityComparer + { + public bool Equals( SpeckleObject x, SpeckleObject y ) + { + return x.Hash == y.Hash; + } + + public int GetHashCode( SpeckleObject obj ) + { + return obj.Hash.GetHashCode(); + } + } + + public partial class SpeckleObject { /// /// Generates a truncated (to 12) md5 hash of an object. From 10da06a3b7ab6f1984fe638a12ae39bf7c27a5a1 Mon Sep 17 00:00:00 2001 From: Matteo Cominetti Date: Mon, 5 Nov 2018 12:44:30 +0000 Subject: [PATCH 16/27] fixed migration logic when same file already exists --- SpeckleCore/LocalData/SpeckleLocalContext.cs | 55 ++++++++++++++++---- 1 file changed, 45 insertions(+), 10 deletions(-) diff --git a/SpeckleCore/LocalData/SpeckleLocalContext.cs b/SpeckleCore/LocalData/SpeckleLocalContext.cs index f495604..9266ace 100755 --- a/SpeckleCore/LocalData/SpeckleLocalContext.cs +++ b/SpeckleCore/LocalData/SpeckleLocalContext.cs @@ -3,8 +3,6 @@ using System.Diagnostics; using System.IO; using System.Linq; -using System.Text; -using System.Threading.Tasks; using SQLite; namespace SpeckleCore @@ -69,12 +67,30 @@ private static void MigrateAccounts() var res = Database.InsertAll(accounts); Directory.CreateDirectory(SettingsFolderPath + @"\MigratedAccounts\"); - int k = 0; + foreach (string file in Directory.EnumerateFiles(SettingsFolderPath, "*.txt")) - File.Move(file, SettingsFolderPath + @"\MigratedAccounts\old_account_" + k++ + ".txt"); + { + try + { + var newName = Path.Combine(SettingsFolderPath, "MigratedAccounts", Path.GetFileName(file)); + if (File.Exists(newName)) + { + File.Delete(newName); + } + + File.Move(file, newName); + } + catch (Exception e) + { + Debug.WriteLine("yolo"); + } + } + } else + { Debug.WriteLine("No existing account text files found."); + } } /// @@ -124,8 +140,14 @@ public static List GetAccountsByEmail(string email) public static Account GetAccountByEmailAndRestApi(string email, string restApi) { var res = Database.Query(String.Format("SELECT * from Account WHERE RestApi = '{0}' AND Email='{1}'", restApi, email)); - if (res.Count >= 1) return res[0]; - else throw new Exception("Could not find account."); + if (res.Count >= 1) + { + return res[0]; + } + else + { + throw new Exception("Could not find account."); + } } /// @@ -135,8 +157,14 @@ public static Account GetAccountByEmailAndRestApi(string email, string restApi) public static Account GetDefaultAccount() { var res = Database.Query("SELECT * FROM Account WHERE IsDefault='true' LIMIT 1"); - if (res.Count == 1) return res[0]; - else throw new Exception("No default account set."); + if (res.Count == 1) + { + return res[0]; + } + else + { + throw new Exception("No default account set."); + } } /// @@ -206,7 +234,10 @@ public static List GetObjects(List objs, string re { var placeholder = objs[i]; var myObject = res.Find(o => o._id == placeholder._id); - if (myObject != null) objs[i] = myObject; + if (myObject != null) + { + objs[i] = myObject; + } } return objs; @@ -285,7 +316,11 @@ public static SpeckleStream GetStream(string streamId, string restApi) { var combinedHash = Converter.getMd5Hash(streamId + restApi); var res = Database.Table().Where(str => str.CombinedHash == combinedHash).ToArray(); - if (res.Length > 0) return res[0].ToSpeckle(); + if (res.Length > 0) + { + return res[0].ToSpeckle(); + } + return null; } From fd4ac1432374245eb9922c30916061a31deeb8d8 Mon Sep 17 00:00:00 2001 From: Dimitrie Stefanescu Date: Tue, 13 Nov 2018 11:58:48 +0000 Subject: [PATCH 17/27] modifies local context with separate table & methods for "sent" objects vs. "received" objects --- SpeckleCore/LocalData/Models.cs | 18 +++++++++++++ SpeckleCore/LocalData/SpeckleLocalContext.cs | 27 ++++++++++++++++++-- 2 files changed, 43 insertions(+), 2 deletions(-) mode change 100755 => 100644 SpeckleCore/LocalData/SpeckleLocalContext.cs diff --git a/SpeckleCore/LocalData/Models.cs b/SpeckleCore/LocalData/Models.cs index e8872f0..3ac8643 100644 --- a/SpeckleCore/LocalData/Models.cs +++ b/SpeckleCore/LocalData/Models.cs @@ -28,6 +28,24 @@ public class Account public bool IsDefault { get; set; } = false; } + /// + /// Special class for efficiently storing sent objects. Why? We do not want to store them fully as they are already stored in the users's file. Kind of duplicates the CachedObject. + /// + public class SentObject + { + /// + /// Represents the api this object came from + /// + [Indexed] + public string RestApi { get; set; } + + [Indexed] + public string DatabaseId { get; set; } + + [PrimaryKey, Indexed] + public string Hash { get; set; } + } + /// /// A class for storing cached objects (that have been retrieved from a database). /// diff --git a/SpeckleCore/LocalData/SpeckleLocalContext.cs b/SpeckleCore/LocalData/SpeckleLocalContext.cs old mode 100755 new mode 100644 index 9266ace..c2979a0 --- a/SpeckleCore/LocalData/SpeckleLocalContext.cs +++ b/SpeckleCore/LocalData/SpeckleLocalContext.cs @@ -34,6 +34,7 @@ public static void Init() Database = new SQLiteConnection(DbPath); Database.CreateTable(); Database.CreateTable(); + Database.CreateTable(); Database.CreateTable(); MigrateAccounts(); @@ -187,7 +188,7 @@ public static void RemoveAccount(Account ac) #endregion - #region Objects + #region Received Objects /// /// Adds a speckle object to the local cache. @@ -243,6 +244,28 @@ public static List GetObjects(List objs, string re return objs; } + #endregion + + #region Sent Objects + + public static void AddSentObject(SpeckleObject obj, string restApi) + { + var sentObj = new SentObject() + { + RestApi = restApi, + DatabaseId = obj._id, + Hash = obj.Hash + }; + + try + { + Database.Insert( sentObj ); + } catch + { + // object was already there, no panic! + } + } + /// /// Replaces any objects in the given list with placeholders if they're found in the local cache, as this means they were sent before and most probably exist on the server. /// @@ -252,7 +275,7 @@ public static List GetObjects(List objs, string re public static List PruneExistingObjects(List objs, string restApi) { var objHashes = objs.Select(obj => true ? obj.Hash : restApi).ToList(); - var res = Database.Table().Where(obj => objHashes.Contains(obj.Hash) && obj.RestApi == restApi).ToList(); + var res = Database.Table().Where(obj => objHashes.Contains(obj.Hash) && obj.RestApi == restApi).ToList(); for (int i = 0; i < objs.Count; i++) { From 3f4232908b905f0dc01f6404e9fcad4851b13314 Mon Sep 17 00:00:00 2001 From: Dimitrie Stefanescu Date: Tue, 13 Nov 2018 13:59:18 +0000 Subject: [PATCH 18/27] renamed some functions for clarity & added purging methods --- SpeckleCore/LocalData/SpeckleLocalContext.cs | 54 ++++++++++++++++++-- 1 file changed, 49 insertions(+), 5 deletions(-) diff --git a/SpeckleCore/LocalData/SpeckleLocalContext.cs b/SpeckleCore/LocalData/SpeckleLocalContext.cs index c2979a0..df7188d 100644 --- a/SpeckleCore/LocalData/SpeckleLocalContext.cs +++ b/SpeckleCore/LocalData/SpeckleLocalContext.cs @@ -10,6 +10,8 @@ namespace SpeckleCore /// /// This class holds the keys to the local sqlite database that acts as a local cache for various speckle things. /// You can access accounts from here across speckle integrations, as well as the local object cache. + /// The cache holds the following tables: Accounts, CachedObjects, SentObjects, CachedStreams. + /// Cached objects are objects that a receiver requested. They are stored fully (with a binary blob of their speckle representation). SentObjects are objects that have been previously sent by a sender and are stored without their speckle representation (they're just refs) as a log against which to diff what to send or not. /// public static partial class LocalContext { @@ -47,6 +49,41 @@ public static void Close() Database?.Close(); } + #region Cleanup & Table Purging + /// + /// Purges the sent objects table. WARNING: Don't do this unless you know what you're doing. + /// + public static void PurgeSentObjects() + { + Database?.Execute( "DELETE FROM SentObject" ); + } + + /// + /// Purges the received objects table. WARNING: Don't do this unless you know what you're doing. + /// + public static void PurgeCachedObjects( ) + { + Database?.Execute( "DELETE FROM CachedObject" ); + } + + /// + /// Purges the accounts. WARNING: Don't do this unless you know what you're doing. + /// + public static void PurgeAccounts( ) + { + Database?.Execute( "DELETE FROM Account" ); + } + + /// + /// Purges the streams table. WARNING: Don't do this unless you know what you're doing. + /// + public static void PurgeCachedStreams() + { + Database?.Execute( "DELETE FROM CachedStream" ); + } + + #endregion + #region Accounts /// /// Migrates existing accounts stored in text files to the sqlite db. @@ -188,14 +225,14 @@ public static void RemoveAccount(Account ac) #endregion - #region Received Objects + #region Received Objects (CachedObject) /// /// Adds a speckle object to the local cache. /// /// The object to add. /// The server url of where it has been persisted. - public static void AddObject(SpeckleObject obj, string restApi) + public static void AddCachedObject(SpeckleObject obj, string restApi) { var bytes = SpeckleCore.Converter.getBytes(obj); var combinedHash = Converter.getMd5Hash(obj._id + restApi); @@ -225,7 +262,7 @@ public static void AddObject(SpeckleObject obj, string restApi) /// Speckle object placeholders to check against the cache. /// The rest api these objects are expected to come from. /// - public static List GetObjects(List objs, string restApi) + public static List GetCachedObjects(List objs, string restApi) { var combinedHashes = objs.Select(obj => Converter.getMd5Hash(obj._id + restApi)).ToList(); var res = Database.Table().Where(obj => combinedHashes.Contains(obj.CombinedHash)).Select(o => o.ToSpeckle()).ToList(); @@ -246,8 +283,15 @@ public static List GetObjects(List objs, string re #endregion - #region Sent Objects + #region Sent Objects (SentObject) + /// + /// Adds an object that has been sent by a sender in the local cache. + /// This does not store the full object, it's just a log that it has been sent + /// to a server so it does not get sent again. + /// + /// Object to store as sent ref in the local database. + /// The server's url. public static void AddSentObject(SpeckleObject obj, string restApi) { var sentObj = new SentObject() @@ -292,7 +336,7 @@ public static List PruneExistingObjects(List objs, #endregion - #region Streams + #region Streams (CachedStream) /// /// Updates or inserts a stream in the local cache. From 3f70c1d68dd6e8afe64e36a3f7732e6be5ea2806 Mon Sep 17 00:00:00 2001 From: Dimitrie Stefanescu Date: Tue, 13 Nov 2018 17:57:51 +0000 Subject: [PATCH 19/27] finally fixes #75 (no more secret token serialisation inside clients) --- SpeckleCore/LocalData/SpeckleLocalContext.cs | 5 +-- SpeckleCore/SpeckleApiClientExtension.cs | 35 ++++++++++++++++++-- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/SpeckleCore/LocalData/SpeckleLocalContext.cs b/SpeckleCore/LocalData/SpeckleLocalContext.cs index df7188d..de15dad 100644 --- a/SpeckleCore/LocalData/SpeckleLocalContext.cs +++ b/SpeckleCore/LocalData/SpeckleLocalContext.cs @@ -174,7 +174,7 @@ public static List GetAccountsByEmail(string email) /// /// /// - /// + /// null if no account is found. public static Account GetAccountByEmailAndRestApi(string email, string restApi) { var res = Database.Query(String.Format("SELECT * from Account WHERE RestApi = '{0}' AND Email='{1}'", restApi, email)); @@ -184,7 +184,8 @@ public static Account GetAccountByEmailAndRestApi(string email, string restApi) } else { - throw new Exception("Could not find account."); + return null; + //throw new Exception("Could not find account."); } } diff --git a/SpeckleCore/SpeckleApiClientExtension.cs b/SpeckleCore/SpeckleApiClientExtension.cs index b31a190..84260b6 100644 --- a/SpeckleCore/SpeckleApiClientExtension.cs +++ b/SpeckleCore/SpeckleApiClientExtension.cs @@ -281,7 +281,7 @@ public void JoinRoom( string streamId ) var eventData = new { eventName = "join", - senderId = ClientId, + senderId = ClientId, streamId = streamId }; @@ -325,9 +325,36 @@ protected SpeckleApiClient( SerializationInfo info, StreamingContext context ) BaseUrl = info.GetString( "BaseUrl" ); StreamId = info.GetString( "StreamId" ); Role = ( ClientRole ) info.GetInt32( "Role" ); - AuthToken = info.GetString( "ApiToken" ); ClientId = info.GetString( "ClientId" ); + //AuthToken = info.GetString( "ApiToken" ); + //string userEmail = null; + + // old clients will not have a user email field :/ + try + { + var userEmail = info.GetString( "UserEmail" ); + var acc = LocalContext.GetAccountByEmailAndRestApi( userEmail, BaseUrl ); + if ( acc != null ) + { + AuthToken = acc.Token; + User = new User() { Email = acc.Email }; + } + } + catch + { + var accs = LocalContext.GetAccountsByRestApi( BaseUrl ); + if ( accs.Count == 0 ) + { + throw new Exception( "You do not have an account that matches this stream's server." ); + } + else + { + AuthToken = accs[ 0 ].Token; + User = new User() { Email = accs[ 0 ].Email }; + } + } + Stream = StreamGetAsync( StreamId, null ).Result.Resource; // does not need waiting for, as we already have a clientid. @@ -340,11 +367,13 @@ protected SpeckleApiClient( SerializationInfo info, StreamingContext context ) public void GetObjectData( SerializationInfo info, StreamingContext context ) { + info.AddValue( "UserEmail", User.Email ); info.AddValue( "BaseUrl", BaseUrl ); info.AddValue( "StreamId", StreamId ); info.AddValue( "Role", Role ); - info.AddValue( "ApiToken", AuthToken ); info.AddValue( "ClientId", ClientId ); + + //info.AddValue( "ApiToken", AuthToken ); } public void Dispose( bool delete = false ) From 35d4a1c72976a671ccf1b6130233582622203474 Mon Sep 17 00:00:00 2001 From: Dimitrie Stefanescu Date: Thu, 15 Nov 2018 12:35:00 +0000 Subject: [PATCH 20/27] made speckle base color inherit from speckle object --- SpeckleCore/Conversion/Converter.cs | 15 +++++++++------ SpeckleCore/Models/ModelResponses.cs | 2 +- SpeckleCore/SpeckleCore.csproj | 1 + 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/SpeckleCore/Conversion/Converter.cs b/SpeckleCore/Conversion/Converter.cs index 5b24a19..deaeb26 100644 --- a/SpeckleCore/Conversion/Converter.cs +++ b/SpeckleCore/Conversion/Converter.cs @@ -3,6 +3,7 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.ComponentModel; +using System.Drawing; using System.Linq; using System.Reflection; using System.Runtime.CompilerServices; @@ -10,6 +11,7 @@ using System.Text; using System.Threading.Tasks; + namespace SpeckleCore { /// @@ -80,7 +82,7 @@ public static byte[ ] base64ToBytes( string str ) /// String from which to generate the hash /// If 0, the full hasdh will be returned, otherwise it will be trimmed to the specified lenght /// - public static string getMd5Hash( string str, int length = 0) + public static string getMd5Hash( string str, int length = 0 ) { using ( System.IO.MemoryStream ms = new System.IO.MemoryStream() ) { @@ -187,12 +189,12 @@ public static object Deserialise( SpeckleObject obj, object root = null ) //var shortName = absObj._assembly.Split( ',' )[ 0 ]; var assembly = System.AppDomain.CurrentDomain.GetAssemblies().FirstOrDefault( a => a.FullName == absObj._assembly ); - + //try again - if(assembly == null) + if ( assembly == null ) { - var shortName = absObj._assembly.Split( ',' )[ 0 ]; - assembly = System.AppDomain.CurrentDomain.GetAssemblies().FirstOrDefault( a => a.FullName.Contains(shortName )); + var shortName = absObj._assembly.Split( ',' )[ 0 ]; + assembly = System.AppDomain.CurrentDomain.GetAssemblies().FirstOrDefault( a => a.FullName.Contains( shortName ) ); } if ( assembly == null ) // we can't deserialise for sure @@ -241,7 +243,7 @@ public static object Deserialise( SpeckleObject obj, object root = null ) try { - if ( (prop!=null && prop.PropertyType.IsArray) || (field!=null && field.FieldType.IsArray) ) + if ( ( prop != null && prop.PropertyType.IsArray ) || ( field != null && field.FieldType.IsArray ) ) { value = ( ( List ) value ).ToArray(); } @@ -664,4 +666,5 @@ private static object WriteValue( object myObject, int recursionDepth, Dictionar } } + } diff --git a/SpeckleCore/Models/ModelResponses.cs b/SpeckleCore/Models/ModelResponses.cs index c7b542b..cd79e6f 100644 --- a/SpeckleCore/Models/ModelResponses.cs +++ b/SpeckleCore/Models/ModelResponses.cs @@ -274,7 +274,7 @@ public static Resource FromJson( string data ) [System.CodeDom.Compiler.GeneratedCode( "NJsonSchema", "9.10.41.0 (Newtonsoft.Json v9.0.0.0)" )] [Serializable] - public partial class SpeckleBaseColor + public partial class SpeckleBaseColor : SpeckleObject { /// alpha value [Newtonsoft.Json.JsonProperty( "a", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore )] diff --git a/SpeckleCore/SpeckleCore.csproj b/SpeckleCore/SpeckleCore.csproj index a210be7..ca3bd4d 100644 --- a/SpeckleCore/SpeckleCore.csproj +++ b/SpeckleCore/SpeckleCore.csproj @@ -33,6 +33,7 @@ + From b7753458e058160915fd85e17b159b88d1b4766b Mon Sep 17 00:00:00 2001 From: Dimitrie Stefanescu Date: Sun, 18 Nov 2018 14:41:25 +0000 Subject: [PATCH 21/27] added step property to speckleinput --- SpeckleCore/Models/ModelObjectsExt.cs | 33 +++++++++++++++------------ SpeckleCore/Models/ModelResponses.cs | 2 +- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/SpeckleCore/Models/ModelObjectsExt.cs b/SpeckleCore/Models/ModelObjectsExt.cs index 6fd0f21..caba07f 100644 --- a/SpeckleCore/Models/ModelObjectsExt.cs +++ b/SpeckleCore/Models/ModelObjectsExt.cs @@ -13,20 +13,20 @@ namespace SpeckleCore { - public class SpeckleObjectComparer : IEqualityComparer + public class SpeckleObjectComparer : IEqualityComparer + { + public bool Equals( SpeckleObject x, SpeckleObject y ) { - public bool Equals( SpeckleObject x, SpeckleObject y ) - { - return x.Hash == y.Hash; - } + return x.Hash == y.Hash; + } - public int GetHashCode( SpeckleObject obj ) - { - return obj.Hash.GetHashCode(); - } + public int GetHashCode( SpeckleObject obj ) + { + return obj.Hash.GetHashCode(); } + } - public partial class SpeckleObject + public partial class SpeckleObject { /// /// Generates a truncated (to 12) md5 hash of an object. @@ -545,8 +545,8 @@ public override void Scale( double factor ) public override void GenerateHash( ) { base.GenerateHash(); - this.GeometryHash += GetMd5FromObject( BasePlane.GeometryHash + XSize.GeometryHash + YSize.GeometryHash + ZSize.GeometryHash ); - this.Hash = GetMd5FromObject( this.GeometryHash + GetMd5FromObject( this.Properties ) ); + this.GeometryHash += GetMd5FromObject( BasePlane.ToJson() + XSize.ToJson() + YSize.ToJson() + ZSize.ToJson() ); + this.Hash = GetMd5FromObject( this ); } } @@ -857,16 +857,19 @@ public SpeckleInput( ) { } public string Guid { get; set; } [Newtonsoft.Json.JsonProperty( "value", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore )] - public float Value { get; set; } + public double Value { get; set; } [Newtonsoft.Json.JsonProperty( "inputType", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore )] public string InputType { get; set; } [Newtonsoft.Json.JsonProperty( "max", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore )] - public float Max { get; set; } + public double Max { get; set; } [Newtonsoft.Json.JsonProperty( "min", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore )] - public float Min { get; set; } + public double Min { get; set; } + + [Newtonsoft.Json.JsonProperty( "step", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore )] + public double Step { get; set; } public SpeckleInput( string name, float min, float max, float value, string inputType, string guid ) { diff --git a/SpeckleCore/Models/ModelResponses.cs b/SpeckleCore/Models/ModelResponses.cs index cd79e6f..c7b542b 100644 --- a/SpeckleCore/Models/ModelResponses.cs +++ b/SpeckleCore/Models/ModelResponses.cs @@ -274,7 +274,7 @@ public static Resource FromJson( string data ) [System.CodeDom.Compiler.GeneratedCode( "NJsonSchema", "9.10.41.0 (Newtonsoft.Json v9.0.0.0)" )] [Serializable] - public partial class SpeckleBaseColor : SpeckleObject + public partial class SpeckleBaseColor { /// alpha value [Newtonsoft.Json.JsonProperty( "a", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore )] From 8a08cf7ab7ee40cc14e36e1a5a9d66f48cb9004c Mon Sep 17 00:00:00 2001 From: Dimitrie Stefanescu Date: Tue, 20 Nov 2018 18:37:31 +0000 Subject: [PATCH 22/27] clients now get really deleted when dispose(true) is called --- SpeckleCore/SpeckleApiClientExtension.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/SpeckleCore/SpeckleApiClientExtension.cs b/SpeckleCore/SpeckleApiClientExtension.cs index 84260b6..f58ff9a 100644 --- a/SpeckleCore/SpeckleApiClientExtension.cs +++ b/SpeckleCore/SpeckleApiClientExtension.cs @@ -386,8 +386,7 @@ public void Dispose( bool delete = false ) WebsocketClient?.Close(); return; } - - ClientUpdateAsync( ClientId, new AppClient() { Online = false, Deleted = true } ); + ClientDeleteAsync( ClientId ); WebsocketClient?.Close(); } } From 8caf649ed97caf6e13745ca2b5cc849eab81ea2f Mon Sep 17 00:00:00 2001 From: Dimitrie Stefanescu Date: Thu, 22 Nov 2018 16:47:12 +0000 Subject: [PATCH 23/27] fixes max_sql_vars error in prune local objects --- SpeckleCore/LocalData/SpeckleLocalContext.cs | 184 ++++++++++--------- 1 file changed, 101 insertions(+), 83 deletions(-) diff --git a/SpeckleCore/LocalData/SpeckleLocalContext.cs b/SpeckleCore/LocalData/SpeckleLocalContext.cs index de15dad..9d98720 100644 --- a/SpeckleCore/LocalData/SpeckleLocalContext.cs +++ b/SpeckleCore/LocalData/SpeckleLocalContext.cs @@ -19,21 +19,21 @@ public static partial class LocalContext private static SQLiteConnection Database; - public static string DbPath = System.Environment.GetFolderPath(System.Environment.SpecialFolder.LocalApplicationData) + @"\SpeckleSettings\SpeckleCache.db"; + public static string DbPath = System.Environment.GetFolderPath( System.Environment.SpecialFolder.LocalApplicationData ) + @"\SpeckleSettings\SpeckleCache.db"; - public static string SettingsFolderPath = System.Environment.GetFolderPath(System.Environment.SpecialFolder.LocalApplicationData) + @"\SpeckleSettings\"; + public static string SettingsFolderPath = System.Environment.GetFolderPath( System.Environment.SpecialFolder.LocalApplicationData ) + @"\SpeckleSettings\"; /// /// Initialises the database context, ensures tables are created and powers up the rocket engines. /// - public static void Init() + public static void Init( ) { - if (IsInit) return; + if ( IsInit ) return; - if (!Directory.Exists(SettingsFolderPath)) - Directory.CreateDirectory(SettingsFolderPath); + if ( !Directory.Exists( SettingsFolderPath ) ) + Directory.CreateDirectory( SettingsFolderPath ); - Database = new SQLiteConnection(DbPath); + Database = new SQLiteConnection( DbPath ); Database.CreateTable(); Database.CreateTable(); Database.CreateTable(); @@ -44,7 +44,7 @@ public static void Init() IsInit = true; } - public static void Close() + public static void Close( ) { Database?.Close(); } @@ -53,7 +53,7 @@ public static void Close() /// /// Purges the sent objects table. WARNING: Don't do this unless you know what you're doing. /// - public static void PurgeSentObjects() + public static void PurgeSentObjects( ) { Database?.Execute( "DELETE FROM SentObject" ); } @@ -77,7 +77,7 @@ public static void PurgeAccounts( ) /// /// Purges the streams table. WARNING: Don't do this unless you know what you're doing. /// - public static void PurgeCachedStreams() + public static void PurgeCachedStreams( ) { Database?.Execute( "DELETE FROM CachedStream" ); } @@ -88,46 +88,46 @@ public static void PurgeCachedStreams() /// /// Migrates existing accounts stored in text files to the sqlite db. /// - private static void MigrateAccounts() + private static void MigrateAccounts( ) { List accounts = new List(); - if (Directory.Exists(SettingsFolderPath) && Directory.EnumerateFiles(SettingsFolderPath, "*.txt").Count() > 0) + if ( Directory.Exists( SettingsFolderPath ) && Directory.EnumerateFiles( SettingsFolderPath, "*.txt" ).Count() > 0 ) { - foreach (string file in Directory.EnumerateFiles(SettingsFolderPath, "*.txt")) + foreach ( string file in Directory.EnumerateFiles( SettingsFolderPath, "*.txt" ) ) { - string content = File.ReadAllText(file); - string[] pieces = content.TrimEnd('\r', '\n').Split(','); + string content = File.ReadAllText( file ); + string[ ] pieces = content.TrimEnd( '\r', '\n' ).Split( ',' ); - accounts.Add(new Account() { Email = pieces[0], Token = pieces[1], ServerName = pieces[2], RestApi = pieces[3] }); + accounts.Add( new Account() { Email = pieces[ 0 ], Token = pieces[ 1 ], ServerName = pieces[ 2 ], RestApi = pieces[ 3 ] } ); } - var res = Database.InsertAll(accounts); + var res = Database.InsertAll( accounts ); - Directory.CreateDirectory(SettingsFolderPath + @"\MigratedAccounts\"); + Directory.CreateDirectory( SettingsFolderPath + @"\MigratedAccounts\" ); - foreach (string file in Directory.EnumerateFiles(SettingsFolderPath, "*.txt")) + foreach ( string file in Directory.EnumerateFiles( SettingsFolderPath, "*.txt" ) ) { try { - var newName = Path.Combine(SettingsFolderPath, "MigratedAccounts", Path.GetFileName(file)); - if (File.Exists(newName)) + var newName = Path.Combine( SettingsFolderPath, "MigratedAccounts", Path.GetFileName( file ) ); + if ( File.Exists( newName ) ) { - File.Delete(newName); + File.Delete( newName ); } - File.Move(file, newName); + File.Move( file, newName ); } - catch (Exception e) + catch ( Exception e ) { - Debug.WriteLine("yolo"); + Debug.WriteLine( "yolo" ); } } } else { - Debug.WriteLine("No existing account text files found."); + Debug.WriteLine( "No existing account text files found." ); } } @@ -135,18 +135,18 @@ private static void MigrateAccounts() /// Adds a new account. /// /// - public static void AddAccount(Account account) + public static void AddAccount( Account account ) { - var res = Database.Insert(account); + var res = Database.Insert( account ); } /// /// Gets all accounts present. /// /// - public static List GetAllAccounts() + public static List GetAllAccounts( ) { - return Database.Query("SELECT * FROM Account"); + return Database.Query( "SELECT * FROM Account" ); } /// @@ -154,9 +154,9 @@ public static List GetAllAccounts() /// /// /// - public static List GetAccountsByRestApi(string RestApi) + public static List GetAccountsByRestApi( string RestApi ) { - return Database.Query("SELECT * from Account WHERE RestApi = ?", RestApi); + return Database.Query( "SELECT * from Account WHERE RestApi = ?", RestApi ); } /// @@ -164,9 +164,9 @@ public static List GetAccountsByRestApi(string RestApi) /// /// /// - public static List GetAccountsByEmail(string email) + public static List GetAccountsByEmail( string email ) { - return Database.Query("SELECT * from Account WHERE Email = ?", email); + return Database.Query( "SELECT * from Account WHERE Email = ?", email ); } /// @@ -175,12 +175,12 @@ public static List GetAccountsByEmail(string email) /// /// /// null if no account is found. - public static Account GetAccountByEmailAndRestApi(string email, string restApi) + public static Account GetAccountByEmailAndRestApi( string email, string restApi ) { - var res = Database.Query(String.Format("SELECT * from Account WHERE RestApi = '{0}' AND Email='{1}'", restApi, email)); - if (res.Count >= 1) + var res = Database.Query( String.Format( "SELECT * from Account WHERE RestApi = '{0}' AND Email='{1}'", restApi, email ) ); + if ( res.Count >= 1 ) { - return res[0]; + return res[ 0 ]; } else { @@ -193,16 +193,16 @@ public static Account GetAccountByEmailAndRestApi(string email, string restApi) /// Returns the default account, if any. Otherwise throws an error. /// /// - public static Account GetDefaultAccount() + public static Account GetDefaultAccount( ) { - var res = Database.Query("SELECT * FROM Account WHERE IsDefault='true' LIMIT 1"); - if (res.Count == 1) + var res = Database.Query( "SELECT * FROM Account WHERE IsDefault='true' LIMIT 1" ); + if ( res.Count == 1 ) { - return res[0]; + return res[ 0 ]; } else { - throw new Exception("No default account set."); + throw new Exception( "No default account set." ); } } @@ -210,18 +210,18 @@ public static Account GetDefaultAccount() /// Sets an account as being the default one, and de-sets defaultness on all others. /// /// - public static void SetDefaultAccount(Account account) + public static void SetDefaultAccount( Account account ) { - Database.Execute("UPDATE Account SET IsDefault=0"); + Database.Execute( "UPDATE Account SET IsDefault=0" ); account.IsDefault = true; - Database.Update(account); + Database.Update( account ); } - public static void RemoveAccount(Account ac) + public static void RemoveAccount( Account ac ) { - Database.Delete(ac.AccountId); + Database.Delete( ac.AccountId ); } #endregion @@ -233,10 +233,10 @@ public static void RemoveAccount(Account ac) /// /// The object to add. /// The server url of where it has been persisted. - public static void AddCachedObject(SpeckleObject obj, string restApi) + public static void AddCachedObject( SpeckleObject obj, string restApi ) { - var bytes = SpeckleCore.Converter.getBytes(obj); - var combinedHash = Converter.getMd5Hash(obj._id + restApi); + var bytes = SpeckleCore.Converter.getBytes( obj ); + var combinedHash = Converter.getMd5Hash( obj._id + restApi ); var cached = new CachedObject() { RestApi = restApi, @@ -249,7 +249,7 @@ public static void AddCachedObject(SpeckleObject obj, string restApi) try { - Database.Insert(cached); + Database.Insert( cached ); } catch { @@ -263,19 +263,19 @@ public static void AddCachedObject(SpeckleObject obj, string restApi) /// Speckle object placeholders to check against the cache. /// The rest api these objects are expected to come from. /// - public static List GetCachedObjects(List objs, string restApi) + public static List GetCachedObjects( List objs, string restApi ) { - var combinedHashes = objs.Select(obj => Converter.getMd5Hash(obj._id + restApi)).ToList(); - var res = Database.Table().Where(obj => combinedHashes.Contains(obj.CombinedHash)).Select(o => o.ToSpeckle()).ToList(); + var combinedHashes = objs.Select( obj => Converter.getMd5Hash( obj._id + restApi ) ).ToList(); + var res = Database.Table().Where( obj => combinedHashes.Contains( obj.CombinedHash ) ).Select( o => o.ToSpeckle() ).ToList(); // populate the original list with whatever objects we found in the database. - for (int i = 0; i < objs.Count; i++) + for ( int i = 0; i < objs.Count; i++ ) { - var placeholder = objs[i]; - var myObject = res.Find(o => o._id == placeholder._id); - if (myObject != null) + var placeholder = objs[ i ]; + var myObject = res.Find( o => o._id == placeholder._id ); + if ( myObject != null ) { - objs[i] = myObject; + objs[ i ] = myObject; } } @@ -293,7 +293,7 @@ public static List GetCachedObjects(List objs, str /// /// Object to store as sent ref in the local database. /// The server's url. - public static void AddSentObject(SpeckleObject obj, string restApi) + public static void AddSentObject( SpeckleObject obj, string restApi ) { var sentObj = new SentObject() { @@ -305,7 +305,8 @@ public static void AddSentObject(SpeckleObject obj, string restApi) try { Database.Insert( sentObj ); - } catch + } + catch { // object was already there, no panic! } @@ -317,18 +318,35 @@ public static void AddSentObject(SpeckleObject obj, string restApi) /// /// /// (Optinoal) The modified list. - public static List PruneExistingObjects(List objs, string restApi) + public static List PruneExistingObjects( List objs, string restApi ) { - var objHashes = objs.Select(obj => true ? obj.Hash : restApi).ToList(); - var res = Database.Table().Where(obj => objHashes.Contains(obj.Hash) && obj.RestApi == restApi).ToList(); + // MAX SQL Vars is 900 + var MaxSqlVars = 900; + + var objHashes = objs.Select( obj => true ? obj.Hash : restApi ).ToList(); + + var partitionedList = new List>(); + + for ( int i = 0; i < objHashes.Count; i += MaxSqlVars ) + { + partitionedList.Add( objHashes.GetRange( i, Math.Min( MaxSqlVars, objHashes.Count - i ) ) ); + } + + var fullRes = new List(); + + foreach ( var subList in partitionedList ) + { + var res = Database.Table().Where( obj => subList.Contains( obj.Hash ) && obj.RestApi == restApi ).ToList(); + fullRes.AddRange( res ); + } - for (int i = 0; i < objs.Count; i++) + for ( int i = 0; i < objs.Count; i++ ) { - var placeholder = objs[i]; - var myObject = res.Find(o => o.Hash == objs[i].Hash); - if (myObject != null) + var placeholder = objs[ i ]; + var myObject = fullRes.Find( o => o.Hash == objs[ i ].Hash ); + if ( myObject != null ) { - objs[i] = new SpecklePlaceholder() { _id = myObject.DatabaseId }; + objs[ i ] = new SpecklePlaceholder() { _id = myObject.DatabaseId }; } } @@ -344,19 +362,19 @@ public static List PruneExistingObjects(List objs, /// /// /// - public static void AddOrUpdateStream(SpeckleStream stream, string restApi) + public static void AddOrUpdateStream( SpeckleStream stream, string restApi ) { - var bytes = SpeckleCore.Converter.getBytes(stream.ToJson()); - var combinedHash = Converter.getMd5Hash(stream._id + restApi); + var bytes = SpeckleCore.Converter.getBytes( stream.ToJson() ); + var combinedHash = Converter.getMd5Hash( stream._id + restApi ); - var cacheRes = Database.Table().Where(existing => existing.CombinedHash == combinedHash).ToList(); + var cacheRes = Database.Table().Where( existing => existing.CombinedHash == combinedHash ).ToList(); - if (cacheRes.Count >= 1) + if ( cacheRes.Count >= 1 ) { - var toUpdate = cacheRes[0]; + var toUpdate = cacheRes[ 0 ]; toUpdate.Bytes = bytes; toUpdate.UpdatedOn = DateTime.Now; - Database.Update(toUpdate); + Database.Update( toUpdate ); } else { @@ -369,7 +387,7 @@ public static void AddOrUpdateStream(SpeckleStream stream, string restApi) AddedOn = DateTime.Now, UpdatedOn = DateTime.Now }; - Database.Insert(toCache); + Database.Insert( toCache ); } //throw new NotImplementedException(); } @@ -380,13 +398,13 @@ public static void AddOrUpdateStream(SpeckleStream stream, string restApi) /// /// /// Null, if nothing found, or the speckle stream. - public static SpeckleStream GetStream(string streamId, string restApi) + public static SpeckleStream GetStream( string streamId, string restApi ) { - var combinedHash = Converter.getMd5Hash(streamId + restApi); - var res = Database.Table().Where(str => str.CombinedHash == combinedHash).ToArray(); - if (res.Length > 0) + var combinedHash = Converter.getMd5Hash( streamId + restApi ); + var res = Database.Table().Where( str => str.CombinedHash == combinedHash ).ToArray(); + if ( res.Length > 0 ) { - return res[0].ToSpeckle(); + return res[ 0 ].ToSpeckle(); } return null; From bf1908c52307bbe6f2e2ac664a35c8ed71cc8f25 Mon Sep 17 00:00:00 2001 From: Matteo Cominetti Date: Thu, 22 Nov 2018 17:00:43 +0000 Subject: [PATCH 24/27] switched to 4.5 --- SpeckleCore/SpeckleCore.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SpeckleCore/SpeckleCore.csproj b/SpeckleCore/SpeckleCore.csproj index ca3bd4d..c0f57f8 100644 --- a/SpeckleCore/SpeckleCore.csproj +++ b/SpeckleCore/SpeckleCore.csproj @@ -9,7 +9,7 @@ Properties SpeckleCore SpeckleCore - v4.6.1 + v4.5 512 From c861912c57fbd7d113c0c2ccd48bbaca8e00db3d Mon Sep 17 00:00:00 2001 From: Dimitrie Stefanescu Date: Thu, 22 Nov 2018 18:15:06 +0000 Subject: [PATCH 25/27] fixed cache misses --- SpeckleCore/LocalData/Models.cs | 2 +- SpeckleCore/LocalData/SpeckleLocalContext.cs | 23 +++++++++++++++++--- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/SpeckleCore/LocalData/Models.cs b/SpeckleCore/LocalData/Models.cs index 3ac8643..e7126fd 100644 --- a/SpeckleCore/LocalData/Models.cs +++ b/SpeckleCore/LocalData/Models.cs @@ -42,7 +42,7 @@ public class SentObject [Indexed] public string DatabaseId { get; set; } - [PrimaryKey, Indexed] + [Indexed] public string Hash { get; set; } } diff --git a/SpeckleCore/LocalData/SpeckleLocalContext.cs b/SpeckleCore/LocalData/SpeckleLocalContext.cs index 9d98720..1d7213d 100644 --- a/SpeckleCore/LocalData/SpeckleLocalContext.cs +++ b/SpeckleCore/LocalData/SpeckleLocalContext.cs @@ -265,14 +265,30 @@ public static void AddCachedObject( SpeckleObject obj, string restApi ) /// public static List GetCachedObjects( List objs, string restApi ) { + var MaxSqlVars = 900; + var combinedHashes = objs.Select( obj => Converter.getMd5Hash( obj._id + restApi ) ).ToList(); - var res = Database.Table().Where( obj => combinedHashes.Contains( obj.CombinedHash ) ).Select( o => o.ToSpeckle() ).ToList(); + + var partitionedList = new List>(); + + for ( int i = 0; i < combinedHashes.Count; i += MaxSqlVars ) + { + partitionedList.Add( combinedHashes.GetRange( i, Math.Min( MaxSqlVars, combinedHashes.Count - i ) ) ); + } + + var fullRes = new List(); + + foreach ( var subList in partitionedList ) + { + var res = Database.Table().Where( obj => subList.Contains( obj.CombinedHash ) ).Select( o => o.ToSpeckle() ).ToList(); + fullRes.AddRange( res ); + } // populate the original list with whatever objects we found in the database. for ( int i = 0; i < objs.Count; i++ ) { var placeholder = objs[ i ]; - var myObject = res.Find( o => o._id == placeholder._id ); + var myObject = fullRes.Find( o => o._id == placeholder._id ); if ( myObject != null ) { objs[ i ] = myObject; @@ -306,8 +322,9 @@ public static void AddSentObject( SpeckleObject obj, string restApi ) { Database.Insert( sentObj ); } - catch + catch(Exception e) { + var dick = e; // object was already there, no panic! } } From 9b4c628f5b85d22875202d9babfc9bcc57e7cbf0 Mon Sep 17 00:00:00 2001 From: Dimitrie Stefanescu Date: Fri, 23 Nov 2018 16:15:08 +0000 Subject: [PATCH 26/27] makes sure db is init on all methods & adds updates w/h new ws methods --- SpeckleCore/LocalData/SpeckleLocalContext.cs | 19 +++ SpeckleCore/SpeckleApiClientExtension.cs | 121 ++++++++++++++++++- 2 files changed, 136 insertions(+), 4 deletions(-) diff --git a/SpeckleCore/LocalData/SpeckleLocalContext.cs b/SpeckleCore/LocalData/SpeckleLocalContext.cs index 1d7213d..3a6e053 100644 --- a/SpeckleCore/LocalData/SpeckleLocalContext.cs +++ b/SpeckleCore/LocalData/SpeckleLocalContext.cs @@ -55,6 +55,7 @@ public static void Close( ) /// public static void PurgeSentObjects( ) { + LocalContext.Init(); Database?.Execute( "DELETE FROM SentObject" ); } @@ -63,6 +64,7 @@ public static void PurgeSentObjects( ) /// public static void PurgeCachedObjects( ) { + LocalContext.Init(); Database?.Execute( "DELETE FROM CachedObject" ); } @@ -71,6 +73,7 @@ public static void PurgeCachedObjects( ) /// public static void PurgeAccounts( ) { + LocalContext.Init(); Database?.Execute( "DELETE FROM Account" ); } @@ -79,6 +82,7 @@ public static void PurgeAccounts( ) /// public static void PurgeCachedStreams( ) { + LocalContext.Init(); Database?.Execute( "DELETE FROM CachedStream" ); } @@ -90,6 +94,7 @@ public static void PurgeCachedStreams( ) /// private static void MigrateAccounts( ) { + LocalContext.Init(); List accounts = new List(); if ( Directory.Exists( SettingsFolderPath ) && Directory.EnumerateFiles( SettingsFolderPath, "*.txt" ).Count() > 0 ) @@ -137,6 +142,7 @@ private static void MigrateAccounts( ) /// public static void AddAccount( Account account ) { + LocalContext.Init(); var res = Database.Insert( account ); } @@ -146,6 +152,7 @@ public static void AddAccount( Account account ) /// public static List GetAllAccounts( ) { + LocalContext.Init(); return Database.Query( "SELECT * FROM Account" ); } @@ -156,6 +163,7 @@ public static List GetAllAccounts( ) /// public static List GetAccountsByRestApi( string RestApi ) { + LocalContext.Init(); return Database.Query( "SELECT * from Account WHERE RestApi = ?", RestApi ); } @@ -166,6 +174,7 @@ public static List GetAccountsByRestApi( string RestApi ) /// public static List GetAccountsByEmail( string email ) { + LocalContext.Init(); return Database.Query( "SELECT * from Account WHERE Email = ?", email ); } @@ -177,6 +186,7 @@ public static List GetAccountsByEmail( string email ) /// null if no account is found. public static Account GetAccountByEmailAndRestApi( string email, string restApi ) { + LocalContext.Init(); var res = Database.Query( String.Format( "SELECT * from Account WHERE RestApi = '{0}' AND Email='{1}'", restApi, email ) ); if ( res.Count >= 1 ) { @@ -195,6 +205,7 @@ public static Account GetAccountByEmailAndRestApi( string email, string restApi /// public static Account GetDefaultAccount( ) { + LocalContext.Init(); var res = Database.Query( "SELECT * FROM Account WHERE IsDefault='true' LIMIT 1" ); if ( res.Count == 1 ) { @@ -212,6 +223,7 @@ public static Account GetDefaultAccount( ) /// public static void SetDefaultAccount( Account account ) { + LocalContext.Init(); Database.Execute( "UPDATE Account SET IsDefault=0" ); @@ -221,6 +233,7 @@ public static void SetDefaultAccount( Account account ) public static void RemoveAccount( Account ac ) { + LocalContext.Init(); Database.Delete( ac.AccountId ); } @@ -235,6 +248,7 @@ public static void RemoveAccount( Account ac ) /// The server url of where it has been persisted. public static void AddCachedObject( SpeckleObject obj, string restApi ) { + LocalContext.Init(); var bytes = SpeckleCore.Converter.getBytes( obj ); var combinedHash = Converter.getMd5Hash( obj._id + restApi ); var cached = new CachedObject() @@ -265,6 +279,7 @@ public static void AddCachedObject( SpeckleObject obj, string restApi ) /// public static List GetCachedObjects( List objs, string restApi ) { + LocalContext.Init(); var MaxSqlVars = 900; var combinedHashes = objs.Select( obj => Converter.getMd5Hash( obj._id + restApi ) ).ToList(); @@ -311,6 +326,7 @@ public static List GetCachedObjects( List objs, st /// The server's url. public static void AddSentObject( SpeckleObject obj, string restApi ) { + LocalContext.Init(); var sentObj = new SentObject() { RestApi = restApi, @@ -337,6 +353,7 @@ public static void AddSentObject( SpeckleObject obj, string restApi ) /// (Optinoal) The modified list. public static List PruneExistingObjects( List objs, string restApi ) { + LocalContext.Init(); // MAX SQL Vars is 900 var MaxSqlVars = 900; @@ -381,6 +398,7 @@ public static List PruneExistingObjects( List objs /// public static void AddOrUpdateStream( SpeckleStream stream, string restApi ) { + LocalContext.Init(); var bytes = SpeckleCore.Converter.getBytes( stream.ToJson() ); var combinedHash = Converter.getMd5Hash( stream._id + restApi ); @@ -417,6 +435,7 @@ public static void AddOrUpdateStream( SpeckleStream stream, string restApi ) /// Null, if nothing found, or the speckle stream. public static SpeckleStream GetStream( string streamId, string restApi ) { + LocalContext.Init(); var combinedHash = Converter.getMd5Hash( streamId + restApi ); var res = Database.Table().Where( str => str.CombinedHash == combinedHash ).ToArray(); if ( res.Length > 0 ) diff --git a/SpeckleCore/SpeckleApiClientExtension.cs b/SpeckleCore/SpeckleApiClientExtension.cs index f58ff9a..986c17b 100644 --- a/SpeckleCore/SpeckleApiClientExtension.cs +++ b/SpeckleCore/SpeckleApiClientExtension.cs @@ -72,6 +72,15 @@ public SpeckleApiClient( string baseUrl, bool isPersistent = false ) : base() SetReadyTimer(); } + /// + /// Initialises this client as a receiver for a specific stream. + /// + /// + /// + /// + /// + /// + /// public async Task IntializeReceiver( string streamId, string documentName, string documentType, string documentGuid, string authToken = null ) { if ( Role != null ) @@ -104,7 +113,14 @@ public async Task IntializeReceiver( string streamId, string documentName, strin } - + /// + /// Initialises this client as a Sender by creating a new stream. + /// + /// + /// + /// + /// + /// public async Task IntializeSender( string authToken, string documentName, string documentType, string documentGuid ) { if ( Role != null ) @@ -186,6 +202,9 @@ private void SetWsReconnectTimer( ) }; } + /// + /// Sets up the websocket client & its events.. + /// public void SetupWebsocket( ) { SetWsReconnectTimer(); @@ -231,6 +250,11 @@ public void SetupWebsocket( ) WebsocketClient.Connect(); } + /// + /// Sends a direct message to another websocket client. + /// + /// The clientId of the socket you want to send the message to. + /// What you want to send. Make it serialisable and small. public void SendMessage( string receipientId, dynamic args ) { if ( !WsConnected ) @@ -250,7 +274,12 @@ public void SendMessage( string receipientId, dynamic args ) WebsocketClient.Send( JsonConvert.SerializeObject( eventData ) ); } - + + /// + /// Broadcasts a message to the default streamId room. + /// + /// The message. Make it serialisable and small. + [Obsolete( "This method will be deprecated in favour of BroadcastMessage( string resourceType, string resourceId, dynamic args )" )] public void BroadcastMessage( dynamic args ) { if ( !WsConnected ) @@ -270,6 +299,37 @@ public void BroadcastMessage( dynamic args ) WebsocketClient.Send( JsonConvert.SerializeObject( eventData ) ); } + /// + /// Broadcasts a message in a specific websocket room, as defined by resourceType and resourceId. + /// + /// Can be stream, object, project, comment, user. + /// The database id of the resource. + /// The message. Make it serialisable and small. + public void BroadcastMessage( string resourceType, string resourceId, dynamic args ) + { + if ( !WsConnected ) + { + OnError?.Invoke( this, new SpeckleEventArgs() { EventName = "Websocket client not connected.", EventData = "Websocket client not connected." } ); + return; + } + + var eventData = new + { + eventName = "broadcast", + senderId = ClientId, + resourceType = resourceType, + resourceId = resourceId, + args = args + }; + + WebsocketClient.Send( JsonConvert.SerializeObject( eventData ) ); + } + + /// + /// Joins a websocket room based on a streamId. + /// + /// The streamId you want to join. + [Obsolete("This method will be deprecated in favour of JoinRoom(resourceType, resourceId).")] public void JoinRoom( string streamId ) { if ( !WsConnected ) @@ -288,6 +348,35 @@ public void JoinRoom( string streamId ) WebsocketClient.Send( JsonConvert.SerializeObject( eventData ) ); } + /// + /// Joins a websocket room based a resource type and its id. This will subscribe you to any broadcasts in that room. + /// + /// Can be stream, object, project, comment, user. + /// The database id of the resource. + public void JoinRoom( string resourceType, string resourceId ) + { + if ( !WsConnected ) + { + OnError?.Invoke( this, new SpeckleEventArgs() { EventName = "Websocket client not connected.", EventData = "Websocket client not connected." } ); + return; + } + + var eventData = new + { + eventName = "join", + senderId = ClientId, + resourceType = resourceType, + resourceId = resourceId + }; + + WebsocketClient.Send( JsonConvert.SerializeObject( eventData ) ); + } + + /// + /// Leaves a room based on its streamId. + /// + /// + [Obsolete( "This method will be deprecated in favour of LeaveRoom(resourceType, resourceId)." )] public void LeaveRoom( string streamId ) { if ( !WsConnected ) @@ -306,6 +395,29 @@ public void LeaveRoom( string streamId ) WebsocketClient.Send( JsonConvert.SerializeObject( eventData ) ); } + /// + /// Leaves a websocket room based a resource type and its id. This will stop you from receiving any broadcasts in that room. + /// + /// Can be stream, object, project, comment, user. + /// The database id of the resource. + public void LeaveRoom( string resourceType, string resourceId ) + { + if ( !WsConnected ) + { + OnError?.Invoke( this, new SpeckleEventArgs() { EventName = "Websocket client not connected.", EventData = "Websocket client not connected." } ); + return; + } + + var eventData = new + { + eventName = "leave", + resourceType = resourceType, + resourceId = resourceId + }; + + WebsocketClient.Send( JsonConvert.SerializeObject( eventData ) ); + } + public void LogError( SpeckleException err ) { OnError?.Invoke( this, new SpeckleEventArgs() { EventName = err.StatusCode.ToString(), EventData = err.Message, EventObject = err } ); @@ -344,14 +456,15 @@ protected SpeckleApiClient( SerializationInfo info, StreamingContext context ) catch { var accs = LocalContext.GetAccountsByRestApi( BaseUrl ); - if ( accs.Count == 0 ) + var sorted = accs.OrderByDescending( acc => acc.IsDefault ).ToList(); + if ( sorted.Count == 0 ) { throw new Exception( "You do not have an account that matches this stream's server." ); } else { AuthToken = accs[ 0 ].Token; - User = new User() { Email = accs[ 0 ].Email }; + User = new User() { Email = sorted[ 0 ].Email }; } } From a901745009c5a02fb74d0f83e0bcb5f91a5cdcbf Mon Sep 17 00:00:00 2001 From: Dimitrie Stefanescu Date: Fri, 23 Nov 2018 16:23:40 +0000 Subject: [PATCH 27/27] thiccc stakkk overflow fix --- SpeckleCore/LocalData/SpeckleLocalContext.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/SpeckleCore/LocalData/SpeckleLocalContext.cs b/SpeckleCore/LocalData/SpeckleLocalContext.cs index 3a6e053..d960b85 100644 --- a/SpeckleCore/LocalData/SpeckleLocalContext.cs +++ b/SpeckleCore/LocalData/SpeckleLocalContext.cs @@ -94,7 +94,6 @@ public static void PurgeCachedStreams( ) /// private static void MigrateAccounts( ) { - LocalContext.Init(); List accounts = new List(); if ( Directory.Exists( SettingsFolderPath ) && Directory.EnumerateFiles( SettingsFolderPath, "*.txt" ).Count() > 0 )