From 44b1001863512e7a15c2f1f4a4568876f4c18cfe Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Mon, 4 Mar 2024 10:51:13 +0100 Subject: [PATCH 001/128] Handle existing sequent schema when using active-record 7.1 The active record 7.1 schema dumper will include all non-`public` schemas. This breaks the sequent table creation since it expect the schema to not exist at all. So now we additionally check for the presence of the `event_records` table or view to determine if the schema file needs to be loaded. --- lib/sequent/migrations/sequent_schema.rb | 3 ++- lib/sequent/support/database.rb | 23 ++++++++++++++++++----- spec/lib/sequent/support/database_spec.rb | 9 ++++++++- 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/lib/sequent/migrations/sequent_schema.rb b/lib/sequent/migrations/sequent_schema.rb index ce94f790..523c9633 100644 --- a/lib/sequent/migrations/sequent_schema.rb +++ b/lib/sequent/migrations/sequent_schema.rb @@ -19,7 +19,8 @@ def create_sequent_schema_if_not_exists(env:, fail_if_exists: true) Sequent::Support::Database.establish_connection(db_config) event_store_schema = Sequent.configuration.event_store_schema_name - schema_exists = Sequent::Support::Database.schema_exists?(event_store_schema) + event_records_table = Sequent.configuration.event_record_class.table_name + schema_exists = Sequent::Support::Database.schema_exists?(event_store_schema, event_records_table) FAIL_IF_EXISTS.call(event_store_schema) if schema_exists && fail_if_exists return if schema_exists diff --git a/lib/sequent/support/database.rb b/lib/sequent/support/database.rb index 115c0f59..fb4e48d9 100644 --- a/lib/sequent/support/database.rb +++ b/lib/sequent/support/database.rb @@ -102,9 +102,22 @@ def self.with_schema_search_path(search_path, db_config, env = ENV['SEQUENT_ENV' establish_connection(db_config) end - def self.schema_exists?(schema) - ActiveRecord::Base.connection.execute( - "SELECT schema_name FROM information_schema.schemata WHERE schema_name like '#{schema}'", + def self.schema_exists?(schema, event_records_table = nil) + schema_exists = ActiveRecord::Base.connection.exec_query( + 'SELECT 1 FROM information_schema.schemata WHERE schema_name LIKE $1', + 'schema_exists?', + [schema], + ).count == 1 + + # The ActiveRecord 7.1 schema_dumper.rb now also adds `create_schema` statements for any schema that + # is not named `public`, and in this case the schema may already be created so we check for the + # existence of the `event_records` table (or view) as well. + return schema_exists unless event_records_table + + ActiveRecord::Base.connection.exec_query( + 'SELECT 1 FROM information_schema.tables WHERE table_schema LIKE $1 AND table_name LIKE $2', + 'schema_exists?', + [schema, event_records_table], ).count == 1 end @@ -116,8 +129,8 @@ def self.configuration_hash end end - def schema_exists?(schema) - self.class.schema_exists?(schema) + def schema_exists?(schema, event_records_table = nil) + self.class.schema_exists?(schema, event_records_table) end def create_schema!(schema) diff --git a/spec/lib/sequent/support/database_spec.rb b/spec/lib/sequent/support/database_spec.rb index dcd604d3..b92afc2c 100644 --- a/spec/lib/sequent/support/database_spec.rb +++ b/spec/lib/sequent/support/database_spec.rb @@ -104,7 +104,7 @@ shared_examples 'instance methods' do describe '#create_schema!' do before do - Sequent::ApplicationRecord.connection.execute('drop schema if exists eventstore') + Sequent::ApplicationRecord.connection.execute('DROP SCHEMA IF EXISTS eventstore CASCADE') end it 'creates the schema' do expect { database.create_schema!('eventstore') }.to change { @@ -116,6 +116,13 @@ database.create_schema!('eventstore') expect { database.create_schema!('eventstore') }.to_not raise_error end + + it 'schema does not exist when specified table is not present' do + database.create_schema!('eventstore') + expect(database.schema_exists?('eventstore', 'event_records')).to eq(false) + database.execute_sql('CREATE VIEW eventstore.event_records (id) AS SELECT 1'); + expect(database.schema_exists?('eventstore', 'event_records')).to eq(true) + end end describe '#drop_schema!' do From 42bf948e27b7f67d4c36a52a4e7a75a7c829f6f8 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Mon, 4 Mar 2024 11:36:26 +0100 Subject: [PATCH 002/128] Rubocop --- lib/sequent/support/database.rb | 1 - spec/lib/sequent/core/aggregate_repository_spec.rb | 4 ++-- spec/lib/sequent/support/database_spec.rb | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/sequent/support/database.rb b/lib/sequent/support/database.rb index fb4e48d9..cf3a8c5e 100644 --- a/lib/sequent/support/database.rb +++ b/lib/sequent/support/database.rb @@ -144,7 +144,6 @@ def drop_schema!(schema) def execute_sql(sql) self.class.execute_sql(sql) end - end end end diff --git a/spec/lib/sequent/core/aggregate_repository_spec.rb b/spec/lib/sequent/core/aggregate_repository_spec.rb index e32b6e63..1fbe1020 100644 --- a/spec/lib/sequent/core/aggregate_repository_spec.rb +++ b/spec/lib/sequent/core/aggregate_repository_spec.rb @@ -150,8 +150,8 @@ def load_from_history(stream, events) repository.add_aggregate aggregate expect { repository.load_aggregate(aggregate.id, String) }.to raise_error { |error| - expect(error).to be_a TypeError - } + expect(error).to be_a TypeError + } end it 'should prevent different aggregates with the same id from being added' do diff --git a/spec/lib/sequent/support/database_spec.rb b/spec/lib/sequent/support/database_spec.rb index b92afc2c..5ada1e58 100644 --- a/spec/lib/sequent/support/database_spec.rb +++ b/spec/lib/sequent/support/database_spec.rb @@ -120,7 +120,7 @@ it 'schema does not exist when specified table is not present' do database.create_schema!('eventstore') expect(database.schema_exists?('eventstore', 'event_records')).to eq(false) - database.execute_sql('CREATE VIEW eventstore.event_records (id) AS SELECT 1'); + database.execute_sql('CREATE VIEW eventstore.event_records (id) AS SELECT 1') expect(database.schema_exists?('eventstore', 'event_records')).to eq(true) end end From 9afbe52a35fe6327b3f1fa857b45895e4e9eca5c Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Mon, 4 Mar 2024 11:41:50 +0100 Subject: [PATCH 003/128] Drop support for active record 6.0 --- .github/workflows/rspec.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/rspec.yml b/.github/workflows/rspec.yml index 5a66b34b..c4cfcda8 100644 --- a/.github/workflows/rspec.yml +++ b/.github/workflows/rspec.yml @@ -29,10 +29,6 @@ jobs: gemfile: 'ar_7_0' - ruby-version: 3.0.5 gemfile: 'ar_7_0' - - ruby-version: 3.0.5 - gemfile: 'ar_6_1' - - ruby-version: 3.0.5 - gemfile: 'ar_6_0' env: BUNDLE_GEMFILE: ${{ github.workspace }}/gemfiles/${{ matrix.gemfile }}.gemfile services: @@ -84,7 +80,7 @@ jobs: - name: Run integration tests simple run: | cd integration-specs/simple - bundle exec rspec + bundle exec rspec integration_rails: env: @@ -119,4 +115,3 @@ jobs: bundle exec rake sequent:db:create_event_store bundle exec rake sequent:db:create_view_schema bundle exec rspec spec - From 959bdcc7bda0bc2742b9f31a3bdba7a0ff70667b Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Mon, 4 Mar 2024 13:51:55 +0100 Subject: [PATCH 004/128] Remove gemfiles related to AR 6.x --- gemfiles/ar_6_0.gemfile | 10 --- gemfiles/ar_6_0.gemfile.lock | 119 ----------------------------------- gemfiles/ar_6_1.gemfile | 10 --- gemfiles/ar_6_1.gemfile.lock | 119 ----------------------------------- 4 files changed, 258 deletions(-) delete mode 100644 gemfiles/ar_6_0.gemfile delete mode 100644 gemfiles/ar_6_0.gemfile.lock delete mode 100644 gemfiles/ar_6_1.gemfile delete mode 100644 gemfiles/ar_6_1.gemfile.lock diff --git a/gemfiles/ar_6_0.gemfile b/gemfiles/ar_6_0.gemfile deleted file mode 100644 index 56f62c4f..00000000 --- a/gemfiles/ar_6_0.gemfile +++ /dev/null @@ -1,10 +0,0 @@ -# frozen_string_literal: true - -source 'https://rubygems.org' - -active_star_version = '= 6.0.6' - -gem 'activemodel', active_star_version -gem 'activerecord', active_star_version - -gemspec path: '../' diff --git a/gemfiles/ar_6_0.gemfile.lock b/gemfiles/ar_6_0.gemfile.lock deleted file mode 100644 index cf2bf61e..00000000 --- a/gemfiles/ar_6_0.gemfile.lock +++ /dev/null @@ -1,119 +0,0 @@ -PATH - remote: .. - specs: - sequent (7.0.1) - activemodel (>= 6.0) - activerecord (>= 6.0) - bcrypt (~> 3.1) - i18n - oj (~> 3) - parallel (~> 1.20) - parser (>= 2.6.5, < 3.3) - pg (~> 1.2) - postgresql_cursor (~> 0.6) - thread_safe (~> 0.3.6) - tzinfo (>= 1.1) - -GEM - remote: https://rubygems.org/ - specs: - activemodel (6.0.6) - activesupport (= 6.0.6) - activerecord (6.0.6) - activemodel (= 6.0.6) - activesupport (= 6.0.6) - activesupport (6.0.6) - concurrent-ruby (~> 1.0, >= 1.0.2) - i18n (>= 0.7, < 2) - minitest (~> 5.1) - tzinfo (~> 1.1) - zeitwerk (~> 2.2, >= 2.2.2) - ast (2.4.2) - bcrypt (3.1.19) - coderay (1.1.3) - concurrent-ruby (1.2.2) - diff-lcs (1.5.0) - docile (1.4.0) - i18n (1.14.1) - concurrent-ruby (~> 1.0) - json (2.6.3) - language_server-protocol (3.17.0.3) - method_source (1.0.0) - minitest (5.20.0) - oj (3.16.1) - parallel (1.23.0) - parser (3.2.2.4) - ast (~> 2.4.1) - racc - pg (1.5.4) - postgresql_cursor (0.6.8) - activerecord (>= 6.0) - pry (0.14.2) - coderay (~> 1.1) - method_source (~> 1.0) - racc (1.7.3) - rainbow (3.1.1) - rake (13.1.0) - regexp_parser (2.8.2) - rexml (3.2.6) - rspec (3.12.0) - rspec-core (~> 3.12.0) - rspec-expectations (~> 3.12.0) - rspec-mocks (~> 3.12.0) - rspec-collection_matchers (1.2.1) - rspec-expectations (>= 2.99.0.beta1) - rspec-core (3.12.2) - rspec-support (~> 3.12.0) - rspec-expectations (3.12.3) - diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.12.0) - rspec-mocks (3.12.6) - diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.12.0) - rspec-support (3.12.1) - rubocop (1.57.2) - json (~> 2.3) - language_server-protocol (>= 3.17.0) - parallel (~> 1.10) - parser (>= 3.2.2.4) - rainbow (>= 2.2.2, < 4.0) - regexp_parser (>= 1.8, < 3.0) - rexml (>= 3.2.5, < 4.0) - rubocop-ast (>= 1.28.1, < 2.0) - ruby-progressbar (~> 1.7) - unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.30.0) - parser (>= 3.2.1.0) - ruby-progressbar (1.13.0) - simplecov (0.22.0) - docile (~> 1.1) - simplecov-html (~> 0.11) - simplecov_json_formatter (~> 0.1) - simplecov-html (0.12.3) - simplecov_json_formatter (0.1.4) - thread_safe (0.3.6) - timecop (0.9.8) - tzinfo (1.2.11) - thread_safe (~> 0.1) - unicode-display_width (2.5.0) - zeitwerk (2.6.12) - -PLATFORMS - arm64-darwin-22 - x86_64-linux - -DEPENDENCIES - activemodel (= 6.0.6) - activerecord (= 6.0.6) - pry (~> 0.13) - rake (~> 13) - rspec (~> 3.10) - rspec-collection_matchers (~> 1.2) - rspec-mocks (~> 3.10) - rubocop (~> 1.56, >= 1.56.3) - sequent! - simplecov (~> 0.21) - timecop (~> 0.9) - -BUNDLED WITH - 2.3.24 diff --git a/gemfiles/ar_6_1.gemfile b/gemfiles/ar_6_1.gemfile deleted file mode 100644 index 2b123e45..00000000 --- a/gemfiles/ar_6_1.gemfile +++ /dev/null @@ -1,10 +0,0 @@ -# frozen_string_literal: true - -source 'https://rubygems.org' - -active_star_version = '= 6.1.7' - -gem 'activemodel', active_star_version -gem 'activerecord', active_star_version - -gemspec path: '../' diff --git a/gemfiles/ar_6_1.gemfile.lock b/gemfiles/ar_6_1.gemfile.lock deleted file mode 100644 index 9a556ccc..00000000 --- a/gemfiles/ar_6_1.gemfile.lock +++ /dev/null @@ -1,119 +0,0 @@ -PATH - remote: .. - specs: - sequent (7.0.1) - activemodel (>= 6.0) - activerecord (>= 6.0) - bcrypt (~> 3.1) - i18n - oj (~> 3) - parallel (~> 1.20) - parser (>= 2.6.5, < 3.3) - pg (~> 1.2) - postgresql_cursor (~> 0.6) - thread_safe (~> 0.3.6) - tzinfo (>= 1.1) - -GEM - remote: https://rubygems.org/ - specs: - activemodel (6.1.7) - activesupport (= 6.1.7) - activerecord (6.1.7) - activemodel (= 6.1.7) - activesupport (= 6.1.7) - activesupport (6.1.7) - concurrent-ruby (~> 1.0, >= 1.0.2) - i18n (>= 1.6, < 2) - minitest (>= 5.1) - tzinfo (~> 2.0) - zeitwerk (~> 2.3) - ast (2.4.2) - bcrypt (3.1.19) - coderay (1.1.3) - concurrent-ruby (1.2.2) - diff-lcs (1.5.0) - docile (1.4.0) - i18n (1.14.1) - concurrent-ruby (~> 1.0) - json (2.6.3) - language_server-protocol (3.17.0.3) - method_source (1.0.0) - minitest (5.20.0) - oj (3.16.1) - parallel (1.23.0) - parser (3.2.2.4) - ast (~> 2.4.1) - racc - pg (1.5.4) - postgresql_cursor (0.6.8) - activerecord (>= 6.0) - pry (0.14.2) - coderay (~> 1.1) - method_source (~> 1.0) - racc (1.7.3) - rainbow (3.1.1) - rake (13.1.0) - regexp_parser (2.8.2) - rexml (3.2.6) - rspec (3.12.0) - rspec-core (~> 3.12.0) - rspec-expectations (~> 3.12.0) - rspec-mocks (~> 3.12.0) - rspec-collection_matchers (1.2.1) - rspec-expectations (>= 2.99.0.beta1) - rspec-core (3.12.2) - rspec-support (~> 3.12.0) - rspec-expectations (3.12.3) - diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.12.0) - rspec-mocks (3.12.6) - diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.12.0) - rspec-support (3.12.1) - rubocop (1.57.2) - json (~> 2.3) - language_server-protocol (>= 3.17.0) - parallel (~> 1.10) - parser (>= 3.2.2.4) - rainbow (>= 2.2.2, < 4.0) - regexp_parser (>= 1.8, < 3.0) - rexml (>= 3.2.5, < 4.0) - rubocop-ast (>= 1.28.1, < 2.0) - ruby-progressbar (~> 1.7) - unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.30.0) - parser (>= 3.2.1.0) - ruby-progressbar (1.13.0) - simplecov (0.22.0) - docile (~> 1.1) - simplecov-html (~> 0.11) - simplecov_json_formatter (~> 0.1) - simplecov-html (0.12.3) - simplecov_json_formatter (0.1.4) - thread_safe (0.3.6) - timecop (0.9.8) - tzinfo (2.0.6) - concurrent-ruby (~> 1.0) - unicode-display_width (2.5.0) - zeitwerk (2.6.12) - -PLATFORMS - arm64-darwin-22 - x86_64-linux - -DEPENDENCIES - activemodel (= 6.1.7) - activerecord (= 6.1.7) - pry (~> 0.13) - rake (~> 13) - rspec (~> 3.10) - rspec-collection_matchers (~> 1.2) - rspec-mocks (~> 3.10) - rubocop (~> 1.56, >= 1.56.3) - sequent! - simplecov (~> 0.21) - timecop (~> 0.9) - -BUNDLED WITH - 2.3.24 From 3809292e2dee67a9d503db58c8a47f0fea41d027 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Mon, 4 Mar 2024 15:18:00 +0100 Subject: [PATCH 005/128] Remove more references to active record before 7.0 --- lib/sequent/configuration.rb | 2 +- lib/sequent/support/database.rb | 12 ++------ sequent.gemspec | 2 +- spec/lib/sequent/support/database_spec.rb | 36 +++++++++++------------ 4 files changed, 21 insertions(+), 31 deletions(-) diff --git a/lib/sequent/configuration.rb b/lib/sequent/configuration.rb index 4c28755f..0fa258e5 100644 --- a/lib/sequent/configuration.rb +++ b/lib/sequent/configuration.rb @@ -129,7 +129,7 @@ def initialize end def can_use_multiple_databases? - enable_multiple_database_support && ActiveRecord.version > Gem::Version.new('6.1.0') + enable_multiple_database_support end def versions_table_name=(table_name) diff --git a/lib/sequent/support/database.rb b/lib/sequent/support/database.rb index cf3a8c5e..1a9fc7ff 100644 --- a/lib/sequent/support/database.rb +++ b/lib/sequent/support/database.rb @@ -23,11 +23,7 @@ def self.read_config(env) database_yml = File.join(Sequent.configuration.database_config_directory, 'database.yml') config = YAML.safe_load(ERB.new(File.read(database_yml)).result, aliases: true)[env] - if Gem.loaded_specs['activerecord'].version >= Gem::Version.create('6.1.0') - ActiveRecord::Base.configurations.resolve(config).configuration_hash.with_indifferent_access - else - ActiveRecord::Base.resolve_config_for_connection(config) - end + ActiveRecord::Base.configurations.resolve(config).configuration_hash.with_indifferent_access end def self.create!(db_config) @@ -122,11 +118,7 @@ def self.schema_exists?(schema, event_records_table = nil) end def self.configuration_hash - if Gem.loaded_specs['activesupport'].version >= Gem::Version.create('6.1.0') - ActiveRecord::Base.connection_db_config.configuration_hash - else - ActiveRecord::Base.connection_config - end + ActiveRecord::Base.connection_db_config.configuration_hash end def schema_exists?(schema, event_records_table = nil) diff --git a/sequent.gemspec b/sequent.gemspec index d86cde4b..8f311870 100644 --- a/sequent.gemspec +++ b/sequent.gemspec @@ -23,7 +23,7 @@ Gem::Specification.new do |s| 'https://github.com/zilverline/sequent' s.license = 'MIT' - active_star_version = '>= 6.0' + active_star_version = '>= 7.0' rspec_version = '~> 3.10' diff --git a/spec/lib/sequent/support/database_spec.rb b/spec/lib/sequent/support/database_spec.rb index 5ada1e58..00b3373e 100644 --- a/spec/lib/sequent/support/database_spec.rb +++ b/spec/lib/sequent/support/database_spec.rb @@ -90,12 +90,11 @@ end describe 'class methods' do - context 'with multiple database support', - skip: Gem.loaded_specs['activerecord'].version < Gem::Version.create('6.1.0') do - before { Sequent.configuration.enable_multiple_database_support = true } - include_examples 'class methods' - after { Sequent.configuration.enable_multiple_database_support = false } - end + context 'with multiple database support' do + before { Sequent.configuration.enable_multiple_database_support = true } + include_examples 'class methods' + after { Sequent.configuration.enable_multiple_database_support = false } + end context 'no multiple database support' do include_examples 'class methods' end @@ -141,19 +140,18 @@ describe 'instance methods' do subject(:database) { Sequent::Support::Database.new } - context 'with multiple database support', - skip: Gem.loaded_specs['activerecord'].version < Gem::Version.create('6.1.0') do - before do - Sequent.configuration.enable_multiple_database_support = true - Sequent::Support::Database.create!(db_config) - Sequent::Support::Database.establish_connection(db_config) - end - after { Sequent::Support::Database.drop!(db_config) } - after :all do - Sequent.configuration.enable_multiple_database_support = false - end - include_examples 'instance methods' - end + context 'with multiple database support' do + before do + Sequent.configuration.enable_multiple_database_support = true + Sequent::Support::Database.create!(db_config) + Sequent::Support::Database.establish_connection(db_config) + end + after { Sequent::Support::Database.drop!(db_config) } + after :all do + Sequent.configuration.enable_multiple_database_support = false + end + include_examples 'instance methods' + end context 'no multiple database support' do before do Sequent::Support::Database.create!(db_config) From d17e0c179334780db4c26378f947e882e6221ad9 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Mon, 4 Mar 2024 15:24:51 +0100 Subject: [PATCH 006/128] Update sequent version and bundle lock files --- gemfiles/ar_7_0.gemfile.lock | 8 ++++---- gemfiles/ar_7_1.gemfile.lock | 13 +++++++------ lib/version.rb | 2 +- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/gemfiles/ar_7_0.gemfile.lock b/gemfiles/ar_7_0.gemfile.lock index cf777706..9d9cac65 100644 --- a/gemfiles/ar_7_0.gemfile.lock +++ b/gemfiles/ar_7_0.gemfile.lock @@ -1,9 +1,9 @@ PATH remote: .. specs: - sequent (7.0.1) - activemodel (>= 6.0) - activerecord (>= 6.0) + sequent (7.1.0) + activemodel (>= 7.0) + activerecord (>= 7.0) bcrypt (~> 3.1) i18n oj (~> 3) @@ -114,4 +114,4 @@ DEPENDENCIES timecop (~> 0.9) BUNDLED WITH - 2.3.24 + 2.4.19 diff --git a/gemfiles/ar_7_1.gemfile.lock b/gemfiles/ar_7_1.gemfile.lock index 3979c13c..b470e672 100644 --- a/gemfiles/ar_7_1.gemfile.lock +++ b/gemfiles/ar_7_1.gemfile.lock @@ -1,9 +1,9 @@ PATH remote: .. specs: - sequent (7.0.1) - activemodel (>= 6.0) - activerecord (>= 6.0) + sequent (7.1.0) + activemodel (>= 7.0) + activerecord (>= 7.0) bcrypt (~> 3.1) i18n oj (~> 3) @@ -35,7 +35,7 @@ GEM tzinfo (~> 2.0) ast (2.4.2) base64 (0.2.0) - bcrypt (3.1.19) + bcrypt (3.1.20) bigdecimal (3.1.4) coderay (1.1.3) concurrent-ruby (1.2.2) @@ -51,12 +51,13 @@ GEM method_source (1.0.0) minitest (5.20.0) mutex_m (0.2.0) - oj (3.16.1) + oj (3.16.3) + bigdecimal (>= 3.0) parallel (1.23.0) parser (3.2.2.4) ast (~> 2.4.1) racc - pg (1.5.4) + pg (1.5.6) postgresql_cursor (0.6.8) activerecord (>= 6.0) pry (0.14.2) diff --git a/lib/version.rb b/lib/version.rb index 51ef9afc..b48ebf8b 100644 --- a/lib/version.rb +++ b/lib/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module Sequent - VERSION = '7.0.1' + VERSION = '7.1.0' end From 34226e6bb38cfb1d13cf071ff3eb712f26986c27 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Thu, 29 Feb 2024 13:23:18 +0100 Subject: [PATCH 007/128] Make serialization work with both TEXT and JSONB columns In the future only JSON/JSONB columns will be supported, but for now command and event JSON is conditionally serialized based on the underlying column type. --- db/sequent_schema.rb | 4 +- lib/sequent/core/command_record.rb | 9 ++- lib/sequent/core/event_record.rb | 9 ++- lib/sequent/core/event_store.rb | 30 ++++--- spec/lib/sequent/core/command_record_spec.rb | 20 ++++- spec/lib/sequent/core/event_record_spec.rb | 78 +++++++++---------- .../sequent/core/serializes_command_spec.rb | 8 ++ 7 files changed, 93 insertions(+), 65 deletions(-) diff --git a/db/sequent_schema.rb b/db/sequent_schema.rb index 38d82b37..87e05eaf 100644 --- a/db/sequent_schema.rb +++ b/db/sequent_schema.rb @@ -5,7 +5,7 @@ t.integer "sequence_number", :null => false t.datetime "created_at", :null => false t.string "event_type", :null => false - t.text "event_json", :null => false + t.jsonb "event_json", :null => false t.integer "command_record_id", :null => false t.integer "stream_record_id", :null => false t.bigint "xact_id" @@ -36,7 +36,7 @@ t.string "command_type", :null => false t.string "event_aggregate_id" t.integer "event_sequence_number" - t.text "command_json", :null => false + t.jsonb "command_json", :null => false t.datetime "created_at", :null => false end diff --git a/lib/sequent/core/command_record.rb b/lib/sequent/core/command_record.rb index 588bb4e3..7e84e3f7 100644 --- a/lib/sequent/core/command_record.rb +++ b/lib/sequent/core/command_record.rb @@ -7,7 +7,7 @@ module Sequent module Core module SerializesCommand def command - args = Sequent::Core::Oj.strict_load(command_json) + args = serialize_json? ? Sequent::Core::Oj.strict_load(command_json) : command_json Class.const_get(command_type).deserialize_from_json(args) end @@ -16,7 +16,7 @@ def command=(command) self.aggregate_id = command.aggregate_id if command.respond_to? :aggregate_id self.user_id = command.user_id if command.respond_to? :user_id self.command_type = command.class.name - self.command_json = Sequent::Core::Oj.dump(command.attributes) + self.command_json = serialize_json? ? Sequent::Core::Oj.dump(command.attributes) : command.attributes # optional attributes (here for historic reasons) # this should be moved to a configurable CommandSerializer @@ -30,6 +30,11 @@ def command=(command) private + def serialize_json? + json_column_type = self.class.columns_hash['command_json'].sql_type_metadata.type + %i[json jsonb].exclude? json_column_type + end + def serialize_attribute?(command, attribute) [self, command].all? { |obj| obj.respond_to?(attribute) } end diff --git a/lib/sequent/core/event_record.rb b/lib/sequent/core/event_record.rb index 9f5e5882..5ebe293b 100644 --- a/lib/sequent/core/event_record.rb +++ b/lib/sequent/core/event_record.rb @@ -46,7 +46,7 @@ def self.after_serialization(event_record, event) module SerializesEvent def event - payload = Sequent::Core::Oj.strict_load(event_json) + payload = serialize_json? ? Sequent::Core::Oj.strict_load(event_json) : event_json Class.const_get(event_type).deserialize_from_json(payload) end @@ -56,7 +56,7 @@ def event=(event) self.organization_id = event.organization_id if event.respond_to?(:organization_id) self.event_type = event.class.name self.created_at = event.created_at - self.event_json = self.class.serialize_to_json(event) + self.event_json = serialize_json? ? self.class.serialize_to_json(event) : event.attributes Sequent.configuration.event_record_hooks_class.after_serialization(self, event) end @@ -70,6 +70,11 @@ def serialize_to_json(event) def self.included(host_class) host_class.extend(ClassMethods) end + + def serialize_json? + json_column_type = self.class.columns_hash['event_json'].sql_type_metadata.type + %i[json jsonb].exclude? json_column_type + end end class EventRecord < Sequent::ApplicationRecord diff --git a/lib/sequent/core/event_store.rb b/lib/sequent/core/event_store.rb index c21ac2f3..7abaae89 100644 --- a/lib/sequent/core/event_store.rb +++ b/lib/sequent/core/event_store.rb @@ -230,9 +230,13 @@ def primary_key_event_records end def deserialize_event(event_hash) - event_type = event_hash.fetch('event_type') - event_json = Sequent::Core::Oj.strict_load(event_hash.fetch('event_json')) - resolve_event_type(event_type).deserialize_from_json(event_json) + record = EventRecord.new + record.event_type = event_hash.fetch('event_type') + record.event_json = event_hash.fetch('event_json') + # When the column type is JSON or JSONB the event record class expects the JSON to be + # deserialized into a hash already. + record.event_json = Sequent::Core::Oj.strict_load(record.event_json) unless record.serialize_json? + record.event rescue StandardError raise DeserializeEventError, event_hash end @@ -255,22 +259,14 @@ def store_events(command, streams_with_events = []) event_stream.stream_record_id = stream_record.id end uncommitted_events.map do |event| - Sequent.configuration.event_record_class.new.tap do |record| - record.command_record_id = command_record.id - record.stream_record_id = event_stream.stream_record_id - record.event = event - end + record = Sequent.configuration.event_record_class.new + record.command_record_id = command_record.id + record.stream_record_id = event_stream.stream_record_id + record.event = event + record.attributes.slice(*column_names) end end - connection = Sequent.configuration.event_record_class.connection - values = event_records - .map { |r| "(#{column_names.map { |c| connection.quote(r[c.to_sym]) }.join(',')})" } - .join(',') - columns = column_names.map { |c| connection.quote_column_name(c) }.join(',') - sql = <<~SQL.chomp - insert into #{connection.quote_table_name(Sequent.configuration.event_record_class.table_name)} (#{columns}) values #{values} - SQL - Sequent.configuration.event_record_class.connection.insert(sql, nil, primary_key_event_records) + Sequent.configuration.event_record_class.insert_all!(event_records) rescue ActiveRecord::RecordNotUnique raise OptimisticLockingError end diff --git a/spec/lib/sequent/core/command_record_spec.rb b/spec/lib/sequent/core/command_record_spec.rb index 5993e72d..641744dc 100644 --- a/spec/lib/sequent/core/command_record_spec.rb +++ b/spec/lib/sequent/core/command_record_spec.rb @@ -3,7 +3,19 @@ require 'spec_helper' describe Sequent::Core::CommandRecord do - let(:command) { Sequent::Core::Command.new(aggregate_id: 'abc') } + class TestCommand < Sequent::Core::Command + attrs field: String + end + + let(:command) { TestCommand.new(aggregate_id: Sequent.new_uuid, field: 'value') } + + before do + # ActiveRecord is doing some timestamp trunctation (microseconds + # to milliseconds) so set the timestamp without microseconds + # here. The JSON timestamp format will not persist these + # microseconds anyway. + command.created_at = Time.parse('2024-02-09T09:09:09.009Z') + end subject { described_class.new } @@ -16,4 +28,10 @@ expect(subject.command).to eq command end end + + it 'should store json in the database' do + Sequent.configuration.event_store.commit_events(command, []) + record = Sequent::Core::CommandRecord.find_by(aggregate_id: command.aggregate_id) + expect(record.command).to eq(command) + end end diff --git a/spec/lib/sequent/core/event_record_spec.rb b/spec/lib/sequent/core/event_record_spec.rb index cc734a4b..66c92bf9 100644 --- a/spec/lib/sequent/core/event_record_spec.rb +++ b/spec/lib/sequent/core/event_record_spec.rb @@ -3,46 +3,23 @@ require 'spec_helper' describe Sequent::Core::EventRecord do - describe Sequent::Core::SerializesEvent do - before do - stub_const('ExampleEvent', Class.new(Sequent::Core::Event)) - stub_const('ExampleRecord', Class.new(Sequent::Core::EventRecord)) + let(:aggregate_id) { Sequent.new_uuid } - ExampleEvent.class_eval do - attrs name: String, age: Integer - end - end + class EventRecordWithOrganizationId < Sequent::Core::EventRecord + attr_accessor :organization_id + end - context 'event' do - it 'initializes an event type from json' do - record = ExampleRecord.new( - { - event_type: ExampleEvent.name, - event_json: { - aggregate_id: 'example-id', - sequence_number: 1, - name: 'example-name', - age: 58, - created_at: DateTime.new(2019, 1, 1), - }.to_json, - }, - ) + before do + stub_const('ExampleEvent', Class.new(Sequent::Core::Event)) - expect(record.event).to eq( - ExampleEvent.new( - aggregate_id: 'example-id', - sequence_number: 1, - name: 'example-name', - age: 58, - created_at: DateTime.new(2019, 1, 1), - ), - ) - end + ExampleEvent.class_eval do + attrs name: String, age: Integer end + end + describe Sequent::Core::SerializesEvent do context 'event=' do it 'assigns attributes from an event' do - aggregate_id = Sequent.new_uuid sequence_number = 1 created_at = DateTime.new(2019, 1, 1) @@ -52,24 +29,20 @@ created_at: created_at, ) - record = ExampleRecord.new + record = Sequent::Core::EventRecord.new record.event = event expect(record.aggregate_id).to eq(aggregate_id) expect(record.sequence_number).to eq(sequence_number) expect(record.event_type).to eq('ExampleEvent') expect(record.created_at).to eq(created_at) - expect(record.event_json).to eq(ExampleRecord.serialize_to_json(event)) + expect(record.event).to eq(event) end it 'conditionally assigns organization_id' do stub_const('EventWithOrganizationId', Class.new(Sequent::Core::Event)) - ExampleRecord.class_eval do - attr_accessor :organization_id - end - - record = ExampleRecord.new + record = EventRecordWithOrganizationId.new event = ExampleEvent.new( aggregate_id: 'aggregate-id', @@ -102,10 +75,33 @@ sequence_number: 1, ) - record = ExampleRecord.new + record = Sequent::Core::EventRecord.new record.event = event expect(event_record_hooks).to have_received(:after_serialization).with(record, event) end end end + + it 'persists events to the database' do + event = ExampleEvent.new( + aggregate_id: aggregate_id, + sequence_number: 1, + created_at: Time.parse('2024-02-29T09:09.009Z'), + ) + + Sequent.configuration.event_store.commit_events( + Sequent::Core::Command.new(aggregate_id: aggregate_id), + [ + [ + Sequent::Core::EventStream.new( + aggregate_type: 'ExampleStream', + aggregate_id: aggregate_id, + ), + [event], + ], + ], + ) + record = Sequent::Core::EventRecord.find_by(aggregate_id: aggregate_id, sequence_number: 1) + expect(record.event).to eq(event) + end end diff --git a/spec/lib/sequent/core/serializes_command_spec.rb b/spec/lib/sequent/core/serializes_command_spec.rb index 6c2cb54c..549b7bcd 100644 --- a/spec/lib/sequent/core/serializes_command_spec.rb +++ b/spec/lib/sequent/core/serializes_command_spec.rb @@ -14,6 +14,10 @@ class RecordMock :event_aggregate_id, :event_sequence_number, :organization_id + + def serialize_json? + true + end end class RecordCommand < Sequent::Core::Command @@ -58,6 +62,10 @@ def initialize(aggregate_id, created_at, user_id, command_type, command_json) @command_type = command_type @command_json = command_json end + + def serialize_json? + true + end end let(:record) do From f1836493f6a456113bdf529395bf30f0f0fe3d23 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Tue, 5 Mar 2024 09:10:29 +0100 Subject: [PATCH 008/128] Change TEXT to JSONB in generator project --- lib/sequent/generator/template_project/db/sequent_schema.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/sequent/generator/template_project/db/sequent_schema.rb b/lib/sequent/generator/template_project/db/sequent_schema.rb index 765c16bc..584c7717 100644 --- a/lib/sequent/generator/template_project/db/sequent_schema.rb +++ b/lib/sequent/generator/template_project/db/sequent_schema.rb @@ -5,7 +5,7 @@ t.integer "sequence_number", :null => false t.datetime "created_at", :null => false t.string "event_type", :null => false - t.text "event_json", :null => false + t.jsonb "event_json", :null => false t.integer "command_record_id", :null => false t.integer "stream_record_id", :null => false end @@ -29,7 +29,7 @@ t.string "user_id" t.uuid "aggregate_id" t.string "command_type", :null => false - t.text "command_json", :null => false + t.jsonb "command_json", :null => false t.datetime "created_at", :null => false end From 513e1d6bafc06ed4c1c827e682e87032005601d1 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Tue, 5 Mar 2024 09:33:05 +0100 Subject: [PATCH 009/128] Do not try to store an empty set of events Fails with Ruby 3.2 and ActiveRecord 7.0. --- lib/sequent/core/event_store.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/sequent/core/event_store.rb b/lib/sequent/core/event_store.rb index 7abaae89..a8435c11 100644 --- a/lib/sequent/core/event_store.rb +++ b/lib/sequent/core/event_store.rb @@ -266,7 +266,7 @@ def store_events(command, streams_with_events = []) record.attributes.slice(*column_names) end end - Sequent.configuration.event_record_class.insert_all!(event_records) + Sequent.configuration.event_record_class.insert_all!(event_records) if event_records.present? rescue ActiveRecord::RecordNotUnique raise OptimisticLockingError end From 01cad19f4c83e0236d38b408e76d315661f0bdd5 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Tue, 5 Mar 2024 14:03:38 +0100 Subject: [PATCH 010/128] Update version to indicate development of next major release --- gemfiles/ar_7_0.gemfile.lock | 2 +- gemfiles/ar_7_1.gemfile.lock | 4 ++-- lib/version.rb | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/gemfiles/ar_7_0.gemfile.lock b/gemfiles/ar_7_0.gemfile.lock index 9d9cac65..5ef09151 100644 --- a/gemfiles/ar_7_0.gemfile.lock +++ b/gemfiles/ar_7_0.gemfile.lock @@ -1,7 +1,7 @@ PATH remote: .. specs: - sequent (7.1.0) + sequent (8.0.0.pre.dev.1) activemodel (>= 7.0) activerecord (>= 7.0) bcrypt (~> 3.1) diff --git a/gemfiles/ar_7_1.gemfile.lock b/gemfiles/ar_7_1.gemfile.lock index b470e672..4215342d 100644 --- a/gemfiles/ar_7_1.gemfile.lock +++ b/gemfiles/ar_7_1.gemfile.lock @@ -1,7 +1,7 @@ PATH remote: .. specs: - sequent (7.1.0) + sequent (8.0.0.pre.dev.1) activemodel (>= 7.0) activerecord (>= 7.0) bcrypt (~> 3.1) @@ -129,4 +129,4 @@ DEPENDENCIES timecop (~> 0.9) BUNDLED WITH - 2.3.24 + 2.4.19 diff --git a/lib/version.rb b/lib/version.rb index b48ebf8b..ee002b3d 100644 --- a/lib/version.rb +++ b/lib/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module Sequent - VERSION = '7.1.0' + VERSION = '8.0.0-dev.1' end From 46a07d75a07a82d41673391922f4487bf0f2854d Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Tue, 5 Mar 2024 15:18:22 +0100 Subject: [PATCH 011/128] Cleanup event deserialization based on column type --- lib/sequent/core/event_store.rb | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/lib/sequent/core/event_store.rb b/lib/sequent/core/event_store.rb index a8435c11..c9e8e68a 100644 --- a/lib/sequent/core/event_store.rb +++ b/lib/sequent/core/event_store.rb @@ -232,10 +232,15 @@ def primary_key_event_records def deserialize_event(event_hash) record = EventRecord.new record.event_type = event_hash.fetch('event_type') - record.event_json = event_hash.fetch('event_json') - # When the column type is JSON or JSONB the event record class expects the JSON to be - # deserialized into a hash already. - record.event_json = Sequent::Core::Oj.strict_load(record.event_json) unless record.serialize_json? + record.event_json = + if record.serialize_json? + event_hash.fetch('event_json') + else + # When the column type is JSON or JSONB the event record + # class expects the JSON to be deserialized into a hash + # already. + Sequent::Core::Oj.strict_load(event_hash.fetch('event_json')) + end record.event rescue StandardError raise DeserializeEventError, event_hash From 1331c8969cd35bf5ea88e78e16b9a148819992cf Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Tue, 5 Mar 2024 15:45:28 +0100 Subject: [PATCH 012/128] Drop active-record 7.0 support Only 7.1 supports composite primary keys. --- .github/workflows/rspec.yml | 6 +- gemfiles/ar_7_0.gemfile | 10 --- gemfiles/ar_7_0.gemfile.lock | 117 ---------------------------- gemfiles/ar_7_1.gemfile.lock | 4 +- integration-specs/rails-app/Gemfile | 2 +- sequent.gemspec | 2 +- 6 files changed, 6 insertions(+), 135 deletions(-) delete mode 100644 gemfiles/ar_7_0.gemfile delete mode 100644 gemfiles/ar_7_0.gemfile.lock diff --git a/.github/workflows/rspec.yml b/.github/workflows/rspec.yml index c4cfcda8..22584112 100644 --- a/.github/workflows/rspec.yml +++ b/.github/workflows/rspec.yml @@ -23,12 +23,10 @@ jobs: include: - ruby-version: 3.2 gemfile: 'ar_7_1' - - ruby-version: 3.2 - gemfile: 'ar_7_0' - ruby-version: 3.1.3 - gemfile: 'ar_7_0' + gemfile: 'ar_7_1' - ruby-version: 3.0.5 - gemfile: 'ar_7_0' + gemfile: 'ar_7_1' env: BUNDLE_GEMFILE: ${{ github.workspace }}/gemfiles/${{ matrix.gemfile }}.gemfile services: diff --git a/gemfiles/ar_7_0.gemfile b/gemfiles/ar_7_0.gemfile deleted file mode 100644 index ac369363..00000000 --- a/gemfiles/ar_7_0.gemfile +++ /dev/null @@ -1,10 +0,0 @@ -# frozen_string_literal: true - -source 'https://rubygems.org' - -active_star_version = '= 7.0.8' - -gem 'activemodel', active_star_version -gem 'activerecord', active_star_version - -gemspec path: '../' diff --git a/gemfiles/ar_7_0.gemfile.lock b/gemfiles/ar_7_0.gemfile.lock deleted file mode 100644 index 5ef09151..00000000 --- a/gemfiles/ar_7_0.gemfile.lock +++ /dev/null @@ -1,117 +0,0 @@ -PATH - remote: .. - specs: - sequent (8.0.0.pre.dev.1) - activemodel (>= 7.0) - activerecord (>= 7.0) - bcrypt (~> 3.1) - i18n - oj (~> 3) - parallel (~> 1.20) - parser (>= 2.6.5, < 3.3) - pg (~> 1.2) - postgresql_cursor (~> 0.6) - thread_safe (~> 0.3.6) - tzinfo (>= 1.1) - -GEM - remote: https://rubygems.org/ - specs: - activemodel (7.0.8) - activesupport (= 7.0.8) - activerecord (7.0.8) - activemodel (= 7.0.8) - activesupport (= 7.0.8) - activesupport (7.0.8) - concurrent-ruby (~> 1.0, >= 1.0.2) - i18n (>= 1.6, < 2) - minitest (>= 5.1) - tzinfo (~> 2.0) - ast (2.4.2) - bcrypt (3.1.19) - coderay (1.1.3) - concurrent-ruby (1.2.2) - diff-lcs (1.5.0) - docile (1.4.0) - i18n (1.14.1) - concurrent-ruby (~> 1.0) - json (2.6.3) - language_server-protocol (3.17.0.3) - method_source (1.0.0) - minitest (5.20.0) - oj (3.16.1) - parallel (1.23.0) - parser (3.2.2.4) - ast (~> 2.4.1) - racc - pg (1.5.4) - postgresql_cursor (0.6.8) - activerecord (>= 6.0) - pry (0.14.2) - coderay (~> 1.1) - method_source (~> 1.0) - racc (1.7.3) - rainbow (3.1.1) - rake (13.1.0) - regexp_parser (2.8.2) - rexml (3.2.6) - rspec (3.12.0) - rspec-core (~> 3.12.0) - rspec-expectations (~> 3.12.0) - rspec-mocks (~> 3.12.0) - rspec-collection_matchers (1.2.1) - rspec-expectations (>= 2.99.0.beta1) - rspec-core (3.12.2) - rspec-support (~> 3.12.0) - rspec-expectations (3.12.3) - diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.12.0) - rspec-mocks (3.12.6) - diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.12.0) - rspec-support (3.12.1) - rubocop (1.57.2) - json (~> 2.3) - language_server-protocol (>= 3.17.0) - parallel (~> 1.10) - parser (>= 3.2.2.4) - rainbow (>= 2.2.2, < 4.0) - regexp_parser (>= 1.8, < 3.0) - rexml (>= 3.2.5, < 4.0) - rubocop-ast (>= 1.28.1, < 2.0) - ruby-progressbar (~> 1.7) - unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.30.0) - parser (>= 3.2.1.0) - ruby-progressbar (1.13.0) - simplecov (0.22.0) - docile (~> 1.1) - simplecov-html (~> 0.11) - simplecov_json_formatter (~> 0.1) - simplecov-html (0.12.3) - simplecov_json_formatter (0.1.4) - thread_safe (0.3.6) - timecop (0.9.8) - tzinfo (2.0.6) - concurrent-ruby (~> 1.0) - unicode-display_width (2.5.0) - -PLATFORMS - arm64-darwin-22 - x86_64-linux - -DEPENDENCIES - activemodel (= 7.0.8) - activerecord (= 7.0.8) - pry (~> 0.13) - rake (~> 13) - rspec (~> 3.10) - rspec-collection_matchers (~> 1.2) - rspec-mocks (~> 3.10) - rubocop (~> 1.56, >= 1.56.3) - sequent! - simplecov (~> 0.21) - timecop (~> 0.9) - -BUNDLED WITH - 2.4.19 diff --git a/gemfiles/ar_7_1.gemfile.lock b/gemfiles/ar_7_1.gemfile.lock index 4215342d..d34cc9d7 100644 --- a/gemfiles/ar_7_1.gemfile.lock +++ b/gemfiles/ar_7_1.gemfile.lock @@ -2,8 +2,8 @@ PATH remote: .. specs: sequent (8.0.0.pre.dev.1) - activemodel (>= 7.0) - activerecord (>= 7.0) + activemodel (>= 7.1) + activerecord (>= 7.1) bcrypt (~> 3.1) i18n oj (~> 3) diff --git a/integration-specs/rails-app/Gemfile b/integration-specs/rails-app/Gemfile index 035716cd..676b944e 100644 --- a/integration-specs/rails-app/Gemfile +++ b/integration-specs/rails-app/Gemfile @@ -4,7 +4,7 @@ source 'https://rubygems.org' git_source(:github) { |repo| "https://github.com/#{repo}.git" } # Bundle edge Rails instead: gem "rails", github: "rails/rails", branch: "main" -gem 'rails', '~> 7.0.8' +gem 'rails', '~> 7.1' # The original asset pipeline for Rails [https://github.com/rails/sprockets-rails] gem 'sprockets-rails' diff --git a/sequent.gemspec b/sequent.gemspec index 8f311870..054e5849 100644 --- a/sequent.gemspec +++ b/sequent.gemspec @@ -23,7 +23,7 @@ Gem::Specification.new do |s| 'https://github.com/zilverline/sequent' s.license = 'MIT' - active_star_version = '>= 7.0' + active_star_version = '>= 7.1' rspec_version = '~> 3.10' From 5fd1b703d015800fdd8a5f80b8d4ba54575d1e85 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Wed, 6 Mar 2024 11:53:11 +0100 Subject: [PATCH 013/128] Use separate snapshot table and composite primary keys Due to the use of `aggregate_id, sequence_number` as the primary key the snapshots can no longer be stored in the event records table, so they now have their own table. The event store API has been extended to allow storing and loading snapshots independently, but snapshots are also returned when loading the events for an aggregate, like before. Since snapshots are now in their own table it is no longer needed to associate a command with a snapshot, but for now the `TakeSnapshot` command is still present. --- db/sequent_schema.rb | 64 ++++---- lib/sequent/configuration.rb | 2 + lib/sequent/core/aggregate_root.rb | 5 +- lib/sequent/core/aggregate_snapshotter.rb | 16 +- lib/sequent/core/event_record.rb | 25 ++- lib/sequent/core/event_store.rb | 69 ++++++--- lib/sequent/core/stream_record.rb | 6 +- lib/sequent/migrations/view_schema.rb | 2 +- .../sequent/core/aggregate_repository_spec.rb | 6 +- spec/lib/sequent/core/aggregate_root_spec.rb | 9 +- .../core/aggregate_snapshotter_spec.rb | 4 +- spec/lib/sequent/core/event_store_spec.rb | 62 +++++--- ...eplay_optimized_postgres_persistor_spec.rb | 144 +++++++++--------- spec/spec_helper.rb | 13 +- 14 files changed, 248 insertions(+), 179 deletions(-) diff --git a/db/sequent_schema.rb b/db/sequent_schema.rb index 87e05eaf..af6fb605 100644 --- a/db/sequent_schema.rb +++ b/db/sequent_schema.rb @@ -1,35 +1,5 @@ ActiveRecord::Schema.define do - create_table "event_records", :force => true do |t| - t.uuid "aggregate_id", :null => false - t.integer "sequence_number", :null => false - t.datetime "created_at", :null => false - t.string "event_type", :null => false - t.jsonb "event_json", :null => false - t.integer "command_record_id", :null => false - t.integer "stream_record_id", :null => false - t.bigint "xact_id" - end - - execute %Q{ -ALTER TABLE event_records ALTER COLUMN xact_id SET DEFAULT pg_current_xact_id()::text::bigint -} - execute %Q{ -CREATE UNIQUE INDEX unique_event_per_aggregate ON event_records ( - aggregate_id, - sequence_number, - (CASE event_type WHEN 'Sequent::Core::SnapshotEvent' THEN 0 ELSE 1 END) -) -} - execute %Q{ -CREATE INDEX snapshot_events ON event_records (aggregate_id, sequence_number DESC) WHERE event_type = 'Sequent::Core::SnapshotEvent' -} - - add_index "event_records", ["command_record_id"], :name => "index_event_records_on_command_record_id" - add_index "event_records", ["event_type"], :name => "index_event_records_on_event_type" - add_index "event_records", ["created_at"], :name => "index_event_records_on_created_at" - add_index "event_records", ["xact_id"], :name => "index_event_records_on_xact_id" - create_table "command_records", :force => true do |t| t.string "user_id" t.uuid "aggregate_id" @@ -42,19 +12,47 @@ add_index "command_records", ["event_aggregate_id", 'event_sequence_number'], :name => "index_command_records_on_event" - create_table "stream_records", :force => true do |t| + create_table "stream_records", :primary_key => ['aggregate_id'], :force => true do |t| t.datetime "created_at", :null => false t.string "aggregate_type", :null => false t.uuid "aggregate_id", :null => false t.integer "snapshot_threshold" end - add_index "stream_records", ["aggregate_id"], :name => "index_stream_records_on_aggregate_id", :unique => true + create_table "event_records", :primary_key => ["aggregate_id", "sequence_number"], :force => true do |t| + t.uuid "aggregate_id", :null => false + t.integer "sequence_number", :null => false + t.datetime "created_at", :null => false + t.string "event_type", :null => false + t.jsonb "event_json", :null => false + t.integer "command_record_id", :null => false + t.bigint "xact_id", :null => false + end + + add_index "event_records", ["command_record_id"], :name => "index_event_records_on_command_record_id" + add_index "event_records", ["event_type"], :name => "index_event_records_on_event_type" + add_index "event_records", ["created_at"], :name => "index_event_records_on_created_at" + add_index "event_records", ["xact_id"], :name => "index_event_records_on_xact_id" + + execute %Q{ +ALTER TABLE event_records ALTER COLUMN xact_id SET DEFAULT pg_current_xact_id()::text::bigint +} execute %q{ ALTER TABLE event_records ADD CONSTRAINT command_fkey FOREIGN KEY (command_record_id) REFERENCES command_records (id) } execute %q{ -ALTER TABLE event_records ADD CONSTRAINT stream_fkey FOREIGN KEY (stream_record_id) REFERENCES stream_records (id) +ALTER TABLE event_records ADD CONSTRAINT stream_fkey FOREIGN KEY (aggregate_id) REFERENCES stream_records (aggregate_id) } + create_table "snapshot_records", :primary_key => ["aggregate_id", "sequence_number"], :force => true do |t| + t.uuid "aggregate_id", :null => false + t.integer "sequence_number", :null => false + t.datetime "created_at", :null => false + t.text "snapshot_type", :null => false + t.jsonb "snapshot_json", :null => false + end + + execute %q{ +ALTER TABLE snapshot_records ADD CONSTRAINT stream_fkey FOREIGN KEY (aggregate_id) REFERENCES stream_records (aggregate_id) +} end diff --git a/lib/sequent/configuration.rb b/lib/sequent/configuration.rb index 0fa258e5..698574af 100644 --- a/lib/sequent/configuration.rb +++ b/lib/sequent/configuration.rb @@ -38,6 +38,7 @@ class Configuration :event_store_cache_event_types, :command_service, :event_record_class, + :snapshot_record_class, :stream_record_class, :snapshot_event_class, :transaction_provider, @@ -95,6 +96,7 @@ def initialize self.event_store = Sequent::Core::EventStore.new self.command_service = Sequent::Core::CommandService.new self.event_record_class = Sequent::Core::EventRecord + self.snapshot_record_class = Sequent::Core::SnapshotRecord self.stream_record_class = Sequent::Core::StreamRecord self.snapshot_event_class = Sequent::Core::SnapshotEvent self.transaction_provider = Sequent::Core::Transactions::ActiveRecordTransactionProvider.new diff --git a/lib/sequent/core/aggregate_root.rb b/lib/sequent/core/aggregate_root.rb index 25c8ecf8..30f3e511 100644 --- a/lib/sequent/core/aggregate_root.rb +++ b/lib/sequent/core/aggregate_root.rb @@ -104,9 +104,8 @@ def clear_events @uncommitted_events = [] end - def take_snapshot! - snapshot = build_event SnapshotEvent, data: Base64.encode64(Marshal.dump(self)) - @uncommitted_events << snapshot + def take_snapshot + build_event SnapshotEvent, data: Base64.encode64(Marshal.dump(self)) end def apply_event(event) diff --git a/lib/sequent/core/aggregate_snapshotter.rb b/lib/sequent/core/aggregate_snapshotter.rb index 9ee42803..847394bd 100644 --- a/lib/sequent/core/aggregate_snapshotter.rb +++ b/lib/sequent/core/aggregate_snapshotter.rb @@ -24,23 +24,27 @@ class AggregateSnapshotter < BaseCommandHandler @last_aggregate_id, command.limit, ) - aggregate_ids.each do |aggregate_id| - take_snapshot!(aggregate_id) - end + snapshots = aggregate_ids.map do |aggregate_id| + take_snapshot(aggregate_id) + end.compact + Sequent.configuration.event_store.store_snapshots(snapshots) + @last_aggregate_id = aggregate_ids.last throw :done if @last_aggregate_id.nil? end on TakeSnapshot do |command| - take_snapshot!(command.aggregate_id) + snapshot = take_snapshot(command.aggregate_id) + Sequent.configuration.event_store.store_snapshots([snapshot]) if snapshot end - def take_snapshot!(aggregate_id) + def take_snapshot(aggregate_id) aggregate = repository.load_aggregate(aggregate_id) Sequent.logger.info "Taking snapshot for aggregate #{aggregate}" - aggregate.take_snapshot! + aggregate.take_snapshot rescue StandardError => e Sequent.logger.error("Failed to take snapshot for aggregate #{aggregate_id}: #{e}, #{e.inspect}") + nil end end end diff --git a/lib/sequent/core/event_record.rb b/lib/sequent/core/event_record.rb index 5ebe293b..646a803f 100644 --- a/lib/sequent/core/event_record.rb +++ b/lib/sequent/core/event_record.rb @@ -83,7 +83,7 @@ class EventRecord < Sequent::ApplicationRecord self.table_name = 'event_records' self.ignored_columns = %w[xact_id] - belongs_to :stream_record + belongs_to :stream_record, foreign_key: :aggregate_id, primary_key: :aggregate_id belongs_to :command_record validates_presence_of :aggregate_id, :sequence_number, :event_type, :event_json, :stream_record, :command_record @@ -107,5 +107,28 @@ def find_origin(record) record end end + + class SnapshotRecord < Sequent::ApplicationRecord + include SerializesEvent + + self.table_name = 'snapshot_records' + + belongs_to :stream_record, foreign_key: :aggregate_id, primary_key: :aggregate_id + + validates_presence_of :aggregate_id, :sequence_number, :snapshot_json, :stream_record + validates_numericality_of :sequence_number, only_integer: true, greater_than: 0 + + def event_type + snapshot_type + end + + def event_json + snapshot_json + end + + def serialize_json? + false + end + end end end diff --git a/lib/sequent/core/event_store.rb b/lib/sequent/core/event_store.rb index c9e8e68a..7471ec60 100644 --- a/lib/sequent/core/event_store.rb +++ b/lib/sequent/core/event_store.rb @@ -115,17 +115,45 @@ def load_events_for_aggregates(aggregate_ids) end end + def store_snapshots(snapshots) + SnapshotRecord.insert_all!( + snapshots.map do |snapshot| + { + aggregate_id: snapshot.aggregate_id, + sequence_number: snapshot.sequence_number, + created_at: snapshot.created_at, + snapshot_type: snapshot.class.name, + snapshot_json: Sequent::Core::Oj.strict_load(Sequent::Core::Oj.dump(snapshot)), + } + end, + ) + end + + def load_latest_snapshot(aggregate_id) + latest_snapshot = SnapshotRecord.where(aggregate_id:).order(:sequence_number).last + latest_snapshot&.event + end + + # Deletes all snapshots for aggregate_id with a sequence_number lower than the specified sequence number. + def delete_snapshots_before(aggregate_id, sequence_number) + SnapshotRecord.where(aggregate_id: aggregate_id).where('sequence_number < ?', sequence_number).delete_all + end + def aggregate_query(aggregate_id) <<~SQL.chomp ( - SELECT event_type, event_json - FROM #{quote_table_name Sequent.configuration.event_record_class.table_name} AS o - WHERE aggregate_id = #{quote(aggregate_id)} - AND sequence_number >= COALESCE((SELECT MAX(sequence_number) - FROM #{quote_table_name Sequent.configuration.event_record_class.table_name} AS i - WHERE event_type = #{quote Sequent.configuration.snapshot_event_class.name} - AND i.aggregate_id = #{quote(aggregate_id)}), 0) - ORDER BY sequence_number ASC, (CASE event_type WHEN #{quote Sequent.configuration.snapshot_event_class.name} THEN 0 ELSE 1 END) ASC + WITH snapshot AS (SELECT * + FROM #{quote_table_name Sequent.configuration.snapshot_record_class.table_name} + WHERE aggregate_id = #{quote(aggregate_id)} + ORDER BY sequence_number DESC + LIMIT 1) + SELECT snapshot_type AS event_type, snapshot_json AS event_json FROM snapshot + UNION ALL + (SELECT event_type, event_json + FROM #{quote_table_name Sequent.configuration.event_record_class.table_name} AS o + WHERE aggregate_id = #{quote(aggregate_id)} + AND sequence_number >= COALESCE((SELECT sequence_number FROM snapshot), 0) + ORDER BY sequence_number ASC) ) SQL end @@ -164,7 +192,7 @@ def replay_events_from_cursor(get_events:, block_size: 2000, event = deserialize_event(record) publish_events([event]) progress += 1 - ids_replayed << record['id'] + ids_replayed << record['aggregate_id'] if progress % block_size == 0 on_progress[progress, false, ids_replayed] ids_replayed.clear @@ -187,14 +215,15 @@ def replay_events_from_cursor(get_events:, block_size: 2000, def aggregates_that_need_snapshots(last_aggregate_id, limit = 10) stream_table = quote_table_name Sequent.configuration.stream_record_class.table_name event_table = quote_table_name Sequent.configuration.event_record_class.table_name + snapshot_table = quote_table_name Sequent.configuration.snapshot_record_class.table_name query = <<~SQL.chomp SELECT aggregate_id FROM #{stream_table} stream WHERE aggregate_id::varchar > COALESCE(#{quote last_aggregate_id}, '') AND snapshot_threshold IS NOT NULL AND snapshot_threshold <= ( - (SELECT MAX(events.sequence_number) FROM #{event_table} events WHERE events.event_type <> #{quote Sequent.configuration.snapshot_event_class.name} AND stream.aggregate_id = events.aggregate_id) - - COALESCE((SELECT MAX(snapshots.sequence_number) FROM #{event_table} snapshots WHERE snapshots.event_type = #{quote Sequent.configuration.snapshot_event_class.name} AND stream.aggregate_id = snapshots.aggregate_id), 0)) + (SELECT MAX(events.sequence_number) FROM #{event_table} events WHERE stream.aggregate_id = events.aggregate_id) - + COALESCE((SELECT MAX(snapshots.sequence_number) FROM #{snapshot_table} snapshots WHERE stream.aggregate_id = snapshots.aggregate_id), 0)) ORDER BY aggregate_id LIMIT #{quote limit} FOR UPDATE @@ -256,17 +285,19 @@ def publish_events(events) def store_events(command, streams_with_events = []) command_record = CommandRecord.create!(command: command) - event_records = streams_with_events.flat_map do |event_stream, uncommitted_events| - unless event_stream.stream_record_id - stream_record = Sequent.configuration.stream_record_class.new - stream_record.event_stream = event_stream - stream_record.save! - event_stream.stream_record_id = stream_record.id - end + streams = streams_with_events.map do |event_stream, _| + { + aggregate_id: event_stream.aggregate_id, + aggregate_type: event_stream.aggregate_type, + snapshot_threshold: event_stream.snapshot_threshold, + } + end + StreamRecord.upsert_all(streams, unique_by: :aggregate_id, update_only: %i[snapshot_threshold]) + + event_records = streams_with_events.flat_map do |_, uncommitted_events| uncommitted_events.map do |event| record = Sequent.configuration.event_record_class.new record.command_record_id = command_record.id - record.stream_record_id = event_stream.stream_record_id record.event = event record.attributes.slice(*column_names) end diff --git a/lib/sequent/core/stream_record.rb b/lib/sequent/core/stream_record.rb index 9aa454a7..a40e8700 100644 --- a/lib/sequent/core/stream_record.rb +++ b/lib/sequent/core/stream_record.rb @@ -5,13 +5,12 @@ module Sequent module Core class EventStream - attr_accessor :aggregate_type, :aggregate_id, :snapshot_threshold, :stream_record_id + attr_accessor :aggregate_type, :aggregate_id, :snapshot_threshold - def initialize(aggregate_type:, aggregate_id:, snapshot_threshold: nil, stream_record_id: nil) + def initialize(aggregate_type:, aggregate_id:, snapshot_threshold: nil) @aggregate_type = aggregate_type @aggregate_id = aggregate_id @snapshot_threshold = snapshot_threshold - @stream_record_id = stream_record_id end end @@ -28,7 +27,6 @@ def event_stream aggregate_type: aggregate_type, aggregate_id: aggregate_id, snapshot_threshold: snapshot_threshold, - stream_record_id: id, ) end diff --git a/lib/sequent/migrations/view_schema.rb b/lib/sequent/migrations/view_schema.rb index 12bf1afd..fad8faf2 100644 --- a/lib/sequent/migrations/view_schema.rb +++ b/lib/sequent/migrations/view_schema.rb @@ -470,7 +470,7 @@ def event_stream(aggregate_prefixes, event_types, minimum_xact_id_inclusive, max end event_stream .order('aggregate_id ASC, sequence_number ASC') - .select('id, event_type, event_json, sequence_number') + .select('aggregate_id, event_type, event_json, sequence_number') end ## shortcut methods diff --git a/spec/lib/sequent/core/aggregate_repository_spec.rb b/spec/lib/sequent/core/aggregate_repository_spec.rb index 1fbe1020..6b7e13e4 100644 --- a/spec/lib/sequent/core/aggregate_repository_spec.rb +++ b/spec/lib/sequent/core/aggregate_repository_spec.rb @@ -448,7 +448,8 @@ def ping Sequent.aggregate_repository.add_aggregate(dummy_aggregate) Sequent.aggregate_repository.commit(DummyCommand.new) - dummy_aggregate.take_snapshot! + snapshot = dummy_aggregate.take_snapshot + Sequent.configuration.event_store.store_snapshots([snapshot]) Timecop.travel(30.minutes) dummy_aggregate.ping @@ -467,7 +468,8 @@ def ping dummy_aggregate.ping Sequent.aggregate_repository.add_aggregate(dummy_aggregate) Sequent.aggregate_repository.commit(DummyCommand.new) - dummy_aggregate.take_snapshot! + snapshot = dummy_aggregate.take_snapshot + Sequent.configuration.event_store.store_snapshots([snapshot]) dummy_aggregate.ping Sequent.aggregate_repository.commit(DummyCommand.new) Sequent.aggregate_repository.clear diff --git a/spec/lib/sequent/core/aggregate_root_spec.rb b/spec/lib/sequent/core/aggregate_root_spec.rb index 5624199f..1b9ffb44 100644 --- a/spec/lib/sequent/core/aggregate_root_spec.rb +++ b/spec/lib/sequent/core/aggregate_root_spec.rb @@ -84,15 +84,8 @@ def event_count context 'snapshotting' do before { subject.generate_event } - it 'adds an uncommitted snapshot event' do - expect do - subject.take_snapshot! - end.to change { subject.uncommitted_events.count }.by(1) - end - it 'restores state from the snapshot' do - subject.take_snapshot! - snapshot_event = subject.uncommitted_events.last + snapshot_event = subject.take_snapshot restored = TestAggregateRoot.load_from_history :stream, [snapshot_event] expect(restored.event_count).to eq 1 end diff --git a/spec/lib/sequent/core/aggregate_snapshotter_spec.rb b/spec/lib/sequent/core/aggregate_snapshotter_spec.rb index 355f2282..77230722 100644 --- a/spec/lib/sequent/core/aggregate_snapshotter_spec.rb +++ b/spec/lib/sequent/core/aggregate_snapshotter_spec.rb @@ -41,9 +41,9 @@ class MyAggregate < Sequent::Core::AggregateRoot; end end it 'can take a snapshot' do - Sequent.command_service.execute_commands(*take_snapshot) + Sequent.command_service.execute_commands(take_snapshot) - expect(Sequent::Core::EventRecord.last.event_type).to eq Sequent::Core::SnapshotEvent.name + expect(Sequent::Core::SnapshotRecord.last.snapshot_type).to eq Sequent::Core::SnapshotEvent.name end context 'loads aggregates with snapshots' do diff --git a/spec/lib/sequent/core/event_store_spec.rb b/spec/lib/sequent/core/event_store_spec.rb index af04e8f1..a10bb46f 100644 --- a/spec/lib/sequent/core/event_store_spec.rb +++ b/spec/lib/sequent/core/event_store_spec.rb @@ -8,6 +8,9 @@ class MyEvent < Sequent::Core::Event end + class MyAggregate < Sequent::Core::AggregateRoot + end + let(:event_store) { Sequent.configuration.event_store } let(:aggregate_id) { Sequent.new_uuid } @@ -40,7 +43,8 @@ class MyEvent < Sequent::Core::Event end context 'snapshotting' do - it 'can store events' do + let(:snapshot_threshold) { 1 } + before do event_store.commit_events( Sequent::Core::CommandRecord.new, [ @@ -48,16 +52,25 @@ class MyEvent < Sequent::Core::Event Sequent::Core::EventStream.new( aggregate_type: 'MyAggregate', aggregate_id: aggregate_id, - snapshot_threshold: 13, + snapshot_threshold: snapshot_threshold, ), - [MyEvent.new(aggregate_id: aggregate_id, sequence_number: 1)], + [ + MyEvent.new( + aggregate_id: aggregate_id, + sequence_number: 1, + created_at: Time.parse('2024-02-29T01:10:12Z'), + data: "with ' unsafe SQL characters;\n", + ), + ], ], ], ) + end + it 'can store events' do stream, events = event_store.load_events aggregate_id - expect(stream.snapshot_threshold).to eq(13) + expect(stream.snapshot_threshold).to eq(1) expect(stream.aggregate_type).to eq('MyAggregate') expect(stream.aggregate_id).to eq(aggregate_id) expect(events.first.aggregate_id).to eq(aggregate_id) @@ -65,22 +78,21 @@ class MyEvent < Sequent::Core::Event end it 'can find streams that need snapshotting' do - event_store.commit_events( - Sequent::Core::CommandRecord.new, - [ - [ - Sequent::Core::EventStream.new( - aggregate_type: 'MyAggregate', - aggregate_id: aggregate_id, - snapshot_threshold: 1, - ), - [MyEvent.new(aggregate_id: aggregate_id, sequence_number: 1)], - ], - ], - ) - expect(event_store.aggregates_that_need_snapshots(nil)).to include(aggregate_id) end + + it 'can store and delete snapshots' do + aggregate = MyAggregate.new(aggregate_id) + snapshot = aggregate.take_snapshot + snapshot.created_at = Time.parse('2024-02-28T04:12:33Z') + + event_store.store_snapshots([snapshot]) + + expect(event_store.load_latest_snapshot(aggregate_id)).to eq(snapshot) + + event_store.delete_snapshots_before(aggregate_id, snapshot.sequence_number + 1) + expect(event_store.load_latest_snapshot(aggregate_id)).to eq(nil) + end end describe '#commit_events' do @@ -247,12 +259,12 @@ class MyEvent < Sequent::Core::Event let(:aggregate_id_1) { Sequent.new_uuid } let(:frozen_time) { Time.parse('2022-02-08 14:15:00 +0200') } let(:event_stream) { instance_of(Sequent::Core::EventStream) } - let(:event_1) { MyEvent.new(id: 3, aggregate_id: aggregate_id_1, sequence_number: 1, created_at: frozen_time) } + let(:event_1) { MyEvent.new(aggregate_id: aggregate_id_1, sequence_number: 1, created_at: frozen_time) } let(:event_2) do - MyEvent.new(id: 2, aggregate_id: aggregate_id_1, sequence_number: 2, created_at: frozen_time + 5.minutes) + MyEvent.new(aggregate_id: aggregate_id_1, sequence_number: 2, created_at: frozen_time + 5.minutes) end let(:event_3) do - MyEvent.new(id: 1, aggregate_id: aggregate_id_1, sequence_number: 3, created_at: frozen_time + 10.minutes) + MyEvent.new(aggregate_id: aggregate_id_1, sequence_number: 3, created_at: frozen_time + 10.minutes) end let(:snapshot_event) do Sequent::Core::SnapshotEvent.new( @@ -275,12 +287,12 @@ class MyEvent < Sequent::Core::Event [ event_1, event_2, - snapshot_event, event_3, ], ], ], ) + event_store.store_snapshots([snapshot_event]) end context 'returning events except snapshot events in order of sequence_number' do @@ -456,9 +468,9 @@ class FailingHandler < Sequent::Core::Projector snapshot_event_type = Sequent.configuration.snapshot_event_class Sequent.configuration.event_record_class .select('event_type, event_json') - .joins("INNER JOIN #{stream_records} ON #{event_records}.stream_record_id = #{stream_records}.id") + .joins("INNER JOIN #{stream_records} ON #{event_records}.aggregate_id = #{stream_records}.aggregate_id") .where('event_type <> ?', snapshot_event_type) - .order!("#{stream_records}.id, #{event_records}.sequence_number") + .order!("#{stream_records}.aggregate_id, #{event_records}.sequence_number") end end @@ -472,7 +484,7 @@ class FailingHandler < Sequent::Core::Projector event_json: '{}', created_at: DateTime.now, command_record_id: command_record.id, - stream_record_id: stream_record.id, + stream_record: stream_record, ) end end diff --git a/spec/lib/sequent/core/persistors/replay_optimized_postgres_persistor_spec.rb b/spec/lib/sequent/core/persistors/replay_optimized_postgres_persistor_spec.rb index cef290f6..01e3d6ae 100644 --- a/spec/lib/sequent/core/persistors/replay_optimized_postgres_persistor_spec.rb +++ b/spec/lib/sequent/core/persistors/replay_optimized_postgres_persistor_spec.rb @@ -27,101 +27,102 @@ def measure_elapsed_time(&block) context '#get_record!' do it 'fails when no object is found' do - expect { persistor.get_record!(record_class, {id: 1}) }.to raise_error(/record #{record_class} not found}*/) + expect { persistor.get_record!(record_class, {aggregate_id: 1}) } + .to raise_error(/record #{record_class} not found}*/) end end context '#update_record' do it 'fails when no object is found' do expect do - persistor.update_record(record_class, mock_event, {id: 1}) + persistor.update_record(record_class, mock_event, {aggregate_id: 1}) end.to raise_error(/record #{record_class} not found}*/) end end context '#get_record' do it 'returns nil when no object is found' do - expect(persistor.get_record(record_class, {id: 1})).to be_nil + expect(persistor.get_record(record_class, {aggregate_id: 1})).to be_nil end end context '#find_records' do it 'returns empty array when no objects are found' do - expect(persistor.find_records(record_class, {id: 1})).to be_empty + expect(persistor.find_records(record_class, {aggregate_id: 1})).to be_empty end end context '#delete_all_records' do it 'does not fail when there is nothing to delete' do - persistor.delete_all_records(record_class, {id: 1}) + persistor.delete_all_records(record_class, {aggregate_id: 1}) end end context '#delete_record' do it 'does not fail when there is nothing to delete' do - persistor.delete_record(record_class, record_class.new(id: 1)) + persistor.delete_record(record_class, record_class.new(aggregate_id: 1)) end end context '#update_all_records' do it 'does not fail when there is nothing to update' do - persistor.update_all_records(record_class, {id: 1}, {sequence_number: 2}) + persistor.update_all_records(record_class, {aggregate_id: 1}, {sequence_number: 2}) end end it 'can save multiple objects at once' do - persistor.create_records(Sequent::Core::EventRecord, [{id: 1}, {id: 2}]) - object = persistor.get_record!(record_class, {id: 1}) - expect(object.id).to eq 1 - object = persistor.get_record!(record_class, {id: 2}) - expect(object.id).to eq 2 + persistor.create_records(Sequent::Core::EventRecord, [{aggregate_id: 1}, {aggregate_id: 2}]) + object = persistor.get_record!(record_class, {aggregate_id: 1}) + expect(object.aggregate_id).to eq 1 + object = persistor.get_record!(record_class, {aggregate_id: 2}) + expect(object.aggregate_id).to eq 2 - objects = persistor.find_records(record_class, {id: [1, 2]}) + objects = persistor.find_records(record_class, {aggregate_id: [1, 2]}) expect(objects).to have(2).items end context 'with an object' do before :each do - persistor.create_record(Sequent::Core::EventRecord, {id: 1}) + persistor.create_record(Sequent::Core::EventRecord, {aggregate_id: 1}) end context '#get_record!' do it 'returns the object' do - object = persistor.get_record!(record_class, {id: 1}) - expect(object.id).to eq 1 + object = persistor.get_record!(record_class, {aggregate_id: 1}) + expect(object.aggregate_id).to eq 1 end end context '#get_record' do it 'returns the object' do - object = persistor.get_record(record_class, {id: 1}) - expect(object.id).to eq 1 + object = persistor.get_record(record_class, {aggregate_id: 1}) + expect(object.aggregate_id).to eq 1 end end context '#find_records' do it 'returns the object' do - objects = persistor.find_records(record_class, {id: 1}) + objects = persistor.find_records(record_class, {aggregate_id: 1}) expect(objects).to have(1).item - expect(objects.first.id).to eq 1 + expect(objects.first.aggregate_id).to eq 1 end end context '#delete_all_records' do it 'deletes the object' do - persistor.delete_all_records(record_class, {id: 1}) + persistor.delete_all_records(record_class, {aggregate_id: 1}) - objects = persistor.find_records(record_class, {id: 1}) + objects = persistor.find_records(record_class, {aggregate_id: 1}) expect(objects).to be_empty end end context '#delete_record' do it 'deletes the object' do - objects = persistor.find_records(record_class, {id: 1}) + objects = persistor.find_records(record_class, {aggregate_id: 1}) persistor.delete_record(record_class, objects.first) - expect(persistor.find_records(record_class, {id: 1})).to be_empty + expect(persistor.find_records(record_class, {aggregate_id: 1})).to be_empty end it 'ignores records that are not present' do @@ -131,11 +132,11 @@ def measure_elapsed_time(&block) context '#update_all_records' do it 'updates the records' do - persistor.update_all_records(record_class, {id: 1}, {sequence_number: 3}) + persistor.update_all_records(record_class, {aggregate_id: 1}, {sequence_number: 3}) - objects = persistor.find_records(record_class, {id: 1}) + objects = persistor.find_records(record_class, {aggregate_id: 1}) expect(objects).to have(1).item - expect(objects.first.id).to eq 1 + expect(objects.first.aggregate_id).to eq 1 expect(objects.first.sequence_number).to eq 3 end end @@ -143,8 +144,8 @@ def measure_elapsed_time(&block) context 'value normalization' do before :each do - persistor.create_record(record_class, {id: 1, event_type: :SymbolEvent}) - persistor.create_record(record_class, {id: 2, event_type: 'StringEvent'}) + persistor.create_record(record_class, {aggregate_id: 1, event_type: :SymbolEvent}) + persistor.create_record(record_class, {aggregate_id: 2, event_type: 'StringEvent'}) end context 'when using an index' do @@ -270,9 +271,9 @@ class ReplayOptimizedPostgresTest < Sequent::ApplicationRecord; end context 'with some records' do let(:aggregate_id) { Sequent.new_uuid } before :each do - persistor.create_record(Sequent::Core::EventRecord, {id: 1, command_record_id: 2}) - persistor.create_record(Sequent::Core::EventRecord, {id: 1, sequence_number: 2}) - persistor.create_record(Sequent::Core::EventRecord, {aggregate_id: aggregate_id, id: 2, command_record_id: 2}) + persistor.create_record(Sequent::Core::EventRecord, {aggregate_id: 1, command_record_id: 2}) + persistor.create_record(Sequent::Core::EventRecord, {aggregate_id: 1, sequence_number: 2}) + persistor.create_record(Sequent::Core::EventRecord, {aggregate_id: aggregate_id, command_record_id: 2}) end let(:persistor) do @@ -288,7 +289,7 @@ class ReplayOptimizedPostgresTest < Sequent::ApplicationRecord; end let(:records) { persistor.find_records(record_class, where_clause) } context 'finding multiple records' do - let(:where_clause) { {id: 1} } + let(:where_clause) { {aggregate_id: 1} } it 'returns the correct number records' do expect(records).to have(2).items @@ -296,7 +297,7 @@ class ReplayOptimizedPostgresTest < Sequent::ApplicationRecord; end end context 'finding array valued where-clause' do - let(:where_clause) { {id: [1, 2]} } + let(:where_clause) { {aggregate_id: [1, aggregate_id]} } it 'returns the correct number records' do expect(records).to have(3).items @@ -304,41 +305,41 @@ class ReplayOptimizedPostgresTest < Sequent::ApplicationRecord; end end context 'with an indexed where clause' do - let(:where_clause) { {id: 1, command_record_id: 2} } + let(:where_clause) { {aggregate_id: 1, command_record_id: 2} } it 'returns the correct number records' do expect(records).to have(1).item end it 'returns the correct record' do - expect(records.first.id).to eq 1 + expect(records.first.aggregate_id).to eq 1 expect(records.first.command_record_id).to eq 2 expect(records.first.sequence_number).to be_nil end end context 'stringified indexed where clause' do - let(:where_clause) { {'id' => 1, 'command_record_id' => 2} } + let(:where_clause) { {'aggregate_id' => 1, 'command_record_id' => 2} } it 'returns the correct number records' do expect(records).to have(1).item end it 'returns the correct record' do - expect(records.first.id).to eq 1 + expect(records.first.aggregate_id).to eq 1 expect(records.first.command_record_id).to eq 2 expect(records.first.sequence_number).to be_nil end end context 'arbitrary order in indexed where clause' do - let(:where_clause) { {'command_record_id' => 2, 'id' => 1} } + let(:where_clause) { {'command_record_id' => 2, 'aggregate_id' => 1} } it 'returns the correct number records' do expect(records).to have(1).item end it 'returns the correct record' do - expect(records.first.id).to eq 1 + expect(records.first.aggregate_id).to eq 1 expect(records.first.command_record_id).to eq 2 expect(records.first.sequence_number).to be_nil end @@ -359,29 +360,29 @@ class ReplayOptimizedPostgresTest < Sequent::ApplicationRecord; end context '#delete_all_records' do it 'deletes the object based on single column' do - expect(persistor.find_records(record_class, {id: 1})).to have(2).items + expect(persistor.find_records(record_class, {aggregate_id: 1})).to have(2).items - persistor.delete_all_records(record_class, {id: 1}) + persistor.delete_all_records(record_class, {aggregate_id: 1}) - expect(persistor.find_records(record_class, {id: 1})).to be_empty + expect(persistor.find_records(record_class, {aggregate_id: 1})).to be_empty end it 'deletes the object based on multiple columns with index' do - expect(persistor.find_records(record_class, {id: 1, command_record_id: 2})).to have(1).item + expect(persistor.find_records(record_class, {aggregate_id: 1, command_record_id: 2})).to have(1).item - persistor.delete_all_records(record_class, {id: 1, command_record_id: 2}) + persistor.delete_all_records(record_class, {aggregate_id: 1, command_record_id: 2}) - expect(persistor.find_records(record_class, {id: 1, command_record_id: 2})).to be_empty - expect(persistor.find_records(record_class, {id: 1, sequence_number: 2})).to have(1).item + expect(persistor.find_records(record_class, {aggregate_id: 1, command_record_id: 2})).to be_empty + expect(persistor.find_records(record_class, {aggregate_id: 1, sequence_number: 2})).to have(1).item end end context '#update_all_records' do it 'only updates the records adhering to the where clause' do - persistor.update_all_records(record_class, {id: 1, sequence_number: 2}, {command_record_id: 10}) + persistor.update_all_records(record_class, {aggregate_id: 1, sequence_number: 2}, {command_record_id: 10}) - object = persistor.get_record!(record_class, {id: 1, sequence_number: 2}) - expect(object.id).to eq 1 + object = persistor.get_record!(record_class, {aggregate_id: 1, sequence_number: 2}) + expect(object.aggregate_id).to eq 1 expect(object.command_record_id).to eq 10 end @@ -390,45 +391,45 @@ class ReplayOptimizedPostgresTest < Sequent::ApplicationRecord; end persistor.update_all_records(record_class, where_clause, {sequence_number: 99}) end context 'in indexed order' do - let(:where_clause) { {id: 1, sequence_number: 2} } + let(:where_clause) { {aggregate_id: 1, sequence_number: 2} } it 'can update an indexed column' do - expect(persistor.get_record(record_class, {id: 1, sequence_number: 2})).to be_nil + expect(persistor.get_record(record_class, {aggregate_id: 1, sequence_number: 2})).to be_nil - object = persistor.get_record!(record_class, {id: 1, sequence_number: 99}) - expect(object.id).to eq 1 + object = persistor.get_record!(record_class, {aggregate_id: 1, sequence_number: 99}) + expect(object.aggregate_id).to eq 1 expect(object.sequence_number).to eq 99 end end context 'in reversed indexed order' do - let(:where_clause) { {sequence_number: 2, 'id' => 1} } + let(:where_clause) { {sequence_number: 2, 'aggregate_id' => 1} } it 'can update an indexed column' do - expect(persistor.get_record(record_class, {id: 1, sequence_number: 2})).to be_nil + expect(persistor.get_record(record_class, {aggregate_id: 1, sequence_number: 2})).to be_nil - object = persistor.get_record!(record_class, {id: 1, sequence_number: 99}) - expect(object.id).to eq 1 + object = persistor.get_record!(record_class, {aggregate_id: 1, sequence_number: 99}) + expect(object.aggregate_id).to eq 1 expect(object.sequence_number).to eq 99 end end end it 'can update an indexed column' do - persistor.update_all_records(record_class, {id: 1, sequence_number: 2}, {sequence_number: 99}) + persistor.update_all_records(record_class, {aggregate_id: 1, sequence_number: 2}, {sequence_number: 99}) - expect(persistor.get_record(record_class, {id: 1, sequence_number: 2})).to be_nil + expect(persistor.get_record(record_class, {aggregate_id: 1, sequence_number: 2})).to be_nil - object = persistor.get_record!(record_class, {id: 1, sequence_number: 99}) - expect(object.id).to eq 1 + object = persistor.get_record!(record_class, {aggregate_id: 1, sequence_number: 99}) + expect(object.aggregate_id).to eq 1 expect(object.sequence_number).to eq 99 end it 'can update an indexed column with reversed where' do - persistor.update_all_records(record_class, {sequence_number: 2, id: 1}, {sequence_number: 99}) + persistor.update_all_records(record_class, {sequence_number: 2, aggregate_id: 1}, {sequence_number: 99}) - expect(persistor.get_record(record_class, {id: 1, sequence_number: 2})).to be_nil + expect(persistor.get_record(record_class, {aggregate_id: 1, sequence_number: 2})).to be_nil - object = persistor.get_record!(record_class, {id: 1, sequence_number: 99}) - expect(object.id).to eq 1 + object = persistor.get_record!(record_class, {aggregate_id: 1, sequence_number: 99}) + expect(object.aggregate_id).to eq 1 expect(object.sequence_number).to eq 99 end end @@ -462,7 +463,7 @@ def hash Sequent::Core::Persistors::ReplayOptimizedPostgresPersistor.new( 50, { - Sequent::Core::EventRecord => [%i[id command_record_id], %i[id sequence_number]], + Sequent::Core::EventRecord => [%i[aggregate_id command_record_id], %i[aggregate_id sequence_number]], }, ) end @@ -472,7 +473,7 @@ def hash aggregate_ids.each_with_index do |aggregate_id, i| persistor.create_record( Sequent::Core::EventRecord, - {id: i, aggregate_id: aggregate_id, command_record_id: i * 7}, + {aggregate_id: aggregate_id, command_record_id: i * 7}, ) end end @@ -492,7 +493,12 @@ def hash elapsed = measure_elapsed_time do ITERATIONS.times do (0...COUNT).each do |i| - expect(persistor.get_record(Sequent::Core::EventRecord, {id: i, command_record_id: i * 7})).to be_present + expect( + persistor.get_record( + Sequent::Core::EventRecord, + {aggregate_id: aggregate_ids[i], command_record_id: i * 7}, + ), + ).to be_present end end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 24f64b6a..3ddaa385 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -33,14 +33,15 @@ def exec_sql(sql) end def insert_events(aggregate_type, events) + streams_with_events = events.group_by(&:aggregate_id).map do |aggregate_id, aggregate_events| + [ + Sequent::Core::EventStream.new(aggregate_type: aggregate_type, aggregate_id: aggregate_id), + aggregate_events, + ] + end Sequent.configuration.event_store.commit_events( Sequent::Core::CommandRecord.new, - [ - [ - Sequent::Core::EventStream.new(aggregate_type: aggregate_type, aggregate_id: events.first.aggregate_id), - events, - ], - ], + streams_with_events, ) end end From c0b16005e0a0baf7b34296e4300be6018a61dc81 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Wed, 6 Mar 2024 12:14:59 +0100 Subject: [PATCH 014/128] Event store API to support permanently deleting events and commands --- lib/sequent/core/event_store.rb | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/lib/sequent/core/event_store.rb b/lib/sequent/core/event_store.rb index 7471ec60..68d6fd21 100644 --- a/lib/sequent/core/event_store.rb +++ b/lib/sequent/core/event_store.rb @@ -236,6 +236,21 @@ def find_event_stream(aggregate_id) record&.event_stream end + def permanently_delete_event_stream(aggregate_id) + SnapshotRecord.where(aggregate_id: aggregate_id).delete_all + EventRecord.where(aggregate_id: aggregate_id).delete_all + StreamRecord.where(aggregate_id: aggregate_id).delete_all + end + + def permanently_delete_commands_without_events(aggregate_id) + connection = Sequent.configuration.event_record_class.connection + connection.exec_update(<<~EOS, 'permanently_delete_commands_without_events', [aggregate_id]) + DELETE FROM command_records + WHERE aggregate_id = $1 + AND NOT EXISTS (SELECT 1 FROM event_records WHERE command_record_id = command_records.id) + EOS + end + private def event_types From 381215aa7402c7bce02639f6af340223b1a8d7e3 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Wed, 6 Mar 2024 14:09:35 +0100 Subject: [PATCH 015/128] Allow passing the same stream multiple times Some tests rely on a specific ordering of storing events so we cannot just group these events by aggregate and pass them to the event store. Instead, each event is associated with a stream and the entire set is passed to the event store. So the event store needs to make sure `upsert_all` does not get the same `aggregate_id` multiple times when upserting the stream records. --- lib/sequent/core/event_store.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/sequent/core/event_store.rb b/lib/sequent/core/event_store.rb index 68d6fd21..24500a9d 100644 --- a/lib/sequent/core/event_store.rb +++ b/lib/sequent/core/event_store.rb @@ -306,7 +306,7 @@ def store_events(command, streams_with_events = []) aggregate_type: event_stream.aggregate_type, snapshot_threshold: event_stream.snapshot_threshold, } - end + end.uniq { |s| s[:aggregate_id] } StreamRecord.upsert_all(streams, unique_by: :aggregate_id, update_only: %i[snapshot_threshold]) event_records = streams_with_events.flat_map do |_, uncommitted_events| From 10e1c89c0a983243d25fb7643bf29334dd82e4f0 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Wed, 6 Mar 2024 14:35:51 +0100 Subject: [PATCH 016/128] Drop Ruby 3.0 support as it is EOL end of March 2024 --- .github/workflows/rspec.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/rspec.yml b/.github/workflows/rspec.yml index 22584112..ab363b83 100644 --- a/.github/workflows/rspec.yml +++ b/.github/workflows/rspec.yml @@ -25,8 +25,6 @@ jobs: gemfile: 'ar_7_1' - ruby-version: 3.1.3 gemfile: 'ar_7_1' - - ruby-version: 3.0.5 - gemfile: 'ar_7_1' env: BUNDLE_GEMFILE: ${{ github.workspace }}/gemfiles/${{ matrix.gemfile }}.gemfile services: From 73b8e8502cdb119daccf34ebbdf099cccf7bcd71 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Wed, 6 Mar 2024 14:39:51 +0100 Subject: [PATCH 017/128] Add Ruby 3.3 to tested releases --- .github/workflows/rspec.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/rspec.yml b/.github/workflows/rspec.yml index ab363b83..36ba99e8 100644 --- a/.github/workflows/rspec.yml +++ b/.github/workflows/rspec.yml @@ -21,7 +21,9 @@ jobs: strategy: matrix: include: - - ruby-version: 3.2 + - ruby-version: '3.3' + gemfile: 'ar_7_1' + - ruby-version: '3.2' gemfile: 'ar_7_1' - ruby-version: 3.1.3 gemfile: 'ar_7_1' From f6cce750709ca10d326fc79e35bcf7c46df45616 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Wed, 6 Mar 2024 14:42:52 +0100 Subject: [PATCH 018/128] Upgrade Puma to v6 for Rack 3 compatibility --- integration-specs/rails-app/Gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-specs/rails-app/Gemfile b/integration-specs/rails-app/Gemfile index 676b944e..71f0629f 100644 --- a/integration-specs/rails-app/Gemfile +++ b/integration-specs/rails-app/Gemfile @@ -12,7 +12,7 @@ gem 'sprockets-rails' gem 'pg' # Use the Puma web server [https://github.com/puma/puma] -gem 'puma', '~> 5.0' +gem 'puma', '~> 6.0' # Use JavaScript with ESM import maps [https://github.com/rails/importmap-rails] gem 'importmap-rails' From 61f75d21208beff2686f9ca36ec5e0b8e1157e3b Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Wed, 6 Mar 2024 14:57:20 +0100 Subject: [PATCH 019/128] Cleanup schema definition (rubocop, etc) and make generator consistent --- db/sequent_schema.rb | 99 ++++++++++--------- .../template_project/db/sequent_schema.rb | 96 ++++++++++-------- 2 files changed, 106 insertions(+), 89 deletions(-) diff --git a/db/sequent_schema.rb b/db/sequent_schema.rb index af6fb605..0ec77115 100644 --- a/db/sequent_schema.rb +++ b/db/sequent_schema.rb @@ -1,58 +1,63 @@ -ActiveRecord::Schema.define do +# frozen_string_literal: true - create_table "command_records", :force => true do |t| - t.string "user_id" - t.uuid "aggregate_id" - t.string "command_type", :null => false - t.string "event_aggregate_id" - t.integer "event_sequence_number" - t.jsonb "command_json", :null => false - t.datetime "created_at", :null => false +ActiveRecord::Schema.define do + create_table 'command_records', force: true do |t| + t.string 'user_id' + t.uuid 'aggregate_id' + t.string 'command_type', null: false + t.string 'event_aggregate_id' + t.integer 'event_sequence_number' + t.jsonb 'command_json', null: false + t.datetime 'created_at', null: false end - add_index "command_records", ["event_aggregate_id", 'event_sequence_number'], :name => "index_command_records_on_event" + add_index 'command_records', %w[event_aggregate_id event_sequence_number], name: 'index_command_records_on_event' - create_table "stream_records", :primary_key => ['aggregate_id'], :force => true do |t| - t.datetime "created_at", :null => false - t.string "aggregate_type", :null => false - t.uuid "aggregate_id", :null => false - t.integer "snapshot_threshold" + create_table 'stream_records', primary_key: ['aggregate_id'], force: true do |t| + t.datetime 'created_at', null: false + t.string 'aggregate_type', null: false + t.uuid 'aggregate_id', null: false + t.integer 'snapshot_threshold' end - create_table "event_records", :primary_key => ["aggregate_id", "sequence_number"], :force => true do |t| - t.uuid "aggregate_id", :null => false - t.integer "sequence_number", :null => false - t.datetime "created_at", :null => false - t.string "event_type", :null => false - t.jsonb "event_json", :null => false - t.integer "command_record_id", :null => false - t.bigint "xact_id", :null => false + create_table 'event_records', primary_key: %w[aggregate_id sequence_number], force: true do |t| + t.uuid 'aggregate_id', null: false + t.integer 'sequence_number', null: false + t.datetime 'created_at', null: false + t.string 'event_type', null: false + t.jsonb 'event_json', null: false + t.integer 'command_record_id', null: false + t.bigint 'xact_id', null: false end - add_index "event_records", ["command_record_id"], :name => "index_event_records_on_command_record_id" - add_index "event_records", ["event_type"], :name => "index_event_records_on_event_type" - add_index "event_records", ["created_at"], :name => "index_event_records_on_created_at" - add_index "event_records", ["xact_id"], :name => "index_event_records_on_xact_id" - - execute %Q{ -ALTER TABLE event_records ALTER COLUMN xact_id SET DEFAULT pg_current_xact_id()::text::bigint -} - execute %q{ -ALTER TABLE event_records ADD CONSTRAINT command_fkey FOREIGN KEY (command_record_id) REFERENCES command_records (id) -} - execute %q{ -ALTER TABLE event_records ADD CONSTRAINT stream_fkey FOREIGN KEY (aggregate_id) REFERENCES stream_records (aggregate_id) -} - - create_table "snapshot_records", :primary_key => ["aggregate_id", "sequence_number"], :force => true do |t| - t.uuid "aggregate_id", :null => false - t.integer "sequence_number", :null => false - t.datetime "created_at", :null => false - t.text "snapshot_type", :null => false - t.jsonb "snapshot_json", :null => false + add_index 'event_records', ['command_record_id'], name: 'index_event_records_on_command_record_id' + add_index 'event_records', ['event_type'], name: 'index_event_records_on_event_type' + add_index 'event_records', ['created_at'], name: 'index_event_records_on_created_at' + add_index 'event_records', ['xact_id'], name: 'index_event_records_on_xact_id' + + execute <<~EOS + ALTER TABLE event_records ALTER COLUMN xact_id SET DEFAULT pg_current_xact_id()::text::bigint + EOS + + create_table 'snapshot_records', primary_key: %w[aggregate_id sequence_number], force: true do |t| + t.uuid 'aggregate_id', null: false + t.integer 'sequence_number', null: false + t.datetime 'created_at', null: false + t.text 'snapshot_type', null: false + t.jsonb 'snapshot_json', null: false end - execute %q{ -ALTER TABLE snapshot_records ADD CONSTRAINT stream_fkey FOREIGN KEY (aggregate_id) REFERENCES stream_records (aggregate_id) -} + add_foreign_key :event_records, + :command_records, + name: 'commands_fkey' + add_foreign_key :event_records, + :stream_records, + column: :aggregate_id, + primary_key: :aggregate_id, + name: 'streams_fkey' + add_foreign_key :snapshot_records, + :stream_records, + column: :aggregate_id, + primary_key: :aggregate_id, + name: 'streams_fkey' end diff --git a/lib/sequent/generator/template_project/db/sequent_schema.rb b/lib/sequent/generator/template_project/db/sequent_schema.rb index 584c7717..0ec77115 100644 --- a/lib/sequent/generator/template_project/db/sequent_schema.rb +++ b/lib/sequent/generator/template_project/db/sequent_schema.rb @@ -1,51 +1,63 @@ -ActiveRecord::Schema.define do +# frozen_string_literal: true - create_table "event_records", :force => true do |t| - t.uuid "aggregate_id", :null => false - t.integer "sequence_number", :null => false - t.datetime "created_at", :null => false - t.string "event_type", :null => false - t.jsonb "event_json", :null => false - t.integer "command_record_id", :null => false - t.integer "stream_record_id", :null => false +ActiveRecord::Schema.define do + create_table 'command_records', force: true do |t| + t.string 'user_id' + t.uuid 'aggregate_id' + t.string 'command_type', null: false + t.string 'event_aggregate_id' + t.integer 'event_sequence_number' + t.jsonb 'command_json', null: false + t.datetime 'created_at', null: false end - execute %Q{ -CREATE UNIQUE INDEX unique_event_per_aggregate ON event_records ( - aggregate_id, - sequence_number, - (CASE event_type WHEN 'Sequent::Core::SnapshotEvent' THEN 0 ELSE 1 END) -) -} - execute %Q{ -CREATE INDEX snapshot_events ON event_records (aggregate_id, sequence_number DESC) WHERE event_type = 'Sequent::Core::SnapshotEvent' -} - - add_index "event_records", ["command_record_id"], :name => "index_event_records_on_command_record_id" - add_index "event_records", ["event_type"], :name => "index_event_records_on_event_type" - add_index "event_records", ["created_at"], :name => "index_event_records_on_created_at" - - create_table "command_records", :force => true do |t| - t.string "user_id" - t.uuid "aggregate_id" - t.string "command_type", :null => false - t.jsonb "command_json", :null => false - t.datetime "created_at", :null => false + add_index 'command_records', %w[event_aggregate_id event_sequence_number], name: 'index_command_records_on_event' + + create_table 'stream_records', primary_key: ['aggregate_id'], force: true do |t| + t.datetime 'created_at', null: false + t.string 'aggregate_type', null: false + t.uuid 'aggregate_id', null: false + t.integer 'snapshot_threshold' end - create_table "stream_records", :force => true do |t| - t.datetime "created_at", :null => false - t.string "aggregate_type", :null => false - t.uuid "aggregate_id", :null => false - t.integer "snapshot_threshold" + create_table 'event_records', primary_key: %w[aggregate_id sequence_number], force: true do |t| + t.uuid 'aggregate_id', null: false + t.integer 'sequence_number', null: false + t.datetime 'created_at', null: false + t.string 'event_type', null: false + t.jsonb 'event_json', null: false + t.integer 'command_record_id', null: false + t.bigint 'xact_id', null: false end - add_index "stream_records", ["aggregate_id"], :name => "index_stream_records_on_aggregate_id", :unique => true - execute %q{ -ALTER TABLE event_records ADD CONSTRAINT command_fkey FOREIGN KEY (command_record_id) REFERENCES command_records (id) -} - execute %q{ -ALTER TABLE event_records ADD CONSTRAINT stream_fkey FOREIGN KEY (stream_record_id) REFERENCES stream_records (id) -} + add_index 'event_records', ['command_record_id'], name: 'index_event_records_on_command_record_id' + add_index 'event_records', ['event_type'], name: 'index_event_records_on_event_type' + add_index 'event_records', ['created_at'], name: 'index_event_records_on_created_at' + add_index 'event_records', ['xact_id'], name: 'index_event_records_on_xact_id' + + execute <<~EOS + ALTER TABLE event_records ALTER COLUMN xact_id SET DEFAULT pg_current_xact_id()::text::bigint + EOS + + create_table 'snapshot_records', primary_key: %w[aggregate_id sequence_number], force: true do |t| + t.uuid 'aggregate_id', null: false + t.integer 'sequence_number', null: false + t.datetime 'created_at', null: false + t.text 'snapshot_type', null: false + t.jsonb 'snapshot_json', null: false + end + add_foreign_key :event_records, + :command_records, + name: 'commands_fkey' + add_foreign_key :event_records, + :stream_records, + column: :aggregate_id, + primary_key: :aggregate_id, + name: 'streams_fkey' + add_foreign_key :snapshot_records, + :stream_records, + column: :aggregate_id, + primary_key: :aggregate_id, + name: 'streams_fkey' end From 62d189764fd65713be54de4f2c647680f13df53b Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Wed, 6 Mar 2024 15:00:59 +0100 Subject: [PATCH 020/128] Serialize snapshots to text if column is not JSON --- lib/sequent/core/event_record.rb | 11 ++++++++++- lib/sequent/core/event_store.rb | 12 +++++++----- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/lib/sequent/core/event_record.rb b/lib/sequent/core/event_record.rb index 646a803f..8e6aee49 100644 --- a/lib/sequent/core/event_record.rb +++ b/lib/sequent/core/event_record.rb @@ -122,12 +122,21 @@ def event_type snapshot_type end + def event_type=(type) + self.snapshot_type = type + end + def event_json snapshot_json end + def event_json=(json) + self.snapshot_json = json + end + def serialize_json? - false + json_column_type = self.class.columns_hash['snapshot_json'].sql_type_metadata.type + %i[json jsonb].exclude? json_column_type end end end diff --git a/lib/sequent/core/event_store.rb b/lib/sequent/core/event_store.rb index 24500a9d..281d2eb5 100644 --- a/lib/sequent/core/event_store.rb +++ b/lib/sequent/core/event_store.rb @@ -116,14 +116,16 @@ def load_events_for_aggregates(aggregate_ids) end def store_snapshots(snapshots) + serializer = SnapshotRecord.new SnapshotRecord.insert_all!( snapshots.map do |snapshot| + serializer.event = snapshot { - aggregate_id: snapshot.aggregate_id, - sequence_number: snapshot.sequence_number, - created_at: snapshot.created_at, - snapshot_type: snapshot.class.name, - snapshot_json: Sequent::Core::Oj.strict_load(Sequent::Core::Oj.dump(snapshot)), + aggregate_id: serializer.aggregate_id, + sequence_number: serializer.sequence_number, + created_at: serializer.created_at, + snapshot_type: serializer.snapshot_type, + snapshot_json: serializer.snapshot_json, } end, ) From 9e99b6d096868a676ea4595b5109e5a3cf847f16 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Wed, 6 Mar 2024 15:01:15 +0100 Subject: [PATCH 021/128] Explicitly define primary key columns, just to be sure --- lib/sequent/core/event_record.rb | 2 ++ lib/sequent/core/stream_record.rb | 1 + 2 files changed, 3 insertions(+) diff --git a/lib/sequent/core/event_record.rb b/lib/sequent/core/event_record.rb index 8e6aee49..7617d8b8 100644 --- a/lib/sequent/core/event_record.rb +++ b/lib/sequent/core/event_record.rb @@ -80,6 +80,7 @@ def serialize_json? class EventRecord < Sequent::ApplicationRecord include SerializesEvent + self.primary_key = %i[aggregate_id sequence_number] self.table_name = 'event_records' self.ignored_columns = %w[xact_id] @@ -111,6 +112,7 @@ def find_origin(record) class SnapshotRecord < Sequent::ApplicationRecord include SerializesEvent + self.primary_key = %i[aggregate_id sequence_number] self.table_name = 'snapshot_records' belongs_to :stream_record, foreign_key: :aggregate_id, primary_key: :aggregate_id diff --git a/lib/sequent/core/stream_record.rb b/lib/sequent/core/stream_record.rb index a40e8700..02d89021 100644 --- a/lib/sequent/core/stream_record.rb +++ b/lib/sequent/core/stream_record.rb @@ -15,6 +15,7 @@ def initialize(aggregate_type:, aggregate_id:, snapshot_threshold: nil) end class StreamRecord < Sequent::ApplicationRecord + self.primary_key = %i[aggregate_id] self.table_name = 'stream_records' validates_presence_of :aggregate_type, :aggregate_id From ce58d877de64b715da8bd54921a64604bbcf73d2 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Wed, 6 Mar 2024 15:38:22 +0100 Subject: [PATCH 022/128] Add `load_event` API to event store for loading single event --- lib/sequent/core/event_store.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/sequent/core/event_store.rb b/lib/sequent/core/event_store.rb index 281d2eb5..60888baf 100644 --- a/lib/sequent/core/event_store.rb +++ b/lib/sequent/core/event_store.rb @@ -85,6 +85,10 @@ def stream_events_for_aggregate(aggregate_id, load_until: nil, &block) fail ArgumentError, 'no events for this aggregate' unless has_events end + def load_event(aggregate_id, sequence_number) + EventRecord.find_by(aggregate_id:, sequence_number:)&.event + end + ## # Returns all events for the aggregate ordered by sequence_number, loading them from the latest snapshot # event onwards, if a snapshot is present From 3210ece80e8b81827fd6a52e10dbd975d38e4c3b Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Tue, 20 Feb 2024 17:26:54 +0100 Subject: [PATCH 023/128] Fix type errors in the specs (Command instead of CommandRecord) --- .../core/aggregate_snapshotter_spec.rb | 4 ++-- spec/lib/sequent/core/event_store_spec.rb | 22 +++++++++---------- spec/spec_helper.rb | 2 +- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/spec/lib/sequent/core/aggregate_snapshotter_spec.rb b/spec/lib/sequent/core/aggregate_snapshotter_spec.rb index 77230722..f1ffaf70 100644 --- a/spec/lib/sequent/core/aggregate_snapshotter_spec.rb +++ b/spec/lib/sequent/core/aggregate_snapshotter_spec.rb @@ -26,7 +26,7 @@ class MyAggregate < Sequent::Core::AggregateRoot; end before :each do Sequent.configuration.command_handlers << described_class.new event_store.commit_events( - Sequent::Core::CommandRecord.new, + Sequent::Core::Command.new(aggregate_id: aggregate_id), [ [ Sequent::Core::EventStream.new( @@ -60,7 +60,7 @@ class MyAggregate < Sequent::Core::AggregateRoot; end before :each do event_store.commit_events( - Sequent::Core::CommandRecord.new, + Sequent::Core::Command.new(aggregate_id: aggregate_id), [ [ Sequent::Core::EventStream.new( diff --git a/spec/lib/sequent/core/event_store_spec.rb b/spec/lib/sequent/core/event_store_spec.rb index a10bb46f..9dd568ab 100644 --- a/spec/lib/sequent/core/event_store_spec.rb +++ b/spec/lib/sequent/core/event_store_spec.rb @@ -46,7 +46,7 @@ class MyAggregate < Sequent::Core::AggregateRoot let(:snapshot_threshold) { 1 } before do event_store.commit_events( - Sequent::Core::CommandRecord.new, + Sequent::Core::Command.new(aggregate_id: aggregate_id), [ [ Sequent::Core::EventStream.new( @@ -99,7 +99,7 @@ class MyAggregate < Sequent::Core::AggregateRoot it 'fails with OptimisticLockingError when RecordNotUnique' do expect do event_store.commit_events( - Sequent::Core::CommandRecord.new, + Sequent::Core::Command.new(aggregate_id: aggregate_id), [ [ Sequent::Core::EventStream.new( @@ -123,7 +123,7 @@ class MyAggregate < Sequent::Core::AggregateRoot describe '#events_exists?' do it 'gets true for an existing aggregate' do event_store.commit_events( - Sequent::Core::CommandRecord.new, + Sequent::Core::Command.new(aggregate_id: aggregate_id), [ [ Sequent::Core::EventStream.new( @@ -146,7 +146,7 @@ class MyAggregate < Sequent::Core::AggregateRoot describe '#stream_exists?' do it 'gets true for an existing aggregate' do event_store.commit_events( - Sequent::Core::CommandRecord.new, + Sequent::Core::Command.new(aggregate_id: aggregate_id), [ [ Sequent::Core::EventStream.new( @@ -175,7 +175,7 @@ class MyAggregate < Sequent::Core::AggregateRoot it 'returns the stream and events for existing aggregates' do event_store.commit_events( - Sequent::Core::CommandRecord.new, + Sequent::Core::Command.new(aggregate_id: aggregate_id), [ [ Sequent::Core::EventStream.new(aggregate_type: 'MyAggregate', aggregate_id: aggregate_id), @@ -204,7 +204,7 @@ class MyAggregate < Sequent::Core::AggregateRoot TestEventForCaching = Class.new(Sequent::Core::Event) event_store.commit_events( - Sequent::Core::CommandRecord.new, + Sequent::Core::Command.new(aggregate_id: aggregate_id), [ [ Sequent::Core::EventStream.new(aggregate_type: 'MyAggregate', aggregate_id: aggregate_id), @@ -233,7 +233,7 @@ class MyAggregate < Sequent::Core::AggregateRoot before :each do event_store.commit_events( - Sequent::Core::CommandRecord.new, + Sequent::Core::Command.new(aggregate_id: aggregate_id), [ [ Sequent::Core::EventStream.new(aggregate_type: 'MyAggregate', aggregate_id: aggregate_id_1), @@ -277,7 +277,7 @@ class MyAggregate < Sequent::Core::AggregateRoot context 'with a snapshot event' do before :each do event_store.commit_events( - Sequent::Core::CommandRecord.new, + Sequent::Core::Command.new(aggregate_id: aggregate_id), [ [ Sequent::Core::EventStream.new( @@ -370,7 +370,7 @@ class FailingHandler < Sequent::Core::Projector it 'calls an event handler that handles the event' do my_event = MyEvent.new(aggregate_id: aggregate_id, sequence_number: 1) event_store.commit_events( - Sequent::Core::CommandRecord.new, + Sequent::Core::Command.new(aggregate_id: aggregate_id), [ [ Sequent::Core::EventStream.new( @@ -390,7 +390,7 @@ class FailingHandler < Sequent::Core::Projector Sequent.configuration.disable_event_handlers = true my_event = MyEvent.new(aggregate_id: aggregate_id, sequence_number: 1) event_store.commit_events( - Sequent::Core::CommandRecord.new, + Sequent::Core::Command.new(aggregate_id: aggregate_id), [ [ Sequent::Core::EventStream.new( @@ -412,7 +412,7 @@ class FailingHandler < Sequent::Core::Projector let(:my_event) { MyEvent.new(aggregate_id: aggregate_id, sequence_number: 1) } subject(:publish_error) do event_store.commit_events( - Sequent::Core::CommandRecord.new, + Sequent::Core::Command.new(aggregate_id: aggregate_id), [ [ Sequent::Core::EventStream.new( diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 3ddaa385..95eebc7e 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -40,7 +40,7 @@ def insert_events(aggregate_type, events) ] end Sequent.configuration.event_store.commit_events( - Sequent::Core::CommandRecord.new, + Sequent::Core::Command.new(events.first.attributes), streams_with_events, ) end From 6cd829f5b73b7985f618a5bfa1da6991ed7f885d Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Wed, 6 Mar 2024 15:46:57 +0100 Subject: [PATCH 024/128] Tests for new event store APIs --- spec/lib/sequent/core/event_store_spec.rb | 31 +++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/spec/lib/sequent/core/event_store_spec.rb b/spec/lib/sequent/core/event_store_spec.rb index 9dd568ab..ca1d027a 100644 --- a/spec/lib/sequent/core/event_store_spec.rb +++ b/spec/lib/sequent/core/event_store_spec.rb @@ -223,6 +223,8 @@ class MyAggregate < Sequent::Core::AggregateRoot stream, events = event_store.load_events(aggregate_id) expect(stream).to be expect(events.first).to be_kind_of(TestEventForCaching) + + expect(event_store.load_event(aggregate_id, events.first.sequence_number)).to eq(events.first) end end end @@ -531,4 +533,33 @@ def initialize @replay_count += 1 end end + + describe '#permanently_delete_commands_without_events' do + before do + event_store.commit_events( + Sequent::Core::Command.new(aggregate_id:), + [ + [ + Sequent::Core::EventStream.new( + aggregate_type: 'MyAggregate', + aggregate_id:, + snapshot_threshold: 13, + ), + [MyEvent.new(aggregate_id:, sequence_number: 1)], + ], + ], + ) + end + + it 'does not delete commands with associated events' do + event_store.permanently_delete_commands_without_events(aggregate_id) + expect(Sequent::Core::CommandRecord.exists?(aggregate_id:)).to be_truthy + end + + it 'deletes commands without associated events' do + event_store.permanently_delete_event_stream(aggregate_id) + event_store.permanently_delete_commands_without_events(aggregate_id) + expect(Sequent::Core::CommandRecord.exists?(aggregate_id:)).to be_falsy + end + end end From 43806b5841db86e9f66a7d8db51e353fc5f04573 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Wed, 6 Mar 2024 16:06:22 +0100 Subject: [PATCH 025/128] Fix foreign key constraint names --- db/sequent_schema.rb | 6 +++--- lib/sequent/generator/template_project/db/sequent_schema.rb | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/db/sequent_schema.rb b/db/sequent_schema.rb index 0ec77115..3cdb015b 100644 --- a/db/sequent_schema.rb +++ b/db/sequent_schema.rb @@ -49,15 +49,15 @@ add_foreign_key :event_records, :command_records, - name: 'commands_fkey' + name: 'command_fkey' add_foreign_key :event_records, :stream_records, column: :aggregate_id, primary_key: :aggregate_id, - name: 'streams_fkey' + name: 'stream_fkey' add_foreign_key :snapshot_records, :stream_records, column: :aggregate_id, primary_key: :aggregate_id, - name: 'streams_fkey' + name: 'stream_fkey' end diff --git a/lib/sequent/generator/template_project/db/sequent_schema.rb b/lib/sequent/generator/template_project/db/sequent_schema.rb index 0ec77115..3cdb015b 100644 --- a/lib/sequent/generator/template_project/db/sequent_schema.rb +++ b/lib/sequent/generator/template_project/db/sequent_schema.rb @@ -49,15 +49,15 @@ add_foreign_key :event_records, :command_records, - name: 'commands_fkey' + name: 'command_fkey' add_foreign_key :event_records, :stream_records, column: :aggregate_id, primary_key: :aggregate_id, - name: 'streams_fkey' + name: 'stream_fkey' add_foreign_key :snapshot_records, :stream_records, column: :aggregate_id, primary_key: :aggregate_id, - name: 'streams_fkey' + name: 'stream_fkey' end From 1b429dbc938cd71569f6b11e70c4daeccc0a402e Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Wed, 6 Mar 2024 16:25:30 +0100 Subject: [PATCH 026/128] Allow arbitrary where conditions when permanently deleting commands --- lib/sequent/core/event_store.rb | 12 +++++------- spec/lib/sequent/core/event_store_spec.rb | 4 ++-- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/lib/sequent/core/event_store.rb b/lib/sequent/core/event_store.rb index 60888baf..8f2eba10 100644 --- a/lib/sequent/core/event_store.rb +++ b/lib/sequent/core/event_store.rb @@ -248,13 +248,11 @@ def permanently_delete_event_stream(aggregate_id) StreamRecord.where(aggregate_id: aggregate_id).delete_all end - def permanently_delete_commands_without_events(aggregate_id) - connection = Sequent.configuration.event_record_class.connection - connection.exec_update(<<~EOS, 'permanently_delete_commands_without_events', [aggregate_id]) - DELETE FROM command_records - WHERE aggregate_id = $1 - AND NOT EXISTS (SELECT 1 FROM event_records WHERE command_record_id = command_records.id) - EOS + def permanently_delete_commands_without_events(where_clause) + CommandRecord + .where(where_clause) + .where('NOT EXISTS (SELECT 1 FROM event_records WHERE command_record_id = command_records.id)') + .delete_all end private diff --git a/spec/lib/sequent/core/event_store_spec.rb b/spec/lib/sequent/core/event_store_spec.rb index ca1d027a..5e5c613c 100644 --- a/spec/lib/sequent/core/event_store_spec.rb +++ b/spec/lib/sequent/core/event_store_spec.rb @@ -552,13 +552,13 @@ def initialize end it 'does not delete commands with associated events' do - event_store.permanently_delete_commands_without_events(aggregate_id) + event_store.permanently_delete_commands_without_events(aggregate_id:) expect(Sequent::Core::CommandRecord.exists?(aggregate_id:)).to be_truthy end it 'deletes commands without associated events' do event_store.permanently_delete_event_stream(aggregate_id) - event_store.permanently_delete_commands_without_events(aggregate_id) + event_store.permanently_delete_commands_without_events(aggregate_id:) expect(Sequent::Core::CommandRecord.exists?(aggregate_id:)).to be_falsy end end From 7c6c7b4503b92f202b1c30d400d80526e8ca67ea Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Wed, 6 Mar 2024 16:56:55 +0100 Subject: [PATCH 027/128] Ensure snapshots are always the first event returned SQL `UNION ALL` (or any query) does not guarantee ordering unless an `ORDER BY` is specified. Since sequence numbers are always greater than 0, the snapshot (if present) can be sorted before the events by using sequence number 0. --- lib/sequent/core/event_store.rb | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/sequent/core/event_store.rb b/lib/sequent/core/event_store.rb index 8f2eba10..34be36e1 100644 --- a/lib/sequent/core/event_store.rb +++ b/lib/sequent/core/event_store.rb @@ -153,13 +153,14 @@ def aggregate_query(aggregate_id) WHERE aggregate_id = #{quote(aggregate_id)} ORDER BY sequence_number DESC LIMIT 1) - SELECT snapshot_type AS event_type, snapshot_json AS event_json FROM snapshot + SELECT snapshot_type AS event_type, snapshot_json AS event_json, 0 AS sequence_number + FROM snapshot UNION ALL - (SELECT event_type, event_json - FROM #{quote_table_name Sequent.configuration.event_record_class.table_name} AS o - WHERE aggregate_id = #{quote(aggregate_id)} - AND sequence_number >= COALESCE((SELECT sequence_number FROM snapshot), 0) - ORDER BY sequence_number ASC) + SELECT event_type, event_json, sequence_number + FROM #{quote_table_name Sequent.configuration.event_record_class.table_name} AS o + WHERE aggregate_id = #{quote(aggregate_id)} + AND sequence_number >= COALESCE((SELECT sequence_number FROM snapshot), 0) + ORDER BY sequence_number ASC ) SQL end From 78658fd514efb6d4b592b60a495a87eb4732cb0d Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Fri, 8 Mar 2024 09:42:28 +0100 Subject: [PATCH 028/128] Cascade deletes to snapshot records Since snapshots are just cached representations of replayed events it is always safe to automatically delete them if the associated stream is deleted. --- db/sequent_schema.rb | 1 + lib/sequent/core/event_store.rb | 1 - lib/sequent/generator/template_project/db/sequent_schema.rb | 1 + 3 files changed, 2 insertions(+), 1 deletion(-) diff --git a/db/sequent_schema.rb b/db/sequent_schema.rb index 3cdb015b..5f0128f5 100644 --- a/db/sequent_schema.rb +++ b/db/sequent_schema.rb @@ -59,5 +59,6 @@ :stream_records, column: :aggregate_id, primary_key: :aggregate_id, + on_delete: :cascade, name: 'stream_fkey' end diff --git a/lib/sequent/core/event_store.rb b/lib/sequent/core/event_store.rb index 34be36e1..2df3ba5b 100644 --- a/lib/sequent/core/event_store.rb +++ b/lib/sequent/core/event_store.rb @@ -244,7 +244,6 @@ def find_event_stream(aggregate_id) end def permanently_delete_event_stream(aggregate_id) - SnapshotRecord.where(aggregate_id: aggregate_id).delete_all EventRecord.where(aggregate_id: aggregate_id).delete_all StreamRecord.where(aggregate_id: aggregate_id).delete_all end diff --git a/lib/sequent/generator/template_project/db/sequent_schema.rb b/lib/sequent/generator/template_project/db/sequent_schema.rb index 3cdb015b..5f0128f5 100644 --- a/lib/sequent/generator/template_project/db/sequent_schema.rb +++ b/lib/sequent/generator/template_project/db/sequent_schema.rb @@ -59,5 +59,6 @@ :stream_records, column: :aggregate_id, primary_key: :aggregate_id, + on_delete: :cascade, name: 'stream_fkey' end From 4490fc518d0fab0c245aca9e41342ca0b97045dd Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Fri, 8 Mar 2024 09:44:10 +0100 Subject: [PATCH 029/128] Use configured event and stream record classes --- lib/sequent/core/event_store.rb | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/sequent/core/event_store.rb b/lib/sequent/core/event_store.rb index 2df3ba5b..7f218d8f 100644 --- a/lib/sequent/core/event_store.rb +++ b/lib/sequent/core/event_store.rb @@ -86,7 +86,7 @@ def stream_events_for_aggregate(aggregate_id, load_until: nil, &block) end def load_event(aggregate_id, sequence_number) - EventRecord.find_by(aggregate_id:, sequence_number:)&.event + Sequent.configuration.event_record_class.find_by(aggregate_id:, sequence_number:)&.event end ## @@ -244,8 +244,8 @@ def find_event_stream(aggregate_id) end def permanently_delete_event_stream(aggregate_id) - EventRecord.where(aggregate_id: aggregate_id).delete_all - StreamRecord.where(aggregate_id: aggregate_id).delete_all + Sequent.configuration.event_record_class.where(aggregate_id: aggregate_id).delete_all + Sequent.configuration.stream_record_class.where(aggregate_id: aggregate_id).delete_all end def permanently_delete_commands_without_events(where_clause) @@ -278,7 +278,7 @@ def primary_key_event_records end def deserialize_event(event_hash) - record = EventRecord.new + record = Sequent.configuration.event_record_class.new record.event_type = event_hash.fetch('event_type') record.event_json = if record.serialize_json? @@ -311,7 +311,8 @@ def store_events(command, streams_with_events = []) snapshot_threshold: event_stream.snapshot_threshold, } end.uniq { |s| s[:aggregate_id] } - StreamRecord.upsert_all(streams, unique_by: :aggregate_id, update_only: %i[snapshot_threshold]) + Sequent.configuration.stream_record_class + .upsert_all(streams, unique_by: :aggregate_id, update_only: %i[snapshot_threshold]) event_records = streams_with_events.flat_map do |_, uncommitted_events| uncommitted_events.map do |event| From faac89d10f6da624c8c2b71e672e0fafae0456c2 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Fri, 8 Mar 2024 09:47:05 +0100 Subject: [PATCH 030/128] Update Rails integration test sequent schema --- .../rails-app/db/sequent_schema.rb | 101 ++++++++++-------- 1 file changed, 55 insertions(+), 46 deletions(-) diff --git a/integration-specs/rails-app/db/sequent_schema.rb b/integration-specs/rails-app/db/sequent_schema.rb index 0d63c37d..5f0128f5 100644 --- a/integration-specs/rails-app/db/sequent_schema.rb +++ b/integration-specs/rails-app/db/sequent_schema.rb @@ -1,55 +1,64 @@ +# frozen_string_literal: true + ActiveRecord::Schema.define do + create_table 'command_records', force: true do |t| + t.string 'user_id' + t.uuid 'aggregate_id' + t.string 'command_type', null: false + t.string 'event_aggregate_id' + t.integer 'event_sequence_number' + t.jsonb 'command_json', null: false + t.datetime 'created_at', null: false + end + + add_index 'command_records', %w[event_aggregate_id event_sequence_number], name: 'index_command_records_on_event' - create_table "event_records", :force => true do |t| - t.uuid "aggregate_id", :null => false - t.integer "sequence_number", :null => false - t.datetime "created_at", :null => false - t.string "event_type", :null => false - t.text "event_json", :null => false - t.integer "command_record_id", :null => false - t.integer "stream_record_id", :null => false + create_table 'stream_records', primary_key: ['aggregate_id'], force: true do |t| + t.datetime 'created_at', null: false + t.string 'aggregate_type', null: false + t.uuid 'aggregate_id', null: false + t.integer 'snapshot_threshold' end - execute %Q{ -CREATE UNIQUE INDEX unique_event_per_aggregate ON event_records ( - aggregate_id, - sequence_number, - (CASE event_type WHEN 'Sequent::Core::SnapshotEvent' THEN 0 ELSE 1 END) -) -} - execute %Q{ -CREATE INDEX snapshot_events ON event_records (aggregate_id, sequence_number DESC) WHERE event_type = 'Sequent::Core::SnapshotEvent' -} - - add_index "event_records", ["command_record_id"], :name => "index_event_records_on_command_record_id" - add_index "event_records", ["event_type"], :name => "index_event_records_on_event_type" - add_index "event_records", ["created_at"], :name => "index_event_records_on_created_at" - - create_table "command_records", :force => true do |t| - t.string "user_id" - t.uuid "aggregate_id" - t.string "command_type", :null => false - t.string "event_aggregate_id" - t.integer "event_sequence_number" - t.text "command_json", :null => false - t.datetime "created_at", :null => false + create_table 'event_records', primary_key: %w[aggregate_id sequence_number], force: true do |t| + t.uuid 'aggregate_id', null: false + t.integer 'sequence_number', null: false + t.datetime 'created_at', null: false + t.string 'event_type', null: false + t.jsonb 'event_json', null: false + t.integer 'command_record_id', null: false + t.bigint 'xact_id', null: false end - add_index "command_records", ["event_aggregate_id", 'event_sequence_number'], :name => "index_command_records_on_event" + add_index 'event_records', ['command_record_id'], name: 'index_event_records_on_command_record_id' + add_index 'event_records', ['event_type'], name: 'index_event_records_on_event_type' + add_index 'event_records', ['created_at'], name: 'index_event_records_on_created_at' + add_index 'event_records', ['xact_id'], name: 'index_event_records_on_xact_id' - create_table "stream_records", :force => true do |t| - t.datetime "created_at", :null => false - t.string "aggregate_type", :null => false - t.uuid "aggregate_id", :null => false - t.integer "snapshot_threshold" - end + execute <<~EOS + ALTER TABLE event_records ALTER COLUMN xact_id SET DEFAULT pg_current_xact_id()::text::bigint + EOS - add_index "stream_records", ["aggregate_id"], :name => "index_stream_records_on_aggregate_id", :unique => true - execute %q{ -ALTER TABLE event_records ADD CONSTRAINT command_fkey FOREIGN KEY (command_record_id) REFERENCES command_records (id) -} - execute %q{ -ALTER TABLE event_records ADD CONSTRAINT stream_fkey FOREIGN KEY (stream_record_id) REFERENCES stream_records (id) -} + create_table 'snapshot_records', primary_key: %w[aggregate_id sequence_number], force: true do |t| + t.uuid 'aggregate_id', null: false + t.integer 'sequence_number', null: false + t.datetime 'created_at', null: false + t.text 'snapshot_type', null: false + t.jsonb 'snapshot_json', null: false + end -end \ No newline at end of file + add_foreign_key :event_records, + :command_records, + name: 'command_fkey' + add_foreign_key :event_records, + :stream_records, + column: :aggregate_id, + primary_key: :aggregate_id, + name: 'stream_fkey' + add_foreign_key :snapshot_records, + :stream_records, + column: :aggregate_id, + primary_key: :aggregate_id, + on_delete: :cascade, + name: 'stream_fkey' +end From e17b74aec59885f32a71b5f81d09a72ee5476083 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Fri, 8 Mar 2024 09:57:17 +0100 Subject: [PATCH 031/128] Require PostgreSQL 14 for Rails integration tests --- .github/workflows/rspec.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rspec.yml b/.github/workflows/rspec.yml index 36ba99e8..acf7b8aa 100644 --- a/.github/workflows/rspec.yml +++ b/.github/workflows/rspec.yml @@ -86,7 +86,7 @@ jobs: runs-on: ubuntu-latest services: postgres: - image: postgres:12 + image: postgres:14 env: POSTGRES_USER: sequent POSTGRES_PASSWORD: sequent From fedff4a5db7aa60d44f25a286449341c4cc62413 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Mon, 11 Mar 2024 16:16:59 +0100 Subject: [PATCH 032/128] Separate SnapshotRecord into its own file --- lib/sequent/core/event_record.rb | 33 ----------------------- lib/sequent/core/event_store.rb | 1 + lib/sequent/core/snapshot_record.rb | 42 +++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 33 deletions(-) create mode 100644 lib/sequent/core/snapshot_record.rb diff --git a/lib/sequent/core/event_record.rb b/lib/sequent/core/event_record.rb index 7617d8b8..d6a12e6b 100644 --- a/lib/sequent/core/event_record.rb +++ b/lib/sequent/core/event_record.rb @@ -108,38 +108,5 @@ def find_origin(record) record end end - - class SnapshotRecord < Sequent::ApplicationRecord - include SerializesEvent - - self.primary_key = %i[aggregate_id sequence_number] - self.table_name = 'snapshot_records' - - belongs_to :stream_record, foreign_key: :aggregate_id, primary_key: :aggregate_id - - validates_presence_of :aggregate_id, :sequence_number, :snapshot_json, :stream_record - validates_numericality_of :sequence_number, only_integer: true, greater_than: 0 - - def event_type - snapshot_type - end - - def event_type=(type) - self.snapshot_type = type - end - - def event_json - snapshot_json - end - - def event_json=(json) - self.snapshot_json = json - end - - def serialize_json? - json_column_type = self.class.columns_hash['snapshot_json'].sql_type_metadata.type - %i[json jsonb].exclude? json_column_type - end - end end end diff --git a/lib/sequent/core/event_store.rb b/lib/sequent/core/event_store.rb index 7f218d8f..28b19cdc 100644 --- a/lib/sequent/core/event_store.rb +++ b/lib/sequent/core/event_store.rb @@ -3,6 +3,7 @@ require 'forwardable' require_relative 'event_record' require_relative 'sequent_oj' +require_relative 'snapshot_record' module Sequent module Core diff --git a/lib/sequent/core/snapshot_record.rb b/lib/sequent/core/snapshot_record.rb new file mode 100644 index 00000000..5c7a7747 --- /dev/null +++ b/lib/sequent/core/snapshot_record.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +require 'active_record' +require_relative 'sequent_oj' +require_relative '../application_record' + +module Sequent + module Core + class SnapshotRecord < Sequent::ApplicationRecord + include SerializesEvent + + self.primary_key = %i[aggregate_id sequence_number] + self.table_name = 'snapshot_records' + + belongs_to :stream_record, foreign_key: :aggregate_id, primary_key: :aggregate_id + + validates_presence_of :aggregate_id, :sequence_number, :snapshot_json, :stream_record + validates_numericality_of :sequence_number, only_integer: true, greater_than: 0 + + def event_type + snapshot_type + end + + def event_type=(type) + self.snapshot_type = type + end + + def event_json + snapshot_json + end + + def event_json=(json) + self.snapshot_json = json + end + + def serialize_json? + json_column_type = self.class.columns_hash['snapshot_json'].sql_type_metadata.type + %i[json jsonb].exclude? json_column_type + end + end + end +end From c62f22885205cfe6c70ccf0659965a00002afc54 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Mon, 11 Mar 2024 16:10:23 +0100 Subject: [PATCH 033/128] Updated changelog --- CHANGELOG.md | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb2350c5..cd8eca93 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,27 @@ -# Changelog 7.0.x (changes since 7.0.0) +# Changelog 8.0.x (changes since 7.0.1) + +- Sequent now requires at least Ruby 3.1 and ActiveRecord 7.1. +- `AggregateRoot#take_snapshot!` has been replaced with + `AggregateRoot#take_snapshot` which returns the snapshot instead of + adding it to the uncommitted events. Snapshots can be stored using + `EventStore#store_snapshots`. +- Events, commands, and snapshot can now be serialized directly to a + `JSON` or `JSONB` PostgreSQL column. Support for the `TEXT` column + type will be removed in a future release. +- Snapshots are now stored into a separate `snapshot_records` table + instead of being mixed in with events in the `event_records` table. + This also makes it possible to store snapshots without an associated + command record. +- The `id` column `event_streams` has been removed and the primary key + is now the `aggregate_id` column. +- The `id` column of `event_records` has been removed and the primary + key are now the `aggregate_id` and `sequence_number` columns. +- New APIs have been added to the event store to prepare for more + internal storage changes. Events streams and commands can now be + permanently deleted, snapshots can be managed, and events can be + loaded individually. + +# Changelog 7.0.1 (changes since 7.0.0) - Replaying all events for the view schema (using `sequent:migrate:online` and `sequent:migrate:offline`) now make use of the PostgreSQL committed transaction id to track events that have already been replayed. The replayed ids table (specified by the removed `Sequent::configuration.replayed_ids_table_name` option) is no longer used and can be dropped from your database. - The `MessageDispatcher` class has been removed. From 7b35c84f06ef7e6e5a48250e12a06323dee4463d Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Mon, 11 Mar 2024 19:53:07 +0100 Subject: [PATCH 034/128] Make temporary bridge methods private --- lib/sequent/core/snapshot_record.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/sequent/core/snapshot_record.rb b/lib/sequent/core/snapshot_record.rb index 5c7a7747..6d6968b7 100644 --- a/lib/sequent/core/snapshot_record.rb +++ b/lib/sequent/core/snapshot_record.rb @@ -17,6 +17,8 @@ class SnapshotRecord < Sequent::ApplicationRecord validates_presence_of :aggregate_id, :sequence_number, :snapshot_json, :stream_record validates_numericality_of :sequence_number, only_integer: true, greater_than: 0 + private + def event_type snapshot_type end From 9a68ac557241c8656f68d44dd50069c70ac80eb1 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Wed, 13 Mar 2024 14:59:52 +0100 Subject: [PATCH 035/128] Workaround for required primary key index for `insert_all!` During migration of the production database there is no proper primary key index yet (since the only unique index also contains a CASE to distinguish between snapshot and normal events). This causes `insert_all!` to fail with error. However, simple bulk insertion does not require finding the primary key index so use a patched version of `ActiveRecord::InsertAll` for this case until the database is able to create the proper primary key index while the system is running. The following SQL can be used for this: ```sql CREATE UNIQUE INDEX CONCURRENTLY event_records_pkey ON event_records (aggregate_id, sequence_number); ALTER TABLE event_records ADD PRIMARY KEY USING INDEX event_records_pkey; DROP INDEX unique_event_per_aggregate; ``` --- lib/sequent/core/event_store.rb | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/lib/sequent/core/event_store.rb b/lib/sequent/core/event_store.rb index 28b19cdc..94bbc708 100644 --- a/lib/sequent/core/event_store.rb +++ b/lib/sequent/core/event_store.rb @@ -323,10 +323,34 @@ def store_events(command, streams_with_events = []) record.attributes.slice(*column_names) end end - Sequent.configuration.event_record_class.insert_all!(event_records) if event_records.present? + + return unless event_records.present? + + PatchedInsertAll.new( + Sequent.configuration.event_record_class, + event_records, + on_duplicate: :raise, + ).execute + + nil rescue ActiveRecord::RecordNotUnique raise OptimisticLockingError end end + + class PatchedInsertAll < ActiveRecord::InsertAll + def find_unique_index_for(_unique_by) + # Find_unique_index_for doesn't work if there is no proper + # primary key index, which is the case while Jortt is + # migrating from the old combined events+snapshots table For + # some reason `ActiveRecord::InsertAll#find_unique_index_for` + # wants to find such an index anyway, even though it is not + # needed for a simple bulk INSERT. Override this method to + # always return nil here, so the `insert_all!` succeeds. Once + # the primary key constraint + index is properly generated we + # can simply use `insert_all` directly again. + nil + end + end end end From 7f00180e82d4bd56b6c67cabee956c9ff87a2975 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Fri, 15 Mar 2024 10:51:31 +0100 Subject: [PATCH 036/128] Remove snapshot_event_class configuration Now that all snapshots are stored in a separate table there is no longer a need to configure the snapshot event class. --- docs/docs/concepts/configuration.md | 2 +- lib/sequent/configuration.rb | 2 -- lib/sequent/core/command_record.rb | 1 - lib/sequent/core/event_store.rb | 1 - lib/sequent/rake/migration_tasks.rb | 2 +- spec/lib/sequent/core/event_store_spec.rb | 2 -- 6 files changed, 2 insertions(+), 8 deletions(-) diff --git a/docs/docs/concepts/configuration.md b/docs/docs/concepts/configuration.md index 569752bd..925321bc 100644 --- a/docs/docs/concepts/configuration.md +++ b/docs/docs/concepts/configuration.md @@ -137,7 +137,7 @@ For the latest configuration possibilities please check the `Sequent::Configurat | online_replay_persistor_class | The class used to persist the `Projector`s. | `Sequent::Core::Persistors::ActiveRecordPersistor` | | primary_database_key | A symbol indicating the primary database if multiple databases are specified within the provided db_config | `:primary` | | primary_database_role | A symbol indicating the primary database role if using multiple databases with active record | `:writing` | -| snapshot_event_class | The event class marking something as a [Snapshot event](snapshotting.html) | `Sequent::Core::SnapshotEvent` | +| snapshot_record_class | The [class](event_store.html) mapped to the `snapshot_records` table | `Sequent::Core::SnapshotRecord` | | stream_record_class | The [class](event_store.html) mapped to the `stream_records` table | `Sequent::Core::StreamRecord` | | strict_check_attributes_on_apply_events | Whether or not sequent should fail on calling `apply` with invalid attributes. | `false`. Will be enabled by default in the next major release. | | time_precision | Sets the precision of encoded time values. Defaults to 3 (equivalent to millisecond precision) | `ActiveSupport::JSON::Encoding.time_precision` | diff --git a/lib/sequent/configuration.rb b/lib/sequent/configuration.rb index 698574af..06876c66 100644 --- a/lib/sequent/configuration.rb +++ b/lib/sequent/configuration.rb @@ -40,7 +40,6 @@ class Configuration :event_record_class, :snapshot_record_class, :stream_record_class, - :snapshot_event_class, :transaction_provider, :event_publisher, :event_record_hooks_class, @@ -98,7 +97,6 @@ def initialize self.event_record_class = Sequent::Core::EventRecord self.snapshot_record_class = Sequent::Core::SnapshotRecord self.stream_record_class = Sequent::Core::StreamRecord - self.snapshot_event_class = Sequent::Core::SnapshotEvent self.transaction_provider = Sequent::Core::Transactions::ActiveRecordTransactionProvider.new self.uuid_generator = Sequent::Core::RandomUuidGenerator self.event_publisher = Sequent::Core::EventPublisher.new diff --git a/lib/sequent/core/command_record.rb b/lib/sequent/core/command_record.rb index 7e84e3f7..97e0a99c 100644 --- a/lib/sequent/core/command_record.rb +++ b/lib/sequent/core/command_record.rb @@ -53,7 +53,6 @@ class CommandRecord < Sequent::ApplicationRecord def parent EventRecord .where(aggregate_id: event_aggregate_id, sequence_number: event_sequence_number) - .where('event_type != ?', Sequent::Core::SnapshotEvent.name) .first end diff --git a/lib/sequent/core/event_store.rb b/lib/sequent/core/event_store.rb index 94bbc708..349d4853 100644 --- a/lib/sequent/core/event_store.rb +++ b/lib/sequent/core/event_store.rb @@ -73,7 +73,6 @@ def stream_events_for_aggregate(aggregate_id, load_until: nil, &block) .configuration .event_record_class .where(aggregate_id: aggregate_id) - .where.not(event_type: Sequent.configuration.snapshot_event_class.name) .order(:sequence_number) q = q.where('created_at < ?', load_until) if load_until.present? has_events = false diff --git a/lib/sequent/rake/migration_tasks.rb b/lib/sequent/rake/migration_tasks.rb index c7164a74..240fde0c 100644 --- a/lib/sequent/rake/migration_tasks.rb +++ b/lib/sequent/rake/migration_tasks.rb @@ -235,7 +235,7 @@ def register_tasks! result = Sequent::ApplicationRecord .connection .execute(<<~EOS) - DELETE FROM #{Sequent.configuration.event_record_class.table_name} WHERE event_type = 'Sequent::Core::SnapshotEvent' + DELETE FROM #{Sequent.configuration.snapshot_record_class.table_name}' EOS Sequent.logger.info "Deleted #{result.cmd_tuples} aggregate snapshots from the event store" end diff --git a/spec/lib/sequent/core/event_store_spec.rb b/spec/lib/sequent/core/event_store_spec.rb index 5e5c613c..7cf07550 100644 --- a/spec/lib/sequent/core/event_store_spec.rb +++ b/spec/lib/sequent/core/event_store_spec.rb @@ -467,11 +467,9 @@ class FailingHandler < Sequent::Core::Projector -> do event_records = Sequent.configuration.event_record_class.table_name stream_records = Sequent.configuration.stream_record_class.table_name - snapshot_event_type = Sequent.configuration.snapshot_event_class Sequent.configuration.event_record_class .select('event_type, event_json') .joins("INNER JOIN #{stream_records} ON #{event_records}.aggregate_id = #{stream_records}.aggregate_id") - .where('event_type <> ?', snapshot_event_type) .order!("#{stream_records}.aggregate_id, #{event_records}.sequence_number") end end From 2c9dad440f07c9231b3abceb3e24951061d4a540 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Mon, 18 Mar 2024 10:29:43 +0100 Subject: [PATCH 037/128] Support deletion of multiple event streams using single call --- lib/sequent/core/event_store.rb | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/sequent/core/event_store.rb b/lib/sequent/core/event_store.rb index 349d4853..d2f9bfdb 100644 --- a/lib/sequent/core/event_store.rb +++ b/lib/sequent/core/event_store.rb @@ -244,8 +244,12 @@ def find_event_stream(aggregate_id) end def permanently_delete_event_stream(aggregate_id) - Sequent.configuration.event_record_class.where(aggregate_id: aggregate_id).delete_all - Sequent.configuration.stream_record_class.where(aggregate_id: aggregate_id).delete_all + permanently_delete_event_streams([aggregate_id]) + end + + def permanently_delete_event_streams(aggregate_ids) + Sequent.configuration.event_record_class.where(aggregate_id: aggregate_ids).delete_all + Sequent.configuration.stream_record_class.where(aggregate_id: aggregate_ids).delete_all end def permanently_delete_commands_without_events(where_clause) From 57434bc947bbda4f1d2fc80d0b135d373ed006ea Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Wed, 20 Mar 2024 14:36:00 +0100 Subject: [PATCH 038/128] Use stored procedures to update event store Commands, streams, and events are now persisted and queried using stored procedures. This allows future database changes without having to modify the Ruby code, improving compatibility during the migration period. --- README.md | 8 +- db/sequent_schema.rb | 3 + db/sequent_schema.sql | 233 +++++++++++++++++ lib/sequent/core/command_record.rb | 2 + lib/sequent/core/event_record.rb | 10 +- lib/sequent/core/event_store.rb | 236 +++++++++--------- lib/sequent/core/stream_record.rb | 7 +- lib/sequent/migrations/view_schema.rb | 2 +- spec/lib/sequent/core/aggregate_root_spec.rb | 7 + spec/lib/sequent/core/event_store_spec.rb | 53 +++- .../sequent/migrations/view_schema_spec.rb | 4 +- 11 files changed, 435 insertions(+), 130 deletions(-) create mode 100644 db/sequent_schema.sql diff --git a/README.md b/README.md index d1791a48..c81bb529 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,13 @@ createuser -D -s -R sequent SEQUENT_ENV=test bundle exec rake sequent:db:create ``` -Run `rspec spec` to run the tests. +Run `rspec spec` to run the tests with the current database schema. + +To ensure the specs use the latest database schema run: + +``` +SEQUENT_ENV=test rake sequent:db:drop sequent:db:create spec +``` ## Changelog diff --git a/db/sequent_schema.rb b/db/sequent_schema.rb index 5f0128f5..80c2e154 100644 --- a/db/sequent_schema.rb +++ b/db/sequent_schema.rb @@ -61,4 +61,7 @@ primary_key: :aggregate_id, on_delete: :cascade, name: 'stream_fkey' + + schema = File.read("#{File.dirname(__FILE__)}/sequent_schema.sql") + execute schema end diff --git a/db/sequent_schema.sql b/db/sequent_schema.sql new file mode 100644 index 00000000..8340f173 --- /dev/null +++ b/db/sequent_schema.sql @@ -0,0 +1,233 @@ +DROP TYPE IF EXISTS aggregate_event_type CASCADE; +CREATE TYPE aggregate_event_type AS ( + aggregate_type text, + aggregate_id uuid, + events_partition_key text, + snapshot_threshold integer, + event_type text, + event_json jsonb +); + +CREATE OR REPLACE FUNCTION load_event( + _aggregate_id uuid, + _sequence_number integer +) RETURNS SETOF aggregate_event_type +LANGUAGE plpgsql AS $$ +BEGIN + RETURN QUERY SELECT aggregate_type::text, _aggregate_id, ''::text, snapshot_threshold, event_type::text, event_json::jsonb + FROM event_records event JOIN stream_records stream ON event.aggregate_id = stream.aggregate_id + WHERE stream.aggregate_id = _aggregate_id + AND sequence_number = _sequence_number; +END; +$$; + +CREATE OR REPLACE FUNCTION load_events( + _aggregate_ids jsonb, + _use_snapshots boolean DEFAULT TRUE, + _until timestamptz DEFAULT NULL +) RETURNS SETOF aggregate_event_type +LANGUAGE plpgsql AS $$ +DECLARE + _aggregate_id event_records.aggregate_id%TYPE; + _snapshot_event snapshot_records; + _snapshot_event_sequence_number integer; + _stream_record stream_records; +BEGIN + FOR _aggregate_id IN SELECT * FROM jsonb_array_elements_text(_aggregate_ids) LOOP + SELECT * + INTO _stream_record + FROM stream_records + WHERE stream_records.aggregate_id = _aggregate_id; + IF NOT FOUND THEN + CONTINUE; + END IF; + + _snapshot_event = NULL; + _snapshot_event_sequence_number = 0; + IF _use_snapshots THEN + SELECT * INTO _snapshot_event + FROM snapshot_records e + WHERE e.aggregate_id = _aggregate_id + AND (_until IS NULL OR e.created_at < _until) + ORDER BY e.sequence_number DESC + LIMIT 1; + IF FOUND THEN + RETURN NEXT ( + _stream_record.aggregate_type::text, + _stream_record.aggregate_id, + ''::text, + _stream_record.snapshot_threshold, + _snapshot_event.snapshot_type::text, + _snapshot_event.snapshot_json::jsonb + ); + _snapshot_event_sequence_number = _snapshot_event.sequence_number; + END IF; + END IF; + + RETURN QUERY SELECT _stream_record.aggregate_type::text, + _stream_record.aggregate_id, + ''::text, + _stream_record.snapshot_threshold, + e.event_type::text, + e.event_json::jsonb + FROM event_records e + WHERE e.aggregate_id = _aggregate_id + AND e.sequence_number >= _snapshot_event_sequence_number + AND (_until IS NULL OR e.created_at < _until) + ORDER BY e.sequence_number; + END LOOP; +END; +$$; + +CREATE OR REPLACE FUNCTION store_command(_command jsonb) RETURNS bigint +LANGUAGE plpgsql AS $$ +DECLARE + _id command_records.id%TYPE; + _command_without_nulls jsonb = jsonb_strip_nulls(_command->'command_json'); +BEGIN + INSERT INTO command_records ( + created_at, user_id, aggregate_id, command_type, command_json, + event_aggregate_id, event_sequence_number + ) VALUES ( + (_command->>'created_at')::timestamp, + (_command_without_nulls->>'user_id')::uuid, + (_command_without_nulls->>'aggregate_id')::uuid, + _command->>'command_type', + _command->'command_json', + (_command_without_nulls->>'event_aggregate_id')::uuid, + (_command_without_nulls->'event_sequence_number')::integer + ) RETURNING id INTO STRICT _id; + RETURN _id; +END; +$$; + +CREATE OR REPLACE PROCEDURE store_events(_command jsonb, _aggregates_with_events jsonb) +LANGUAGE plpgsql AS $$ +DECLARE + _command_record_id command_records.id%TYPE; + _aggregate jsonb; + _aggregate_without_nulls jsonb; + _events jsonb; + _event jsonb; + _aggregate_id stream_records.aggregate_id%TYPE; + _created_at stream_records.created_at%TYPE; + _snapshot_threshold stream_records.snapshot_threshold%TYPE; + _sequence_number event_records.sequence_number%TYPE; +BEGIN + _command_record_id = store_command(_command); + + FOR _aggregate, _events IN SELECT row->0, row->1 FROM jsonb_array_elements(_aggregates_with_events) AS row LOOP + _aggregate_id = (_aggregate->>'aggregate_id')::uuid; + _aggregate_without_nulls = jsonb_strip_nulls(_aggregate); + _snapshot_threshold = _aggregate_without_nulls->'snapshot_threshold'; + + IF NOT EXISTS (SELECT 1 FROM stream_records WHERE aggregate_id = _aggregate_id) THEN + _created_at = _events->0->>'created_at'; + _sequence_number = _events->0->'event_json'->'sequence_number'; + IF _sequence_number <> 1 THEN + RAISE EXCEPTION 'sequence number of first event new aggregate must be 1, was %', _sequence_number; + END IF; + + INSERT INTO stream_records (created_at, aggregate_type, aggregate_id, snapshot_threshold) + VALUES (_created_at, _aggregate->>'aggregate_type', _aggregate_id, _snapshot_threshold); + END IF; + + FOR _event IN SELECT * FROM jsonb_array_elements(_events) LOOP + _created_at = _event->'created_at'; + _sequence_number = _event->'event_json'->'sequence_number'; + INSERT INTO event_records (aggregate_id, sequence_number, created_at, event_type, event_json, command_record_id) + VALUES ( + (_event->'event_json'->>'aggregate_id')::uuid, + _sequence_number, + _created_at, + _event->>'event_type', + _event->'event_json', + _command_record_id + ); + END LOOP; + END LOOP; +END; +$$; + +CREATE OR REPLACE PROCEDURE store_snapshots(_snapshots jsonb) +LANGUAGE plpgsql AS $$ +DECLARE + _aggregate_id uuid; + _snapshot jsonb; +BEGIN + FOR _snapshot IN SELECT * FROM jsonb_array_elements(_snapshots) LOOP + _aggregate_id = _snapshot->>'aggregate_id'; + INSERT INTO snapshot_records (aggregate_id, sequence_number, created_at, snapshot_type, snapshot_json) + VALUES ( + _aggregate_id, + (_snapshot->'sequence_number')::integer, + (_snapshot->>'created_at')::timestamptz, + _snapshot->>'snapshot_type', + _snapshot->'snapshot_json' + ); + END LOOP; +END; +$$; + +CREATE OR REPLACE FUNCTION load_latest_snapshot(_aggregate_id uuid) RETURNS aggregate_event_type +LANGUAGE SQL AS $$ + SELECT a.aggregate_type, a.aggregate_id, '', a.snapshot_threshold, s.snapshot_type, s.snapshot_json::jsonb + FROM snapshot_records s JOIN stream_records a ON s.aggregate_id = a.aggregate_id + WHERE s.aggregate_id = _aggregate_id + ORDER BY sequence_number DESC + LIMIT 1; +$$; + +CREATE OR REPLACE PROCEDURE delete_snapshots_before(_aggregate_id uuid, _sequence_number integer) +LANGUAGE plpgsql AS $$ +BEGIN + DELETE FROM snapshot_records + WHERE aggregate_id = _aggregate_id + AND sequence_number < _sequence_number; +END; +$$; + +CREATE OR REPLACE FUNCTION aggregates_that_need_snapshots(_last_aggregate_id uuid, _limit integer) + RETURNS TABLE (aggregate_id uuid) +LANGUAGE plpgsql AS $$ +BEGIN + RETURN QUERY SELECT stream.aggregate_id + FROM stream_records stream + WHERE (_last_aggregate_id IS NULL OR stream.aggregate_id > _last_aggregate_id) + AND snapshot_threshold IS NOT NULL + AND snapshot_threshold <= ( + (SELECT MAX(events.sequence_number) FROM event_records events WHERE stream.aggregate_id = events.aggregate_id) - + COALESCE((SELECT MAX(snapshots.sequence_number) FROM snapshot_records snapshots WHERE stream.aggregate_id = snapshots.aggregate_id), 0)) + ORDER BY 1 + LIMIT _limit; +END; +$$; + +CREATE OR REPLACE PROCEDURE permanently_delete_commands_without_events(_aggregate_id uuid, _organization_id uuid) +LANGUAGE plpgsql AS $$ +BEGIN + IF _organization_id IS NOT NULL THEN + RAISE EXCEPTION 'deleting by organization_id is not supported by this version of Sequent'; + END IF; + IF _aggregate_id IS NULL AND _organization_id IS NULL THEN + RAISE EXCEPTION 'aggregate_id or organization_id must be specified to delete commands'; + END IF; + + DELETE FROM command_records + WHERE (_aggregate_id IS NULL OR aggregate_id = _aggregate_id) + --AND (_organization_id IS NULL OR organization_id = _organization_id) + AND NOT EXISTS (SELECT 1 FROM event_records WHERE command_record_id = command_records.id); +END; +$$; + +CREATE OR REPLACE PROCEDURE permanently_delete_event_streams(_aggregate_ids jsonb) +LANGUAGE plpgsql AS $$ +BEGIN + DELETE FROM event_records + USING jsonb_array_elements_text(_aggregate_ids) AS ids (id) + WHERE event_records.aggregate_id = ids.id::uuid; + DELETE FROM stream_records + USING jsonb_array_elements_text(_aggregate_ids) AS ids (id) + WHERE stream_records.aggregate_id = ids.id::uuid; +END; +$$; diff --git a/lib/sequent/core/command_record.rb b/lib/sequent/core/command_record.rb index 97e0a99c..3652c5e5 100644 --- a/lib/sequent/core/command_record.rb +++ b/lib/sequent/core/command_record.rb @@ -31,6 +31,8 @@ def command=(command) private def serialize_json? + return true unless self.class.respond_to? :columns_hash + json_column_type = self.class.columns_hash['command_json'].sql_type_metadata.type %i[json jsonb].exclude? json_column_type end diff --git a/lib/sequent/core/event_record.rb b/lib/sequent/core/event_record.rb index d6a12e6b..98d0c8d5 100644 --- a/lib/sequent/core/event_record.rb +++ b/lib/sequent/core/event_record.rb @@ -65,6 +65,13 @@ module ClassMethods def serialize_to_json(event) Sequent::Core::Oj.dump(event) end + + def serialize_json? + return true unless respond_to? :columns_hash + + json_column_type = columns_hash['event_json'].sql_type_metadata.type + %i[json jsonb].exclude? json_column_type + end end def self.included(host_class) @@ -72,8 +79,7 @@ def self.included(host_class) end def serialize_json? - json_column_type = self.class.columns_hash['event_json'].sql_type_metadata.type - %i[json jsonb].exclude? json_column_type + self.class.serialize_json? end end diff --git a/lib/sequent/core/event_store.rb b/lib/sequent/core/event_store.rb index d2f9bfdb..0e8876d2 100644 --- a/lib/sequent/core/event_store.rb +++ b/lib/sequent/core/event_store.rb @@ -57,7 +57,7 @@ def commit_events(command, streams_with_events) end ## - # Returns all events for the AggregateRoot ordered by sequence_number, disregarding snapshot events. + # Returns all events for the AggregateRoot ordered by sequence_number, disregarding snapshots. # # This streaming is done in batches to prevent loading many events in memory all at once. A usecase for ignoring # the snapshots is when events of a nested AggregateRoot need to be loaded up until a certain moment in time. @@ -69,15 +69,20 @@ def stream_events_for_aggregate(aggregate_id, load_until: nil, &block) stream = find_event_stream(aggregate_id) fail ArgumentError, 'no stream found for this aggregate' if stream.blank? - q = Sequent - .configuration - .event_record_class - .where(aggregate_id: aggregate_id) - .order(:sequence_number) - q = q.where('created_at < ?', load_until) if load_until.present? has_events = false - q.select('event_type, event_json').each_row do |event_hash| + # PostgreSQLCursor::Cursor does not support bind parameters, so bind parameters manually instead. + sql = ActiveRecord::Base.sanitize_sql_array( + [ + 'SELECT * FROM load_events(:aggregate_ids, FALSE, :load_until)', + { + aggregate_ids: [aggregate_id].to_json, + load_until: load_until, + }, + ], + ) + + PostgreSQLCursor::Cursor.new(sql, {connection: connection}).each_row do |event_hash| has_events = true event = deserialize_event(event_hash) block.call([stream, event]) @@ -86,7 +91,12 @@ def stream_events_for_aggregate(aggregate_id, load_until: nil, &block) end def load_event(aggregate_id, sequence_number) - Sequent.configuration.event_record_class.find_by(aggregate_id:, sequence_number:)&.event + event_hash = connection.exec_query( + 'SELECT * FROM load_event($1, $2)', + 'load_event', + [aggregate_id, sequence_number], + ).first + deserialize_event(event_hash) if event_hash end ## @@ -100,69 +110,57 @@ def load_events(aggregate_id) def load_events_for_aggregates(aggregate_ids) return [] if aggregate_ids.none? - streams = Sequent.configuration.stream_record_class.where(aggregate_id: aggregate_ids) - - query = aggregate_ids.uniq.map { |aggregate_id| aggregate_query(aggregate_id) }.join(' UNION ALL ') - events = Sequent.configuration.event_record_class.connection.select_all(query).map do |event_hash| - deserialize_event(event_hash) - end - - events - .group_by(&:aggregate_id) - .map do |aggregate_id, es| + query_events(aggregate_ids) + .group_by { |row| row['aggregate_id'] } + .values + .map do |rows| [ - streams.find do |stream_record| - stream_record.aggregate_id == aggregate_id - end.event_stream, - es, + EventStream.new( + aggregate_type: rows.first['aggregate_type'], + aggregate_id: rows.first['aggregate_id'], + events_partition_key: rows.first['events_partition_key'], + snapshot_threshold: rows.first['snapshot_threshold'], + ), + rows.map { |row| deserialize_event(row) }, ] end end def store_snapshots(snapshots) - serializer = SnapshotRecord.new - SnapshotRecord.insert_all!( + json = Sequent::Core::Oj.dump( snapshots.map do |snapshot| - serializer.event = snapshot { - aggregate_id: serializer.aggregate_id, - sequence_number: serializer.sequence_number, - created_at: serializer.created_at, - snapshot_type: serializer.snapshot_type, - snapshot_json: serializer.snapshot_json, + aggregate_id: snapshot.aggregate_id, + sequence_number: snapshot.sequence_number, + created_at: snapshot.created_at, + snapshot_type: snapshot.class.name, + snapshot_json: snapshot, } end, ) + connection.exec_update( + 'CALL store_snapshots($1)', + 'store_snapshots', + [json], + ) end def load_latest_snapshot(aggregate_id) - latest_snapshot = SnapshotRecord.where(aggregate_id:).order(:sequence_number).last - latest_snapshot&.event + snapshot_hash = connection.exec_query( + 'SELECT * FROM load_latest_snapshot($1)', + 'load_latest_snapshot', + [aggregate_id], + ).first + deserialize_event(snapshot_hash) unless snapshot_hash['aggregate_id'].nil? end # Deletes all snapshots for aggregate_id with a sequence_number lower than the specified sequence number. def delete_snapshots_before(aggregate_id, sequence_number) - SnapshotRecord.where(aggregate_id: aggregate_id).where('sequence_number < ?', sequence_number).delete_all - end - - def aggregate_query(aggregate_id) - <<~SQL.chomp - ( - WITH snapshot AS (SELECT * - FROM #{quote_table_name Sequent.configuration.snapshot_record_class.table_name} - WHERE aggregate_id = #{quote(aggregate_id)} - ORDER BY sequence_number DESC - LIMIT 1) - SELECT snapshot_type AS event_type, snapshot_json AS event_json, 0 AS sequence_number - FROM snapshot - UNION ALL - SELECT event_type, event_json, sequence_number - FROM #{quote_table_name Sequent.configuration.event_record_class.table_name} AS o - WHERE aggregate_id = #{quote(aggregate_id)} - AND sequence_number >= COALESCE((SELECT sequence_number FROM snapshot), 0) - ORDER BY sequence_number ASC - ) - SQL + connection.exec_update( + 'CALL delete_snapshots_before($1, $2)', + 'delete_snapshots_before', + [aggregate_id, sequence_number], + ) end def stream_exists?(aggregate_id) @@ -172,6 +170,7 @@ def stream_exists?(aggregate_id) def events_exists?(aggregate_id) Sequent.configuration.event_record_class.exists?(aggregate_id: aggregate_id) end + ## # Replays all events in the event store to the registered event_handlers. # @@ -220,22 +219,11 @@ def replay_events_from_cursor(get_events:, block_size: 2000, # Returns the ids of aggregates that need a new snapshot. # def aggregates_that_need_snapshots(last_aggregate_id, limit = 10) - stream_table = quote_table_name Sequent.configuration.stream_record_class.table_name - event_table = quote_table_name Sequent.configuration.event_record_class.table_name - snapshot_table = quote_table_name Sequent.configuration.snapshot_record_class.table_name - query = <<~SQL.chomp - SELECT aggregate_id - FROM #{stream_table} stream - WHERE aggregate_id::varchar > COALESCE(#{quote last_aggregate_id}, '') - AND snapshot_threshold IS NOT NULL - AND snapshot_threshold <= ( - (SELECT MAX(events.sequence_number) FROM #{event_table} events WHERE stream.aggregate_id = events.aggregate_id) - - COALESCE((SELECT MAX(snapshots.sequence_number) FROM #{snapshot_table} snapshots WHERE stream.aggregate_id = snapshots.aggregate_id), 0)) - ORDER BY aggregate_id - LIMIT #{quote limit} - FOR UPDATE - SQL - Sequent.configuration.event_record_class.connection.select_all(query).map { |x| x['aggregate_id'] } + connection.exec_query( + 'SELECT aggregate_id FROM aggregates_that_need_snapshots($1, $2)', + 'aggregates_that_need_snapshots', + [last_aggregate_id, limit], + ).map { |x| x['aggregate_id'] } end def find_event_stream(aggregate_id) @@ -248,15 +236,23 @@ def permanently_delete_event_stream(aggregate_id) end def permanently_delete_event_streams(aggregate_ids) - Sequent.configuration.event_record_class.where(aggregate_id: aggregate_ids).delete_all - Sequent.configuration.stream_record_class.where(aggregate_id: aggregate_ids).delete_all + connection.exec_update( + 'CALL permanently_delete_event_streams($1)', + 'permanently_delete_event_streams', + [aggregate_ids.to_json], + ) end - def permanently_delete_commands_without_events(where_clause) - CommandRecord - .where(where_clause) - .where('NOT EXISTS (SELECT 1 FROM event_records WHERE command_record_id = command_records.id)') - .delete_all + def permanently_delete_commands_without_events(aggregate_id: nil, organization_id: nil) + unless aggregate_id || organization_id + fail ArgumentError, 'aggregate_id and/or organization_id must be specified' + end + + connection.exec_update( + 'CALL permanently_delete_commands_without_events($1, $2)', + 'permanently_delete_commands_without_events', + [aggregate_id, organization_id], + ) end private @@ -269,6 +265,18 @@ def event_types end end + def connection + Sequent.configuration.event_record_class.connection + end + + def query_events(aggregate_ids, use_snapshots = true, load_until = nil) + connection.exec_query( + 'SELECT * FROM load_events($1::JSONB, $2, $3)', + 'load_events', + [aggregate_ids.to_json, use_snapshots, load_until], + ) + end + def column_names @column_names ||= Sequent .configuration @@ -282,10 +290,11 @@ def primary_key_event_records end def deserialize_event(event_hash) + should_serialize_json = Sequent.configuration.event_record_class.serialize_json? record = Sequent.configuration.event_record_class.new record.event_type = event_hash.fetch('event_type') record.event_json = - if record.serialize_json? + if should_serialize_json event_hash.fetch('event_json') else # When the column type is JSON or JSONB the event record @@ -307,52 +316,41 @@ def publish_events(events) end def store_events(command, streams_with_events = []) - command_record = CommandRecord.create!(command: command) - streams = streams_with_events.map do |event_stream, _| - { - aggregate_id: event_stream.aggregate_id, - aggregate_type: event_stream.aggregate_type, - snapshot_threshold: event_stream.snapshot_threshold, - } - end.uniq { |s| s[:aggregate_id] } - Sequent.configuration.stream_record_class - .upsert_all(streams, unique_by: :aggregate_id, update_only: %i[snapshot_threshold]) - - event_records = streams_with_events.flat_map do |_, uncommitted_events| - uncommitted_events.map do |event| - record = Sequent.configuration.event_record_class.new - record.command_record_id = command_record.id - record.event = event - record.attributes.slice(*column_names) - end + command_record = { + created_at: convert_timestamp(command.created_at&.to_time || Time.now), + command_type: command.class.name, + command_json: command, + } + + events = streams_with_events.map do |stream, uncommitted_events| + [ + Sequent::Core::Oj.strict_load(Sequent::Core::Oj.dump(stream)), + uncommitted_events.map do |event| + { + created_at: convert_timestamp(event.created_at.to_time), + event_type: event.class.name, + event_json: event, + } + end, + ] end - - return unless event_records.present? - - PatchedInsertAll.new( - Sequent.configuration.event_record_class, - event_records, - on_duplicate: :raise, - ).execute - - nil + connection.exec_update( + 'CALL store_events($1, $2)', + 'store_events', + [ + Sequent::Core::Oj.dump(command_record), + Sequent::Core::Oj.dump(events), + ], + ) rescue ActiveRecord::RecordNotUnique raise OptimisticLockingError end - end - class PatchedInsertAll < ActiveRecord::InsertAll - def find_unique_index_for(_unique_by) - # Find_unique_index_for doesn't work if there is no proper - # primary key index, which is the case while Jortt is - # migrating from the old combined events+snapshots table For - # some reason `ActiveRecord::InsertAll#find_unique_index_for` - # wants to find such an index anyway, even though it is not - # needed for a simple bulk INSERT. Override this method to - # always return nil here, so the `insert_all!` succeeds. Once - # the primary key constraint + index is properly generated we - # can simply use `insert_all` directly again. - nil + def convert_timestamp(timestamp) + # Since ActiveRecord uses `TIMESTAMP WITHOUT TIME ZONE` + # we need to manually convert database timestamps to the + # ActiveRecord default time zone on serialization. + ActiveRecord.default_timezone == :utc ? timestamp.getutc : timestamp.getlocal end end end diff --git a/lib/sequent/core/stream_record.rb b/lib/sequent/core/stream_record.rb index 02d89021..f9748f60 100644 --- a/lib/sequent/core/stream_record.rb +++ b/lib/sequent/core/stream_record.rb @@ -5,11 +5,12 @@ module Sequent module Core class EventStream - attr_accessor :aggregate_type, :aggregate_id, :snapshot_threshold + attr_accessor :aggregate_type, :aggregate_id, :events_partition_key, :snapshot_threshold - def initialize(aggregate_type:, aggregate_id:, snapshot_threshold: nil) + def initialize(aggregate_type:, aggregate_id:, events_partition_key: '', snapshot_threshold: nil) @aggregate_type = aggregate_type @aggregate_id = aggregate_id + @events_partition_key = events_partition_key @snapshot_threshold = snapshot_threshold end end @@ -21,7 +22,7 @@ class StreamRecord < Sequent::ApplicationRecord validates_presence_of :aggregate_type, :aggregate_id validates_numericality_of :snapshot_threshold, only_integer: true, greater_than: 0, allow_nil: true - has_many :event_records + has_many :event_records, foreign_key: :aggregate_id, primary_key: :aggregate_id def event_stream EventStream.new( diff --git a/lib/sequent/migrations/view_schema.rb b/lib/sequent/migrations/view_schema.rb index fad8faf2..8c6f4438 100644 --- a/lib/sequent/migrations/view_schema.rb +++ b/lib/sequent/migrations/view_schema.rb @@ -470,7 +470,7 @@ def event_stream(aggregate_prefixes, event_types, minimum_xact_id_inclusive, max end event_stream .order('aggregate_id ASC, sequence_number ASC') - .select('aggregate_id, event_type, event_json, sequence_number') + .select('event_type, event_json') end ## shortcut methods diff --git a/spec/lib/sequent/core/aggregate_root_spec.rb b/spec/lib/sequent/core/aggregate_root_spec.rb index 1b9ffb44..7db4fccc 100644 --- a/spec/lib/sequent/core/aggregate_root_spec.rb +++ b/spec/lib/sequent/core/aggregate_root_spec.rb @@ -84,6 +84,13 @@ def event_count context 'snapshotting' do before { subject.generate_event } + it 'returns a snapshot event' do + snapshot = subject.take_snapshot + expect(snapshot.aggregate_id).to be(subject.id) + expect(snapshot.sequence_number).to eq(2) + expect(snapshot.data).to be_present + end + it 'restores state from the snapshot' do snapshot_event = subject.take_snapshot restored = TestAggregateRoot.load_from_history :stream, [snapshot_event] diff --git a/spec/lib/sequent/core/event_store_spec.rb b/spec/lib/sequent/core/event_store_spec.rb index 7cf07550..8d94a8d5 100644 --- a/spec/lib/sequent/core/event_store_spec.rb +++ b/spec/lib/sequent/core/event_store_spec.rb @@ -6,6 +6,7 @@ describe Sequent::Core::EventStore do class MyEvent < Sequent::Core::Event + attrs data: String end class MyAggregate < Sequent::Core::AggregateRoot @@ -75,6 +76,7 @@ class MyAggregate < Sequent::Core::AggregateRoot expect(stream.aggregate_id).to eq(aggregate_id) expect(events.first.aggregate_id).to eq(aggregate_id) expect(events.first.sequence_number).to eq(1) + expect(events.first.data).to eq("with ' unsafe SQL characters;\n") end it 'can find streams that need snapshotting' do @@ -121,7 +123,7 @@ class MyAggregate < Sequent::Core::AggregateRoot end describe '#events_exists?' do - it 'gets true for an existing aggregate' do + before do event_store.commit_events( Sequent::Core::Command.new(aggregate_id: aggregate_id), [ @@ -135,16 +137,24 @@ class MyAggregate < Sequent::Core::AggregateRoot ], ], ) + end + + it 'gets true for an existing aggregate' do expect(event_store.events_exists?(aggregate_id)).to eq(true) end it 'gets false for an non-existing aggregate' do + expect(event_store.events_exists?(Sequent.new_uuid)).to eq(false) + end + + it 'gets false after deletion' do + event_store.permanently_delete_event_stream(aggregate_id) expect(event_store.events_exists?(aggregate_id)).to eq(false) end end describe '#stream_exists?' do - it 'gets true for an existing aggregate' do + before do event_store.commit_events( Sequent::Core::Command.new(aggregate_id: aggregate_id), [ @@ -158,10 +168,18 @@ class MyAggregate < Sequent::Core::AggregateRoot ], ], ) + end + + it 'gets true for an existing aggregate' do expect(event_store.stream_exists?(aggregate_id)).to eq(true) end it 'gets false for an non-existing aggregate' do + expect(event_store.stream_exists?(Sequent.new_uuid)).to eq(false) + end + + it 'gets false after deletion' do + event_store.permanently_delete_event_stream(aggregate_id) expect(event_store.stream_exists?(aggregate_id)).to eq(false) end end @@ -186,6 +204,8 @@ class MyAggregate < Sequent::Core::AggregateRoot stream, events = event_store.load_events(aggregate_id) expect(stream).to be expect(events).to be + + expect(event_store.load_event(aggregate_id, events.first.sequence_number)).to eq(events.first) end context 'and event type caching disabled' do @@ -518,6 +538,35 @@ class FailingHandler < Sequent::Core::Projector end end + describe '#delete_commands_without_events' do + before do + event_store.commit_events( + Sequent::Core::Command.new(aggregate_id: aggregate_id), + [ + [ + Sequent::Core::EventStream.new( + aggregate_type: 'MyAggregate', + aggregate_id: aggregate_id, + snapshot_threshold: 13, + ), + [MyEvent.new(aggregate_id: aggregate_id, sequence_number: 1)], + ], + ], + ) + end + + it 'does not delete commands with associated events' do + event_store.permanently_delete_commands_without_events(aggregate_id: aggregate_id) + expect(Sequent::Core::CommandRecord.exists?(aggregate_id: aggregate_id)).to be_truthy + end + + it 'deletes commands without associated events' do + event_store.permanently_delete_event_stream(aggregate_id) + event_store.permanently_delete_commands_without_events(aggregate_id: aggregate_id) + expect(Sequent::Core::CommandRecord.exists?(aggregate_id: aggregate_id)).to be_falsy + end + end + class ReplayCounter < Sequent::Core::Projector attr_reader :replay_count diff --git a/spec/lib/sequent/migrations/view_schema_spec.rb b/spec/lib/sequent/migrations/view_schema_spec.rb index 22d42ff6..837a4bee 100644 --- a/spec/lib/sequent/migrations/view_schema_spec.rb +++ b/spec/lib/sequent/migrations/view_schema_spec.rb @@ -437,8 +437,8 @@ insert_events( 'Account', [ + AccountCreated.new(aggregate_id: account_id_2, sequence_number: 1), AccountCreated.new(aggregate_id: account_id_2, sequence_number: 2), - AccountCreated.new(aggregate_id: account_id_2, sequence_number: 3), ], ) @@ -483,8 +483,8 @@ insert_events( 'Account', [ + AccountCreated.new(aggregate_id: account_id_2, sequence_number: 1), AccountCreated.new(aggregate_id: account_id_2, sequence_number: 2), - AccountCreated.new(aggregate_id: account_id_2, sequence_number: 3), ], ) From ac54b506c5e3e14e2f76c6cc152469652b02ce33 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Wed, 20 Mar 2024 14:36:26 +0100 Subject: [PATCH 039/128] Fix flaky specs by ensuring running transactions have terminated If PostgreSQL or someone else is running a background transaction while inserting events the newly inserted events may not yet be visible to the online migration. Ensure all transactions that were running during the insertion are completed before starting the online migration. --- .../sequent/migrations/view_schema_spec.rb | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/spec/lib/sequent/migrations/view_schema_spec.rb b/spec/lib/sequent/migrations/view_schema_spec.rb index 837a4bee..bf2322fb 100644 --- a/spec/lib/sequent/migrations/view_schema_spec.rb +++ b/spec/lib/sequent/migrations/view_schema_spec.rb @@ -5,6 +5,17 @@ require_relative '../fixtures/spec_migrations' describe Sequent::Migrations::ViewSchema do + wait_for_persisted_events_to_become_visible_for_online_migration = -> do + query = <<~EOS + SELECT max(xact_id) IS NULL OR + max(xact_id) < pg_snapshot_xmin(pg_current_snapshot())::text::bigint AS done + FROM event_records + EOS + until ActiveRecord::Base.connection.exec_query(query).first['done'] + Sequent.logger.info 'Waiting for transactions to finish so test events are visible for online migration' + end + end + let(:opts) { {db_config: db_config} } let(:migrator) { Sequent::Migrations::ViewSchema.new(**opts) } let(:database_name) { Sequent.new_uuid } @@ -181,6 +192,7 @@ MessageSet.new(aggregate_id: message_aggregate_id, sequence_number: 2, message: 'Foobar'), ], ) + wait_for_persisted_events_to_become_visible_for_online_migration[] before_migration_xact_id = Sequent::Migrations::Versions.current_snapshot_xmin_xact_id @@ -224,6 +236,7 @@ MessageSet.new(aggregate_id: message_aggregate_id, sequence_number: 2, message: 'Foobar'), ], ) + wait_for_persisted_events_to_become_visible_for_online_migration[] migrator.migrate_online @@ -272,6 +285,7 @@ AccountCreated.new(aggregate_id: account_id, sequence_number: 2), ], ) + wait_for_persisted_events_to_become_visible_for_online_migration[] expect { migrator.migrate_online }.to raise_error(Parallel::UndumpableException) @@ -293,6 +307,7 @@ AccountCreated.new(aggregate_id: Sequent.new_uuid, sequence_number: 1), ], ) + wait_for_persisted_events_to_become_visible_for_online_migration[] result = Parallel.map([1, 2], in_processes: 2) do |_id| @connected ||= Sequent::Support::Database.establish_connection(db_config) @@ -371,6 +386,7 @@ insert_events('Account', [AccountCreated.new(aggregate_id: account_id, sequence_number: 1)]) insert_events('Message', [MessageCreated.new(aggregate_id: message_id, sequence_number: 1)]) + wait_for_persisted_events_to_become_visible_for_online_migration[] migrator.migrate_online @@ -430,6 +446,8 @@ it 'stops and does a rollback' do insert_events('Account', [AccountCreated.new(aggregate_id: account_id, sequence_number: 1)]) + wait_for_persisted_events_to_become_visible_for_online_migration[] + migrator.migrate_online account_id_2 = Sequent.new_uuid @@ -458,6 +476,7 @@ before :each do insert_events('Account', [AccountCreated.new(aggregate_id: account_id, sequence_number: 1)]) insert_events('Message', [MessageCreated.new(aggregate_id: message_id, sequence_number: 1)]) + wait_for_persisted_events_to_become_visible_for_online_migration[] migrator.migrate_online migrator.migrate_offline From 825ce46eeb81d8d4a9d7e312e9c27adfc03b6b55 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Thu, 21 Mar 2024 15:57:50 +0100 Subject: [PATCH 040/128] Specify primary key so command record also works with a view Views are used by the partitioned Sequent implementation to provide some level of backwards compatibility and easier query-ability. --- lib/sequent/core/command_record.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/sequent/core/command_record.rb b/lib/sequent/core/command_record.rb index 3652c5e5..8ac7836d 100644 --- a/lib/sequent/core/command_record.rb +++ b/lib/sequent/core/command_record.rb @@ -46,6 +46,7 @@ def serialize_attribute?(command, attribute) class CommandRecord < Sequent::ApplicationRecord include SerializesCommand + self.primary_key = :id self.table_name = 'command_records' has_many :event_records From a2a23a52fe217e4ba757e112d2fe5e6a4afe1933 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Fri, 22 Mar 2024 09:41:16 +0100 Subject: [PATCH 041/128] Test that event is stored as JSON object Uses a direct database query to verify that the event is not accidentally doubly-encoded. --- spec/lib/sequent/core/event_store_spec.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/spec/lib/sequent/core/event_store_spec.rb b/spec/lib/sequent/core/event_store_spec.rb index 8d94a8d5..057c8a12 100644 --- a/spec/lib/sequent/core/event_store_spec.rb +++ b/spec/lib/sequent/core/event_store_spec.rb @@ -68,6 +68,22 @@ class MyAggregate < Sequent::Core::AggregateRoot ) end + it 'stores the event as JSON object' do + # Test to ensure stored data is not accidentally doubly-encoded, + # so query database directly instead of using `load_event`. + row = ActiveRecord::Base.connection.exec_query( + "SELECT event_json, event_json::jsonb->>'data' AS data FROM event_records \ + WHERE aggregate_id = $1 and sequence_number = $2", + 'query_event', + [aggregate_id, 1], + ).first + + expect(row['data']).to eq("with ' unsafe SQL characters;\n") + json = Sequent::Core::Oj.strict_load(row['event_json']) + expect(json['aggregate_id']).to eq(aggregate_id) + expect(json['sequence_number']).to eq(1) + end + it 'can store events' do stream, events = event_store.load_events aggregate_id From 02f4bc99a52978124fe81a78601646454f5593e5 Mon Sep 17 00:00:00 2001 From: Lars Vonk Date: Mon, 25 Mar 2024 14:54:37 +0100 Subject: [PATCH 042/128] Debug correct information when logging --- lib/sequent/core/helpers/message_handler.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/sequent/core/helpers/message_handler.rb b/lib/sequent/core/helpers/message_handler.rb index 494c1f22..520ee7be 100644 --- a/lib/sequent/core/helpers/message_handler.rb +++ b/lib/sequent/core/helpers/message_handler.rb @@ -107,7 +107,9 @@ def handle_message(message) def dispatch_message(message, handlers) handlers.each do |handler| - Sequent.logger.debug("[MessageHandler] Handler #{@context.class} handling #{message.class}") + if Sequent.logger.debug? + Sequent.logger.debug("[MessageHandler] Handler #{self.class} handling #{message.class}") + end instance_exec(message, &handler) end end From adbd21ff0a83216323cae79123e849a2b025c1cf Mon Sep 17 00:00:00 2001 From: Lars Vonk Date: Mon, 25 Mar 2024 14:54:57 +0100 Subject: [PATCH 043/128] Wrap debugging in if statement to prevent unneeded string creations --- lib/sequent/configuration.rb | 8 +++++--- lib/sequent/core/command_service.rb | 2 +- lib/sequent/core/event_publisher.rb | 2 +- lib/sequent/core/event_store.rb | 6 ++++-- lib/sequent/util/timer.rb | 2 +- 5 files changed, 12 insertions(+), 8 deletions(-) diff --git a/lib/sequent/configuration.rb b/lib/sequent/configuration.rb index 06876c66..10bb3ce7 100644 --- a/lib/sequent/configuration.rb +++ b/lib/sequent/configuration.rb @@ -159,18 +159,20 @@ def autoregister! self.class.instance.command_handlers ||= [] for_each_autoregisterable_descenant_of(Sequent::CommandHandler) do |command_handler_class| - Sequent.logger.debug("[Configuration] Autoregistering CommandHandler #{command_handler_class}") + if Sequent.logger.debug? + Sequent.logger.debug("[Configuration] Autoregistering CommandHandler #{command_handler_class}") + end self.class.instance.command_handlers << command_handler_class.new end self.class.instance.event_handlers ||= [] for_each_autoregisterable_descenant_of(Sequent::Projector) do |projector_class| - Sequent.logger.debug("[Configuration] Autoregistering Projector #{projector_class}") + Sequent.logger.debug("[Configuration] Autoregistering Projector #{projector_class}") if Sequent.logger.debug? self.class.instance.event_handlers << projector_class.new end for_each_autoregisterable_descenant_of(Sequent::Workflow) do |workflow_class| - Sequent.logger.debug("[Configuration] Autoregistering Workflow #{workflow_class}") + Sequent.logger.debug("[Configuration] Autoregistering Workflow #{workflow_class}") if Sequent.logger.debug? self.class.instance.event_handlers << workflow_class.new end diff --git a/lib/sequent/core/command_service.rb b/lib/sequent/core/command_service.rb index 64254326..39771344 100644 --- a/lib/sequent/core/command_service.rb +++ b/lib/sequent/core/command_service.rb @@ -67,7 +67,7 @@ def process_commands def process_command(command) fail ArgumentError, 'command is required' if command.nil? - Sequent.logger.debug("[CommandService] Processing command #{command.class}") + Sequent.logger.debug("[CommandService] Processing command #{command.class}") if Sequent.logger.debug? filters.each { |filter| filter.execute(command) } diff --git a/lib/sequent/core/event_publisher.rb b/lib/sequent/core/event_publisher.rb index 9de25843..f64f997d 100644 --- a/lib/sequent/core/event_publisher.rb +++ b/lib/sequent/core/event_publisher.rb @@ -51,7 +51,7 @@ def process_events def process_event(event) fail ArgumentError, 'event is required' if event.nil? - Sequent.logger.debug("[EventPublisher] Publishing event #{event.class}") + Sequent.logger.debug("[EventPublisher] Publishing event #{event.class}") if Sequent.logger.debug? configuration.event_handlers.each do |handler| handler.handle_message event diff --git a/lib/sequent/core/event_store.rb b/lib/sequent/core/event_store.rb index d2f9bfdb..84601f15 100644 --- a/lib/sequent/core/event_store.rb +++ b/lib/sequent/core/event_store.rb @@ -209,10 +209,12 @@ def replay_events_from_cursor(get_events:, block_size: 2000, end PRINT_PROGRESS = ->(progress, done, _) do + next unless Sequent.logger.debug? + if done - Sequent.logger.debug "Done replaying #{progress} events" + Sequent.logger.debug("Done replaying #{progress} events") else - Sequent.logger.debug "Replayed #{progress} events" + Sequent.logger.debug("Replayed #{progress} events") end end diff --git a/lib/sequent/util/timer.rb b/lib/sequent/util/timer.rb index f86cac1f..a9d02298 100644 --- a/lib/sequent/util/timer.rb +++ b/lib/sequent/util/timer.rb @@ -9,7 +9,7 @@ def time(msg) ensure stop = Time.now seconds = stop - start - Sequent.logger.debug("#{msg} in #{seconds} seconds") if seconds > 1 + Sequent.logger.debug("#{msg} in #{seconds} seconds") if seconds > 1 && Sequent.logger.debug? end end end From 4acf40e558970ed3522e923d4553dab54e87905b Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Wed, 27 Mar 2024 15:46:59 +0100 Subject: [PATCH 044/128] Aggregates can now specify the event partition key The `EventStream` class is now a `Data` class, so Ruby 3.1 support is dropped. This fixes tests that otherwise expect the exact same instance to be returned. --- .github/workflows/rspec.yml | 2 -- lib/sequent/core/aggregate_root.rb | 23 +++++++++++++++++++---- lib/sequent/core/stream_record.rb | 9 ++------- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/.github/workflows/rspec.yml b/.github/workflows/rspec.yml index acf7b8aa..298207bd 100644 --- a/.github/workflows/rspec.yml +++ b/.github/workflows/rspec.yml @@ -25,8 +25,6 @@ jobs: gemfile: 'ar_7_1' - ruby-version: '3.2' gemfile: 'ar_7_1' - - ruby-version: 3.1.3 - gemfile: 'ar_7_1' env: BUNDLE_GEMFILE: ${{ github.workspace }}/gemfiles/${{ matrix.gemfile }}.gemfile services: diff --git a/lib/sequent/core/aggregate_root.rb b/lib/sequent/core/aggregate_root.rb index 30f3e511..f0d0e8e6 100644 --- a/lib/sequent/core/aggregate_root.rb +++ b/lib/sequent/core/aggregate_root.rb @@ -41,7 +41,7 @@ class AggregateRoot include SnapshotConfiguration extend ActiveSupport::DescendantsTracker - attr_reader :id, :uncommitted_events, :sequence_number, :event_stream + attr_reader :id, :uncommitted_events, :sequence_number def self.load_from_history(stream, events) first, *rest = events @@ -61,9 +61,6 @@ def initialize(id) @id = id @uncommitted_events = [] @sequence_number = 1 - @event_stream = EventStream.new aggregate_type: self.class.name, - aggregate_id: id, - snapshot_threshold: self.class.snapshot_default_threshold end def load_from_history(stream, events) @@ -100,6 +97,24 @@ def to_s "#{self.class.name}: #{@id}" end + def event_stream + EventStream.new aggregate_type: self.class.name, + aggregate_id: id, + events_partition_key: events_partition_key, + snapshot_threshold: self.class.snapshot_default_threshold + end + + # Provide the partitioning key for storing events. This value + # must be a string and will be used by PostgreSQL to store the + # events in the right partition. + # + # The value may change over the lifetime of the aggregate, old + # events will be moved to the correct partition after a + # change. This can be an expensive database operation. + def events_partition_key + nil + end + def clear_events @uncommitted_events = [] end diff --git a/lib/sequent/core/stream_record.rb b/lib/sequent/core/stream_record.rb index f9748f60..a3127e9f 100644 --- a/lib/sequent/core/stream_record.rb +++ b/lib/sequent/core/stream_record.rb @@ -4,14 +4,9 @@ module Sequent module Core - class EventStream - attr_accessor :aggregate_type, :aggregate_id, :events_partition_key, :snapshot_threshold - + EventStream = Data.define(:aggregate_type, :aggregate_id, :events_partition_key, :snapshot_threshold) do def initialize(aggregate_type:, aggregate_id:, events_partition_key: '', snapshot_threshold: nil) - @aggregate_type = aggregate_type - @aggregate_id = aggregate_id - @events_partition_key = events_partition_key - @snapshot_threshold = snapshot_threshold + super(aggregate_type:, aggregate_id:, events_partition_key:, snapshot_threshold:) end end From 4c8c6b7e1101786ea276dd82a5ca0f88dfd3812c Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Thu, 28 Mar 2024 09:22:13 +0100 Subject: [PATCH 045/128] Use partitioned table schema --- db/sequent_schema.rb | 4 +- db/sequent_schema.sql | 641 ++++++++++++++++++++++++++++++++++++------ spec/spec_helper.rb | 2 +- 3 files changed, 561 insertions(+), 86 deletions(-) diff --git a/db/sequent_schema.rb b/db/sequent_schema.rb index 80c2e154..4c8d9003 100644 --- a/db/sequent_schema.rb +++ b/db/sequent_schema.rb @@ -2,10 +2,10 @@ ActiveRecord::Schema.define do create_table 'command_records', force: true do |t| - t.string 'user_id' + t.uuid 'user_id' t.uuid 'aggregate_id' t.string 'command_type', null: false - t.string 'event_aggregate_id' + t.uuid 'event_aggregate_id' t.integer 'event_sequence_number' t.jsonb 'command_json', null: false t.datetime 'created_at', null: false diff --git a/db/sequent_schema.sql b/db/sequent_schema.sql index 8340f173..3ae74b6e 100644 --- a/db/sequent_schema.sql +++ b/db/sequent_schema.sql @@ -8,15 +8,458 @@ CREATE TYPE aggregate_event_type AS ( event_json jsonb ); +CREATE TABLE command_types (id SMALLINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, type text UNIQUE NOT NULL); +CREATE TABLE aggregate_types (id SMALLINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, type text UNIQUE NOT NULL); +CREATE TABLE event_types (id SMALLINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, type text UNIQUE NOT NULL); + +CREATE TABLE commands ( + id bigint DEFAULT nextval('command_records_id_seq'), + created_at timestamp with time zone NOT NULL, + user_id uuid, + aggregate_id uuid, + command_type_id SMALLINT NOT NULL REFERENCES command_types (id), + command_json jsonb NOT NULL, + event_aggregate_id uuid, + event_sequence_number integer, + PRIMARY KEY (id) +) PARTITION BY RANGE (id); +CREATE INDEX commands_command_type_id_idx ON commands (command_type_id); +CREATE INDEX commands_aggregate_id_idx ON commands (aggregate_id); +CREATE INDEX commands_event_idx ON commands (event_aggregate_id, event_sequence_number); + +CREATE TABLE commands_default PARTITION OF commands DEFAULT; + +CREATE TABLE aggregates ( + aggregate_id uuid PRIMARY KEY, + events_partition_key text NOT NULL DEFAULT '', + aggregate_type_id SMALLINT NOT NULL REFERENCES aggregate_types (id), + snapshot_threshold integer, + created_at timestamp with time zone NOT NULL DEFAULT NOW(), + UNIQUE (events_partition_key, aggregate_id) +) PARTITION BY RANGE (aggregate_id); +CREATE INDEX aggregates_aggregate_type_id_idx ON aggregates (aggregate_type_id); + +CREATE TABLE aggregates_0 PARTITION OF aggregates FOR VALUES FROM (MINVALUE) TO ('40000000-0000-0000-0000-000000000000'); +ALTER TABLE aggregates_0 CLUSTER ON aggregates_0_events_partition_key_aggregate_id_key; +CREATE TABLE aggregates_4 PARTITION OF aggregates FOR VALUES FROM ('40000000-0000-0000-0000-000000000000') TO ('80000000-0000-0000-0000-000000000000'); +ALTER TABLE aggregates_4 CLUSTER ON aggregates_4_events_partition_key_aggregate_id_key; +CREATE TABLE aggregates_8 PARTITION OF aggregates FOR VALUES FROM ('80000000-0000-0000-0000-000000000000') TO ('c0000000-0000-0000-0000-000000000000'); +ALTER TABLE aggregates_8 CLUSTER ON aggregates_8_events_partition_key_aggregate_id_key; +CREATE TABLE aggregates_c PARTITION OF aggregates FOR VALUES FROM ('c0000000-0000-0000-0000-000000000000') TO (MAXVALUE); +ALTER TABLE aggregates_c CLUSTER ON aggregates_c_events_partition_key_aggregate_id_key; + +CREATE TABLE events ( + aggregate_id uuid NOT NULL, + partition_key text DEFAULT '', + sequence_number integer NOT NULL, + created_at timestamp with time zone NOT NULL, + command_id bigint NOT NULL, + event_type_id SMALLINT NOT NULL REFERENCES event_types (id), + event_json jsonb NOT NULL, + xact_id bigint DEFAULT pg_current_xact_id()::text::bigint, + PRIMARY KEY (partition_key, aggregate_id, sequence_number), + FOREIGN KEY (partition_key, aggregate_id) + REFERENCES aggregates (events_partition_key, aggregate_id) + ON UPDATE CASCADE ON DELETE RESTRICT, + FOREIGN KEY (command_id) REFERENCES commands (id) +) PARTITION BY RANGE (partition_key); +CREATE INDEX events_command_id_idx ON events (command_id); +CREATE INDEX events_event_type_id_idx ON events (event_type_id); +CREATE INDEX events_xact_id_idx ON events (xact_id) WHERE xact_id IS NOT NULL; + +CREATE TABLE events_default PARTITION OF events DEFAULT; +ALTER TABLE events_default CLUSTER ON events_default_pkey; +CREATE TABLE events_2023_and_earlier PARTITION OF events FOR VALUES FROM ('Y00') TO ('Y24'); +ALTER TABLE events_2023_and_earlier CLUSTER ON events_2023_and_earlier_pkey; +CREATE TABLE events_2024 PARTITION OF events FOR VALUES FROM ('Y24') TO ('Y25'); +ALTER TABLE events_2024 CLUSTER ON events_2024_pkey; +CREATE TABLE events_2025_and_later PARTITION OF events FOR VALUES FROM ('Y25') TO ('Y99'); +ALTER TABLE events_2025_and_later CLUSTER ON events_2025_and_later_pkey; +CREATE TABLE events_organizations PARTITION OF events FOR VALUES FROM ('O') TO ('Og'); +ALTER TABLE events_organizations CLUSTER ON events_organizations_pkey; +CREATE TABLE events_aggregate PARTITION OF events FOR VALUES FROM ('A') TO ('Ag'); +ALTER TABLE events_aggregate CLUSTER ON events_aggregate_pkey; + +TRUNCATE TABLE snapshot_records; +ALTER TABLE snapshot_records + ALTER COLUMN created_at TYPE timestamptz USING created_at AT TIME ZONE 'Europe/Amsterdam', + ALTER COLUMN snapshot_type TYPE text, + ALTER COLUMN snapshot_json TYPE jsonb USING snapshot_json::jsonb, + DROP CONSTRAINT stream_fkey, + ADD CONSTRAINT aggregate_fkey FOREIGN KEY (aggregate_id) REFERENCES aggregates (aggregate_id) + ON UPDATE CASCADE ON DELETE CASCADE; + +ALTER TABLE command_records RENAME TO old_command_records; +ALTER TABLE event_records RENAME TO old_event_records; +ALTER TABLE stream_records RENAME TO old_stream_records; + +CREATE OR REPLACE FUNCTION enrich_command_json(command commands) RETURNS jsonb +LANGUAGE plpgsql AS $$ +BEGIN + RETURN jsonb_build_object( + 'command_type', (SELECT type FROM command_types WHERE command_types.id = command.command_type_id), + 'created_at', command.created_at, + 'user_id', command.user_id, + 'aggregate_id', command.aggregate_id, + 'event_aggregate_id', command.event_aggregate_id, + 'event_sequence_number', command.event_sequence_number + ) + || command.command_json; +END +$$; + +CREATE VIEW command_records (id, user_id, aggregate_id, command_type, command_json, created_at, event_aggregate_id, event_sequence_number) AS + SELECT id, + user_id, + aggregate_id, + (SELECT type FROM command_types WHERE command_types.id = command.command_type_id), + enrich_command_json(command), + created_at, + event_aggregate_id, + event_sequence_number + FROM commands command + UNION ALL + SELECT id, + user_id, + aggregate_id, + command_type, + command_json::jsonb, + created_at, + event_aggregate_id, + event_sequence_number + FROM old_command_records; + +CREATE OR REPLACE FUNCTION enrich_event_json(event events) RETURNS jsonb +LANGUAGE plpgsql AS $$ +BEGIN + RETURN jsonb_build_object( + 'aggregate_id', event.aggregate_id, + 'sequence_number', event.sequence_number, + 'created_at', event.created_at + ) + || event.event_json; +END +$$; + +CREATE VIEW event_records (aggregate_id, partition_key, sequence_number, created_at, event_type, event_json, command_record_id, xact_id) AS + SELECT aggregate.aggregate_id, + event.partition_key, + event.sequence_number, + event.created_at, + type.type, + enrich_event_json(event.*)::text AS event_json, + command_id, + event.xact_id + FROM aggregates aggregate + JOIN events event ON aggregate.aggregate_id = event.aggregate_id AND aggregate.events_partition_key = event.partition_key + JOIN event_types type ON event.event_type_id = type.id + UNION ALL + SELECT aggregate_id, + '', + sequence_number, + (event_json::jsonb->>'created_at')::timestamptz AS created_at, + event_type::text, + event_json::text, + command_record_id, + xact_id + FROM old_event_records; + +CREATE VIEW stream_records (aggregate_id, events_partition_key, aggregate_type, snapshot_threshold, created_at) AS + SELECT aggregates.aggregate_id, + aggregates.events_partition_key, + aggregate_types.type, + aggregates.snapshot_threshold, + aggregates.created_at + FROM aggregates JOIN aggregate_types ON aggregates.aggregate_type_id = aggregate_types.id + UNION ALL + SELECT aggregate_id, + NULL, + aggregate_type, + snapshot_threshold, + created_at + FROM old_stream_records; + +INSERT INTO command_types (type) SELECT DISTINCT command_type FROM command_records ORDER BY 1; +INSERT INTO aggregate_types (type) SELECT DISTINCT aggregate_type FROM stream_records ORDER BY 1; +INSERT INTO event_types (type) SELECT DISTINCT event_type FROM old_event_records ORDER BY 1; + +CREATE OR REPLACE FUNCTION determine_events_partition_key(_aggregate_id uuid, _organization_id uuid, _event_json jsonb) RETURNS text AS $$ +DECLARE + _event_json_without_nulls jsonb = jsonb_strip_nulls(_event_json); + _date text; + _partition_key text; +BEGIN + _date = (CASE + WHEN _event_json_without_nulls->'booking_date' IS NOT NULL THEN + regexp_replace(_event_json_without_nulls->>'booking_date', '[ _-]', '', 'g') + WHEN _event_json_without_nulls->'year_of_delivery' IS NOT NULL AND _event_json_without_nulls->'month_of_delivery' IS NOT NULL THEN + trim(to_char((_event_json_without_nulls->>'year_of_delivery')::integer, '0000')) || trim(to_char((_event_json_without_nulls->>'month_of_delivery')::integer, '00')) + WHEN _event_json_without_nulls->'happened_at' IS NOT NULL THEN + regexp_replace(LEFT((_event_json_without_nulls->>'happened_at')::text, 10), '[ _-]', '', 'g') + WHEN _event_json->'book_date' IS NOT NULL THEN + regexp_replace(LEFT((_event_json_without_nulls->>'book_date')::text, 10), '[ _-]', '', 'g') + WHEN _event_json->>'event_type' NOT IN ( + 'BankCreatedEvent', + 'CompleteOrganizationCreatedEvent', + 'CustomerCreatedEvent', + 'CustomerLedgerAccountCreated', + 'EmailProxyCreated', + 'EstimateNumbersCreated', + 'ExpenseNumbersCreatedEvent', + 'InboxNamesCreated', + 'InvoiceNumbersCreatedEvent', + 'LabelableCreated', + 'LabelCreated', + 'OrganizationBankCreated', + 'OrganizationCreatedAccountEvent', + 'OrganizationCreatedByAccountantEvent', + 'OrganizationCreatedEvent', + 'OrganizationEstimateNumbersCreated', + 'OrganizationExpenseNumbersCreated', + 'OrganizationInvoiceNumbersCreated', + 'OrganizationLedgerCreated', + 'PortalConnectionCreated', + 'PortalConnectionsCreated', + 'ProWowOrganizationCreatedEvent', + 'RootRuleCreated', + 'RuleCreated', + 'UserCreatedEvent', + 'UserCreatedByAccountantEvent' + ) THEN + to_char((_event_json->>'created_at')::timestamptz, 'YYYYMMDD') + ELSE + NULL + END); + + IF _date IS NOT NULL AND _organization_id IS NOT NULL THEN + _partition_key = 'Y' || SUBSTRING(_date FROM 3 FOR 2) || 'O' || LEFT(_organization_id::text, 4); + ELSIF _organization_id IS NOT NULL THEN + _partition_key = 'O' || LEFT(_organization_id::text, 4); + ELSE + _partition_key = 'A' || LEFT(_aggregate_id::text, 2); + END IF; + + RETURN _partition_key; +END +$$ +LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION upsert_aggregate_type(_type aggregate_types.type%TYPE) RETURNS SMALLINT +LANGUAGE plpgsql AS +$$ +DECLARE + _id aggregate_types.id%TYPE; +BEGIN + SELECT id INTO _id FROM aggregate_types WHERE type = _type; + IF NOT FOUND THEN + INSERT INTO aggregate_types (type) VALUES (_type) RETURNING id INTO STRICT _id; + END IF; + RETURN _id; +END +$$; + +CREATE OR REPLACE FUNCTION upsert_command_type(_type command_types.type%TYPE) RETURNS SMALLINT +LANGUAGE plpgsql AS +$$ +DECLARE + _id command_types.id%TYPE; +BEGIN + SELECT id INTO _id FROM command_types WHERE type = _type; + IF NOT FOUND THEN + RAISE NOTICE 'command type % not found, inserting', _type; + INSERT INTO command_types (type) VALUES (_type) RETURNING id INTO STRICT _id; + END IF; + RETURN _id; +END +$$; + +CREATE OR REPLACE FUNCTION upsert_event_type(_type event_types.type%TYPE) RETURNS SMALLINT +LANGUAGE plpgsql AS +$$ +DECLARE + _id event_types.id%TYPE; +BEGIN + SELECT id INTO _id FROM event_types WHERE type = _type; + IF NOT FOUND THEN + INSERT INTO event_types (type) VALUES (_type) RETURNING id INTO STRICT _id; + END IF; + RETURN _id; +END +$$; + +CREATE OR REPLACE PROCEDURE migrate_command(_command_id commands.id%TYPE) +LANGUAGE plpgsql AS $$ +DECLARE + _command_type text; + _command_record command_records; + _command_without_nulls jsonb; +BEGIN + IF EXISTS (SELECT 1 FROM commands WHERE id = _command_id) THEN + RETURN; + END IF; + + SELECT * + INTO _command_record + FROM command_records + WHERE id = _command_id; + IF NOT FOUND THEN + -- Event without command + RETURN; + END IF; + + _command_without_nulls = jsonb_strip_nulls(_command_record.command_json::jsonb); + + INSERT INTO commands ( + id, created_at, user_id, aggregate_id, command_type_id, command_json, + event_aggregate_id, event_sequence_number + ) VALUES ( + _command_id, + COALESCE((_command_without_nulls->>'created_at')::timestamptz, _command_record.created_at AT TIME ZONE 'Europe/Amsterdam'), + (_command_without_nulls->>'user_id')::uuid, + (_command_without_nulls->>'aggregate_id')::uuid, + upsert_command_type(_command_record.command_type), + _command_record.command_json::jsonb - '{created_at,organization_id,user_id,aggregate_id,event_aggregate_id,event_sequence_number}'::text[], + (_command_without_nulls->>'event_aggregate_id')::uuid, + (_command_without_nulls->'event_sequence_number')::integer + ); +END +$$; + +CREATE OR REPLACE FUNCTION migrate_aggregate(_aggregate_id uuid, _provided_events_partition_key text) RETURNS boolean AS $$ +DECLARE + _aggregate_with_first_event RECORD; + _events_partition_key text; + _event_json jsonb; + _event old_event_records; +BEGIN + SELECT s.*, e.event_type, e.event_json + INTO _aggregate_with_first_event + FROM old_stream_records s JOIN old_event_records e ON s.aggregate_id = e.aggregate_id + WHERE s.aggregate_id = _aggregate_id + AND e.sequence_number = 1; + IF NOT FOUND THEN + RETURN FALSE; + END IF; + + _event_json = _aggregate_with_first_event.event_json::jsonb; + _events_partition_key = COALESCE( + _provided_events_partition_key, + determine_events_partition_key(_aggregate_with_first_event.aggregate_id, NULL, _event_json) + ); + + INSERT INTO aggregates (aggregate_id, created_at, aggregate_type_id, events_partition_key, snapshot_threshold) + VALUES ( + _aggregate_id, + _aggregate_with_first_event.created_at AT TIME ZONE 'Europe/Amsterdam', + upsert_aggregate_type(_aggregate_with_first_event.aggregate_type), + COALESCE(_events_partition_key, ''), + _aggregate_with_first_event.snapshot_threshold + ); + + FOR _event IN SELECT * FROM old_event_records WHERE aggregate_id = _aggregate_id ORDER BY sequence_number LOOP + _event_json = _event.event_json::jsonb; + + CALL migrate_command(_event.command_record_id); + + INSERT INTO events (partition_key, aggregate_id, sequence_number, created_at, command_id, event_type_id, event_json, xact_id) + VALUES ( + _events_partition_key, + _aggregate_id, + _event.sequence_number, + (_event_json->>'created_at')::timestamptz, + _event.command_record_id, + upsert_event_type(_event.event_type), + _event_json - '{aggregate_id,organization_id,created_at,sequence_number}'::text[], + _event.xact_id + ); + END LOOP; + + DELETE FROM old_event_records WHERE aggregate_id = _aggregate_id; + DELETE FROM old_stream_records WHERE aggregate_id = _aggregate_id; + + RETURN TRUE; +END; +$$ +LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION load_old_events( + _aggregate_id old_stream_records.aggregate_id%TYPE, + _use_snapshots boolean DEFAULT TRUE, + _until event_records.created_at%TYPE DEFAULT NULL +) RETURNS SETOF aggregate_event_type AS $$ +DECLARE + _snapshot_event snapshot_records; + _snapshot_event_sequence_number integer = 0; + _stream_record old_stream_records; +BEGIN + SELECT * + INTO _stream_record + FROM old_stream_records + WHERE old_stream_records.aggregate_id = _aggregate_id; + IF NOT FOUND THEN + RETURN; + END IF; + + IF _use_snapshots THEN + SELECT * + INTO _snapshot_event + FROM snapshot_records snapshot + WHERE snapshot.aggregate_id = _aggregate_id + ORDER BY snapshot.sequence_number DESC + LIMIT 1; + IF FOUND THEN + RETURN NEXT (_stream_record.aggregate_type, + _stream_record.aggregate_id, + '', + _stream_record.snapshot_threshold, + _snapshot_event.snapshot_type::text, + _snapshot_event.snapshot_json::jsonb); + _snapshot_event_sequence_number = _snapshot_event.sequence_number; + END IF; + END IF; + + RETURN QUERY SELECT _stream_record.aggregate_type, + _stream_record.aggregate_id, + '', + _stream_record.snapshot_threshold, + event.event_type::text, + event.event_json::jsonb + FROM old_event_records event + WHERE aggregate_id = _aggregate_id::uuid + AND sequence_number >= _snapshot_event_sequence_number + AND (_until IS NULL OR created_at < _until) + ORDER BY sequence_number; +END; +$$ +LANGUAGE plpgsql; + CREATE OR REPLACE FUNCTION load_event( _aggregate_id uuid, _sequence_number integer ) RETURNS SETOF aggregate_event_type LANGUAGE plpgsql AS $$ +DECLARE + _aggregate aggregates; + _aggregate_type text; BEGIN - RETURN QUERY SELECT aggregate_type::text, _aggregate_id, ''::text, snapshot_threshold, event_type::text, event_json::jsonb - FROM event_records event JOIN stream_records stream ON event.aggregate_id = stream.aggregate_id - WHERE stream.aggregate_id = _aggregate_id + SELECT * INTO _aggregate + FROM aggregates + WHERE aggregate_id = _aggregate_id; + IF NOT FOUND THEN + RETURN QUERY SELECT aggregate_type::text, _aggregate_id, ''::text, snapshot_threshold, event_type::text, event_json::jsonb + FROM old_event_records event JOIN old_stream_records stream ON event.aggregate_id = stream.aggregate_id + WHERE stream.aggregate_id = _aggregate_id + AND sequence_number = _sequence_number; + RETURN; + END IF; + + SELECT type INTO STRICT _aggregate_type + FROM aggregate_types + WHERE id = _aggregate.aggregate_type_id; + + RETURN QUERY SELECT _aggregate_type, aggregate_id, _aggregate.events_partition_key, _aggregate.snapshot_threshold, event_type, event_json::jsonb + FROM event_records + WHERE aggregate_id = _aggregate_id AND sequence_number = _sequence_number; END; $$; @@ -28,53 +471,49 @@ CREATE OR REPLACE FUNCTION load_events( ) RETURNS SETOF aggregate_event_type LANGUAGE plpgsql AS $$ DECLARE - _aggregate_id event_records.aggregate_id%TYPE; - _snapshot_event snapshot_records; - _snapshot_event_sequence_number integer; - _stream_record stream_records; + _aggregate_type text; + _aggregate_id aggregates.aggregate_id%TYPE; + _aggregate aggregates; + _snapshot snapshot_records; + _start_sequence_number events.sequence_number%TYPE; BEGIN FOR _aggregate_id IN SELECT * FROM jsonb_array_elements_text(_aggregate_ids) LOOP - SELECT * - INTO _stream_record - FROM stream_records - WHERE stream_records.aggregate_id = _aggregate_id; + SELECT * INTO _aggregate FROM aggregates WHERE aggregates.aggregate_id = _aggregate_id; IF NOT FOUND THEN + RETURN QUERY SELECT * FROM load_old_events(_aggregate_id::uuid, _use_snapshots, _until); CONTINUE; END IF; - _snapshot_event = NULL; - _snapshot_event_sequence_number = 0; + SELECT type INTO STRICT _aggregate_type + FROM aggregate_types + WHERE id = _aggregate.aggregate_type_id; + + _start_sequence_number = 0; IF _use_snapshots THEN - SELECT * INTO _snapshot_event - FROM snapshot_records e - WHERE e.aggregate_id = _aggregate_id - AND (_until IS NULL OR e.created_at < _until) - ORDER BY e.sequence_number DESC - LIMIT 1; + SELECT * INTO _snapshot FROM snapshot_records snapshots WHERE snapshots.aggregate_id = _aggregate.aggregate_id ORDER BY sequence_number DESC LIMIT 1; IF FOUND THEN - RETURN NEXT ( - _stream_record.aggregate_type::text, - _stream_record.aggregate_id, - ''::text, - _stream_record.snapshot_threshold, - _snapshot_event.snapshot_type::text, - _snapshot_event.snapshot_json::jsonb - ); - _snapshot_event_sequence_number = _snapshot_event.sequence_number; + _start_sequence_number := _snapshot.sequence_number; + RETURN NEXT (_aggregate_type, + _aggregate.aggregate_id, + _aggregate.events_partition_key, + _aggregate.snapshot_threshold, + _snapshot.snapshot_type, + _snapshot.snapshot_json); END IF; END IF; - - RETURN QUERY SELECT _stream_record.aggregate_type::text, - _stream_record.aggregate_id, - ''::text, - _stream_record.snapshot_threshold, - e.event_type::text, - e.event_json::jsonb - FROM event_records e - WHERE e.aggregate_id = _aggregate_id - AND e.sequence_number >= _snapshot_event_sequence_number - AND (_until IS NULL OR e.created_at < _until) - ORDER BY e.sequence_number; + RETURN QUERY SELECT _aggregate_type, + _aggregate.aggregate_id, + _aggregate.events_partition_key, + _aggregate.snapshot_threshold, + event_types.type, + enrich_event_json(events) + FROM events + INNER JOIN event_types ON events.event_type_id = event_types.id + WHERE events.partition_key = _aggregate.events_partition_key + AND events.aggregate_id = _aggregate.aggregate_id + AND events.sequence_number >= _start_sequence_number + AND (_until IS NULL OR events.created_at < _until) + ORDER BY events.sequence_number; END LOOP; END; $$; @@ -82,18 +521,18 @@ $$; CREATE OR REPLACE FUNCTION store_command(_command jsonb) RETURNS bigint LANGUAGE plpgsql AS $$ DECLARE - _id command_records.id%TYPE; + _id commands.id%TYPE; _command_without_nulls jsonb = jsonb_strip_nulls(_command->'command_json'); BEGIN - INSERT INTO command_records ( - created_at, user_id, aggregate_id, command_type, command_json, + INSERT INTO commands ( + created_at, user_id, aggregate_id, command_type_id, command_json, event_aggregate_id, event_sequence_number ) VALUES ( - (_command->>'created_at')::timestamp, + (_command->>'created_at')::timestamptz, (_command_without_nulls->>'user_id')::uuid, (_command_without_nulls->>'aggregate_id')::uuid, - _command->>'command_type', - _command->'command_json', + upsert_command_type(_command->>'command_type'), + (_command->'command_json') - '{command_type,created_at,organization_id,user_id,aggregate_id,event_aggregate_id,event_sequence_number}'::text[], (_command_without_nulls->>'event_aggregate_id')::uuid, (_command_without_nulls->'event_sequence_number')::integer ) RETURNING id INTO STRICT _id; @@ -104,45 +543,62 @@ $$; CREATE OR REPLACE PROCEDURE store_events(_command jsonb, _aggregates_with_events jsonb) LANGUAGE plpgsql AS $$ DECLARE - _command_record_id command_records.id%TYPE; + _command_id commands.id%TYPE; _aggregate jsonb; _aggregate_without_nulls jsonb; _events jsonb; _event jsonb; - _aggregate_id stream_records.aggregate_id%TYPE; - _created_at stream_records.created_at%TYPE; - _snapshot_threshold stream_records.snapshot_threshold%TYPE; - _sequence_number event_records.sequence_number%TYPE; + _aggregate_id aggregates.aggregate_id%TYPE; + _created_at aggregates.created_at%TYPE; + _provided_events_partition_key aggregates.events_partition_key%TYPE; + _existing_events_partition_key aggregates.events_partition_key%TYPE; + _events_partition_key aggregates.events_partition_key%TYPE; + _snapshot_threshold aggregates.snapshot_threshold%TYPE; + _sequence_number events.sequence_number%TYPE; BEGIN - _command_record_id = store_command(_command); + _command_id = store_command(_command); FOR _aggregate, _events IN SELECT row->0, row->1 FROM jsonb_array_elements(_aggregates_with_events) AS row LOOP - _aggregate_id = (_aggregate->>'aggregate_id')::uuid; + _aggregate_id = _aggregate->>'aggregate_id'; _aggregate_without_nulls = jsonb_strip_nulls(_aggregate); _snapshot_threshold = _aggregate_without_nulls->'snapshot_threshold'; + _provided_events_partition_key = _aggregate_without_nulls->>'events_partition_key'; - IF NOT EXISTS (SELECT 1 FROM stream_records WHERE aggregate_id = _aggregate_id) THEN - _created_at = _events->0->>'created_at'; - _sequence_number = _events->0->'event_json'->'sequence_number'; - IF _sequence_number <> 1 THEN - RAISE EXCEPTION 'sequence number of first event new aggregate must be 1, was %', _sequence_number; - END IF; + PERFORM migrate_aggregate(_aggregate_id, _provided_events_partition_key); - INSERT INTO stream_records (created_at, aggregate_type, aggregate_id, snapshot_threshold) - VALUES (_created_at, _aggregate->>'aggregate_type', _aggregate_id, _snapshot_threshold); + SELECT events_partition_key INTO _existing_events_partition_key FROM aggregates WHERE aggregate_id = _aggregate_id; + IF NOT FOUND THEN + _events_partition_key = COALESCE(_provided_events_partition_key, determine_events_partition_key(_aggregate_id, NULL, _events->0->'event_json')); + ELSE + _events_partition_key = COALESCE(_provided_events_partition_key, _existing_events_partition_key); END IF; + + INSERT INTO aggregates (aggregate_id, created_at, aggregate_type_id, events_partition_key, snapshot_threshold) + VALUES ( + _aggregate_id, + (_events->0->>'created_at')::timestamptz, + upsert_aggregate_type(_aggregate->>'aggregate_type'), + COALESCE(_events_partition_key, ''), + _snapshot_threshold + ) ON CONFLICT (aggregate_id) + DO UPDATE SET events_partition_key = EXCLUDED.events_partition_key, + snapshot_threshold = EXCLUDED.snapshot_threshold + WHERE aggregates.events_partition_key <> EXCLUDED.events_partition_key + OR aggregates.snapshot_threshold <> EXCLUDED.snapshot_threshold; + FOR _event IN SELECT * FROM jsonb_array_elements(_events) LOOP - _created_at = _event->'created_at'; - _sequence_number = _event->'event_json'->'sequence_number'; - INSERT INTO event_records (aggregate_id, sequence_number, created_at, event_type, event_json, command_record_id) - VALUES ( - (_event->'event_json'->>'aggregate_id')::uuid, + _created_at = (_event->>'created_at')::timestamptz; + _sequence_number = _event->'event_json'->>'sequence_number'; + INSERT INTO events (partition_key, aggregate_id, sequence_number, created_at, command_id, event_type_id, event_json) + VALUES ( + _events_partition_key, + _aggregate_id, _sequence_number, _created_at, - _event->>'event_type', - _event->'event_json', - _command_record_id + _command_id, + upsert_event_type(_event->>'event_type'), + (_event->'event_json') - '{aggregate_id,created_at,event_type,organization_id,sequence_number,stream_record_id}'::text[] ); END LOOP; END LOOP; @@ -153,10 +609,14 @@ CREATE OR REPLACE PROCEDURE store_snapshots(_snapshots jsonb) LANGUAGE plpgsql AS $$ DECLARE _aggregate_id uuid; + _events_partition_key text; _snapshot jsonb; BEGIN FOR _snapshot IN SELECT * FROM jsonb_array_elements(_snapshots) LOOP _aggregate_id = _snapshot->>'aggregate_id'; + + PERFORM migrate_aggregate(_aggregate_id, NULL); + INSERT INTO snapshot_records (aggregate_id, sequence_number, created_at, snapshot_type, snapshot_json) VALUES ( _aggregate_id, @@ -171,10 +631,15 @@ $$; CREATE OR REPLACE FUNCTION load_latest_snapshot(_aggregate_id uuid) RETURNS aggregate_event_type LANGUAGE SQL AS $$ - SELECT a.aggregate_type, a.aggregate_id, '', a.snapshot_threshold, s.snapshot_type, s.snapshot_json::jsonb - FROM snapshot_records s JOIN stream_records a ON s.aggregate_id = a.aggregate_id - WHERE s.aggregate_id = _aggregate_id - ORDER BY sequence_number DESC + SELECT (SELECT type FROM aggregate_types WHERE id = a.aggregate_type_id), + a.aggregate_id, + a.events_partition_key, + a.snapshot_threshold, + s.snapshot_type, + s.snapshot_json + FROM aggregates a JOIN snapshot_records s ON a.aggregate_id = s.aggregate_id + WHERE a.aggregate_id = _aggregate_id + ORDER BY s.sequence_number DESC LIMIT 1; $$; @@ -206,28 +671,38 @@ $$; CREATE OR REPLACE PROCEDURE permanently_delete_commands_without_events(_aggregate_id uuid, _organization_id uuid) LANGUAGE plpgsql AS $$ BEGIN - IF _organization_id IS NOT NULL THEN - RAISE EXCEPTION 'deleting by organization_id is not supported by this version of Sequent'; - END IF; IF _aggregate_id IS NULL AND _organization_id IS NULL THEN RAISE EXCEPTION 'aggregate_id or organization_id must be specified to delete commands'; END IF; - DELETE FROM command_records + DELETE FROM old_command_records + WHERE (_aggregate_id IS NULL OR aggregate_id = _aggregate_id) + AND NOT EXISTS (SELECT 1 FROM events WHERE command_id = old_command_records.id) + AND NOT EXISTS (SELECT 1 FROM old_event_records WHERE command_record_id = old_command_records.id); + DELETE FROM commands WHERE (_aggregate_id IS NULL OR aggregate_id = _aggregate_id) - --AND (_organization_id IS NULL OR organization_id = _organization_id) - AND NOT EXISTS (SELECT 1 FROM event_records WHERE command_record_id = command_records.id); + AND NOT EXISTS (SELECT 1 FROM events WHERE command_id = commands.id) + AND NOT EXISTS (SELECT 1 FROM old_event_records WHERE command_record_id = commands.id); END; $$; CREATE OR REPLACE PROCEDURE permanently_delete_event_streams(_aggregate_ids jsonb) LANGUAGE plpgsql AS $$ BEGIN - DELETE FROM event_records + DELETE FROM old_event_records + USING jsonb_array_elements_text(_aggregate_ids) AS ids (id) + WHERE old_event_records.aggregate_id = ids.id::uuid; + DELETE FROM old_stream_records + USING jsonb_array_elements_text(_aggregate_ids) AS ids (id) + WHERE old_stream_records.aggregate_id = ids.id::uuid; + + DELETE FROM events USING jsonb_array_elements_text(_aggregate_ids) AS ids (id) - WHERE event_records.aggregate_id = ids.id::uuid; - DELETE FROM stream_records + JOIN aggregates ON ids.id::uuid = aggregates.aggregate_id + WHERE events.partition_key = aggregates.events_partition_key + AND events.aggregate_id = aggregates.aggregate_id; + DELETE FROM aggregates USING jsonb_array_elements_text(_aggregate_ids) AS ids (id) - WHERE stream_records.aggregate_id = ids.id::uuid; + WHERE aggregates.aggregate_id = ids.id::uuid; END; $$; diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 95eebc7e..185cd549 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -23,7 +23,7 @@ RSpec.configure do |c| c.before do Database.establish_connection - Sequent::ApplicationRecord.connection.execute('TRUNCATE command_records, stream_records CASCADE') + Sequent::ApplicationRecord.connection.execute('TRUNCATE commands, aggregates CASCADE') Sequent::Configuration.reset Sequent.configuration.database_config_directory = 'tmp' end From 22179fec12c1cf5542a06b1b57568200a2021c8e Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Tue, 2 Apr 2024 12:45:28 +0200 Subject: [PATCH 046/128] Group partitioned events into approximately equal parts --- lib/sequent/migrations/grouper.rb | 72 ++++++++++++++++++ spec/lib/sequent/migrations/grouper_spec.rb | 84 +++++++++++++++++++++ 2 files changed, 156 insertions(+) create mode 100644 lib/sequent/migrations/grouper.rb create mode 100644 spec/lib/sequent/migrations/grouper_spec.rb diff --git a/lib/sequent/migrations/grouper.rb b/lib/sequent/migrations/grouper.rb new file mode 100644 index 00000000..fbef6631 --- /dev/null +++ b/lib/sequent/migrations/grouper.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +module Sequent + module Migrations + module Grouper + # Generate approximately equally sized groups based on the + # events partition keys and the number of events per partition + # key. Each group is defined by a lower bound (partition-key, + # aggregate-id) and upper bound (partition-key, aggregate-id) + # (inclusive). + # + # For splitting a partition into equal sized groups the + # assumption is made that aggregate-ids and their events are + # equally distributed. + def self.group_partitions(partitions, target_group_size, _minimum_group_count) + return [] unless partitions.present? + + partitions = partitions.sort.map do |key, count| + PartitionData.new(key:, original_size: count, remaining_size: count, lower_bound: 0) + end + + # total_count = partitions.reduce(0) { |acc, (_, count)| acc + count } + result = [] + + partition = partitions.shift + current_start = [partition.key, LOWEST_UUID] + current_size = 0 + + while partition.present? + if current_size + partition.remaining_size < target_group_size + current_size += partition.remaining_size + if partitions.empty? + result << (current_start .. [partition.key, HIGHEST_UUID]) + break + end + partition = partitions.shift + elsif current_size + partition.remaining_size == target_group_size + result << (current_start .. [partition.key, HIGHEST_UUID]) + + partition = partitions.shift + current_start = [partition&.key, LOWEST_UUID] + current_size = 0 + else + taken = target_group_size - current_size + upper_bound = partition.lower_bound + UUID_COUNT * taken / partition.original_size + + result << (current_start .. [partition.key, number_to_uuid(upper_bound - 1)]) + + remaining_size = partition.remaining_size - taken + partition = partition.with(remaining_size:, lower_bound: upper_bound) + current_start = [partition.key, number_to_uuid(upper_bound)] + current_size = 0 + end + end + result + end + + PartitionData = Data.define(:key, :original_size, :remaining_size, :lower_bound) + + def self.number_to_uuid(number) + fail ArgumentError, number unless (0..2**128 - 1).include? number + + s = format('%032x', number) + "#{s[0..7]}-#{s[8..11]}-#{s[12..15]}-#{s[16..19]}-#{s[20..]}" + end + + UUID_COUNT = 2**128 + LOWEST_UUID = number_to_uuid(0) + HIGHEST_UUID = number_to_uuid(UUID_COUNT - 1) + end + end +end diff --git a/spec/lib/sequent/migrations/grouper_spec.rb b/spec/lib/sequent/migrations/grouper_spec.rb new file mode 100644 index 00000000..c533f338 --- /dev/null +++ b/spec/lib/sequent/migrations/grouper_spec.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_relative '../../../../lib/sequent/migrations/grouper' + +describe Sequent::Migrations::Grouper do + let(:subject) { Sequent::Migrations::Grouper } + let(:partitions) do + {a: 200, b: 600, c: 200} + end + + def ensure_invariants(groups) + groups.each do |range| + # start must be <= to end_inclusive for each group + expect(range.begin <=> range.end).to be <= 0 + end + groups.each_cons(2).each do |prev_group, next_group| + # end of previous group must be < start of next group + expect(prev_group.end <=> next_group.end).to be < 0 + end + end + + it 'creates a single group when all partitions fit' do + expect(subject.group_partitions(partitions, 1000, 1)) + .to eq([[:a, subject::LOWEST_UUID] .. [:c, subject::HIGHEST_UUID]]) +end + + it 'creates multiple groups from a single large partition' do + expect(subject.group_partitions({a: 100}, 40, 1)) + .to eq( + [ + [:a, subject::LOWEST_UUID] .. [:a, '66666666-6666-6666-6666-666666666665'], + [:a, '66666666-6666-6666-6666-666666666666'] .. [:a, 'cccccccc-cccc-cccc-cccc-cccccccccccb'], + [:a, 'cccccccc-cccc-cccc-cccc-cccccccccccc'] .. [:a, subject::HIGHEST_UUID], + ], + ) + ensure_invariants(subject.group_partitions({a: 100}, 40, 1)) + end + + context 'splits groups assuming an uniform distribution' do + it 'splits group in half' do + expect(subject.group_partitions(partitions, 500, 1)) + .to eq( + [ + [:a, subject::LOWEST_UUID] .. [:b, '7fffffff-ffff-ffff-ffff-ffffffffffff'], + [:b, '80000000-0000-0000-0000-000000000000'] .. [:c, subject::HIGHEST_UUID], + ], + ) + end + + it 'splits group in three unequal parts' do + expect(subject.group_partitions(partitions, 400, 1)) + .to eq( + [ + [:a, subject::LOWEST_UUID] .. [:b, '55555555-5555-5555-5555-555555555554'], + [:b, '55555555-5555-5555-5555-555555555555'] .. [:b, 'ffffffff-ffff-ffff-ffff-ffffffffffff'], + [:c, '00000000-0000-0000-0000-000000000000'] .. [:c, subject::HIGHEST_UUID], + ], + ) + end + + it 'splits group in three equal parts' do + expect(subject.group_partitions({a: 200, b: 500, c: 200}, 300, 1)) + .to eq( + [ + [:a, subject::LOWEST_UUID] .. [:b, '33333333-3333-3333-3333-333333333332'], + [:b, '33333333-3333-3333-3333-333333333333'] .. [:b, 'cccccccc-cccc-cccc-cccc-cccccccccccb'], + [:b, 'cccccccc-cccc-cccc-cccc-cccccccccccc'] .. [:c, subject::HIGHEST_UUID], + ], + ) + end + + it 'splits group in many equal parts' do + expect(subject.group_partitions({a: 200, b: 500, c: 200}, 300, 1)) + .to eq( + [ + [:a, subject::LOWEST_UUID] .. [:b, '33333333-3333-3333-3333-333333333332'], + [:b, '33333333-3333-3333-3333-333333333333'] .. [:b, 'cccccccc-cccc-cccc-cccc-cccccccccccb'], + [:b, 'cccccccc-cccc-cccc-cccc-cccccccccccc'] .. [:c, subject::HIGHEST_UUID], + ], + ) + end + end +end From 7db8e8fd852227702c3e709fb6eec8d370a59625 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Tue, 2 Apr 2024 13:39:00 +0200 Subject: [PATCH 047/128] Use data class to define group endpoints The data class includes Comparable for easier rspec assertions. --- lib/sequent/migrations/grouper.rb | 28 +++++++--- spec/lib/sequent/migrations/grouper_spec.rb | 58 +++++++++++++-------- 2 files changed, 58 insertions(+), 28 deletions(-) diff --git a/lib/sequent/migrations/grouper.rb b/lib/sequent/migrations/grouper.rb index fbef6631..cfa4b727 100644 --- a/lib/sequent/migrations/grouper.rb +++ b/lib/sequent/migrations/grouper.rb @@ -3,6 +3,16 @@ module Sequent module Migrations module Grouper + GroupEndpoint = Data.define(:partition_key, :aggregate_id) do + def <=>(other) + return unless other.is_a?(self.class) + + [partition_key, aggregate_id] <=> [other.partition_key, other.aggregate_id] + end + + include Comparable + end + # Generate approximately equally sized groups based on the # events partition keys and the number of events per partition # key. Each group is defined by a lower bound (partition-key, @@ -23,32 +33,34 @@ def self.group_partitions(partitions, target_group_size, _minimum_group_count) result = [] partition = partitions.shift - current_start = [partition.key, LOWEST_UUID] + current_start = GroupEndpoint.new(partition.key, LOWEST_UUID) current_size = 0 while partition.present? if current_size + partition.remaining_size < target_group_size current_size += partition.remaining_size if partitions.empty? - result << (current_start .. [partition.key, HIGHEST_UUID]) + result << (current_start..GroupEndpoint.new(partition.key, HIGHEST_UUID)) break end partition = partitions.shift elsif current_size + partition.remaining_size == target_group_size - result << (current_start .. [partition.key, HIGHEST_UUID]) + result << (current_start..GroupEndpoint.new(partition.key, HIGHEST_UUID)) partition = partitions.shift - current_start = [partition&.key, LOWEST_UUID] + break unless partition + + current_start = GroupEndpoint.new(partition.key, LOWEST_UUID) current_size = 0 else taken = target_group_size - current_size upper_bound = partition.lower_bound + UUID_COUNT * taken / partition.original_size - result << (current_start .. [partition.key, number_to_uuid(upper_bound - 1)]) + result << (current_start..GroupEndpoint.new(partition.key, number_to_uuid(upper_bound - 1))) remaining_size = partition.remaining_size - taken partition = partition.with(remaining_size:, lower_bound: upper_bound) - current_start = [partition.key, number_to_uuid(upper_bound)] + current_start = GroupEndpoint.new(partition.key, number_to_uuid(upper_bound)) current_size = 0 end end @@ -64,6 +76,10 @@ def self.number_to_uuid(number) "#{s[0..7]}-#{s[8..11]}-#{s[12..15]}-#{s[16..19]}-#{s[20..]}" end + def self.uuid_to_number(uuid) + Integer(uuid.gsub(/-/, ''), 16) + end + UUID_COUNT = 2**128 LOWEST_UUID = number_to_uuid(0) HIGHEST_UUID = number_to_uuid(UUID_COUNT - 1) diff --git a/spec/lib/sequent/migrations/grouper_spec.rb b/spec/lib/sequent/migrations/grouper_spec.rb index c533f338..ce37338a 100644 --- a/spec/lib/sequent/migrations/grouper_spec.rb +++ b/spec/lib/sequent/migrations/grouper_spec.rb @@ -4,34 +4,48 @@ require_relative '../../../../lib/sequent/migrations/grouper' describe Sequent::Migrations::Grouper do + EP = ->(partition_key, aggregate_id) { Sequent::Migrations::Grouper::GroupEndpoint.new(partition_key, aggregate_id) } + NEXT_UUID = ->(uuid) { + Sequent::Migrations::Grouper.number_to_uuid( + (Sequent::Migrations::Grouper.uuid_to_number(uuid) + 1) % Sequent::Migrations::Grouper::UUID_COUNT, + ) + } + let(:subject) { Sequent::Migrations::Grouper } let(:partitions) do {a: 200, b: 600, c: 200} end def ensure_invariants(groups) - groups.each do |range| - # start must be <= to end_inclusive for each group - expect(range.begin <=> range.end).to be <= 0 + groups.each do |group| + # begin must be before end for each group + expect(group.exclude_end?).to be_falsey + expect(group.begin).to be <= group.end end groups.each_cons(2).each do |prev_group, next_group| - # end of previous group must be < start of next group - expect(prev_group.end <=> next_group.end).to be < 0 + # end of previous group must be before begin of next group + expect(prev_group.end).to be < next_group.begin + if prev_group.end.partition_key == next_group.begin.partition_key + expect(NEXT_UUID[prev_group.end.aggregate_id]).to eq(next_group.begin.aggregate_id) + else + expect(prev_group.end.aggregate_id).to eq(Sequent::Migrations::Grouper::HIGHEST_UUID) + expect(next_group.begin.aggregate_id).to eq(Sequent::Migrations::Grouper::LOWEST_UUID) + end end end - + it 'creates a single group when all partitions fit' do expect(subject.group_partitions(partitions, 1000, 1)) - .to eq([[:a, subject::LOWEST_UUID] .. [:c, subject::HIGHEST_UUID]]) -end + .to eq([EP[:a, subject::LOWEST_UUID]..EP[:c, subject::HIGHEST_UUID]]) + end it 'creates multiple groups from a single large partition' do expect(subject.group_partitions({a: 100}, 40, 1)) .to eq( [ - [:a, subject::LOWEST_UUID] .. [:a, '66666666-6666-6666-6666-666666666665'], - [:a, '66666666-6666-6666-6666-666666666666'] .. [:a, 'cccccccc-cccc-cccc-cccc-cccccccccccb'], - [:a, 'cccccccc-cccc-cccc-cccc-cccccccccccc'] .. [:a, subject::HIGHEST_UUID], + EP[:a, subject::LOWEST_UUID]..EP[:a, '66666666-6666-6666-6666-666666666665'], + EP[:a, '66666666-6666-6666-6666-666666666666']..EP[:a, 'cccccccc-cccc-cccc-cccc-cccccccccccb'], + EP[:a, 'cccccccc-cccc-cccc-cccc-cccccccccccc']..EP[:a, subject::HIGHEST_UUID], ], ) ensure_invariants(subject.group_partitions({a: 100}, 40, 1)) @@ -42,8 +56,8 @@ def ensure_invariants(groups) expect(subject.group_partitions(partitions, 500, 1)) .to eq( [ - [:a, subject::LOWEST_UUID] .. [:b, '7fffffff-ffff-ffff-ffff-ffffffffffff'], - [:b, '80000000-0000-0000-0000-000000000000'] .. [:c, subject::HIGHEST_UUID], + EP[:a, subject::LOWEST_UUID]..EP[:b, '7fffffff-ffff-ffff-ffff-ffffffffffff'], + EP[:b, '80000000-0000-0000-0000-000000000000']..EP[:c, subject::HIGHEST_UUID], ], ) end @@ -52,9 +66,9 @@ def ensure_invariants(groups) expect(subject.group_partitions(partitions, 400, 1)) .to eq( [ - [:a, subject::LOWEST_UUID] .. [:b, '55555555-5555-5555-5555-555555555554'], - [:b, '55555555-5555-5555-5555-555555555555'] .. [:b, 'ffffffff-ffff-ffff-ffff-ffffffffffff'], - [:c, '00000000-0000-0000-0000-000000000000'] .. [:c, subject::HIGHEST_UUID], + EP[:a, subject::LOWEST_UUID]..EP[:b, '55555555-5555-5555-5555-555555555554'], + EP[:b, '55555555-5555-5555-5555-555555555555']..EP[:b, 'ffffffff-ffff-ffff-ffff-ffffffffffff'], + EP[:c, '00000000-0000-0000-0000-000000000000']..EP[:c, subject::HIGHEST_UUID], ], ) end @@ -63,9 +77,9 @@ def ensure_invariants(groups) expect(subject.group_partitions({a: 200, b: 500, c: 200}, 300, 1)) .to eq( [ - [:a, subject::LOWEST_UUID] .. [:b, '33333333-3333-3333-3333-333333333332'], - [:b, '33333333-3333-3333-3333-333333333333'] .. [:b, 'cccccccc-cccc-cccc-cccc-cccccccccccb'], - [:b, 'cccccccc-cccc-cccc-cccc-cccccccccccc'] .. [:c, subject::HIGHEST_UUID], + EP[:a, subject::LOWEST_UUID]..EP[:b, '33333333-3333-3333-3333-333333333332'], + EP[:b, '33333333-3333-3333-3333-333333333333']..EP[:b, 'cccccccc-cccc-cccc-cccc-cccccccccccb'], + EP[:b, 'cccccccc-cccc-cccc-cccc-cccccccccccc']..EP[:c, subject::HIGHEST_UUID], ], ) end @@ -74,9 +88,9 @@ def ensure_invariants(groups) expect(subject.group_partitions({a: 200, b: 500, c: 200}, 300, 1)) .to eq( [ - [:a, subject::LOWEST_UUID] .. [:b, '33333333-3333-3333-3333-333333333332'], - [:b, '33333333-3333-3333-3333-333333333333'] .. [:b, 'cccccccc-cccc-cccc-cccc-cccccccccccb'], - [:b, 'cccccccc-cccc-cccc-cccc-cccccccccccc'] .. [:c, subject::HIGHEST_UUID], + EP[:a, subject::LOWEST_UUID]..EP[:b, '33333333-3333-3333-3333-333333333332'], + EP[:b, '33333333-3333-3333-3333-333333333333']..EP[:b, 'cccccccc-cccc-cccc-cccc-cccccccccccb'], + EP[:b, 'cccccccc-cccc-cccc-cccc-cccccccccccc']..EP[:c, subject::HIGHEST_UUID], ], ) end From 00261c0c44a30109e25c6231988d1e0d0ede4a83 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Tue, 2 Apr 2024 15:13:53 +0200 Subject: [PATCH 048/128] Replay events ordered by partition key for improved locality --- lib/sequent/configuration.rb | 3 + lib/sequent/dry_run/view_schema.rb | 5 +- lib/sequent/migrations/grouper.rb | 10 +- lib/sequent/migrations/view_schema.rb | 100 ++++++------------ lib/sequent/rake/migration_tasks.rb | 11 +- spec/lib/sequent/core/event_store_spec.rb | 48 ++++----- spec/lib/sequent/migrations/grouper_spec.rb | 14 +-- .../sequent/migrations/view_schema_spec.rb | 2 + spec/spec_helper.rb | 4 +- 9 files changed, 78 insertions(+), 119 deletions(-) diff --git a/lib/sequent/configuration.rb b/lib/sequent/configuration.rb index 06876c66..c7a8b35a 100644 --- a/lib/sequent/configuration.rb +++ b/lib/sequent/configuration.rb @@ -21,6 +21,7 @@ class Configuration MIGRATIONS_CLASS_NAME = 'Sequent::Migrations::Projectors' DEFAULT_NUMBER_OF_REPLAY_PROCESSES = 4 + DEFAULT_REPLAY_GROUP_TARGET_SIZE = 250_000 DEFAULT_OFFLINE_REPLAY_PERSISTOR_CLASS = Sequent::Core::Persistors::ActiveRecordPersistor DEFAULT_ONLINE_REPLAY_PERSISTOR_CLASS = Sequent::Core::Persistors::ActiveRecordPersistor @@ -56,6 +57,7 @@ class Configuration :offline_replay_persistor_class, :online_replay_persistor_class, :number_of_replay_processes, + :replay_group_target_size, :database_config_directory, :database_schema_directory, :event_store_schema_name, @@ -107,6 +109,7 @@ def initialize self.event_store_schema_name = DEFAULT_EVENT_STORE_SCHEMA_NAME self.migrations_class_name = MIGRATIONS_CLASS_NAME self.number_of_replay_processes = DEFAULT_NUMBER_OF_REPLAY_PROCESSES + self.replay_group_target_size = DEFAULT_REPLAY_GROUP_TARGET_SIZE self.event_record_hooks_class = DEFAULT_EVENT_RECORD_HOOKS_CLASS diff --git a/lib/sequent/dry_run/view_schema.rb b/lib/sequent/dry_run/view_schema.rb index 9e6a69bf..63b89f7d 100644 --- a/lib/sequent/dry_run/view_schema.rb +++ b/lib/sequent/dry_run/view_schema.rb @@ -9,7 +9,7 @@ module DryRun # This migration does not insert anything into the database, mainly usefull # for performance testing migrations. class ViewSchema < Migrations::ViewSchema - def migrate_dryrun(regex:, group_exponent: 3, limit: nil, offset: nil) + def migrate_dryrun(regex:) persistor = DryRun::ReadOnlyReplayOptimizedPostgresPersistor.new projectors = Sequent::Core::Migratable.all.select { |p| p.replay_persistor.nil? && p.name.match(regex || /.*/) } @@ -17,8 +17,7 @@ def migrate_dryrun(regex:, group_exponent: 3, limit: nil, offset: nil) Sequent.logger.info "Dry run using the following projectors: #{projectors.map(&:name).join(', ')}" starting = Process.clock_gettime(Process::CLOCK_MONOTONIC) - groups = groups(group_exponent: group_exponent, limit: limit, offset: offset) - replay!(persistor, projectors: projectors, groups: groups) + replay!(persistor, projectors:) ending = Process.clock_gettime(Process::CLOCK_MONOTONIC) Sequent.logger.info("Done migrate_dryrun for version #{Sequent.new_version} in #{ending - starting} s") diff --git a/lib/sequent/migrations/grouper.rb b/lib/sequent/migrations/grouper.rb index cfa4b727..b939473c 100644 --- a/lib/sequent/migrations/grouper.rb +++ b/lib/sequent/migrations/grouper.rb @@ -11,6 +11,10 @@ def <=>(other) end include Comparable + + def to_s + "(#{partition_key}, #{aggregate_id})" + end end # Generate approximately equally sized groups based on the @@ -22,20 +26,18 @@ def <=>(other) # For splitting a partition into equal sized groups the # assumption is made that aggregate-ids and their events are # equally distributed. - def self.group_partitions(partitions, target_group_size, _minimum_group_count) + def self.group_partitions(partitions, target_group_size) return [] unless partitions.present? partitions = partitions.sort.map do |key, count| PartitionData.new(key:, original_size: count, remaining_size: count, lower_bound: 0) end - # total_count = partitions.reduce(0) { |acc, (_, count)| acc + count } - result = [] - partition = partitions.shift current_start = GroupEndpoint.new(partition.key, LOWEST_UUID) current_size = 0 + result = [] while partition.present? if current_size + partition.remaining_size < target_group_size current_size += partition.remaining_size diff --git a/lib/sequent/migrations/view_schema.rb b/lib/sequent/migrations/view_schema.rb index 8c6f4438..8f265ea2 100644 --- a/lib/sequent/migrations/view_schema.rb +++ b/lib/sequent/migrations/view_schema.rb @@ -13,6 +13,7 @@ require_relative 'executor' require_relative 'sql' require_relative 'versions' +require_relative 'grouper' module Sequent module Migrations @@ -60,16 +61,6 @@ module Migrations # # end class ViewSchema - # Corresponds with the index on aggregate_id column in the event_records table - # - # Since we replay in batches of the first 3 chars of the uuid we created an index on - # these 3 characters. Hence the name ;-) - # - # This also means that the online replay is divided up into 16**3 groups - # This might seem a lot for starting event store, but when you will get more - # events, you will see that this is pretty good partitioned. - LENGTH_OF_SUBSTRING_INDEX_ON_AGGREGATE_ID_IN_EVENT_STORE = 3 - include Sequent::Util::Timer include Sequent::Util::Printer include Sql @@ -143,11 +134,10 @@ def create_view_tables # Utility method that replays events for all managed_tables from all Sequent::Core::Projector's # # This method is mainly useful in test scenario's or development tasks - def replay_all!(group_exponent: 1) + def replay_all! replay!( Sequent.configuration.online_replay_persistor_class.new, projectors: Core::Migratable.projectors, - groups: groups(group_exponent: group_exponent), ) end @@ -205,7 +195,6 @@ def migrate_online if plan.projectors.any? replay!( Sequent.configuration.online_replay_persistor_class.new, - groups: groups, maximum_xact_id_exclusive: Versions.running.first.xmin_xact_id, ) end @@ -259,7 +248,6 @@ def migrate_offline if plan.projectors.any? replay!( Sequent.configuration.offline_replay_persistor_class.new, - groups: groups(group_exponent: 1), minimum_xact_id_inclusive: Versions.running.first.xmin_xact_id, ) end @@ -308,18 +296,23 @@ def ensure_version_correct! def replay!( replay_persistor, - groups:, projectors: plan.projectors, minimum_xact_id_inclusive: nil, maximum_xact_id_exclusive: nil ) - logger.info "groups: #{groups.size}" + event_types = projectors.flat_map { |projector| projector.message_mapping.keys }.uniq.map(&:name) + group_target_size = Sequent.configuration.replay_group_target_size + partitions = Sequent.configuration.event_record_class.where(event_type: event_types) + .group(:partition_key) + .order(:partition_key) + .count + event_count = partitions.map(&:count).sum + groups = Sequent::Migrations::Grouper.group_partitions(partitions, group_target_size) with_sequent_config(replay_persistor, projectors) do - logger.info 'Start replaying events' + logger.info "Start replaying #{event_count} events in #{groups.size} groups" - time("#{groups.size} groups replayed") do - event_types = projectors.flat_map { |projector| projector.message_mapping.keys }.uniq.map(&:name) + time("#{event_count} events in #{groups.size} groups replayed") do disconnect! @connected = false @@ -327,24 +320,23 @@ def replay!( result = Parallel.map_with_index( groups, in_processes: Sequent.configuration.number_of_replay_processes, - ) do |aggregate_prefixes, index| + ) do |group, index| @connected ||= establish_connection msg = <<~EOS.chomp - Group (#{aggregate_prefixes.first}-#{aggregate_prefixes.last}) #{index + 1}/#{groups.size} replayed + Group #{group} (#{index + 1}/#{groups.size}) replayed EOS time(msg) do replay_events( - aggregate_prefixes, - event_types, - minimum_xact_id_inclusive, - maximum_xact_id_exclusive, + -> { + event_stream(group, event_types, minimum_xact_id_inclusive, maximum_xact_id_exclusive) + }, replay_persistor, &on_progress ) end nil rescue StandardError => e - logger.error "Replaying failed for ids: ^#{aggregate_prefixes.first} - #{aggregate_prefixes.last}" + logger.error "Replaying failed for group: #{group}" logger.error '+++++++++++++++ ERROR +++++++++++++++' recursively_print(e) raise Parallel::Kill # immediately kill all sub-processes @@ -356,19 +348,14 @@ def replay!( end def replay_events( - aggregate_prefixes, - event_types, - minimum_xact_id_inclusive, - maximum_xact_id_exclusive, + get_events, replay_persistor, &on_progress ) Sequent.configuration.event_store.replay_events_from_cursor( block_size: 1000, - get_events: -> { - event_stream(aggregate_prefixes, event_types, minimum_xact_id_inclusive, maximum_xact_id_exclusive) - }, - on_progress: on_progress, + get_events:, + on_progress:, ) replay_persistor.commit @@ -386,30 +373,6 @@ def rollback_migration Versions.rollback!(Sequent.new_version) end - def groups(group_exponent: 3, limit: nil, offset: nil) - number_of_groups = 16**group_exponent - groups = groups_of_aggregate_id_prefixes(number_of_groups) - groups = groups.drop(offset) unless offset.nil? - groups = groups.take(limit) unless limit.nil? - groups - end - - def groups_of_aggregate_id_prefixes(number_of_groups) - all_prefixes = (0...16**LENGTH_OF_SUBSTRING_INDEX_ON_AGGREGATE_ID_IN_EVENT_STORE).to_a.map do |i| - i.to_s(16) - end - all_prefixes = all_prefixes.map { |s| s.length == 3 ? s : "#{'0' * (3 - s.length)}#{s}" } - - logger.info "Number of groups #{number_of_groups}" - - logger.debug "Prefixes: #{all_prefixes.length}" - if number_of_groups > all_prefixes.length - fail "Can not have more groups #{number_of_groups} than number of prefixes #{all_prefixes.length}" - end - - all_prefixes.each_slice(all_prefixes.length / number_of_groups).to_a - end - def in_view_schema(&block) Sequent::Support::Database.with_schema_search_path(view_schema, db_config, &block) end @@ -450,13 +413,18 @@ def with_sequent_config(replay_persistor, projectors, &block) Sequent::Configuration.restore(old_config) end - def event_stream(aggregate_prefixes, event_types, minimum_xact_id_inclusive, maximum_xact_id_exclusive) - fail ArgumentError, 'aggregate_prefixes is mandatory' unless aggregate_prefixes.present? - - event_stream = Sequent.configuration.event_record_class.where(event_type: event_types) - event_stream = event_stream.where(<<~SQL, aggregate_prefixes) - substring(aggregate_id::text from 1 for #{LENGTH_OF_SUBSTRING_INDEX_ON_AGGREGATE_ID_IN_EVENT_STORE}) in (?) - SQL + def event_stream(group, event_types, minimum_xact_id_inclusive, maximum_xact_id_exclusive) + fail ArgumentError, 'group is mandatory' if group.nil? + + event_stream = Sequent.configuration.event_record_class.where( + event_type: event_types, + ).where( + '(partition_key, aggregate_id) BETWEEN (?, ?) AND (?, ?)', + group.begin.partition_key, + group.begin.aggregate_id, + group.end.partition_key, + group.end.aggregate_id, + ) if minimum_xact_id_inclusive && maximum_xact_id_exclusive event_stream = event_stream.where( 'xact_id >= ? AND xact_id < ?', @@ -469,7 +437,7 @@ def event_stream(aggregate_prefixes, event_types, minimum_xact_id_inclusive, max event_stream = event_stream.where('xact_id IS NULL OR xact_id < ?', maximum_xact_id_exclusive) end event_stream - .order('aggregate_id ASC, sequence_number ASC') + .order(:partition_key, :aggregate_id, :sequence_number) .select('event_type, event_json') end diff --git a/lib/sequent/rake/migration_tasks.rb b/lib/sequent/rake/migration_tasks.rb index 240fde0c..e4dc8481 100644 --- a/lib/sequent/rake/migration_tasks.rb +++ b/lib/sequent/rake/migration_tasks.rb @@ -192,18 +192,15 @@ def register_tasks! Pass a regular expression as parameter to select the projectors to run, otherwise all projectors are selected. EOS - task :dryrun, %i[regex group_exponent limit offset] => ['sequent:init', :init] do |_task, args| + task :dryrun, %i[regex group_target_size] => ['sequent:init', :init] do |_task, args| ensure_sequent_env_set! db_config = Sequent::Support::Database.read_config(@env) view_schema = Sequent::DryRun::ViewSchema.new(db_config: db_config) - view_schema.migrate_dryrun( - regex: args[:regex], - group_exponent: (args[:group_exponent] || 3).to_i, - limit: args[:limit]&.to_i, - offset: args[:offset]&.to_i, - ) + Sequent.configuration.replay_group_target_size = group_target_size + + view_schema.migrate_dryrun(regex: args[:regex]) end end diff --git a/spec/lib/sequent/core/event_store_spec.rb b/spec/lib/sequent/core/event_store_spec.rb index 057c8a12..25a1fe64 100644 --- a/spec/lib/sequent/core/event_store_spec.rb +++ b/spec/lib/sequent/core/event_store_spec.rb @@ -484,44 +484,32 @@ class FailingHandler < Sequent::Core::Projector end describe '#replay_events_from_cursor' do - let(:stream_record) do - Sequent::Core::StreamRecord.create!( - aggregate_type: 'Sequent::Core::AggregateRoot', - aggregate_id: aggregate_id, - created_at: DateTime.now, - ) + let(:events) do + 5.times.map { |n| Sequent::Core::Event.new(aggregate_id:, sequence_number: n + 1) } end - let(:command_record) do - Sequent::Core::CommandRecord.create!( - command_type: 'Sequent::Core::Command', - command_json: '{}', - aggregate_id: stream_record.aggregate_id, + + before do + ActiveRecord::Base.connection.exec_update('TRUNCATE TABLE aggregates CASCADE') + event_store.commit_events( + Sequent::Core::Command.new(aggregate_id:), + [ + [ + Sequent::Core::EventStream.new( + aggregate_type: 'Sequent::Core::AggregateRoot', + aggregate_id:, + events_partition_key: 'Y24', + ), + events, + ], + ], ) end let(:get_events) do -> do - event_records = Sequent.configuration.event_record_class.table_name - stream_records = Sequent.configuration.stream_record_class.table_name Sequent.configuration.event_record_class .select('event_type, event_json') - .joins("INNER JOIN #{stream_records} ON #{event_records}.aggregate_id = #{stream_records}.aggregate_id") - .order!("#{stream_records}.aggregate_id, #{event_records}.sequence_number") - end - end - - before do - Sequent::Core::EventRecord.delete_all - 5.times do |n| - Sequent::Core::EventRecord.create!( - aggregate_id: stream_record.aggregate_id, - sequence_number: n + 1, - event_type: 'Sequent::Core::Event', - event_json: '{}', - created_at: DateTime.now, - command_record_id: command_record.id, - stream_record: stream_record, - ) + .order(:aggregate_id, :sequence_number) end end diff --git a/spec/lib/sequent/migrations/grouper_spec.rb b/spec/lib/sequent/migrations/grouper_spec.rb index ce37338a..bf8137f4 100644 --- a/spec/lib/sequent/migrations/grouper_spec.rb +++ b/spec/lib/sequent/migrations/grouper_spec.rb @@ -35,12 +35,12 @@ def ensure_invariants(groups) end it 'creates a single group when all partitions fit' do - expect(subject.group_partitions(partitions, 1000, 1)) + expect(subject.group_partitions(partitions, 1000)) .to eq([EP[:a, subject::LOWEST_UUID]..EP[:c, subject::HIGHEST_UUID]]) end it 'creates multiple groups from a single large partition' do - expect(subject.group_partitions({a: 100}, 40, 1)) + expect(subject.group_partitions({a: 100}, 40)) .to eq( [ EP[:a, subject::LOWEST_UUID]..EP[:a, '66666666-6666-6666-6666-666666666665'], @@ -48,12 +48,12 @@ def ensure_invariants(groups) EP[:a, 'cccccccc-cccc-cccc-cccc-cccccccccccc']..EP[:a, subject::HIGHEST_UUID], ], ) - ensure_invariants(subject.group_partitions({a: 100}, 40, 1)) + ensure_invariants(subject.group_partitions({a: 100}, 40)) end context 'splits groups assuming an uniform distribution' do it 'splits group in half' do - expect(subject.group_partitions(partitions, 500, 1)) + expect(subject.group_partitions(partitions, 500)) .to eq( [ EP[:a, subject::LOWEST_UUID]..EP[:b, '7fffffff-ffff-ffff-ffff-ffffffffffff'], @@ -63,7 +63,7 @@ def ensure_invariants(groups) end it 'splits group in three unequal parts' do - expect(subject.group_partitions(partitions, 400, 1)) + expect(subject.group_partitions(partitions, 400)) .to eq( [ EP[:a, subject::LOWEST_UUID]..EP[:b, '55555555-5555-5555-5555-555555555554'], @@ -74,7 +74,7 @@ def ensure_invariants(groups) end it 'splits group in three equal parts' do - expect(subject.group_partitions({a: 200, b: 500, c: 200}, 300, 1)) + expect(subject.group_partitions({a: 200, b: 500, c: 200}, 300)) .to eq( [ EP[:a, subject::LOWEST_UUID]..EP[:b, '33333333-3333-3333-3333-333333333332'], @@ -85,7 +85,7 @@ def ensure_invariants(groups) end it 'splits group in many equal parts' do - expect(subject.group_partitions({a: 200, b: 500, c: 200}, 300, 1)) + expect(subject.group_partitions({a: 200, b: 500, c: 200}, 300)) .to eq( [ EP[:a, subject::LOWEST_UUID]..EP[:b, '33333333-3333-3333-3333-333333333332'], diff --git a/spec/lib/sequent/migrations/view_schema_spec.rb b/spec/lib/sequent/migrations/view_schema_spec.rb index bf2322fb..48d7c53d 100644 --- a/spec/lib/sequent/migrations/view_schema_spec.rb +++ b/spec/lib/sequent/migrations/view_schema_spec.rb @@ -182,6 +182,7 @@ AccountCreated.new(aggregate_id: Sequent.new_uuid, sequence_number: 1), AccountCreated.new(aggregate_id: Sequent.new_uuid, sequence_number: 1), ], + events_partition_key: 'a', ) message_aggregate_id = Sequent.new_uuid @@ -191,6 +192,7 @@ MessageCreated.new(aggregate_id: message_aggregate_id, sequence_number: 1), MessageSet.new(aggregate_id: message_aggregate_id, sequence_number: 2, message: 'Foobar'), ], + events_partition_key: 'b', ) wait_for_persisted_events_to_become_visible_for_online_migration[] diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 185cd549..b8c822aa 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -32,10 +32,10 @@ def exec_sql(sql) Sequent::ApplicationRecord.connection.execute(sql) end - def insert_events(aggregate_type, events) + def insert_events(aggregate_type, events, events_partition_key: '') streams_with_events = events.group_by(&:aggregate_id).map do |aggregate_id, aggregate_events| [ - Sequent::Core::EventStream.new(aggregate_type: aggregate_type, aggregate_id: aggregate_id), + Sequent::Core::EventStream.new(aggregate_type:, aggregate_id:, events_partition_key:), aggregate_events, ] end From d7ecc156f7aa06a4a5b51494d3429f0c845cbc2e Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Tue, 2 Apr 2024 18:00:38 +0200 Subject: [PATCH 049/128] Add property based test to grouping of partitions for replay --- gemfiles/ar_7_1.gemfile.lock | 3 ++ sequent.gemspec | 1 + spec/lib/sequent/migrations/grouper_spec.rb | 46 ++++++++++++++------- 3 files changed, 35 insertions(+), 15 deletions(-) diff --git a/gemfiles/ar_7_1.gemfile.lock b/gemfiles/ar_7_1.gemfile.lock index d34cc9d7..76911c51 100644 --- a/gemfiles/ar_7_1.gemfile.lock +++ b/gemfiles/ar_7_1.gemfile.lock @@ -60,6 +60,7 @@ GEM pg (1.5.6) postgresql_cursor (0.6.8) activerecord (>= 6.0) + prop_check (1.0.0) pry (0.14.2) coderay (~> 1.1) method_source (~> 1.0) @@ -113,11 +114,13 @@ GEM PLATFORMS arm64-darwin-22 + arm64-darwin-23 x86_64-linux DEPENDENCIES activemodel (= 7.1.1) activerecord (= 7.1.1) + prop_check (~> 1.0) pry (~> 0.13) rake (~> 13) rspec (~> 3.10) diff --git a/sequent.gemspec b/sequent.gemspec index 054e5849..d57cf907 100644 --- a/sequent.gemspec +++ b/sequent.gemspec @@ -38,6 +38,7 @@ Gem::Specification.new do |s| s.add_dependency 'postgresql_cursor', '~> 0.6' s.add_dependency 'thread_safe', '~> 0.3.6' s.add_dependency 'tzinfo', '>= 1.1' + s.add_development_dependency 'prop_check', '~> 1.0' s.add_development_dependency 'pry', '~> 0.13' s.add_development_dependency 'rake', '~> 13' s.add_development_dependency 'rspec', rspec_version diff --git a/spec/lib/sequent/migrations/grouper_spec.rb b/spec/lib/sequent/migrations/grouper_spec.rb index bf8137f4..823289b5 100644 --- a/spec/lib/sequent/migrations/grouper_spec.rb +++ b/spec/lib/sequent/migrations/grouper_spec.rb @@ -2,8 +2,11 @@ require 'spec_helper' require_relative '../../../../lib/sequent/migrations/grouper' +require 'prop_check' describe Sequent::Migrations::Grouper do + G = PropCheck::Generators + EP = ->(partition_key, aggregate_id) { Sequent::Migrations::Grouper::GroupEndpoint.new(partition_key, aggregate_id) } NEXT_UUID = ->(uuid) { Sequent::Migrations::Grouper.number_to_uuid( @@ -16,20 +19,34 @@ {a: 200, b: 600, c: 200} end - def ensure_invariants(groups) - groups.each do |group| - # begin must be before end for each group - expect(group.exclude_end?).to be_falsey - expect(group.begin).to be <= group.end - end - groups.each_cons(2).each do |prev_group, next_group| - # end of previous group must be before begin of next group - expect(prev_group.end).to be < next_group.begin - if prev_group.end.partition_key == next_group.begin.partition_key - expect(NEXT_UUID[prev_group.end.aggregate_id]).to eq(next_group.begin.aggregate_id) - else - expect(prev_group.end.aggregate_id).to eq(Sequent::Migrations::Grouper::HIGHEST_UUID) - expect(next_group.begin.aggregate_id).to eq(Sequent::Migrations::Grouper::LOWEST_UUID) + it 'groups partitions into a sorted list covering all partitions without gaps' do + PropCheck.forall( + G.hash(G.alphanumeric_string, G.positive_integer), + G.positive_integer, + ) do |partitions, group_target_size| + next unless partitions.present? + + groups = subject.group_partitions(partitions, group_target_size) + + # The groups must cover all partitions + expect(groups.first.begin).to eq(EP[partitions.keys.min, Sequent::Migrations::Grouper::LOWEST_UUID]) + expect(groups.last.end).to eq(EP[partitions.keys.max, Sequent::Migrations::Grouper::HIGHEST_UUID]) + + groups.each do |group| + # begin must be before end for each group + expect(group).not_to be_exclude_end + expect(group.begin).to be <= group.end + end + groups.each_cons(2).each do |prev_group, next_group| + # end of previous group must be before begin of next group + expect(prev_group.end).to be < next_group.begin + # groups must be consecutive + if prev_group.end.partition_key == next_group.begin.partition_key + expect(NEXT_UUID[prev_group.end.aggregate_id]).to eq(next_group.begin.aggregate_id) + else + expect(prev_group.end.aggregate_id).to eq(Sequent::Migrations::Grouper::HIGHEST_UUID) + expect(next_group.begin.aggregate_id).to eq(Sequent::Migrations::Grouper::LOWEST_UUID) + end end end end @@ -48,7 +65,6 @@ def ensure_invariants(groups) EP[:a, 'cccccccc-cccc-cccc-cccc-cccccccccccc']..EP[:a, subject::HIGHEST_UUID], ], ) - ensure_invariants(subject.group_partitions({a: 100}, 40)) end context 'splits groups assuming an uniform distribution' do From 71422c5d98dd258a1c0d5d418588fad27de2818f Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Wed, 17 Apr 2024 18:29:43 +0200 Subject: [PATCH 050/128] The origin of a command or event is now always a command record Normally this is also enforced by the foreign key constraints of the database. In case the links are broken we still return the command record closest to the origin that we can find, instead of returning an event record in this case. --- lib/sequent/core/command_record.rb | 8 +------- lib/sequent/core/event_record.rb | 8 +------- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/lib/sequent/core/command_record.rb b/lib/sequent/core/command_record.rb index 8ac7836d..d4e4ccd5 100644 --- a/lib/sequent/core/command_record.rb +++ b/lib/sequent/core/command_record.rb @@ -64,13 +64,7 @@ def children end def origin - parent.present? ? find_origin(parent) : self - end - - def find_origin(record) - return find_origin(record.parent) if record.parent.present? - - record + parent&.origin || self end end end diff --git a/lib/sequent/core/event_record.rb b/lib/sequent/core/event_record.rb index 98d0c8d5..96995a8e 100644 --- a/lib/sequent/core/event_record.rb +++ b/lib/sequent/core/event_record.rb @@ -105,13 +105,7 @@ def children end def origin - parent.present? ? find_origin(parent) : self - end - - def find_origin(record) - return find_origin(record.parent) if record.parent.present? - - record + parent&.origin end end end From d93f08ca2703829b9091401590a1aea68c928e2a Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Wed, 17 Apr 2024 18:28:22 +0200 Subject: [PATCH 051/128] Simplified schema without backwards compatibility concerns --- db/sequent_schema.rb | 53 ------ db/sequent_schema.sql | 428 +++++++----------------------------------- 2 files changed, 70 insertions(+), 411 deletions(-) diff --git a/db/sequent_schema.rb b/db/sequent_schema.rb index 4c8d9003..97364112 100644 --- a/db/sequent_schema.rb +++ b/db/sequent_schema.rb @@ -1,44 +1,6 @@ # frozen_string_literal: true ActiveRecord::Schema.define do - create_table 'command_records', force: true do |t| - t.uuid 'user_id' - t.uuid 'aggregate_id' - t.string 'command_type', null: false - t.uuid 'event_aggregate_id' - t.integer 'event_sequence_number' - t.jsonb 'command_json', null: false - t.datetime 'created_at', null: false - end - - add_index 'command_records', %w[event_aggregate_id event_sequence_number], name: 'index_command_records_on_event' - - create_table 'stream_records', primary_key: ['aggregate_id'], force: true do |t| - t.datetime 'created_at', null: false - t.string 'aggregate_type', null: false - t.uuid 'aggregate_id', null: false - t.integer 'snapshot_threshold' - end - - create_table 'event_records', primary_key: %w[aggregate_id sequence_number], force: true do |t| - t.uuid 'aggregate_id', null: false - t.integer 'sequence_number', null: false - t.datetime 'created_at', null: false - t.string 'event_type', null: false - t.jsonb 'event_json', null: false - t.integer 'command_record_id', null: false - t.bigint 'xact_id', null: false - end - - add_index 'event_records', ['command_record_id'], name: 'index_event_records_on_command_record_id' - add_index 'event_records', ['event_type'], name: 'index_event_records_on_event_type' - add_index 'event_records', ['created_at'], name: 'index_event_records_on_created_at' - add_index 'event_records', ['xact_id'], name: 'index_event_records_on_xact_id' - - execute <<~EOS - ALTER TABLE event_records ALTER COLUMN xact_id SET DEFAULT pg_current_xact_id()::text::bigint - EOS - create_table 'snapshot_records', primary_key: %w[aggregate_id sequence_number], force: true do |t| t.uuid 'aggregate_id', null: false t.integer 'sequence_number', null: false @@ -47,21 +9,6 @@ t.jsonb 'snapshot_json', null: false end - add_foreign_key :event_records, - :command_records, - name: 'command_fkey' - add_foreign_key :event_records, - :stream_records, - column: :aggregate_id, - primary_key: :aggregate_id, - name: 'stream_fkey' - add_foreign_key :snapshot_records, - :stream_records, - column: :aggregate_id, - primary_key: :aggregate_id, - on_delete: :cascade, - name: 'stream_fkey' - schema = File.read("#{File.dirname(__FILE__)}/sequent_schema.sql") execute schema end diff --git a/db/sequent_schema.sql b/db/sequent_schema.sql index 3ae74b6e..88daf4f6 100644 --- a/db/sequent_schema.sql +++ b/db/sequent_schema.sql @@ -13,15 +13,14 @@ CREATE TABLE aggregate_types (id SMALLINT GENERATED ALWAYS AS IDENTITY PRIMARY K CREATE TABLE event_types (id SMALLINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, type text UNIQUE NOT NULL); CREATE TABLE commands ( - id bigint DEFAULT nextval('command_records_id_seq'), + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, created_at timestamp with time zone NOT NULL, user_id uuid, aggregate_id uuid, command_type_id SMALLINT NOT NULL REFERENCES command_types (id), command_json jsonb NOT NULL, event_aggregate_id uuid, - event_sequence_number integer, - PRIMARY KEY (id) + event_sequence_number integer ) PARTITION BY RANGE (id); CREATE INDEX commands_command_type_id_idx ON commands (command_type_id); CREATE INDEX commands_aggregate_id_idx ON commands (aggregate_id); @@ -82,17 +81,9 @@ ALTER TABLE events_aggregate CLUSTER ON events_aggregate_pkey; TRUNCATE TABLE snapshot_records; ALTER TABLE snapshot_records - ALTER COLUMN created_at TYPE timestamptz USING created_at AT TIME ZONE 'Europe/Amsterdam', - ALTER COLUMN snapshot_type TYPE text, - ALTER COLUMN snapshot_json TYPE jsonb USING snapshot_json::jsonb, - DROP CONSTRAINT stream_fkey, ADD CONSTRAINT aggregate_fkey FOREIGN KEY (aggregate_id) REFERENCES aggregates (aggregate_id) ON UPDATE CASCADE ON DELETE CASCADE; -ALTER TABLE command_records RENAME TO old_command_records; -ALTER TABLE event_records RENAME TO old_event_records; -ALTER TABLE stream_records RENAME TO old_stream_records; - CREATE OR REPLACE FUNCTION enrich_command_json(command commands) RETURNS jsonb LANGUAGE plpgsql AS $$ BEGIN @@ -108,27 +99,6 @@ BEGIN END $$; -CREATE VIEW command_records (id, user_id, aggregate_id, command_type, command_json, created_at, event_aggregate_id, event_sequence_number) AS - SELECT id, - user_id, - aggregate_id, - (SELECT type FROM command_types WHERE command_types.id = command.command_type_id), - enrich_command_json(command), - created_at, - event_aggregate_id, - event_sequence_number - FROM commands command - UNION ALL - SELECT id, - user_id, - aggregate_id, - command_type, - command_json::jsonb, - created_at, - event_aggregate_id, - event_sequence_number - FROM old_command_records; - CREATE OR REPLACE FUNCTION enrich_event_json(event events) RETURNS jsonb LANGUAGE plpgsql AS $$ BEGIN @@ -141,298 +111,6 @@ BEGIN END $$; -CREATE VIEW event_records (aggregate_id, partition_key, sequence_number, created_at, event_type, event_json, command_record_id, xact_id) AS - SELECT aggregate.aggregate_id, - event.partition_key, - event.sequence_number, - event.created_at, - type.type, - enrich_event_json(event.*)::text AS event_json, - command_id, - event.xact_id - FROM aggregates aggregate - JOIN events event ON aggregate.aggregate_id = event.aggregate_id AND aggregate.events_partition_key = event.partition_key - JOIN event_types type ON event.event_type_id = type.id - UNION ALL - SELECT aggregate_id, - '', - sequence_number, - (event_json::jsonb->>'created_at')::timestamptz AS created_at, - event_type::text, - event_json::text, - command_record_id, - xact_id - FROM old_event_records; - -CREATE VIEW stream_records (aggregate_id, events_partition_key, aggregate_type, snapshot_threshold, created_at) AS - SELECT aggregates.aggregate_id, - aggregates.events_partition_key, - aggregate_types.type, - aggregates.snapshot_threshold, - aggregates.created_at - FROM aggregates JOIN aggregate_types ON aggregates.aggregate_type_id = aggregate_types.id - UNION ALL - SELECT aggregate_id, - NULL, - aggregate_type, - snapshot_threshold, - created_at - FROM old_stream_records; - -INSERT INTO command_types (type) SELECT DISTINCT command_type FROM command_records ORDER BY 1; -INSERT INTO aggregate_types (type) SELECT DISTINCT aggregate_type FROM stream_records ORDER BY 1; -INSERT INTO event_types (type) SELECT DISTINCT event_type FROM old_event_records ORDER BY 1; - -CREATE OR REPLACE FUNCTION determine_events_partition_key(_aggregate_id uuid, _organization_id uuid, _event_json jsonb) RETURNS text AS $$ -DECLARE - _event_json_without_nulls jsonb = jsonb_strip_nulls(_event_json); - _date text; - _partition_key text; -BEGIN - _date = (CASE - WHEN _event_json_without_nulls->'booking_date' IS NOT NULL THEN - regexp_replace(_event_json_without_nulls->>'booking_date', '[ _-]', '', 'g') - WHEN _event_json_without_nulls->'year_of_delivery' IS NOT NULL AND _event_json_without_nulls->'month_of_delivery' IS NOT NULL THEN - trim(to_char((_event_json_without_nulls->>'year_of_delivery')::integer, '0000')) || trim(to_char((_event_json_without_nulls->>'month_of_delivery')::integer, '00')) - WHEN _event_json_without_nulls->'happened_at' IS NOT NULL THEN - regexp_replace(LEFT((_event_json_without_nulls->>'happened_at')::text, 10), '[ _-]', '', 'g') - WHEN _event_json->'book_date' IS NOT NULL THEN - regexp_replace(LEFT((_event_json_without_nulls->>'book_date')::text, 10), '[ _-]', '', 'g') - WHEN _event_json->>'event_type' NOT IN ( - 'BankCreatedEvent', - 'CompleteOrganizationCreatedEvent', - 'CustomerCreatedEvent', - 'CustomerLedgerAccountCreated', - 'EmailProxyCreated', - 'EstimateNumbersCreated', - 'ExpenseNumbersCreatedEvent', - 'InboxNamesCreated', - 'InvoiceNumbersCreatedEvent', - 'LabelableCreated', - 'LabelCreated', - 'OrganizationBankCreated', - 'OrganizationCreatedAccountEvent', - 'OrganizationCreatedByAccountantEvent', - 'OrganizationCreatedEvent', - 'OrganizationEstimateNumbersCreated', - 'OrganizationExpenseNumbersCreated', - 'OrganizationInvoiceNumbersCreated', - 'OrganizationLedgerCreated', - 'PortalConnectionCreated', - 'PortalConnectionsCreated', - 'ProWowOrganizationCreatedEvent', - 'RootRuleCreated', - 'RuleCreated', - 'UserCreatedEvent', - 'UserCreatedByAccountantEvent' - ) THEN - to_char((_event_json->>'created_at')::timestamptz, 'YYYYMMDD') - ELSE - NULL - END); - - IF _date IS NOT NULL AND _organization_id IS NOT NULL THEN - _partition_key = 'Y' || SUBSTRING(_date FROM 3 FOR 2) || 'O' || LEFT(_organization_id::text, 4); - ELSIF _organization_id IS NOT NULL THEN - _partition_key = 'O' || LEFT(_organization_id::text, 4); - ELSE - _partition_key = 'A' || LEFT(_aggregate_id::text, 2); - END IF; - - RETURN _partition_key; -END -$$ -LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION upsert_aggregate_type(_type aggregate_types.type%TYPE) RETURNS SMALLINT -LANGUAGE plpgsql AS -$$ -DECLARE - _id aggregate_types.id%TYPE; -BEGIN - SELECT id INTO _id FROM aggregate_types WHERE type = _type; - IF NOT FOUND THEN - INSERT INTO aggregate_types (type) VALUES (_type) RETURNING id INTO STRICT _id; - END IF; - RETURN _id; -END -$$; - -CREATE OR REPLACE FUNCTION upsert_command_type(_type command_types.type%TYPE) RETURNS SMALLINT -LANGUAGE plpgsql AS -$$ -DECLARE - _id command_types.id%TYPE; -BEGIN - SELECT id INTO _id FROM command_types WHERE type = _type; - IF NOT FOUND THEN - RAISE NOTICE 'command type % not found, inserting', _type; - INSERT INTO command_types (type) VALUES (_type) RETURNING id INTO STRICT _id; - END IF; - RETURN _id; -END -$$; - -CREATE OR REPLACE FUNCTION upsert_event_type(_type event_types.type%TYPE) RETURNS SMALLINT -LANGUAGE plpgsql AS -$$ -DECLARE - _id event_types.id%TYPE; -BEGIN - SELECT id INTO _id FROM event_types WHERE type = _type; - IF NOT FOUND THEN - INSERT INTO event_types (type) VALUES (_type) RETURNING id INTO STRICT _id; - END IF; - RETURN _id; -END -$$; - -CREATE OR REPLACE PROCEDURE migrate_command(_command_id commands.id%TYPE) -LANGUAGE plpgsql AS $$ -DECLARE - _command_type text; - _command_record command_records; - _command_without_nulls jsonb; -BEGIN - IF EXISTS (SELECT 1 FROM commands WHERE id = _command_id) THEN - RETURN; - END IF; - - SELECT * - INTO _command_record - FROM command_records - WHERE id = _command_id; - IF NOT FOUND THEN - -- Event without command - RETURN; - END IF; - - _command_without_nulls = jsonb_strip_nulls(_command_record.command_json::jsonb); - - INSERT INTO commands ( - id, created_at, user_id, aggregate_id, command_type_id, command_json, - event_aggregate_id, event_sequence_number - ) VALUES ( - _command_id, - COALESCE((_command_without_nulls->>'created_at')::timestamptz, _command_record.created_at AT TIME ZONE 'Europe/Amsterdam'), - (_command_without_nulls->>'user_id')::uuid, - (_command_without_nulls->>'aggregate_id')::uuid, - upsert_command_type(_command_record.command_type), - _command_record.command_json::jsonb - '{created_at,organization_id,user_id,aggregate_id,event_aggregate_id,event_sequence_number}'::text[], - (_command_without_nulls->>'event_aggregate_id')::uuid, - (_command_without_nulls->'event_sequence_number')::integer - ); -END -$$; - -CREATE OR REPLACE FUNCTION migrate_aggregate(_aggregate_id uuid, _provided_events_partition_key text) RETURNS boolean AS $$ -DECLARE - _aggregate_with_first_event RECORD; - _events_partition_key text; - _event_json jsonb; - _event old_event_records; -BEGIN - SELECT s.*, e.event_type, e.event_json - INTO _aggregate_with_first_event - FROM old_stream_records s JOIN old_event_records e ON s.aggregate_id = e.aggregate_id - WHERE s.aggregate_id = _aggregate_id - AND e.sequence_number = 1; - IF NOT FOUND THEN - RETURN FALSE; - END IF; - - _event_json = _aggregate_with_first_event.event_json::jsonb; - _events_partition_key = COALESCE( - _provided_events_partition_key, - determine_events_partition_key(_aggregate_with_first_event.aggregate_id, NULL, _event_json) - ); - - INSERT INTO aggregates (aggregate_id, created_at, aggregate_type_id, events_partition_key, snapshot_threshold) - VALUES ( - _aggregate_id, - _aggregate_with_first_event.created_at AT TIME ZONE 'Europe/Amsterdam', - upsert_aggregate_type(_aggregate_with_first_event.aggregate_type), - COALESCE(_events_partition_key, ''), - _aggregate_with_first_event.snapshot_threshold - ); - - FOR _event IN SELECT * FROM old_event_records WHERE aggregate_id = _aggregate_id ORDER BY sequence_number LOOP - _event_json = _event.event_json::jsonb; - - CALL migrate_command(_event.command_record_id); - - INSERT INTO events (partition_key, aggregate_id, sequence_number, created_at, command_id, event_type_id, event_json, xact_id) - VALUES ( - _events_partition_key, - _aggregate_id, - _event.sequence_number, - (_event_json->>'created_at')::timestamptz, - _event.command_record_id, - upsert_event_type(_event.event_type), - _event_json - '{aggregate_id,organization_id,created_at,sequence_number}'::text[], - _event.xact_id - ); - END LOOP; - - DELETE FROM old_event_records WHERE aggregate_id = _aggregate_id; - DELETE FROM old_stream_records WHERE aggregate_id = _aggregate_id; - - RETURN TRUE; -END; -$$ -LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION load_old_events( - _aggregate_id old_stream_records.aggregate_id%TYPE, - _use_snapshots boolean DEFAULT TRUE, - _until event_records.created_at%TYPE DEFAULT NULL -) RETURNS SETOF aggregate_event_type AS $$ -DECLARE - _snapshot_event snapshot_records; - _snapshot_event_sequence_number integer = 0; - _stream_record old_stream_records; -BEGIN - SELECT * - INTO _stream_record - FROM old_stream_records - WHERE old_stream_records.aggregate_id = _aggregate_id; - IF NOT FOUND THEN - RETURN; - END IF; - - IF _use_snapshots THEN - SELECT * - INTO _snapshot_event - FROM snapshot_records snapshot - WHERE snapshot.aggregate_id = _aggregate_id - ORDER BY snapshot.sequence_number DESC - LIMIT 1; - IF FOUND THEN - RETURN NEXT (_stream_record.aggregate_type, - _stream_record.aggregate_id, - '', - _stream_record.snapshot_threshold, - _snapshot_event.snapshot_type::text, - _snapshot_event.snapshot_json::jsonb); - _snapshot_event_sequence_number = _snapshot_event.sequence_number; - END IF; - END IF; - - RETURN QUERY SELECT _stream_record.aggregate_type, - _stream_record.aggregate_id, - '', - _stream_record.snapshot_threshold, - event.event_type::text, - event.event_json::jsonb - FROM old_event_records event - WHERE aggregate_id = _aggregate_id::uuid - AND sequence_number >= _snapshot_event_sequence_number - AND (_until IS NULL OR created_at < _until) - ORDER BY sequence_number; -END; -$$ -LANGUAGE plpgsql; - CREATE OR REPLACE FUNCTION load_event( _aggregate_id uuid, _sequence_number integer @@ -446,10 +124,6 @@ BEGIN FROM aggregates WHERE aggregate_id = _aggregate_id; IF NOT FOUND THEN - RETURN QUERY SELECT aggregate_type::text, _aggregate_id, ''::text, snapshot_threshold, event_type::text, event_json::jsonb - FROM old_event_records event JOIN old_stream_records stream ON event.aggregate_id = stream.aggregate_id - WHERE stream.aggregate_id = _aggregate_id - AND sequence_number = _sequence_number; RETURN; END IF; @@ -480,7 +154,6 @@ BEGIN FOR _aggregate_id IN SELECT * FROM jsonb_array_elements_text(_aggregate_ids) LOOP SELECT * INTO _aggregate FROM aggregates WHERE aggregates.aggregate_id = _aggregate_id; IF NOT FOUND THEN - RETURN QUERY SELECT * FROM load_old_events(_aggregate_id::uuid, _use_snapshots, _until); CONTINUE; END IF; @@ -524,6 +197,13 @@ DECLARE _id commands.id%TYPE; _command_without_nulls jsonb = jsonb_strip_nulls(_command->'command_json'); BEGIN + IF NOT EXISTS (SELECT 1 FROM command_types t WHERE t.type = _command->>'command_type') THEN + -- Only try inserting if it doesn't exist to avoid exhausting the id sequence + INSERT INTO command_types (type) + VALUES (_command->>'command_type') + ON CONFLICT DO NOTHING; + END IF; + INSERT INTO commands ( created_at, user_id, aggregate_id, command_type_id, command_json, event_aggregate_id, event_sequence_number @@ -531,10 +211,10 @@ BEGIN (_command->>'created_at')::timestamptz, (_command_without_nulls->>'user_id')::uuid, (_command_without_nulls->>'aggregate_id')::uuid, - upsert_command_type(_command->>'command_type'), + (SELECT id FROM command_types WHERE type = _command->>'command_type'), (_command->'command_json') - '{command_type,created_at,organization_id,user_id,aggregate_id,event_aggregate_id,event_sequence_number}'::text[], (_command_without_nulls->>'event_aggregate_id')::uuid, - (_command_without_nulls->'event_sequence_number')::integer + (_command_without_nulls->>'event_sequence_number')::integer ) RETURNING id INTO STRICT _id; RETURN _id; END; @@ -558,28 +238,42 @@ DECLARE BEGIN _command_id = store_command(_command); + WITH types AS ( + SELECT DISTINCT row->0->>'aggregate_type' AS type + FROM jsonb_array_elements(_aggregates_with_events) AS row + ) + INSERT INTO aggregate_types (type) + SELECT type FROM types + WHERE type NOT IN (SELECT type FROM aggregate_types) + ORDER BY 1 + ON CONFLICT DO NOTHING; + + WITH types AS ( + SELECT DISTINCT events->>'event_type' AS type + FROM jsonb_array_elements(_aggregates_with_events) AS row + CROSS JOIN LATERAL jsonb_array_elements(row->1) AS events + ) + INSERT INTO event_types (type) + SELECT type FROM types + WHERE type NOT IN (SELECT type FROM event_types) + ORDER BY 1 + ON CONFLICT DO NOTHING; + FOR _aggregate, _events IN SELECT row->0, row->1 FROM jsonb_array_elements(_aggregates_with_events) AS row LOOP _aggregate_id = _aggregate->>'aggregate_id'; _aggregate_without_nulls = jsonb_strip_nulls(_aggregate); _snapshot_threshold = _aggregate_without_nulls->'snapshot_threshold'; _provided_events_partition_key = _aggregate_without_nulls->>'events_partition_key'; - PERFORM migrate_aggregate(_aggregate_id, _provided_events_partition_key); - SELECT events_partition_key INTO _existing_events_partition_key FROM aggregates WHERE aggregate_id = _aggregate_id; - IF NOT FOUND THEN - _events_partition_key = COALESCE(_provided_events_partition_key, determine_events_partition_key(_aggregate_id, NULL, _events->0->'event_json')); - ELSE - _events_partition_key = COALESCE(_provided_events_partition_key, _existing_events_partition_key); - END IF; - + _events_partition_key = COALESCE(_provided_events_partition_key, _existing_events_partition_key, ''); INSERT INTO aggregates (aggregate_id, created_at, aggregate_type_id, events_partition_key, snapshot_threshold) VALUES ( _aggregate_id, (_events->0->>'created_at')::timestamptz, - upsert_aggregate_type(_aggregate->>'aggregate_type'), - COALESCE(_events_partition_key, ''), + (SELECT id FROM aggregate_types WHERE type = _aggregate->>'aggregate_type'), + _events_partition_key, _snapshot_threshold ) ON CONFLICT (aggregate_id) DO UPDATE SET events_partition_key = EXCLUDED.events_partition_key, @@ -597,8 +291,8 @@ BEGIN _sequence_number, _created_at, _command_id, - upsert_event_type(_event->>'event_type'), - (_event->'event_json') - '{aggregate_id,created_at,event_type,organization_id,sequence_number,stream_record_id}'::text[] + (SELECT id FROM event_types WHERE type = _event->>'event_type'), + (_event->'event_json') - '{aggregate_id,created_at,event_type,sequence_number,stream_record_id}'::text[] ); END LOOP; END LOOP; @@ -615,8 +309,6 @@ BEGIN FOR _snapshot IN SELECT * FROM jsonb_array_elements(_snapshots) LOOP _aggregate_id = _snapshot->>'aggregate_id'; - PERFORM migrate_aggregate(_aggregate_id, NULL); - INSERT INTO snapshot_records (aggregate_id, sequence_number, created_at, snapshot_type, snapshot_json) VALUES ( _aggregate_id, @@ -675,27 +367,15 @@ BEGIN RAISE EXCEPTION 'aggregate_id or organization_id must be specified to delete commands'; END IF; - DELETE FROM old_command_records - WHERE (_aggregate_id IS NULL OR aggregate_id = _aggregate_id) - AND NOT EXISTS (SELECT 1 FROM events WHERE command_id = old_command_records.id) - AND NOT EXISTS (SELECT 1 FROM old_event_records WHERE command_record_id = old_command_records.id); DELETE FROM commands WHERE (_aggregate_id IS NULL OR aggregate_id = _aggregate_id) - AND NOT EXISTS (SELECT 1 FROM events WHERE command_id = commands.id) - AND NOT EXISTS (SELECT 1 FROM old_event_records WHERE command_record_id = commands.id); + AND NOT EXISTS (SELECT 1 FROM events WHERE command_id = commands.id); END; $$; CREATE OR REPLACE PROCEDURE permanently_delete_event_streams(_aggregate_ids jsonb) LANGUAGE plpgsql AS $$ BEGIN - DELETE FROM old_event_records - USING jsonb_array_elements_text(_aggregate_ids) AS ids (id) - WHERE old_event_records.aggregate_id = ids.id::uuid; - DELETE FROM old_stream_records - USING jsonb_array_elements_text(_aggregate_ids) AS ids (id) - WHERE old_stream_records.aggregate_id = ids.id::uuid; - DELETE FROM events USING jsonb_array_elements_text(_aggregate_ids) AS ids (id) JOIN aggregates ON ids.id::uuid = aggregates.aggregate_id @@ -706,3 +386,35 @@ BEGIN WHERE aggregates.aggregate_id = ids.id::uuid; END; $$; + +CREATE VIEW command_records (id, user_id, aggregate_id, command_type, command_json, created_at, event_aggregate_id, event_sequence_number) AS + SELECT id, + user_id, + aggregate_id, + (SELECT type FROM command_types WHERE command_types.id = command.command_type_id), + enrich_command_json(command), + created_at, + event_aggregate_id, + event_sequence_number + FROM commands command; + +CREATE VIEW event_records (aggregate_id, partition_key, sequence_number, created_at, event_type, event_json, command_record_id, xact_id) AS + SELECT aggregate.aggregate_id, + event.partition_key, + event.sequence_number, + event.created_at, + type.type, + enrich_event_json(event.*)::text AS event_json, + command_id, + event.xact_id + FROM aggregates aggregate + JOIN events event ON aggregate.aggregate_id = event.aggregate_id AND aggregate.events_partition_key = event.partition_key + JOIN event_types type ON event.event_type_id = type.id; + +CREATE VIEW stream_records (aggregate_id, events_partition_key, aggregate_type, snapshot_threshold, created_at) AS + SELECT aggregates.aggregate_id, + aggregates.events_partition_key, + aggregate_types.type, + aggregates.snapshot_threshold, + aggregates.created_at + FROM aggregates JOIN aggregate_types ON aggregates.aggregate_type_id = aggregate_types.id; From 800412f5da5cedb3c8e74940df51f570f3a1cfc9 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Fri, 19 Apr 2024 15:41:45 +0200 Subject: [PATCH 052/128] Clarified method names and test for commands and events audit trail --- docs/docs/concepts/advanced.md | 14 ++-- lib/sequent/core/command_record.rb | 26 ++++--- lib/sequent/core/event_record.rb | 25 ++++--- spec/lib/sequent/core/audit_trail_spec.rb | 83 +++++++++++++++++++++++ 4 files changed, 122 insertions(+), 26 deletions(-) create mode 100644 spec/lib/sequent/core/audit_trail_spec.rb diff --git a/docs/docs/concepts/advanced.md b/docs/docs/concepts/advanced.md index 68d544c8..5e65f692 100644 --- a/docs/docs/concepts/advanced.md +++ b/docs/docs/concepts/advanced.md @@ -22,14 +22,14 @@ From a `Sequent::Core::CommandRecord` command_record = Sequent::Core::CommandRecord.find(1) # The EventRecord that 'caused' this Command -command_record.parent +command_record.parent_event # Returns the top level Sequent::Core::CommandRecord that 'caused' # this CommandRecord. -command_record.origin +command_record.origin_command # Returns the EventRecords caused by this command -command_record.children +command_record.child_events ``` From a `Sequent::Core::EventRecord` @@ -37,17 +37,17 @@ From a `Sequent::Core::EventRecord` event_record = Sequent::Core::EventRecord.find(1) # Returns the Sequent::Core::CommandRecord that 'caused' this Event -event_record.parent +event_record.parent_command # Returns the top level Sequent::Core::CommandRecord that 'caused' # this Event. This traverses all the way up. # When coming from Sequent < 3.2 this can also # be an EventRecord. -event_record.origin +event_record.origin_command # Returns the Sequent::Core::CommandRecord's that were execute because # of this event -event_record.children +event_record.child_commands ``` ## Upcasting @@ -98,7 +98,7 @@ end After upcasting update all other references to the renamed attribute in `AggregateRoot`s or any other Ruby object. Upcasting does not apply to `Command`s as they are never deserialized, at least not by Sequent. So in a `Command` it -is safe to rename the attribute (and update references to that attribute). +is safe to rename the attribute (and update references to that attribute). ## What-if scenarios diff --git a/lib/sequent/core/command_record.rb b/lib/sequent/core/command_record.rb index d4e4ccd5..38100283 100644 --- a/lib/sequent/core/command_record.rb +++ b/lib/sequent/core/command_record.rb @@ -49,23 +49,29 @@ class CommandRecord < Sequent::ApplicationRecord self.primary_key = :id self.table_name = 'command_records' - has_many :event_records + has_many :child_events, + inverse_of: :parent_command, + class_name: :EventRecord, + foreign_key: :command_record_id validates_presence_of :command_type, :command_json - def parent - EventRecord - .where(aggregate_id: event_aggregate_id, sequence_number: event_sequence_number) - .first + # A `belongs_to` association fails in weird ways with ActiveRecord 7.1, probably due to the use of composite + # primary keys so use an explicit query here and cache the result. + def parent_event + @parent_event ||= EventRecord.find_by(aggregate_id: event_aggregate_id, sequence_number: event_sequence_number) end - def children - event_records + def origin_command + parent_event&.parent_command&.origin_command || self end - def origin - parent&.origin || self - end + # @deprecated + alias parent parent_event + # @deprecated + alias children child_events + # @deprecated + alias origin origin_command end end end diff --git a/lib/sequent/core/event_record.rb b/lib/sequent/core/event_record.rb index 96995a8e..6df1a7f6 100644 --- a/lib/sequent/core/event_record.rb +++ b/lib/sequent/core/event_record.rb @@ -91,22 +91,29 @@ class EventRecord < Sequent::ApplicationRecord self.ignored_columns = %w[xact_id] belongs_to :stream_record, foreign_key: :aggregate_id, primary_key: :aggregate_id - belongs_to :command_record - validates_presence_of :aggregate_id, :sequence_number, :event_type, :event_json, :stream_record, :command_record + belongs_to :parent_command, class_name: :CommandRecord, foreign_key: :command_record_id + + has_many :child_commands, + class_name: :CommandRecord, + query_constraints: %i[event_aggregate_id event_sequence_number], + primary_key: %i[aggregate_id sequence_number] + + validates_presence_of :aggregate_id, :sequence_number, :event_type, :event_json, :stream_record, :parent_command validates_numericality_of :sequence_number, only_integer: true, greater_than: 0 - def parent - command_record + def self.find_by_event(event) + where(aggregate_id: event.aggregate_id, sequence_number: event.sequence_number).first end - def children - CommandRecord.where(event_aggregate_id: aggregate_id, event_sequence_number: sequence_number) + def origin_command + parent_command&.origin_command end - def origin - parent&.origin - end + # @deprecated + alias parent parent_command + alias children child_commands + alias origin origin_command end end end diff --git a/spec/lib/sequent/core/audit_trail_spec.rb b/spec/lib/sequent/core/audit_trail_spec.rb new file mode 100644 index 00000000..eebeb76c --- /dev/null +++ b/spec/lib/sequent/core/audit_trail_spec.rb @@ -0,0 +1,83 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Sequent::Core do + let(:aggregate_id) { Sequent.new_uuid } + + class TestAuditCommand < Sequent::Core::Command; end + class TestAuditEvent < Sequent::Core::Event; end + class TestCausedByCommand < Sequent::Core::Command; end + class TestCausedByEvent < Sequent::Core::Event; end + + let(:aggregate_id_1) { Sequent.new_uuid } + let(:aggregate_id_2) { Sequent.new_uuid } + + it 'tracks cause-and-effect for commands and events' do + audit_event = TestAuditEvent.new( + aggregate_id: aggregate_id_1, + sequence_number: 1, + created_at: Time.parse('2024-02-29T09:00.009Z'), + ) + + Sequent.configuration.event_store.commit_events( + TestAuditCommand.new(aggregate_id: aggregate_id_1), + [ + [ + Sequent::Core::EventStream.new( + aggregate_type: 'AuditTest', + aggregate_id: aggregate_id_1, + ), + [audit_event], + ], + ], + ) + + caused_by_event = TestCausedByEvent.new( + aggregate_id: aggregate_id_2, + sequence_number: 1, + created_at: Time.parse('2024-02-29T09:10.009Z'), + ) + Sequent.configuration.event_store.commit_events( + TestCausedByCommand.new( + aggregate_id: aggregate_id_2, + event_aggregate_id: audit_event.aggregate_id, + event_sequence_number: audit_event.sequence_number, + ), + [ + [ + Sequent::Core::EventStream.new( + aggregate_type: 'CausedByTest', + aggregate_id: aggregate_id_2, + ), + [caused_by_event], + ], + ], + ) + + audit_command_record = Sequent::Core::CommandRecord.where(aggregate_id: aggregate_id_1).first + audit_event_record = Sequent::Core::EventRecord.find_by_event(audit_event) + caused_by_command_record = Sequent::Core::CommandRecord.where(aggregate_id: aggregate_id_2).first + caused_by_event_record = Sequent::Core::EventRecord.find_by_event(caused_by_event) + + expect(audit_command_record.parent_event).to be_nil + expect(audit_command_record.origin_command).to eq(audit_command_record) + expect(audit_command_record.child_events.to_a).to include(audit_event_record) + expect(audit_command_record.child_events.to_a).not_to include(caused_by_event_record) + + expect(audit_event_record.parent_command).to eq(audit_command_record) + expect(audit_event_record.origin_command).to eq(audit_command_record) + expect(audit_event_record.child_commands.to_a).to include(caused_by_command_record) + + expect(caused_by_command_record.parent_event).to eq(audit_event_record) + expect(caused_by_command_record.origin_command).to eq(audit_command_record) + expect(caused_by_command_record.child_events.to_a).to include(caused_by_event_record) + expect(caused_by_command_record.child_events.to_a).not_to include(audit_event_record) + + expect(caused_by_event_record.parent_command).to eq(caused_by_command_record) + expect(caused_by_event_record.parent_command.parent_event).to eq(audit_event_record) + expect(caused_by_event_record.origin_command).to eq(audit_command_record) + + expect(caused_by_event_record.child_commands.to_a).to be_empty + end +end From e72e3a40615804c3808eddff740f092f5b12baed Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Fri, 19 Apr 2024 16:26:47 +0200 Subject: [PATCH 053/128] Minor cleanups --- docs/docs/concepts/advanced.md | 2 +- lib/sequent/core/event_record.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/docs/concepts/advanced.md b/docs/docs/concepts/advanced.md index 5e65f692..8c3cda6a 100644 --- a/docs/docs/concepts/advanced.md +++ b/docs/docs/concepts/advanced.md @@ -34,7 +34,7 @@ command_record.child_events From a `Sequent::Core::EventRecord` ```ruby -event_record = Sequent::Core::EventRecord.find(1) +event_record = Sequent::Core::EventRecord.first # Returns the Sequent::Core::CommandRecord that 'caused' this Event event_record.parent_command diff --git a/lib/sequent/core/event_record.rb b/lib/sequent/core/event_record.rb index 6df1a7f6..89e2d3fe 100644 --- a/lib/sequent/core/event_record.rb +++ b/lib/sequent/core/event_record.rb @@ -103,7 +103,7 @@ class EventRecord < Sequent::ApplicationRecord validates_numericality_of :sequence_number, only_integer: true, greater_than: 0 def self.find_by_event(event) - where(aggregate_id: event.aggregate_id, sequence_number: event.sequence_number).first + find_by(aggregate_id: event.aggregate_id, sequence_number: event.sequence_number) end def origin_command From 6661fcac49ac43a7e28a5799e9b135e3b3283dbc Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Wed, 1 May 2024 12:06:38 +0200 Subject: [PATCH 054/128] Optimize replay of events by scanning partitioned events directly This avoids using the `event_records` view which joins with the `aggregates` table, which is unnecessary in this case. Also add "before" and "after" groups to ensure we capture all possible events, in case statistical scanning is used to count the number of events to determine the event replay groups. --- db/sequent_schema.sql | 2 +- lib/sequent/core/aggregate_type.rb | 13 +++++ lib/sequent/core/command_type.rb | 13 +++++ lib/sequent/core/core.rb | 6 +++ lib/sequent/core/event_type.rb | 13 +++++ lib/sequent/core/partitioned_aggregate.rb | 20 +++++++ lib/sequent/core/partitioned_command.rb | 17 ++++++ lib/sequent/core/partitioned_event.rb | 21 ++++++++ lib/sequent/migrations/view_schema.rb | 53 ++++++++++++++----- .../sequent/core/partitioned_event_spec.rb | 46 ++++++++++++++++ 10 files changed, 189 insertions(+), 15 deletions(-) create mode 100644 lib/sequent/core/aggregate_type.rb create mode 100644 lib/sequent/core/command_type.rb create mode 100644 lib/sequent/core/event_type.rb create mode 100644 lib/sequent/core/partitioned_aggregate.rb create mode 100644 lib/sequent/core/partitioned_command.rb create mode 100644 lib/sequent/core/partitioned_event.rb create mode 100644 spec/lib/sequent/core/partitioned_event_spec.rb diff --git a/db/sequent_schema.sql b/db/sequent_schema.sql index 88daf4f6..85306cde 100644 --- a/db/sequent_schema.sql +++ b/db/sequent_schema.sql @@ -404,7 +404,7 @@ CREATE VIEW event_records (aggregate_id, partition_key, sequence_number, created event.sequence_number, event.created_at, type.type, - enrich_event_json(event.*)::text AS event_json, + enrich_event_json(event) AS event_json, command_id, event.xact_id FROM aggregates aggregate diff --git a/lib/sequent/core/aggregate_type.rb b/lib/sequent/core/aggregate_type.rb new file mode 100644 index 00000000..7c579b59 --- /dev/null +++ b/lib/sequent/core/aggregate_type.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +require 'active_record' +require_relative 'sequent_oj' +require_relative '../application_record' + +module Sequent + module Core + class AggregateType < Sequent::ApplicationRecord + self.inheritance_column = nil + end + end +end diff --git a/lib/sequent/core/command_type.rb b/lib/sequent/core/command_type.rb new file mode 100644 index 00000000..d502116e --- /dev/null +++ b/lib/sequent/core/command_type.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +require 'active_record' +require_relative 'sequent_oj' +require_relative '../application_record' + +module Sequent + module Core + class CommandType < Sequent::ApplicationRecord + self.inheritance_column = nil + end + end +end diff --git a/lib/sequent/core/core.rb b/lib/sequent/core/core.rb index 8603b572..7d11f28f 100644 --- a/lib/sequent/core/core.rb +++ b/lib/sequent/core/core.rb @@ -20,3 +20,9 @@ require_relative 'random_uuid_generator' require_relative 'event_publisher' require_relative 'aggregate_snapshotter' +require_relative 'aggregate_type' +require_relative 'command_type' +require_relative 'event_type' +require_relative 'partitioned_aggregate' +require_relative 'partitioned_command' +require_relative 'partitioned_event' diff --git a/lib/sequent/core/event_type.rb b/lib/sequent/core/event_type.rb new file mode 100644 index 00000000..f93164a8 --- /dev/null +++ b/lib/sequent/core/event_type.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +require 'active_record' +require_relative 'sequent_oj' +require_relative '../application_record' + +module Sequent + module Core + class EventType < Sequent::ApplicationRecord + self.inheritance_column = nil + end + end +end diff --git a/lib/sequent/core/partitioned_aggregate.rb b/lib/sequent/core/partitioned_aggregate.rb new file mode 100644 index 00000000..c6e45b1d --- /dev/null +++ b/lib/sequent/core/partitioned_aggregate.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +require 'active_record' +require_relative '../application_record' + +module Sequent + module Core + class PartitionedAggregate < Sequent::ApplicationRecord + self.table_name = :aggregates + self.primary_key = %i[aggregate_id] + + belongs_to :aggregate_type + has_many :events, + inverse_of: :aggregate, + class_name: :PartitionedEvent, + primary_key: %i[events_partition_key aggregate_id], + query_constraints: %i[partition_key aggregate_id] + end + end +end diff --git a/lib/sequent/core/partitioned_command.rb b/lib/sequent/core/partitioned_command.rb new file mode 100644 index 00000000..8a85f7c4 --- /dev/null +++ b/lib/sequent/core/partitioned_command.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +require 'active_record' +require_relative '../application_record' + +module Sequent + module Core + class PartitionedCommand < Sequent::ApplicationRecord + self.table_name = :commands + + belongs_to :command_type + has_many :events, + inverse_of: :command, + class_name: :PartitionedEvent + end + end +end diff --git a/lib/sequent/core/partitioned_event.rb b/lib/sequent/core/partitioned_event.rb new file mode 100644 index 00000000..e1b53d63 --- /dev/null +++ b/lib/sequent/core/partitioned_event.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +require 'active_record' +require_relative '../application_record' + +module Sequent + module Core + class PartitionedEvent < Sequent::ApplicationRecord + self.table_name = :events + self.primary_key = %i[partition_key aggregate_id sequence_number] + + belongs_to :event_type + belongs_to :command, + inverse_of: :events, + class_name: :PartitionedCommand + belongs_to :aggregate, + inverse_of: :events, + class_name: :PartitionedAggregate + end + end +end diff --git a/lib/sequent/migrations/view_schema.rb b/lib/sequent/migrations/view_schema.rb index 8f265ea2..c09773d9 100644 --- a/lib/sequent/migrations/view_schema.rb +++ b/lib/sequent/migrations/view_schema.rb @@ -302,13 +302,21 @@ def replay!( ) event_types = projectors.flat_map { |projector| projector.message_mapping.keys }.uniq.map(&:name) group_target_size = Sequent.configuration.replay_group_target_size - partitions = Sequent.configuration.event_record_class.where(event_type: event_types) + event_type_ids = Sequent::Core::EventType.where(type: event_types).pluck(:id) + partitions = Sequent::Core::PartitionedEvent.where(event_type_id: event_type_ids) .group(:partition_key) .order(:partition_key) .count event_count = partitions.map(&:count).sum groups = Sequent::Migrations::Grouper.group_partitions(partitions, group_target_size) + if groups.empty? + groups = [nil..nil] + else + groups.prepend(nil..groups.first.begin) + groups.append(groups.last.end..nil) + end + with_sequent_config(replay_persistor, projectors) do logger.info "Start replaying #{event_count} events in #{groups.size} groups" @@ -328,7 +336,7 @@ def replay!( time(msg) do replay_events( -> { - event_stream(group, event_types, minimum_xact_id_inclusive, maximum_xact_id_exclusive) + event_stream(group, event_type_ids, minimum_xact_id_inclusive, maximum_xact_id_exclusive) }, replay_persistor, &on_progress @@ -413,18 +421,35 @@ def with_sequent_config(replay_persistor, projectors, &block) Sequent::Configuration.restore(old_config) end - def event_stream(group, event_types, minimum_xact_id_inclusive, maximum_xact_id_exclusive) + def event_stream(group, event_type_ids, minimum_xact_id_inclusive, maximum_xact_id_exclusive) fail ArgumentError, 'group is mandatory' if group.nil? - event_stream = Sequent.configuration.event_record_class.where( - event_type: event_types, - ).where( - '(partition_key, aggregate_id) BETWEEN (?, ?) AND (?, ?)', - group.begin.partition_key, - group.begin.aggregate_id, - group.end.partition_key, - group.end.aggregate_id, - ) + event_stream = Core::PartitionedEvent + .joins('JOIN event_types ON events.event_type_id = event_types.id') + .where( + event_type_id: event_type_ids, + ) + if group.begin && group.end + event_stream = event_stream.where( + '(events.partition_key, events.aggregate_id) BETWEEN (?, ?) AND (?, ?)', + group.begin.partition_key, + group.begin.aggregate_id, + group.end.partition_key, + group.end.aggregate_id, + ) + elsif group.end + event_stream = event_stream.where( + '(events.partition_key, events.aggregate_id) < (?, ?)', + group.end.partition_key, + group.end.aggregate_id, + ) + elsif group.begin + event_stream = event_stream.where( + '(events.partition_key, events.aggregate_id) > (?, ?)', + group.begin.partition_key, + group.begin.aggregate_id, + ) + end if minimum_xact_id_inclusive && maximum_xact_id_exclusive event_stream = event_stream.where( 'xact_id >= ? AND xact_id < ?', @@ -437,8 +462,8 @@ def event_stream(group, event_types, minimum_xact_id_inclusive, maximum_xact_id_ event_stream = event_stream.where('xact_id IS NULL OR xact_id < ?', maximum_xact_id_exclusive) end event_stream - .order(:partition_key, :aggregate_id, :sequence_number) - .select('event_type, event_json') + .order('events.partition_key', 'events.aggregate_id', 'events.sequence_number') + .select('event_types.type AS event_type, enrich_event_json(events) AS event_json') end ## shortcut methods diff --git a/spec/lib/sequent/core/partitioned_event_spec.rb b/spec/lib/sequent/core/partitioned_event_spec.rb new file mode 100644 index 00000000..c6544ef6 --- /dev/null +++ b/spec/lib/sequent/core/partitioned_event_spec.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Sequent::Core::PartitionedEvent do + let(:aggregate_id) { Sequent.new_uuid } + let(:events_partition_key) { 'abc' } + + let(:event_store) { Sequent.configuration.event_store } + + before do + event_store.commit_events( + Sequent::Core::Command.new(aggregate_id:), + [ + [ + Sequent::Core::EventStream.new(aggregate_type: 'Aggregate', aggregate_id:, events_partition_key:), + [ + Sequent::Core::Event.new(aggregate_id:, sequence_number: 1), + ], + ], + ], + ) + end + + it 'persists to the partitioned tables' do + aggregate = Sequent::Core::PartitionedAggregate.first + expect(aggregate).to be_present + expect(aggregate.aggregate_id).to eq(aggregate_id) + expect(aggregate.events_partition_key).to eq(events_partition_key) + expect(aggregate.aggregate_type.type).to eq('Aggregate') + + events = aggregate.events.to_a + expect(events.size).to eq(1) + + event = events[0] + expect(event.aggregate).to be(aggregate) + expect(event.aggregate_id).to eq(aggregate_id) + expect(event.partition_key).to eq(events_partition_key) + expect(event.event_type.type).to eq('Sequent::Core::Event') + + command = event.command + expect(command).to be_present + expect(command.command_type.type).to eq('Sequent::Core::Command') + expect(command.events).to eq(events) + end +end From 80469f0becfa43fec3d0d9ee17b7f64fbe273a1b Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Wed, 1 May 2024 17:08:37 +0200 Subject: [PATCH 055/128] Move partitioned storage classes to an internal module --- lib/sequent/core/core.rb | 6 --- .../{core => internal}/aggregate_type.rb | 3 +- .../{core => internal}/command_type.rb | 3 +- lib/sequent/{core => internal}/event_type.rb | 3 +- lib/sequent/internal/internal.rb | 14 ++++++ .../partitioned_aggregate.rb | 2 +- .../{core => internal}/partitioned_command.rb | 2 +- .../{core => internal}/partitioned_event.rb | 2 +- lib/sequent/migrations/view_schema.rb | 6 +-- lib/sequent/sequent.rb | 1 + .../sequent/core/partitioned_event_spec.rb | 46 ----------------- .../internal/partitioned_storage_spec.rb | 50 +++++++++++++++++++ 12 files changed, 74 insertions(+), 64 deletions(-) rename lib/sequent/{core => internal}/aggregate_type.rb (83%) rename lib/sequent/{core => internal}/command_type.rb (83%) rename lib/sequent/{core => internal}/event_type.rb (83%) create mode 100644 lib/sequent/internal/internal.rb rename lib/sequent/{core => internal}/partitioned_aggregate.rb (96%) rename lib/sequent/{core => internal}/partitioned_command.rb (95%) rename lib/sequent/{core => internal}/partitioned_event.rb (96%) delete mode 100644 spec/lib/sequent/core/partitioned_event_spec.rb create mode 100644 spec/lib/sequent/internal/partitioned_storage_spec.rb diff --git a/lib/sequent/core/core.rb b/lib/sequent/core/core.rb index 7d11f28f..8603b572 100644 --- a/lib/sequent/core/core.rb +++ b/lib/sequent/core/core.rb @@ -20,9 +20,3 @@ require_relative 'random_uuid_generator' require_relative 'event_publisher' require_relative 'aggregate_snapshotter' -require_relative 'aggregate_type' -require_relative 'command_type' -require_relative 'event_type' -require_relative 'partitioned_aggregate' -require_relative 'partitioned_command' -require_relative 'partitioned_event' diff --git a/lib/sequent/core/aggregate_type.rb b/lib/sequent/internal/aggregate_type.rb similarity index 83% rename from lib/sequent/core/aggregate_type.rb rename to lib/sequent/internal/aggregate_type.rb index 7c579b59..9e2ce9c0 100644 --- a/lib/sequent/core/aggregate_type.rb +++ b/lib/sequent/internal/aggregate_type.rb @@ -1,11 +1,10 @@ # frozen_string_literal: true require 'active_record' -require_relative 'sequent_oj' require_relative '../application_record' module Sequent - module Core + module Internal class AggregateType < Sequent::ApplicationRecord self.inheritance_column = nil end diff --git a/lib/sequent/core/command_type.rb b/lib/sequent/internal/command_type.rb similarity index 83% rename from lib/sequent/core/command_type.rb rename to lib/sequent/internal/command_type.rb index d502116e..53593740 100644 --- a/lib/sequent/core/command_type.rb +++ b/lib/sequent/internal/command_type.rb @@ -1,11 +1,10 @@ # frozen_string_literal: true require 'active_record' -require_relative 'sequent_oj' require_relative '../application_record' module Sequent - module Core + module Internal class CommandType < Sequent::ApplicationRecord self.inheritance_column = nil end diff --git a/lib/sequent/core/event_type.rb b/lib/sequent/internal/event_type.rb similarity index 83% rename from lib/sequent/core/event_type.rb rename to lib/sequent/internal/event_type.rb index f93164a8..63d9ca9c 100644 --- a/lib/sequent/core/event_type.rb +++ b/lib/sequent/internal/event_type.rb @@ -1,11 +1,10 @@ # frozen_string_literal: true require 'active_record' -require_relative 'sequent_oj' require_relative '../application_record' module Sequent - module Core + module Internal class EventType < Sequent::ApplicationRecord self.inheritance_column = nil end diff --git a/lib/sequent/internal/internal.rb b/lib/sequent/internal/internal.rb new file mode 100644 index 00000000..b1b7ab78 --- /dev/null +++ b/lib/sequent/internal/internal.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +require_relative 'aggregate_type' +require_relative 'command_type' +require_relative 'event_type' +require_relative 'partitioned_aggregate' +require_relative 'partitioned_command' +require_relative 'partitioned_event' + +module Sequent + module Internal + end + private_constant :Internal +end diff --git a/lib/sequent/core/partitioned_aggregate.rb b/lib/sequent/internal/partitioned_aggregate.rb similarity index 96% rename from lib/sequent/core/partitioned_aggregate.rb rename to lib/sequent/internal/partitioned_aggregate.rb index c6e45b1d..8cf6f279 100644 --- a/lib/sequent/core/partitioned_aggregate.rb +++ b/lib/sequent/internal/partitioned_aggregate.rb @@ -4,7 +4,7 @@ require_relative '../application_record' module Sequent - module Core + module Internal class PartitionedAggregate < Sequent::ApplicationRecord self.table_name = :aggregates self.primary_key = %i[aggregate_id] diff --git a/lib/sequent/core/partitioned_command.rb b/lib/sequent/internal/partitioned_command.rb similarity index 95% rename from lib/sequent/core/partitioned_command.rb rename to lib/sequent/internal/partitioned_command.rb index 8a85f7c4..eeb0740e 100644 --- a/lib/sequent/core/partitioned_command.rb +++ b/lib/sequent/internal/partitioned_command.rb @@ -4,7 +4,7 @@ require_relative '../application_record' module Sequent - module Core + module Internal class PartitionedCommand < Sequent::ApplicationRecord self.table_name = :commands diff --git a/lib/sequent/core/partitioned_event.rb b/lib/sequent/internal/partitioned_event.rb similarity index 96% rename from lib/sequent/core/partitioned_event.rb rename to lib/sequent/internal/partitioned_event.rb index e1b53d63..f40c79cd 100644 --- a/lib/sequent/core/partitioned_event.rb +++ b/lib/sequent/internal/partitioned_event.rb @@ -4,7 +4,7 @@ require_relative '../application_record' module Sequent - module Core + module Internal class PartitionedEvent < Sequent::ApplicationRecord self.table_name = :events self.primary_key = %i[partition_key aggregate_id sequence_number] diff --git a/lib/sequent/migrations/view_schema.rb b/lib/sequent/migrations/view_schema.rb index c09773d9..2fbfc16d 100644 --- a/lib/sequent/migrations/view_schema.rb +++ b/lib/sequent/migrations/view_schema.rb @@ -302,8 +302,8 @@ def replay!( ) event_types = projectors.flat_map { |projector| projector.message_mapping.keys }.uniq.map(&:name) group_target_size = Sequent.configuration.replay_group_target_size - event_type_ids = Sequent::Core::EventType.where(type: event_types).pluck(:id) - partitions = Sequent::Core::PartitionedEvent.where(event_type_id: event_type_ids) + event_type_ids = Internal::EventType.where(type: event_types).pluck(:id) + partitions = Internal::PartitionedEvent.where(event_type_id: event_type_ids) .group(:partition_key) .order(:partition_key) .count @@ -424,7 +424,7 @@ def with_sequent_config(replay_persistor, projectors, &block) def event_stream(group, event_type_ids, minimum_xact_id_inclusive, maximum_xact_id_exclusive) fail ArgumentError, 'group is mandatory' if group.nil? - event_stream = Core::PartitionedEvent + event_stream = Internal::PartitionedEvent .joins('JOIN event_types ON events.event_type_id = event_types.id') .where( event_type_id: event_type_ids, diff --git a/lib/sequent/sequent.rb b/lib/sequent/sequent.rb index 087e748c..9a464b92 100644 --- a/lib/sequent/sequent.rb +++ b/lib/sequent/sequent.rb @@ -8,6 +8,7 @@ require_relative 'core/projector' require_relative 'core/workflow' require_relative 'core/value_object' +require_relative 'internal/internal' require_relative 'migrations/migrations' module Sequent diff --git a/spec/lib/sequent/core/partitioned_event_spec.rb b/spec/lib/sequent/core/partitioned_event_spec.rb deleted file mode 100644 index c6544ef6..00000000 --- a/spec/lib/sequent/core/partitioned_event_spec.rb +++ /dev/null @@ -1,46 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -describe Sequent::Core::PartitionedEvent do - let(:aggregate_id) { Sequent.new_uuid } - let(:events_partition_key) { 'abc' } - - let(:event_store) { Sequent.configuration.event_store } - - before do - event_store.commit_events( - Sequent::Core::Command.new(aggregate_id:), - [ - [ - Sequent::Core::EventStream.new(aggregate_type: 'Aggregate', aggregate_id:, events_partition_key:), - [ - Sequent::Core::Event.new(aggregate_id:, sequence_number: 1), - ], - ], - ], - ) - end - - it 'persists to the partitioned tables' do - aggregate = Sequent::Core::PartitionedAggregate.first - expect(aggregate).to be_present - expect(aggregate.aggregate_id).to eq(aggregate_id) - expect(aggregate.events_partition_key).to eq(events_partition_key) - expect(aggregate.aggregate_type.type).to eq('Aggregate') - - events = aggregate.events.to_a - expect(events.size).to eq(1) - - event = events[0] - expect(event.aggregate).to be(aggregate) - expect(event.aggregate_id).to eq(aggregate_id) - expect(event.partition_key).to eq(events_partition_key) - expect(event.event_type.type).to eq('Sequent::Core::Event') - - command = event.command - expect(command).to be_present - expect(command.command_type.type).to eq('Sequent::Core::Command') - expect(command.events).to eq(events) - end -end diff --git a/spec/lib/sequent/internal/partitioned_storage_spec.rb b/spec/lib/sequent/internal/partitioned_storage_spec.rb new file mode 100644 index 00000000..5728278a --- /dev/null +++ b/spec/lib/sequent/internal/partitioned_storage_spec.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +require 'spec_helper' + +module Sequent + module Internal + describe 'partitioned storage' do + let(:aggregate_id) { Sequent.new_uuid } + let(:events_partition_key) { 'abc' } + + let(:event_store) { Sequent.configuration.event_store } + + before do + event_store.commit_events( + Sequent::Core::Command.new(aggregate_id:), + [ + [ + Sequent::Core::EventStream.new(aggregate_type: 'Aggregate', aggregate_id:, events_partition_key:), + [ + Sequent::Core::Event.new(aggregate_id:, sequence_number: 1), + ], + ], + ], + ) + end + + it 'persists to the partitioned tables' do + aggregate = PartitionedAggregate.first + expect(aggregate).to be_present + expect(aggregate.aggregate_id).to eq(aggregate_id) + expect(aggregate.events_partition_key).to eq(events_partition_key) + expect(aggregate.aggregate_type.type).to eq('Aggregate') + + events = aggregate.events.to_a + expect(events.size).to eq(1) + + event = events[0] + expect(event.aggregate).to be(aggregate) + expect(event.aggregate_id).to eq(aggregate_id) + expect(event.partition_key).to eq(events_partition_key) + expect(event.event_type.type).to eq('Sequent::Core::Event') + + command = event.command + expect(command).to be_present + expect(command.command_type.type).to eq('Sequent::Core::Command') + expect(command.events).to eq(events) + end + end + end +end From a524274a85de68be2eda020e470c7f80df76dd45 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Fri, 3 May 2024 14:01:59 +0200 Subject: [PATCH 056/128] Replace loop with bulk insert --- db/sequent_schema.sql | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/db/sequent_schema.sql b/db/sequent_schema.sql index 85306cde..f0e82b9d 100644 --- a/db/sequent_schema.sql +++ b/db/sequent_schema.sql @@ -281,20 +281,15 @@ BEGIN WHERE aggregates.events_partition_key <> EXCLUDED.events_partition_key OR aggregates.snapshot_threshold <> EXCLUDED.snapshot_threshold; - FOR _event IN SELECT * FROM jsonb_array_elements(_events) LOOP - _created_at = (_event->>'created_at')::timestamptz; - _sequence_number = _event->'event_json'->>'sequence_number'; - INSERT INTO events (partition_key, aggregate_id, sequence_number, created_at, command_id, event_type_id, event_json) - VALUES ( - _events_partition_key, - _aggregate_id, - _sequence_number, - _created_at, - _command_id, - (SELECT id FROM event_types WHERE type = _event->>'event_type'), - (_event->'event_json') - '{aggregate_id,created_at,event_type,sequence_number,stream_record_id}'::text[] - ); - END LOOP; + INSERT INTO events (partition_key, aggregate_id, sequence_number, created_at, command_id, event_type_id, event_json) + SELECT _events_partition_key, + _aggregate_id, + (event->'event_json'->'sequence_number')::integer, + (event->>'created_at')::timestamptz, + _command_id, + (SELECT id FROM event_types WHERE type = event->>'event_type'), + (event->'event_json') - '{aggregate_id,created_at,event_type,sequence_number}'::text[] + FROM jsonb_array_elements(_events) AS event; END LOOP; END; $$; From 321911ed62e6b7207bd51249585816850c4cea6b Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Fri, 3 May 2024 14:08:29 +0200 Subject: [PATCH 057/128] Replace jsonb_strip_nulls with NULLIF --- db/sequent_schema.sql | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/db/sequent_schema.sql b/db/sequent_schema.sql index f0e82b9d..bee5f7f0 100644 --- a/db/sequent_schema.sql +++ b/db/sequent_schema.sql @@ -195,7 +195,7 @@ CREATE OR REPLACE FUNCTION store_command(_command jsonb) RETURNS bigint LANGUAGE plpgsql AS $$ DECLARE _id commands.id%TYPE; - _command_without_nulls jsonb = jsonb_strip_nulls(_command->'command_json'); + _command_json jsonb = _command->'command_json'; BEGIN IF NOT EXISTS (SELECT 1 FROM command_types t WHERE t.type = _command->>'command_type') THEN -- Only try inserting if it doesn't exist to avoid exhausting the id sequence @@ -209,12 +209,12 @@ BEGIN event_aggregate_id, event_sequence_number ) VALUES ( (_command->>'created_at')::timestamptz, - (_command_without_nulls->>'user_id')::uuid, - (_command_without_nulls->>'aggregate_id')::uuid, + (_command_json->>'user_id')::uuid, + (_command_json->>'aggregate_id')::uuid, (SELECT id FROM command_types WHERE type = _command->>'command_type'), (_command->'command_json') - '{command_type,created_at,organization_id,user_id,aggregate_id,event_aggregate_id,event_sequence_number}'::text[], - (_command_without_nulls->>'event_aggregate_id')::uuid, - (_command_without_nulls->>'event_sequence_number')::integer + (_command_json->>'event_aggregate_id')::uuid, + NULLIF(_command_json->'event_sequence_number', 'null'::jsonb)::integer ) RETURNING id INTO STRICT _id; RETURN _id; END; @@ -225,16 +225,12 @@ LANGUAGE plpgsql AS $$ DECLARE _command_id commands.id%TYPE; _aggregate jsonb; - _aggregate_without_nulls jsonb; _events jsonb; - _event jsonb; _aggregate_id aggregates.aggregate_id%TYPE; - _created_at aggregates.created_at%TYPE; _provided_events_partition_key aggregates.events_partition_key%TYPE; _existing_events_partition_key aggregates.events_partition_key%TYPE; _events_partition_key aggregates.events_partition_key%TYPE; _snapshot_threshold aggregates.snapshot_threshold%TYPE; - _sequence_number events.sequence_number%TYPE; BEGIN _command_id = store_command(_command); @@ -261,9 +257,8 @@ BEGIN FOR _aggregate, _events IN SELECT row->0, row->1 FROM jsonb_array_elements(_aggregates_with_events) AS row LOOP _aggregate_id = _aggregate->>'aggregate_id'; - _aggregate_without_nulls = jsonb_strip_nulls(_aggregate); - _snapshot_threshold = _aggregate_without_nulls->'snapshot_threshold'; - _provided_events_partition_key = _aggregate_without_nulls->>'events_partition_key'; + _snapshot_threshold = NULLIF(_aggregate->'snapshot_threshold', 'null'::jsonb); + _provided_events_partition_key = _aggregate->>'events_partition_key'; SELECT events_partition_key INTO _existing_events_partition_key FROM aggregates WHERE aggregate_id = _aggregate_id; _events_partition_key = COALESCE(_provided_events_partition_key, _existing_events_partition_key, ''); From e6ca38d361517d3a9aacd4316af4165e3b7c22ee Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Fri, 3 May 2024 14:11:24 +0200 Subject: [PATCH 058/128] Update project template schema --- .../template_project/db/sequent_schema.rb | 54 +-- .../template_project/db/sequent_schema.sql | 410 ++++++++++++++++++ 2 files changed, 412 insertions(+), 52 deletions(-) create mode 100644 lib/sequent/generator/template_project/db/sequent_schema.sql diff --git a/lib/sequent/generator/template_project/db/sequent_schema.rb b/lib/sequent/generator/template_project/db/sequent_schema.rb index 5f0128f5..97364112 100644 --- a/lib/sequent/generator/template_project/db/sequent_schema.rb +++ b/lib/sequent/generator/template_project/db/sequent_schema.rb @@ -1,44 +1,6 @@ # frozen_string_literal: true ActiveRecord::Schema.define do - create_table 'command_records', force: true do |t| - t.string 'user_id' - t.uuid 'aggregate_id' - t.string 'command_type', null: false - t.string 'event_aggregate_id' - t.integer 'event_sequence_number' - t.jsonb 'command_json', null: false - t.datetime 'created_at', null: false - end - - add_index 'command_records', %w[event_aggregate_id event_sequence_number], name: 'index_command_records_on_event' - - create_table 'stream_records', primary_key: ['aggregate_id'], force: true do |t| - t.datetime 'created_at', null: false - t.string 'aggregate_type', null: false - t.uuid 'aggregate_id', null: false - t.integer 'snapshot_threshold' - end - - create_table 'event_records', primary_key: %w[aggregate_id sequence_number], force: true do |t| - t.uuid 'aggregate_id', null: false - t.integer 'sequence_number', null: false - t.datetime 'created_at', null: false - t.string 'event_type', null: false - t.jsonb 'event_json', null: false - t.integer 'command_record_id', null: false - t.bigint 'xact_id', null: false - end - - add_index 'event_records', ['command_record_id'], name: 'index_event_records_on_command_record_id' - add_index 'event_records', ['event_type'], name: 'index_event_records_on_event_type' - add_index 'event_records', ['created_at'], name: 'index_event_records_on_created_at' - add_index 'event_records', ['xact_id'], name: 'index_event_records_on_xact_id' - - execute <<~EOS - ALTER TABLE event_records ALTER COLUMN xact_id SET DEFAULT pg_current_xact_id()::text::bigint - EOS - create_table 'snapshot_records', primary_key: %w[aggregate_id sequence_number], force: true do |t| t.uuid 'aggregate_id', null: false t.integer 'sequence_number', null: false @@ -47,18 +9,6 @@ t.jsonb 'snapshot_json', null: false end - add_foreign_key :event_records, - :command_records, - name: 'command_fkey' - add_foreign_key :event_records, - :stream_records, - column: :aggregate_id, - primary_key: :aggregate_id, - name: 'stream_fkey' - add_foreign_key :snapshot_records, - :stream_records, - column: :aggregate_id, - primary_key: :aggregate_id, - on_delete: :cascade, - name: 'stream_fkey' + schema = File.read("#{File.dirname(__FILE__)}/sequent_schema.sql") + execute schema end diff --git a/lib/sequent/generator/template_project/db/sequent_schema.sql b/lib/sequent/generator/template_project/db/sequent_schema.sql new file mode 100644 index 00000000..bee5f7f0 --- /dev/null +++ b/lib/sequent/generator/template_project/db/sequent_schema.sql @@ -0,0 +1,410 @@ +DROP TYPE IF EXISTS aggregate_event_type CASCADE; +CREATE TYPE aggregate_event_type AS ( + aggregate_type text, + aggregate_id uuid, + events_partition_key text, + snapshot_threshold integer, + event_type text, + event_json jsonb +); + +CREATE TABLE command_types (id SMALLINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, type text UNIQUE NOT NULL); +CREATE TABLE aggregate_types (id SMALLINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, type text UNIQUE NOT NULL); +CREATE TABLE event_types (id SMALLINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, type text UNIQUE NOT NULL); + +CREATE TABLE commands ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + created_at timestamp with time zone NOT NULL, + user_id uuid, + aggregate_id uuid, + command_type_id SMALLINT NOT NULL REFERENCES command_types (id), + command_json jsonb NOT NULL, + event_aggregate_id uuid, + event_sequence_number integer +) PARTITION BY RANGE (id); +CREATE INDEX commands_command_type_id_idx ON commands (command_type_id); +CREATE INDEX commands_aggregate_id_idx ON commands (aggregate_id); +CREATE INDEX commands_event_idx ON commands (event_aggregate_id, event_sequence_number); + +CREATE TABLE commands_default PARTITION OF commands DEFAULT; + +CREATE TABLE aggregates ( + aggregate_id uuid PRIMARY KEY, + events_partition_key text NOT NULL DEFAULT '', + aggregate_type_id SMALLINT NOT NULL REFERENCES aggregate_types (id), + snapshot_threshold integer, + created_at timestamp with time zone NOT NULL DEFAULT NOW(), + UNIQUE (events_partition_key, aggregate_id) +) PARTITION BY RANGE (aggregate_id); +CREATE INDEX aggregates_aggregate_type_id_idx ON aggregates (aggregate_type_id); + +CREATE TABLE aggregates_0 PARTITION OF aggregates FOR VALUES FROM (MINVALUE) TO ('40000000-0000-0000-0000-000000000000'); +ALTER TABLE aggregates_0 CLUSTER ON aggregates_0_events_partition_key_aggregate_id_key; +CREATE TABLE aggregates_4 PARTITION OF aggregates FOR VALUES FROM ('40000000-0000-0000-0000-000000000000') TO ('80000000-0000-0000-0000-000000000000'); +ALTER TABLE aggregates_4 CLUSTER ON aggregates_4_events_partition_key_aggregate_id_key; +CREATE TABLE aggregates_8 PARTITION OF aggregates FOR VALUES FROM ('80000000-0000-0000-0000-000000000000') TO ('c0000000-0000-0000-0000-000000000000'); +ALTER TABLE aggregates_8 CLUSTER ON aggregates_8_events_partition_key_aggregate_id_key; +CREATE TABLE aggregates_c PARTITION OF aggregates FOR VALUES FROM ('c0000000-0000-0000-0000-000000000000') TO (MAXVALUE); +ALTER TABLE aggregates_c CLUSTER ON aggregates_c_events_partition_key_aggregate_id_key; + +CREATE TABLE events ( + aggregate_id uuid NOT NULL, + partition_key text DEFAULT '', + sequence_number integer NOT NULL, + created_at timestamp with time zone NOT NULL, + command_id bigint NOT NULL, + event_type_id SMALLINT NOT NULL REFERENCES event_types (id), + event_json jsonb NOT NULL, + xact_id bigint DEFAULT pg_current_xact_id()::text::bigint, + PRIMARY KEY (partition_key, aggregate_id, sequence_number), + FOREIGN KEY (partition_key, aggregate_id) + REFERENCES aggregates (events_partition_key, aggregate_id) + ON UPDATE CASCADE ON DELETE RESTRICT, + FOREIGN KEY (command_id) REFERENCES commands (id) +) PARTITION BY RANGE (partition_key); +CREATE INDEX events_command_id_idx ON events (command_id); +CREATE INDEX events_event_type_id_idx ON events (event_type_id); +CREATE INDEX events_xact_id_idx ON events (xact_id) WHERE xact_id IS NOT NULL; + +CREATE TABLE events_default PARTITION OF events DEFAULT; +ALTER TABLE events_default CLUSTER ON events_default_pkey; +CREATE TABLE events_2023_and_earlier PARTITION OF events FOR VALUES FROM ('Y00') TO ('Y24'); +ALTER TABLE events_2023_and_earlier CLUSTER ON events_2023_and_earlier_pkey; +CREATE TABLE events_2024 PARTITION OF events FOR VALUES FROM ('Y24') TO ('Y25'); +ALTER TABLE events_2024 CLUSTER ON events_2024_pkey; +CREATE TABLE events_2025_and_later PARTITION OF events FOR VALUES FROM ('Y25') TO ('Y99'); +ALTER TABLE events_2025_and_later CLUSTER ON events_2025_and_later_pkey; +CREATE TABLE events_organizations PARTITION OF events FOR VALUES FROM ('O') TO ('Og'); +ALTER TABLE events_organizations CLUSTER ON events_organizations_pkey; +CREATE TABLE events_aggregate PARTITION OF events FOR VALUES FROM ('A') TO ('Ag'); +ALTER TABLE events_aggregate CLUSTER ON events_aggregate_pkey; + +TRUNCATE TABLE snapshot_records; +ALTER TABLE snapshot_records + ADD CONSTRAINT aggregate_fkey FOREIGN KEY (aggregate_id) REFERENCES aggregates (aggregate_id) + ON UPDATE CASCADE ON DELETE CASCADE; + +CREATE OR REPLACE FUNCTION enrich_command_json(command commands) RETURNS jsonb +LANGUAGE plpgsql AS $$ +BEGIN + RETURN jsonb_build_object( + 'command_type', (SELECT type FROM command_types WHERE command_types.id = command.command_type_id), + 'created_at', command.created_at, + 'user_id', command.user_id, + 'aggregate_id', command.aggregate_id, + 'event_aggregate_id', command.event_aggregate_id, + 'event_sequence_number', command.event_sequence_number + ) + || command.command_json; +END +$$; + +CREATE OR REPLACE FUNCTION enrich_event_json(event events) RETURNS jsonb +LANGUAGE plpgsql AS $$ +BEGIN + RETURN jsonb_build_object( + 'aggregate_id', event.aggregate_id, + 'sequence_number', event.sequence_number, + 'created_at', event.created_at + ) + || event.event_json; +END +$$; + +CREATE OR REPLACE FUNCTION load_event( + _aggregate_id uuid, + _sequence_number integer +) RETURNS SETOF aggregate_event_type +LANGUAGE plpgsql AS $$ +DECLARE + _aggregate aggregates; + _aggregate_type text; +BEGIN + SELECT * INTO _aggregate + FROM aggregates + WHERE aggregate_id = _aggregate_id; + IF NOT FOUND THEN + RETURN; + END IF; + + SELECT type INTO STRICT _aggregate_type + FROM aggregate_types + WHERE id = _aggregate.aggregate_type_id; + + RETURN QUERY SELECT _aggregate_type, aggregate_id, _aggregate.events_partition_key, _aggregate.snapshot_threshold, event_type, event_json::jsonb + FROM event_records + WHERE aggregate_id = _aggregate_id + AND sequence_number = _sequence_number; +END; +$$; + +CREATE OR REPLACE FUNCTION load_events( + _aggregate_ids jsonb, + _use_snapshots boolean DEFAULT TRUE, + _until timestamptz DEFAULT NULL +) RETURNS SETOF aggregate_event_type +LANGUAGE plpgsql AS $$ +DECLARE + _aggregate_type text; + _aggregate_id aggregates.aggregate_id%TYPE; + _aggregate aggregates; + _snapshot snapshot_records; + _start_sequence_number events.sequence_number%TYPE; +BEGIN + FOR _aggregate_id IN SELECT * FROM jsonb_array_elements_text(_aggregate_ids) LOOP + SELECT * INTO _aggregate FROM aggregates WHERE aggregates.aggregate_id = _aggregate_id; + IF NOT FOUND THEN + CONTINUE; + END IF; + + SELECT type INTO STRICT _aggregate_type + FROM aggregate_types + WHERE id = _aggregate.aggregate_type_id; + + _start_sequence_number = 0; + IF _use_snapshots THEN + SELECT * INTO _snapshot FROM snapshot_records snapshots WHERE snapshots.aggregate_id = _aggregate.aggregate_id ORDER BY sequence_number DESC LIMIT 1; + IF FOUND THEN + _start_sequence_number := _snapshot.sequence_number; + RETURN NEXT (_aggregate_type, + _aggregate.aggregate_id, + _aggregate.events_partition_key, + _aggregate.snapshot_threshold, + _snapshot.snapshot_type, + _snapshot.snapshot_json); + END IF; + END IF; + RETURN QUERY SELECT _aggregate_type, + _aggregate.aggregate_id, + _aggregate.events_partition_key, + _aggregate.snapshot_threshold, + event_types.type, + enrich_event_json(events) + FROM events + INNER JOIN event_types ON events.event_type_id = event_types.id + WHERE events.partition_key = _aggregate.events_partition_key + AND events.aggregate_id = _aggregate.aggregate_id + AND events.sequence_number >= _start_sequence_number + AND (_until IS NULL OR events.created_at < _until) + ORDER BY events.sequence_number; + END LOOP; +END; +$$; + +CREATE OR REPLACE FUNCTION store_command(_command jsonb) RETURNS bigint +LANGUAGE plpgsql AS $$ +DECLARE + _id commands.id%TYPE; + _command_json jsonb = _command->'command_json'; +BEGIN + IF NOT EXISTS (SELECT 1 FROM command_types t WHERE t.type = _command->>'command_type') THEN + -- Only try inserting if it doesn't exist to avoid exhausting the id sequence + INSERT INTO command_types (type) + VALUES (_command->>'command_type') + ON CONFLICT DO NOTHING; + END IF; + + INSERT INTO commands ( + created_at, user_id, aggregate_id, command_type_id, command_json, + event_aggregate_id, event_sequence_number + ) VALUES ( + (_command->>'created_at')::timestamptz, + (_command_json->>'user_id')::uuid, + (_command_json->>'aggregate_id')::uuid, + (SELECT id FROM command_types WHERE type = _command->>'command_type'), + (_command->'command_json') - '{command_type,created_at,organization_id,user_id,aggregate_id,event_aggregate_id,event_sequence_number}'::text[], + (_command_json->>'event_aggregate_id')::uuid, + NULLIF(_command_json->'event_sequence_number', 'null'::jsonb)::integer + ) RETURNING id INTO STRICT _id; + RETURN _id; +END; +$$; + +CREATE OR REPLACE PROCEDURE store_events(_command jsonb, _aggregates_with_events jsonb) +LANGUAGE plpgsql AS $$ +DECLARE + _command_id commands.id%TYPE; + _aggregate jsonb; + _events jsonb; + _aggregate_id aggregates.aggregate_id%TYPE; + _provided_events_partition_key aggregates.events_partition_key%TYPE; + _existing_events_partition_key aggregates.events_partition_key%TYPE; + _events_partition_key aggregates.events_partition_key%TYPE; + _snapshot_threshold aggregates.snapshot_threshold%TYPE; +BEGIN + _command_id = store_command(_command); + + WITH types AS ( + SELECT DISTINCT row->0->>'aggregate_type' AS type + FROM jsonb_array_elements(_aggregates_with_events) AS row + ) + INSERT INTO aggregate_types (type) + SELECT type FROM types + WHERE type NOT IN (SELECT type FROM aggregate_types) + ORDER BY 1 + ON CONFLICT DO NOTHING; + + WITH types AS ( + SELECT DISTINCT events->>'event_type' AS type + FROM jsonb_array_elements(_aggregates_with_events) AS row + CROSS JOIN LATERAL jsonb_array_elements(row->1) AS events + ) + INSERT INTO event_types (type) + SELECT type FROM types + WHERE type NOT IN (SELECT type FROM event_types) + ORDER BY 1 + ON CONFLICT DO NOTHING; + + FOR _aggregate, _events IN SELECT row->0, row->1 FROM jsonb_array_elements(_aggregates_with_events) AS row LOOP + _aggregate_id = _aggregate->>'aggregate_id'; + _snapshot_threshold = NULLIF(_aggregate->'snapshot_threshold', 'null'::jsonb); + _provided_events_partition_key = _aggregate->>'events_partition_key'; + + SELECT events_partition_key INTO _existing_events_partition_key FROM aggregates WHERE aggregate_id = _aggregate_id; + _events_partition_key = COALESCE(_provided_events_partition_key, _existing_events_partition_key, ''); + + INSERT INTO aggregates (aggregate_id, created_at, aggregate_type_id, events_partition_key, snapshot_threshold) + VALUES ( + _aggregate_id, + (_events->0->>'created_at')::timestamptz, + (SELECT id FROM aggregate_types WHERE type = _aggregate->>'aggregate_type'), + _events_partition_key, + _snapshot_threshold + ) ON CONFLICT (aggregate_id) + DO UPDATE SET events_partition_key = EXCLUDED.events_partition_key, + snapshot_threshold = EXCLUDED.snapshot_threshold + WHERE aggregates.events_partition_key <> EXCLUDED.events_partition_key + OR aggregates.snapshot_threshold <> EXCLUDED.snapshot_threshold; + + INSERT INTO events (partition_key, aggregate_id, sequence_number, created_at, command_id, event_type_id, event_json) + SELECT _events_partition_key, + _aggregate_id, + (event->'event_json'->'sequence_number')::integer, + (event->>'created_at')::timestamptz, + _command_id, + (SELECT id FROM event_types WHERE type = event->>'event_type'), + (event->'event_json') - '{aggregate_id,created_at,event_type,sequence_number}'::text[] + FROM jsonb_array_elements(_events) AS event; + END LOOP; +END; +$$; + +CREATE OR REPLACE PROCEDURE store_snapshots(_snapshots jsonb) +LANGUAGE plpgsql AS $$ +DECLARE + _aggregate_id uuid; + _events_partition_key text; + _snapshot jsonb; +BEGIN + FOR _snapshot IN SELECT * FROM jsonb_array_elements(_snapshots) LOOP + _aggregate_id = _snapshot->>'aggregate_id'; + + INSERT INTO snapshot_records (aggregate_id, sequence_number, created_at, snapshot_type, snapshot_json) + VALUES ( + _aggregate_id, + (_snapshot->'sequence_number')::integer, + (_snapshot->>'created_at')::timestamptz, + _snapshot->>'snapshot_type', + _snapshot->'snapshot_json' + ); + END LOOP; +END; +$$; + +CREATE OR REPLACE FUNCTION load_latest_snapshot(_aggregate_id uuid) RETURNS aggregate_event_type +LANGUAGE SQL AS $$ + SELECT (SELECT type FROM aggregate_types WHERE id = a.aggregate_type_id), + a.aggregate_id, + a.events_partition_key, + a.snapshot_threshold, + s.snapshot_type, + s.snapshot_json + FROM aggregates a JOIN snapshot_records s ON a.aggregate_id = s.aggregate_id + WHERE a.aggregate_id = _aggregate_id + ORDER BY s.sequence_number DESC + LIMIT 1; +$$; + +CREATE OR REPLACE PROCEDURE delete_snapshots_before(_aggregate_id uuid, _sequence_number integer) +LANGUAGE plpgsql AS $$ +BEGIN + DELETE FROM snapshot_records + WHERE aggregate_id = _aggregate_id + AND sequence_number < _sequence_number; +END; +$$; + +CREATE OR REPLACE FUNCTION aggregates_that_need_snapshots(_last_aggregate_id uuid, _limit integer) + RETURNS TABLE (aggregate_id uuid) +LANGUAGE plpgsql AS $$ +BEGIN + RETURN QUERY SELECT stream.aggregate_id + FROM stream_records stream + WHERE (_last_aggregate_id IS NULL OR stream.aggregate_id > _last_aggregate_id) + AND snapshot_threshold IS NOT NULL + AND snapshot_threshold <= ( + (SELECT MAX(events.sequence_number) FROM event_records events WHERE stream.aggregate_id = events.aggregate_id) - + COALESCE((SELECT MAX(snapshots.sequence_number) FROM snapshot_records snapshots WHERE stream.aggregate_id = snapshots.aggregate_id), 0)) + ORDER BY 1 + LIMIT _limit; +END; +$$; + +CREATE OR REPLACE PROCEDURE permanently_delete_commands_without_events(_aggregate_id uuid, _organization_id uuid) +LANGUAGE plpgsql AS $$ +BEGIN + IF _aggregate_id IS NULL AND _organization_id IS NULL THEN + RAISE EXCEPTION 'aggregate_id or organization_id must be specified to delete commands'; + END IF; + + DELETE FROM commands + WHERE (_aggregate_id IS NULL OR aggregate_id = _aggregate_id) + AND NOT EXISTS (SELECT 1 FROM events WHERE command_id = commands.id); +END; +$$; + +CREATE OR REPLACE PROCEDURE permanently_delete_event_streams(_aggregate_ids jsonb) +LANGUAGE plpgsql AS $$ +BEGIN + DELETE FROM events + USING jsonb_array_elements_text(_aggregate_ids) AS ids (id) + JOIN aggregates ON ids.id::uuid = aggregates.aggregate_id + WHERE events.partition_key = aggregates.events_partition_key + AND events.aggregate_id = aggregates.aggregate_id; + DELETE FROM aggregates + USING jsonb_array_elements_text(_aggregate_ids) AS ids (id) + WHERE aggregates.aggregate_id = ids.id::uuid; +END; +$$; + +CREATE VIEW command_records (id, user_id, aggregate_id, command_type, command_json, created_at, event_aggregate_id, event_sequence_number) AS + SELECT id, + user_id, + aggregate_id, + (SELECT type FROM command_types WHERE command_types.id = command.command_type_id), + enrich_command_json(command), + created_at, + event_aggregate_id, + event_sequence_number + FROM commands command; + +CREATE VIEW event_records (aggregate_id, partition_key, sequence_number, created_at, event_type, event_json, command_record_id, xact_id) AS + SELECT aggregate.aggregate_id, + event.partition_key, + event.sequence_number, + event.created_at, + type.type, + enrich_event_json(event) AS event_json, + command_id, + event.xact_id + FROM aggregates aggregate + JOIN events event ON aggregate.aggregate_id = event.aggregate_id AND aggregate.events_partition_key = event.partition_key + JOIN event_types type ON event.event_type_id = type.id; + +CREATE VIEW stream_records (aggregate_id, events_partition_key, aggregate_type, snapshot_threshold, created_at) AS + SELECT aggregates.aggregate_id, + aggregates.events_partition_key, + aggregate_types.type, + aggregates.snapshot_threshold, + aggregates.created_at + FROM aggregates JOIN aggregate_types ON aggregates.aggregate_type_id = aggregate_types.id; From 3ade1cc2a48fa4991e692225b34aa126c32456ef Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Fri, 3 May 2024 14:14:36 +0200 Subject: [PATCH 059/128] Simplify require --- spec/lib/sequent/migrations/grouper_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/lib/sequent/migrations/grouper_spec.rb b/spec/lib/sequent/migrations/grouper_spec.rb index 823289b5..aa7926e9 100644 --- a/spec/lib/sequent/migrations/grouper_spec.rb +++ b/spec/lib/sequent/migrations/grouper_spec.rb @@ -1,8 +1,8 @@ # frozen_string_literal: true require 'spec_helper' -require_relative '../../../../lib/sequent/migrations/grouper' require 'prop_check' +require 'sequent/migrations/grouper' describe Sequent::Migrations::Grouper do G = PropCheck::Generators From 0b94acc0203415ab04fdd8fa45041d4ae2cf004d Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Tue, 7 May 2024 11:28:44 +0200 Subject: [PATCH 060/128] Do not query database if event identifier is null --- lib/sequent/core/command_record.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/sequent/core/command_record.rb b/lib/sequent/core/command_record.rb index 38100283..91007a64 100644 --- a/lib/sequent/core/command_record.rb +++ b/lib/sequent/core/command_record.rb @@ -59,6 +59,8 @@ class CommandRecord < Sequent::ApplicationRecord # A `belongs_to` association fails in weird ways with ActiveRecord 7.1, probably due to the use of composite # primary keys so use an explicit query here and cache the result. def parent_event + return nil unless event_aggregate_id && event_sequence_number + @parent_event ||= EventRecord.find_by(aggregate_id: event_aggregate_id, sequence_number: event_sequence_number) end From 1883857d399d2b7685d655591dfea6e75cd4bd61 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Tue, 7 May 2024 17:54:28 +0200 Subject: [PATCH 061/128] Code cleanup --- lib/sequent/migrations/grouper.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/sequent/migrations/grouper.rb b/lib/sequent/migrations/grouper.rb index b939473c..6ba14a8b 100644 --- a/lib/sequent/migrations/grouper.rb +++ b/lib/sequent/migrations/grouper.rb @@ -72,7 +72,7 @@ def self.group_partitions(partitions, target_group_size) PartitionData = Data.define(:key, :original_size, :remaining_size, :lower_bound) def self.number_to_uuid(number) - fail ArgumentError, number unless (0..2**128 - 1).include? number + fail ArgumentError, number unless (0..UUID_COUNT - 1).include? number s = format('%032x', number) "#{s[0..7]}-#{s[8..11]}-#{s[12..15]}-#{s[16..19]}-#{s[20..]}" From 440c97c86c7df94f29251adca3bce27a15d22b39 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Tue, 7 May 2024 17:55:49 +0200 Subject: [PATCH 062/128] Replace .map(&proc).sum with .sum(&proc) --- .../dry_run/read_only_replay_optimized_postgres_persistor.rb | 2 +- lib/sequent/migrations/view_schema.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/sequent/dry_run/read_only_replay_optimized_postgres_persistor.rb b/lib/sequent/dry_run/read_only_replay_optimized_postgres_persistor.rb index f264cca9..756dab2f 100644 --- a/lib/sequent/dry_run/read_only_replay_optimized_postgres_persistor.rb +++ b/lib/sequent/dry_run/read_only_replay_optimized_postgres_persistor.rb @@ -15,7 +15,7 @@ def commit # Running in dryrun mode, not committing anything. ending = Process.clock_gettime(Process::CLOCK_MONOTONIC) elapsed = ending - @starting - count = @record_store.values.map(&:size).sum + count = @record_store.values.sum(&:size) Sequent.logger.info( "dryrun: processed #{count} records in #{elapsed.round(2)} s (#{(count / elapsed).round(2)} records/s)", ) diff --git a/lib/sequent/migrations/view_schema.rb b/lib/sequent/migrations/view_schema.rb index 2fbfc16d..810f388c 100644 --- a/lib/sequent/migrations/view_schema.rb +++ b/lib/sequent/migrations/view_schema.rb @@ -307,7 +307,7 @@ def replay!( .group(:partition_key) .order(:partition_key) .count - event_count = partitions.map(&:count).sum + event_count = partitions.sum(&:count) groups = Sequent::Migrations::Grouper.group_partitions(partitions, group_target_size) if groups.empty? From 1e2372b21b3d48725adbbb1aa41640e74e563ad0 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Tue, 7 May 2024 17:59:26 +0200 Subject: [PATCH 063/128] Make the default schema a bit more generic --- db/sequent_schema.sql | 2 -- lib/sequent/generator/template_project/db/sequent_schema.sql | 2 -- 2 files changed, 4 deletions(-) diff --git a/db/sequent_schema.sql b/db/sequent_schema.sql index bee5f7f0..162fd85b 100644 --- a/db/sequent_schema.sql +++ b/db/sequent_schema.sql @@ -74,8 +74,6 @@ CREATE TABLE events_2024 PARTITION OF events FOR VALUES FROM ('Y24') TO ('Y25'); ALTER TABLE events_2024 CLUSTER ON events_2024_pkey; CREATE TABLE events_2025_and_later PARTITION OF events FOR VALUES FROM ('Y25') TO ('Y99'); ALTER TABLE events_2025_and_later CLUSTER ON events_2025_and_later_pkey; -CREATE TABLE events_organizations PARTITION OF events FOR VALUES FROM ('O') TO ('Og'); -ALTER TABLE events_organizations CLUSTER ON events_organizations_pkey; CREATE TABLE events_aggregate PARTITION OF events FOR VALUES FROM ('A') TO ('Ag'); ALTER TABLE events_aggregate CLUSTER ON events_aggregate_pkey; diff --git a/lib/sequent/generator/template_project/db/sequent_schema.sql b/lib/sequent/generator/template_project/db/sequent_schema.sql index bee5f7f0..162fd85b 100644 --- a/lib/sequent/generator/template_project/db/sequent_schema.sql +++ b/lib/sequent/generator/template_project/db/sequent_schema.sql @@ -74,8 +74,6 @@ CREATE TABLE events_2024 PARTITION OF events FOR VALUES FROM ('Y24') TO ('Y25'); ALTER TABLE events_2024 CLUSTER ON events_2024_pkey; CREATE TABLE events_2025_and_later PARTITION OF events FOR VALUES FROM ('Y25') TO ('Y99'); ALTER TABLE events_2025_and_later CLUSTER ON events_2025_and_later_pkey; -CREATE TABLE events_organizations PARTITION OF events FOR VALUES FROM ('O') TO ('Og'); -ALTER TABLE events_organizations CLUSTER ON events_organizations_pkey; CREATE TABLE events_aggregate PARTITION OF events FOR VALUES FROM ('A') TO ('Ag'); ALTER TABLE events_aggregate CLUSTER ON events_aggregate_pkey; From 1c431cb29d686797325a4b76fba6e96d57eaee7c Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Fri, 17 May 2024 14:00:01 +0200 Subject: [PATCH 064/128] Take into account xact_id when counting events to determine groups --- lib/sequent/migrations/view_schema.rb | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/lib/sequent/migrations/view_schema.rb b/lib/sequent/migrations/view_schema.rb index 810f388c..f993781e 100644 --- a/lib/sequent/migrations/view_schema.rb +++ b/lib/sequent/migrations/view_schema.rb @@ -303,11 +303,13 @@ def replay!( event_types = projectors.flat_map { |projector| projector.message_mapping.keys }.uniq.map(&:name) group_target_size = Sequent.configuration.replay_group_target_size event_type_ids = Internal::EventType.where(type: event_types).pluck(:id) - partitions = Internal::PartitionedEvent.where(event_type_id: event_type_ids) - .group(:partition_key) - .order(:partition_key) - .count + + partitions_query = Internal::PartitionedEvent.where(event_type_id: event_type_ids) + partitions_query = xact_id_filter(partitions_query, minimum_xact_id_inclusive, maximum_xact_id_exclusive) + + partitions = partitions_query.group(:partition_key).order(:partition_key).count event_count = partitions.sum(&:count) + groups = Sequent::Migrations::Grouper.group_partitions(partitions, group_target_size) if groups.empty? @@ -450,20 +452,24 @@ def event_stream(group, event_type_ids, minimum_xact_id_inclusive, maximum_xact_ group.begin.aggregate_id, ) end + event_stream = xact_id_filter(event_stream, minimum_xact_id_inclusive, maximum_xact_id_exclusive) + event_stream + .order('events.partition_key', 'events.aggregate_id', 'events.sequence_number') + .select('event_types.type AS event_type, enrich_event_json(events) AS event_json') + end + + def xact_id_filter(events_query, minimum_xact_id_inclusive, maximum_xact_id_exclusive) if minimum_xact_id_inclusive && maximum_xact_id_exclusive - event_stream = event_stream.where( + events_query.where( 'xact_id >= ? AND xact_id < ?', minimum_xact_id_inclusive, maximum_xact_id_exclusive, ) elsif minimum_xact_id_inclusive - event_stream = event_stream.where('xact_id >= ?', minimum_xact_id_inclusive) + events_query.where('xact_id >= ?', minimum_xact_id_inclusive) elsif maximum_xact_id_exclusive - event_stream = event_stream.where('xact_id IS NULL OR xact_id < ?', maximum_xact_id_exclusive) + events_query.where('xact_id IS NULL OR xact_id < ?', maximum_xact_id_exclusive) end - event_stream - .order('events.partition_key', 'events.aggregate_id', 'events.sequence_number') - .select('event_types.type AS event_type, enrich_event_json(events) AS event_json') end ## shortcut methods From 8d668b3cf76e6e4cd516a531b838227b7132741f Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Fri, 17 May 2024 15:49:22 +0200 Subject: [PATCH 065/128] Return unmodified query when no xact_id limits specified --- lib/sequent/migrations/view_schema.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/sequent/migrations/view_schema.rb b/lib/sequent/migrations/view_schema.rb index f993781e..faf71cfc 100644 --- a/lib/sequent/migrations/view_schema.rb +++ b/lib/sequent/migrations/view_schema.rb @@ -469,6 +469,8 @@ def xact_id_filter(events_query, minimum_xact_id_inclusive, maximum_xact_id_excl events_query.where('xact_id >= ?', minimum_xact_id_inclusive) elsif maximum_xact_id_exclusive events_query.where('xact_id IS NULL OR xact_id < ?', maximum_xact_id_exclusive) + else + events_query end end From 2d60f6a2b3b8f1c58c8d08f243817f644c029974 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Wed, 5 Jun 2024 08:58:29 +0200 Subject: [PATCH 066/128] Query events table directly instead of using view The view requires joining against the `aggregates` table again, but we already have all the information we need from that table. --- db/sequent_schema.sql | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/db/sequent_schema.sql b/db/sequent_schema.sql index 162fd85b..e3813b38 100644 --- a/db/sequent_schema.sql +++ b/db/sequent_schema.sql @@ -129,10 +129,17 @@ BEGIN FROM aggregate_types WHERE id = _aggregate.aggregate_type_id; - RETURN QUERY SELECT _aggregate_type, aggregate_id, _aggregate.events_partition_key, _aggregate.snapshot_threshold, event_type, event_json::jsonb - FROM event_records - WHERE aggregate_id = _aggregate_id - AND sequence_number = _sequence_number; + RETURN QUERY SELECT _aggregate_type, + _aggregate_id, + _aggregate.events_partition_key, + _aggregate.snapshot_threshold, + event_types.type, + enrich_event_json(events) + FROM events + INNER JOIN event_types ON events.event_type_id = event_types.id + WHERE events.partition_key = _aggregate.events_partition_key + AND events.aggregate_id = _aggregate_id + AND events.sequence_number = _sequence_number; END; $$; From 00d6001c96a1738b202689a5225aef06fd17ed41 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Thu, 6 Jun 2024 11:10:28 +0200 Subject: [PATCH 067/128] Update template project --- .../template_project/db/sequent_schema.sql | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/lib/sequent/generator/template_project/db/sequent_schema.sql b/lib/sequent/generator/template_project/db/sequent_schema.sql index 162fd85b..e3813b38 100644 --- a/lib/sequent/generator/template_project/db/sequent_schema.sql +++ b/lib/sequent/generator/template_project/db/sequent_schema.sql @@ -129,10 +129,17 @@ BEGIN FROM aggregate_types WHERE id = _aggregate.aggregate_type_id; - RETURN QUERY SELECT _aggregate_type, aggregate_id, _aggregate.events_partition_key, _aggregate.snapshot_threshold, event_type, event_json::jsonb - FROM event_records - WHERE aggregate_id = _aggregate_id - AND sequence_number = _sequence_number; + RETURN QUERY SELECT _aggregate_type, + _aggregate_id, + _aggregate.events_partition_key, + _aggregate.snapshot_threshold, + event_types.type, + enrich_event_json(events) + FROM events + INNER JOIN event_types ON events.event_type_id = event_types.id + WHERE events.partition_key = _aggregate.events_partition_key + AND events.aggregate_id = _aggregate_id + AND events.sequence_number = _sequence_number; END; $$; From cd60195e4e367ed6819871081ad08764ad2a2a48 Mon Sep 17 00:00:00 2001 From: Lars Vonk Date: Mon, 10 Jun 2024 10:28:46 +0200 Subject: [PATCH 068/128] Correctly sum number of events to replay --- lib/sequent/migrations/view_schema.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/sequent/migrations/view_schema.rb b/lib/sequent/migrations/view_schema.rb index faf71cfc..adb09eab 100644 --- a/lib/sequent/migrations/view_schema.rb +++ b/lib/sequent/migrations/view_schema.rb @@ -308,7 +308,7 @@ def replay!( partitions_query = xact_id_filter(partitions_query, minimum_xact_id_inclusive, maximum_xact_id_exclusive) partitions = partitions_query.group(:partition_key).order(:partition_key).count - event_count = partitions.sum(&:count) + event_count = partitions.values.sum groups = Sequent::Migrations::Grouper.group_partitions(partitions, group_target_size) From a81a2d7736b637d245ddac68d7214b005421bcbb Mon Sep 17 00:00:00 2001 From: Lars Vonk Date: Mon, 10 Jun 2024 15:30:49 +0200 Subject: [PATCH 069/128] Add spec to break concurrent read/update on partition key --- spec/lib/sequent/core/event_store_spec.rb | 71 +++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/spec/lib/sequent/core/event_store_spec.rb b/spec/lib/sequent/core/event_store_spec.rb index 25a1fe64..82f55ce6 100644 --- a/spec/lib/sequent/core/event_store_spec.rb +++ b/spec/lib/sequent/core/event_store_spec.rb @@ -224,6 +224,77 @@ class MyAggregate < Sequent::Core::AggregateRoot expect(event_store.load_event(aggregate_id, events.first.sequence_number)).to eq(events.first) end + context 'changing the partition_key and loading concurrently' do + before do + event_store.commit_events( + Sequent::Core::Command.new(aggregate_id: aggregate_id), + [ + [ + Sequent::Core::EventStream.new( + aggregate_type: 'MyAggregate', + aggregate_id: aggregate_id, + events_partition_key: 'Y24', + ), + [MyEvent.new(aggregate_id: aggregate_id, sequence_number: 1)], + ], + ], + ) + end + let(:thread_stopper) do + Class.new do + def initialize + @stop = false + end + def stopped? + @stop + end + def stop + @stop = true + end + end + end + + it 'will still allow to load the events' do + stopper = thread_stopper.new + + reader_thread = Thread.new do + events = [] + ActiveRecord::Base.connection_pool.with_connection do + until stopper.stopped? + ActiveRecord::Base.transaction do + events << event_store.load_events(aggregate_id)&.first + end + sleep(0.001) + end + end + events + end + updater_thread = Thread.new do + 1000.times do |_i| + ActiveRecord::Base.connection_pool.with_connection do |c| + c.exec_update( + 'UPDATE aggregates SET events_partition_key = $1 WHERE aggregate_id = $2', + 'aggregates', + [%w[Y23 Y24].sample, aggregate_id], + ) + sleep(0.0005) + end + end + end + updater_thread.join + stopper.stop + # wait for t1 to stop and collect its return value + events = reader_thread.value + # check that our test pool has some meaningful size + expect(events.length > 100).to be_truthy + + misses = events.select(&:nil?).length + expect(misses).to eq(0), <<~EOS + Expected the events can always be loaded when the partition key is changed. But there are #{misses} misses. + EOS + end + end + context 'and event type caching disabled' do around do |example| current = Sequent.configuration.event_store_cache_event_types From 68c24ebe4d00007042cffb9a446210981e0a5bf6 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Mon, 10 Jun 2024 16:13:48 +0200 Subject: [PATCH 070/128] Protect against concurrency problems at read committed isolation level With read committed isolation level it is possible for the `load_events` function to fail when another session updates the aggregate's `events_partition_key`, since reads are not repeatable at this isolation level. By using a single query to load an aggregate, its snapshot, and its event's all queries use the same database snapshot ensuring a consistent set of data. When storing the events order lock the aggregate row and inserts in `aggregate_id` order to concurrency issues (deadlocks or foreign-key violations). --- db/sequent_schema.sql | 111 +++++++++------------- spec/lib/sequent/core/event_store_spec.rb | 4 +- 2 files changed, 45 insertions(+), 70 deletions(-) diff --git a/db/sequent_schema.sql b/db/sequent_schema.sql index e3813b38..7831fe6e 100644 --- a/db/sequent_schema.sql +++ b/db/sequent_schema.sql @@ -114,32 +114,19 @@ CREATE OR REPLACE FUNCTION load_event( _sequence_number integer ) RETURNS SETOF aggregate_event_type LANGUAGE plpgsql AS $$ -DECLARE - _aggregate aggregates; - _aggregate_type text; BEGIN - SELECT * INTO _aggregate - FROM aggregates - WHERE aggregate_id = _aggregate_id; - IF NOT FOUND THEN - RETURN; - END IF; - - SELECT type INTO STRICT _aggregate_type - FROM aggregate_types - WHERE id = _aggregate.aggregate_type_id; - - RETURN QUERY SELECT _aggregate_type, - _aggregate_id, - _aggregate.events_partition_key, - _aggregate.snapshot_threshold, - event_types.type, - enrich_event_json(events) - FROM events - INNER JOIN event_types ON events.event_type_id = event_types.id - WHERE events.partition_key = _aggregate.events_partition_key - AND events.aggregate_id = _aggregate_id - AND events.sequence_number = _sequence_number; + RETURN QUERY SELECT aggregate_types.type, + a.aggregate_id, + a.events_partition_key, + a.snapshot_threshold, + event_types.type, + enrich_event_json(e) + FROM aggregates a + INNER JOIN events e ON (a.events_partition_key, a.aggregate_id) = (e.partition_key, e.aggregate_id) + INNER JOIN aggregate_types ON a.aggregate_type_id = aggregate_types.id + INNER JOIN event_types ON e.event_type_id = event_types.id + WHERE a.aggregate_id = _aggregate_id + AND e.sequence_number = _sequence_number; END; $$; @@ -150,48 +137,36 @@ CREATE OR REPLACE FUNCTION load_events( ) RETURNS SETOF aggregate_event_type LANGUAGE plpgsql AS $$ DECLARE - _aggregate_type text; _aggregate_id aggregates.aggregate_id%TYPE; - _aggregate aggregates; - _snapshot snapshot_records; - _start_sequence_number events.sequence_number%TYPE; BEGIN FOR _aggregate_id IN SELECT * FROM jsonb_array_elements_text(_aggregate_ids) LOOP - SELECT * INTO _aggregate FROM aggregates WHERE aggregates.aggregate_id = _aggregate_id; - IF NOT FOUND THEN - CONTINUE; - END IF; - - SELECT type INTO STRICT _aggregate_type - FROM aggregate_types - WHERE id = _aggregate.aggregate_type_id; - - _start_sequence_number = 0; - IF _use_snapshots THEN - SELECT * INTO _snapshot FROM snapshot_records snapshots WHERE snapshots.aggregate_id = _aggregate.aggregate_id ORDER BY sequence_number DESC LIMIT 1; - IF FOUND THEN - _start_sequence_number := _snapshot.sequence_number; - RETURN NEXT (_aggregate_type, - _aggregate.aggregate_id, - _aggregate.events_partition_key, - _aggregate.snapshot_threshold, - _snapshot.snapshot_type, - _snapshot.snapshot_json); - END IF; - END IF; - RETURN QUERY SELECT _aggregate_type, - _aggregate.aggregate_id, - _aggregate.events_partition_key, - _aggregate.snapshot_threshold, - event_types.type, - enrich_event_json(events) - FROM events - INNER JOIN event_types ON events.event_type_id = event_types.id - WHERE events.partition_key = _aggregate.events_partition_key - AND events.aggregate_id = _aggregate.aggregate_id - AND events.sequence_number >= _start_sequence_number - AND (_until IS NULL OR events.created_at < _until) - ORDER BY events.sequence_number; + -- Use a single query to avoid race condition with UPDATEs to the events partition key + -- in case transaction isolation level is lower than repeatable read (the default of + -- PostgreSQL is read committed). + RETURN QUERY WITH + aggregate AS ( + SELECT aggregate_types.type, aggregate_id, events_partition_key, snapshot_threshold + FROM aggregates + JOIN aggregate_types ON aggregate_type_id = aggregate_types.id + WHERE aggregate_id = _aggregate_id + ), + snapshot AS ( + SELECT * + FROM snapshot_records + WHERE _use_snapshots + AND aggregate_id = _aggregate_id + AND (_until IS NULL OR created_at < _until) + ORDER BY sequence_number DESC LIMIT 1 + ) + (SELECT a.*, s.snapshot_type, s.snapshot_json FROM aggregate a, snapshot s) + UNION ALL + (SELECT a.*, event_types.type, enrich_event_json(e) + FROM aggregate a + JOIN events e ON (a.events_partition_key, a.aggregate_id) = (e.partition_key, e.aggregate_id) + JOIN event_types ON e.event_type_id = event_types.id + WHERE e.sequence_number >= COALESCE((SELECT sequence_number FROM snapshot), 0) + AND (_until IS NULL OR e.created_at < _until) + ORDER BY e.sequence_number ASC); END LOOP; END; $$; @@ -260,12 +235,12 @@ BEGIN ORDER BY 1 ON CONFLICT DO NOTHING; - FOR _aggregate, _events IN SELECT row->0, row->1 FROM jsonb_array_elements(_aggregates_with_events) AS row LOOP + FOR _aggregate, _events IN SELECT row->0, row->1 FROM jsonb_array_elements(_aggregates_with_events) AS row ORDER BY 1 LOOP _aggregate_id = _aggregate->>'aggregate_id'; _snapshot_threshold = NULLIF(_aggregate->'snapshot_threshold', 'null'::jsonb); _provided_events_partition_key = _aggregate->>'events_partition_key'; - SELECT events_partition_key INTO _existing_events_partition_key FROM aggregates WHERE aggregate_id = _aggregate_id; + SELECT events_partition_key INTO _existing_events_partition_key FROM aggregates WHERE aggregate_id = _aggregate_id FOR UPDATE; _events_partition_key = COALESCE(_provided_events_partition_key, _existing_events_partition_key, ''); INSERT INTO aggregates (aggregate_id, created_at, aggregate_type_id, events_partition_key, snapshot_threshold) @@ -278,8 +253,8 @@ BEGIN ) ON CONFLICT (aggregate_id) DO UPDATE SET events_partition_key = EXCLUDED.events_partition_key, snapshot_threshold = EXCLUDED.snapshot_threshold - WHERE aggregates.events_partition_key <> EXCLUDED.events_partition_key - OR aggregates.snapshot_threshold <> EXCLUDED.snapshot_threshold; + WHERE aggregates.events_partition_key IS DISTINCT FROM EXCLUDED.events_partition_key + OR aggregates.snapshot_threshold IS DISTINCT FROM EXCLUDED.snapshot_threshold; INSERT INTO events (partition_key, aggregate_id, sequence_number, created_at, command_id, event_type_id, event_json) SELECT _events_partition_key, diff --git a/spec/lib/sequent/core/event_store_spec.rb b/spec/lib/sequent/core/event_store_spec.rb index 82f55ce6..ff0aed0b 100644 --- a/spec/lib/sequent/core/event_store_spec.rb +++ b/spec/lib/sequent/core/event_store_spec.rb @@ -275,7 +275,7 @@ def stop c.exec_update( 'UPDATE aggregates SET events_partition_key = $1 WHERE aggregate_id = $2', 'aggregates', - [%w[Y23 Y24].sample, aggregate_id], + [('aa'..'zz').to_a.sample, aggregate_id], ) sleep(0.0005) end @@ -286,7 +286,7 @@ def stop # wait for t1 to stop and collect its return value events = reader_thread.value # check that our test pool has some meaningful size - expect(events.length > 100).to be_truthy + expect(events.length).to be > 100 misses = events.select(&:nil?).length expect(misses).to eq(0), <<~EOS From 76d096c8408b7a43b95b00baece13e20cebb2fce Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Mon, 10 Jun 2024 17:21:18 +0200 Subject: [PATCH 071/128] Correctly order passed in aggregates with events --- db/sequent_schema.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/db/sequent_schema.sql b/db/sequent_schema.sql index 7831fe6e..33fbb9b4 100644 --- a/db/sequent_schema.sql +++ b/db/sequent_schema.sql @@ -235,7 +235,9 @@ BEGIN ORDER BY 1 ON CONFLICT DO NOTHING; - FOR _aggregate, _events IN SELECT row->0, row->1 FROM jsonb_array_elements(_aggregates_with_events) AS row ORDER BY 1 LOOP + FOR _aggregate, _events IN SELECT row->0, row->1 FROM jsonb_array_elements(_aggregates_with_events) AS row + ORDER BY row->0->'aggregate_id', row->1->0->'event_json'->'sequence_number' + LOOP _aggregate_id = _aggregate->>'aggregate_id'; _snapshot_threshold = NULLIF(_aggregate->'snapshot_threshold', 'null'::jsonb); _provided_events_partition_key = _aggregate->>'events_partition_key'; From 69ad65d7a06b1a11cca76a95408db0cffa07dca3 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Thu, 13 Jun 2024 15:36:06 +0200 Subject: [PATCH 072/128] Use pure SQL to define Sequent schema Separate creating tables and indexes (can only be run once) from stored procedures and views (can be run at any time to update to the latest definitions). Cleanup output when creating the database by suppressing output of SQL statements. --- db/sequent_pgsql.sql | 318 ++++++++++++++++++++++++++++++++++++++++ db/sequent_schema.rb | 14 +- db/sequent_schema.sql | 333 ++---------------------------------------- 3 files changed, 333 insertions(+), 332 deletions(-) create mode 100644 db/sequent_pgsql.sql diff --git a/db/sequent_pgsql.sql b/db/sequent_pgsql.sql new file mode 100644 index 00000000..b0ac83a2 --- /dev/null +++ b/db/sequent_pgsql.sql @@ -0,0 +1,318 @@ +DROP TYPE IF EXISTS aggregate_event_type CASCADE; +CREATE TYPE aggregate_event_type AS ( + aggregate_type text, + aggregate_id uuid, + events_partition_key text, + snapshot_threshold integer, + event_type text, + event_json jsonb +); + +CREATE OR REPLACE FUNCTION enrich_command_json(command commands) RETURNS jsonb +LANGUAGE plpgsql AS $$ +BEGIN + RETURN jsonb_build_object( + 'command_type', (SELECT type FROM command_types WHERE command_types.id = command.command_type_id), + 'created_at', command.created_at, + 'user_id', command.user_id, + 'aggregate_id', command.aggregate_id, + 'event_aggregate_id', command.event_aggregate_id, + 'event_sequence_number', command.event_sequence_number + ) + || command.command_json; +END +$$; + +CREATE OR REPLACE FUNCTION enrich_event_json(event events) RETURNS jsonb +LANGUAGE plpgsql AS $$ +BEGIN + RETURN jsonb_build_object( + 'aggregate_id', event.aggregate_id, + 'sequence_number', event.sequence_number, + 'created_at', event.created_at + ) + || event.event_json; +END +$$; + +CREATE OR REPLACE FUNCTION load_event( + _aggregate_id uuid, + _sequence_number integer +) RETURNS SETOF aggregate_event_type +LANGUAGE plpgsql AS $$ +BEGIN + RETURN QUERY SELECT aggregate_types.type, + a.aggregate_id, + a.events_partition_key, + a.snapshot_threshold, + event_types.type, + enrich_event_json(e) + FROM aggregates a + INNER JOIN events e ON (a.events_partition_key, a.aggregate_id) = (e.partition_key, e.aggregate_id) + INNER JOIN aggregate_types ON a.aggregate_type_id = aggregate_types.id + INNER JOIN event_types ON e.event_type_id = event_types.id + WHERE a.aggregate_id = _aggregate_id + AND e.sequence_number = _sequence_number; +END; +$$; + +CREATE OR REPLACE FUNCTION load_events( + _aggregate_ids jsonb, + _use_snapshots boolean DEFAULT TRUE, + _until timestamptz DEFAULT NULL +) RETURNS SETOF aggregate_event_type +LANGUAGE plpgsql AS $$ +DECLARE + _aggregate_id aggregates.aggregate_id%TYPE; +BEGIN + FOR _aggregate_id IN SELECT * FROM jsonb_array_elements_text(_aggregate_ids) LOOP + -- Use a single query to avoid race condition with UPDATEs to the events partition key + -- in case transaction isolation level is lower than repeatable read (the default of + -- PostgreSQL is read committed). + RETURN QUERY WITH + aggregate AS ( + SELECT aggregate_types.type, aggregate_id, events_partition_key, snapshot_threshold + FROM aggregates + JOIN aggregate_types ON aggregate_type_id = aggregate_types.id + WHERE aggregate_id = _aggregate_id + ), + snapshot AS ( + SELECT * + FROM snapshot_records + WHERE _use_snapshots + AND aggregate_id = _aggregate_id + AND (_until IS NULL OR created_at < _until) + ORDER BY sequence_number DESC LIMIT 1 + ) + (SELECT a.*, s.snapshot_type, s.snapshot_json FROM aggregate a, snapshot s) + UNION ALL + (SELECT a.*, event_types.type, enrich_event_json(e) + FROM aggregate a + JOIN events e ON (a.events_partition_key, a.aggregate_id) = (e.partition_key, e.aggregate_id) + JOIN event_types ON e.event_type_id = event_types.id + WHERE e.sequence_number >= COALESCE((SELECT sequence_number FROM snapshot), 0) + AND (_until IS NULL OR e.created_at < _until) + ORDER BY e.sequence_number ASC); + END LOOP; +END; +$$; + +CREATE OR REPLACE FUNCTION store_command(_command jsonb) RETURNS bigint +LANGUAGE plpgsql AS $$ +DECLARE + _id commands.id%TYPE; + _command_json jsonb = _command->'command_json'; +BEGIN + IF NOT EXISTS (SELECT 1 FROM command_types t WHERE t.type = _command->>'command_type') THEN + -- Only try inserting if it doesn't exist to avoid exhausting the id sequence + INSERT INTO command_types (type) + VALUES (_command->>'command_type') + ON CONFLICT DO NOTHING; + END IF; + + INSERT INTO commands ( + created_at, user_id, aggregate_id, command_type_id, command_json, + event_aggregate_id, event_sequence_number + ) VALUES ( + (_command->>'created_at')::timestamptz, + (_command_json->>'user_id')::uuid, + (_command_json->>'aggregate_id')::uuid, + (SELECT id FROM command_types WHERE type = _command->>'command_type'), + (_command->'command_json') - '{command_type,created_at,organization_id,user_id,aggregate_id,event_aggregate_id,event_sequence_number}'::text[], + (_command_json->>'event_aggregate_id')::uuid, + NULLIF(_command_json->'event_sequence_number', 'null'::jsonb)::integer + ) RETURNING id INTO STRICT _id; + RETURN _id; +END; +$$; + +CREATE OR REPLACE PROCEDURE store_events(_command jsonb, _aggregates_with_events jsonb) +LANGUAGE plpgsql AS $$ +DECLARE + _command_id commands.id%TYPE; + _aggregate jsonb; + _events jsonb; + _aggregate_id aggregates.aggregate_id%TYPE; + _provided_events_partition_key aggregates.events_partition_key%TYPE; + _existing_events_partition_key aggregates.events_partition_key%TYPE; + _events_partition_key aggregates.events_partition_key%TYPE; + _snapshot_threshold aggregates.snapshot_threshold%TYPE; +BEGIN + _command_id = store_command(_command); + + WITH types AS ( + SELECT DISTINCT row->0->>'aggregate_type' AS type + FROM jsonb_array_elements(_aggregates_with_events) AS row + ) + INSERT INTO aggregate_types (type) + SELECT type FROM types + WHERE type NOT IN (SELECT type FROM aggregate_types) + ORDER BY 1 + ON CONFLICT DO NOTHING; + + WITH types AS ( + SELECT DISTINCT events->>'event_type' AS type + FROM jsonb_array_elements(_aggregates_with_events) AS row + CROSS JOIN LATERAL jsonb_array_elements(row->1) AS events + ) + INSERT INTO event_types (type) + SELECT type FROM types + WHERE type NOT IN (SELECT type FROM event_types) + ORDER BY 1 + ON CONFLICT DO NOTHING; + + FOR _aggregate, _events IN SELECT row->0, row->1 FROM jsonb_array_elements(_aggregates_with_events) AS row + ORDER BY row->0->'aggregate_id', row->1->0->'event_json'->'sequence_number' + LOOP + _aggregate_id = _aggregate->>'aggregate_id'; + _snapshot_threshold = NULLIF(_aggregate->'snapshot_threshold', 'null'::jsonb); + _provided_events_partition_key = _aggregate->>'events_partition_key'; + + SELECT events_partition_key INTO _existing_events_partition_key FROM aggregates WHERE aggregate_id = _aggregate_id FOR UPDATE; + _events_partition_key = COALESCE(_provided_events_partition_key, _existing_events_partition_key, ''); + + INSERT INTO aggregates (aggregate_id, created_at, aggregate_type_id, events_partition_key, snapshot_threshold) + VALUES ( + _aggregate_id, + (_events->0->>'created_at')::timestamptz, + (SELECT id FROM aggregate_types WHERE type = _aggregate->>'aggregate_type'), + _events_partition_key, + _snapshot_threshold + ) ON CONFLICT (aggregate_id) + DO UPDATE SET events_partition_key = EXCLUDED.events_partition_key, + snapshot_threshold = EXCLUDED.snapshot_threshold + WHERE aggregates.events_partition_key IS DISTINCT FROM EXCLUDED.events_partition_key + OR aggregates.snapshot_threshold IS DISTINCT FROM EXCLUDED.snapshot_threshold; + + INSERT INTO events (partition_key, aggregate_id, sequence_number, created_at, command_id, event_type_id, event_json) + SELECT _events_partition_key, + _aggregate_id, + (event->'event_json'->'sequence_number')::integer, + (event->>'created_at')::timestamptz, + _command_id, + (SELECT id FROM event_types WHERE type = event->>'event_type'), + (event->'event_json') - '{aggregate_id,created_at,event_type,sequence_number}'::text[] + FROM jsonb_array_elements(_events) AS event; + END LOOP; +END; +$$; + +CREATE OR REPLACE PROCEDURE store_snapshots(_snapshots jsonb) +LANGUAGE plpgsql AS $$ +DECLARE + _aggregate_id uuid; + _events_partition_key text; + _snapshot jsonb; +BEGIN + FOR _snapshot IN SELECT * FROM jsonb_array_elements(_snapshots) LOOP + _aggregate_id = _snapshot->>'aggregate_id'; + + INSERT INTO snapshot_records (aggregate_id, sequence_number, created_at, snapshot_type, snapshot_json) + VALUES ( + _aggregate_id, + (_snapshot->'sequence_number')::integer, + (_snapshot->>'created_at')::timestamptz, + _snapshot->>'snapshot_type', + _snapshot->'snapshot_json' + ); + END LOOP; +END; +$$; + +CREATE OR REPLACE FUNCTION load_latest_snapshot(_aggregate_id uuid) RETURNS aggregate_event_type +LANGUAGE SQL AS $$ + SELECT (SELECT type FROM aggregate_types WHERE id = a.aggregate_type_id), + a.aggregate_id, + a.events_partition_key, + a.snapshot_threshold, + s.snapshot_type, + s.snapshot_json + FROM aggregates a JOIN snapshot_records s ON a.aggregate_id = s.aggregate_id + WHERE a.aggregate_id = _aggregate_id + ORDER BY s.sequence_number DESC + LIMIT 1; +$$; + +CREATE OR REPLACE PROCEDURE delete_snapshots_before(_aggregate_id uuid, _sequence_number integer) +LANGUAGE plpgsql AS $$ +BEGIN + DELETE FROM snapshot_records + WHERE aggregate_id = _aggregate_id + AND sequence_number < _sequence_number; +END; +$$; + +CREATE OR REPLACE FUNCTION aggregates_that_need_snapshots(_last_aggregate_id uuid, _limit integer) + RETURNS TABLE (aggregate_id uuid) +LANGUAGE plpgsql AS $$ +BEGIN + RETURN QUERY SELECT stream.aggregate_id + FROM stream_records stream + WHERE (_last_aggregate_id IS NULL OR stream.aggregate_id > _last_aggregate_id) + AND snapshot_threshold IS NOT NULL + AND snapshot_threshold <= ( + (SELECT MAX(events.sequence_number) FROM event_records events WHERE stream.aggregate_id = events.aggregate_id) - + COALESCE((SELECT MAX(snapshots.sequence_number) FROM snapshot_records snapshots WHERE stream.aggregate_id = snapshots.aggregate_id), 0)) + ORDER BY 1 + LIMIT _limit; +END; +$$; + +CREATE OR REPLACE PROCEDURE permanently_delete_commands_without_events(_aggregate_id uuid, _organization_id uuid) +LANGUAGE plpgsql AS $$ +BEGIN + IF _aggregate_id IS NULL AND _organization_id IS NULL THEN + RAISE EXCEPTION 'aggregate_id or organization_id must be specified to delete commands'; + END IF; + + DELETE FROM commands + WHERE (_aggregate_id IS NULL OR aggregate_id = _aggregate_id) + AND NOT EXISTS (SELECT 1 FROM events WHERE command_id = commands.id); +END; +$$; + +CREATE OR REPLACE PROCEDURE permanently_delete_event_streams(_aggregate_ids jsonb) +LANGUAGE plpgsql AS $$ +BEGIN + DELETE FROM events + USING jsonb_array_elements_text(_aggregate_ids) AS ids (id) + JOIN aggregates ON ids.id::uuid = aggregates.aggregate_id + WHERE events.partition_key = aggregates.events_partition_key + AND events.aggregate_id = aggregates.aggregate_id; + DELETE FROM aggregates + USING jsonb_array_elements_text(_aggregate_ids) AS ids (id) + WHERE aggregates.aggregate_id = ids.id::uuid; +END; +$$; + +CREATE OR REPLACE VIEW command_records (id, user_id, aggregate_id, command_type, command_json, created_at, event_aggregate_id, event_sequence_number) AS + SELECT id, + user_id, + aggregate_id, + (SELECT type FROM command_types WHERE command_types.id = command.command_type_id), + enrich_command_json(command), + created_at, + event_aggregate_id, + event_sequence_number + FROM commands command; + +CREATE OR REPLACE VIEW event_records (aggregate_id, partition_key, sequence_number, created_at, event_type, event_json, command_record_id, xact_id) AS + SELECT aggregate.aggregate_id, + event.partition_key, + event.sequence_number, + event.created_at, + type.type, + enrich_event_json(event) AS event_json, + command_id, + event.xact_id + FROM aggregates aggregate + JOIN events event ON aggregate.aggregate_id = event.aggregate_id AND aggregate.events_partition_key = event.partition_key + JOIN event_types type ON event.event_type_id = type.id; + +CREATE OR REPLACE VIEW stream_records (aggregate_id, events_partition_key, aggregate_type, snapshot_threshold, created_at) AS + SELECT aggregates.aggregate_id, + aggregates.events_partition_key, + aggregate_types.type, + aggregates.snapshot_threshold, + aggregates.created_at + FROM aggregates JOIN aggregate_types ON aggregates.aggregate_type_id = aggregate_types.id; diff --git a/db/sequent_schema.rb b/db/sequent_schema.rb index 97364112..4a4ed00a 100644 --- a/db/sequent_schema.rb +++ b/db/sequent_schema.rb @@ -1,14 +1,10 @@ # frozen_string_literal: true ActiveRecord::Schema.define do - create_table 'snapshot_records', primary_key: %w[aggregate_id sequence_number], force: true do |t| - t.uuid 'aggregate_id', null: false - t.integer 'sequence_number', null: false - t.datetime 'created_at', null: false - t.text 'snapshot_type', null: false - t.jsonb 'snapshot_json', null: false + say_with_time 'Installing Sequent schema' do + say 'Creating tables and indexes', true + suppress_messages { execute File.read("#{File.dirname(__FILE__)}/sequent_schema.sql") } + say 'Creating stored procedures and views', true + suppress_messages { execute File.read("#{File.dirname(__FILE__)}/sequent_pgsql.sql") } end - - schema = File.read("#{File.dirname(__FILE__)}/sequent_schema.sql") - execute schema end diff --git a/db/sequent_schema.sql b/db/sequent_schema.sql index 33fbb9b4..faaebbe6 100644 --- a/db/sequent_schema.sql +++ b/db/sequent_schema.sql @@ -1,13 +1,3 @@ -DROP TYPE IF EXISTS aggregate_event_type CASCADE; -CREATE TYPE aggregate_event_type AS ( - aggregate_type text, - aggregate_id uuid, - events_partition_key text, - snapshot_threshold integer, - event_type text, - event_json jsonb -); - CREATE TABLE command_types (id SMALLINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, type text UNIQUE NOT NULL); CREATE TABLE aggregate_types (id SMALLINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, type text UNIQUE NOT NULL); CREATE TABLE event_types (id SMALLINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, type text UNIQUE NOT NULL); @@ -77,316 +67,13 @@ ALTER TABLE events_2025_and_later CLUSTER ON events_2025_and_later_pkey; CREATE TABLE events_aggregate PARTITION OF events FOR VALUES FROM ('A') TO ('Ag'); ALTER TABLE events_aggregate CLUSTER ON events_aggregate_pkey; -TRUNCATE TABLE snapshot_records; -ALTER TABLE snapshot_records - ADD CONSTRAINT aggregate_fkey FOREIGN KEY (aggregate_id) REFERENCES aggregates (aggregate_id) - ON UPDATE CASCADE ON DELETE CASCADE; - -CREATE OR REPLACE FUNCTION enrich_command_json(command commands) RETURNS jsonb -LANGUAGE plpgsql AS $$ -BEGIN - RETURN jsonb_build_object( - 'command_type', (SELECT type FROM command_types WHERE command_types.id = command.command_type_id), - 'created_at', command.created_at, - 'user_id', command.user_id, - 'aggregate_id', command.aggregate_id, - 'event_aggregate_id', command.event_aggregate_id, - 'event_sequence_number', command.event_sequence_number - ) - || command.command_json; -END -$$; - -CREATE OR REPLACE FUNCTION enrich_event_json(event events) RETURNS jsonb -LANGUAGE plpgsql AS $$ -BEGIN - RETURN jsonb_build_object( - 'aggregate_id', event.aggregate_id, - 'sequence_number', event.sequence_number, - 'created_at', event.created_at - ) - || event.event_json; -END -$$; - -CREATE OR REPLACE FUNCTION load_event( - _aggregate_id uuid, - _sequence_number integer -) RETURNS SETOF aggregate_event_type -LANGUAGE plpgsql AS $$ -BEGIN - RETURN QUERY SELECT aggregate_types.type, - a.aggregate_id, - a.events_partition_key, - a.snapshot_threshold, - event_types.type, - enrich_event_json(e) - FROM aggregates a - INNER JOIN events e ON (a.events_partition_key, a.aggregate_id) = (e.partition_key, e.aggregate_id) - INNER JOIN aggregate_types ON a.aggregate_type_id = aggregate_types.id - INNER JOIN event_types ON e.event_type_id = event_types.id - WHERE a.aggregate_id = _aggregate_id - AND e.sequence_number = _sequence_number; -END; -$$; - -CREATE OR REPLACE FUNCTION load_events( - _aggregate_ids jsonb, - _use_snapshots boolean DEFAULT TRUE, - _until timestamptz DEFAULT NULL -) RETURNS SETOF aggregate_event_type -LANGUAGE plpgsql AS $$ -DECLARE - _aggregate_id aggregates.aggregate_id%TYPE; -BEGIN - FOR _aggregate_id IN SELECT * FROM jsonb_array_elements_text(_aggregate_ids) LOOP - -- Use a single query to avoid race condition with UPDATEs to the events partition key - -- in case transaction isolation level is lower than repeatable read (the default of - -- PostgreSQL is read committed). - RETURN QUERY WITH - aggregate AS ( - SELECT aggregate_types.type, aggregate_id, events_partition_key, snapshot_threshold - FROM aggregates - JOIN aggregate_types ON aggregate_type_id = aggregate_types.id - WHERE aggregate_id = _aggregate_id - ), - snapshot AS ( - SELECT * - FROM snapshot_records - WHERE _use_snapshots - AND aggregate_id = _aggregate_id - AND (_until IS NULL OR created_at < _until) - ORDER BY sequence_number DESC LIMIT 1 - ) - (SELECT a.*, s.snapshot_type, s.snapshot_json FROM aggregate a, snapshot s) - UNION ALL - (SELECT a.*, event_types.type, enrich_event_json(e) - FROM aggregate a - JOIN events e ON (a.events_partition_key, a.aggregate_id) = (e.partition_key, e.aggregate_id) - JOIN event_types ON e.event_type_id = event_types.id - WHERE e.sequence_number >= COALESCE((SELECT sequence_number FROM snapshot), 0) - AND (_until IS NULL OR e.created_at < _until) - ORDER BY e.sequence_number ASC); - END LOOP; -END; -$$; - -CREATE OR REPLACE FUNCTION store_command(_command jsonb) RETURNS bigint -LANGUAGE plpgsql AS $$ -DECLARE - _id commands.id%TYPE; - _command_json jsonb = _command->'command_json'; -BEGIN - IF NOT EXISTS (SELECT 1 FROM command_types t WHERE t.type = _command->>'command_type') THEN - -- Only try inserting if it doesn't exist to avoid exhausting the id sequence - INSERT INTO command_types (type) - VALUES (_command->>'command_type') - ON CONFLICT DO NOTHING; - END IF; - - INSERT INTO commands ( - created_at, user_id, aggregate_id, command_type_id, command_json, - event_aggregate_id, event_sequence_number - ) VALUES ( - (_command->>'created_at')::timestamptz, - (_command_json->>'user_id')::uuid, - (_command_json->>'aggregate_id')::uuid, - (SELECT id FROM command_types WHERE type = _command->>'command_type'), - (_command->'command_json') - '{command_type,created_at,organization_id,user_id,aggregate_id,event_aggregate_id,event_sequence_number}'::text[], - (_command_json->>'event_aggregate_id')::uuid, - NULLIF(_command_json->'event_sequence_number', 'null'::jsonb)::integer - ) RETURNING id INTO STRICT _id; - RETURN _id; -END; -$$; - -CREATE OR REPLACE PROCEDURE store_events(_command jsonb, _aggregates_with_events jsonb) -LANGUAGE plpgsql AS $$ -DECLARE - _command_id commands.id%TYPE; - _aggregate jsonb; - _events jsonb; - _aggregate_id aggregates.aggregate_id%TYPE; - _provided_events_partition_key aggregates.events_partition_key%TYPE; - _existing_events_partition_key aggregates.events_partition_key%TYPE; - _events_partition_key aggregates.events_partition_key%TYPE; - _snapshot_threshold aggregates.snapshot_threshold%TYPE; -BEGIN - _command_id = store_command(_command); - - WITH types AS ( - SELECT DISTINCT row->0->>'aggregate_type' AS type - FROM jsonb_array_elements(_aggregates_with_events) AS row - ) - INSERT INTO aggregate_types (type) - SELECT type FROM types - WHERE type NOT IN (SELECT type FROM aggregate_types) - ORDER BY 1 - ON CONFLICT DO NOTHING; - - WITH types AS ( - SELECT DISTINCT events->>'event_type' AS type - FROM jsonb_array_elements(_aggregates_with_events) AS row - CROSS JOIN LATERAL jsonb_array_elements(row->1) AS events - ) - INSERT INTO event_types (type) - SELECT type FROM types - WHERE type NOT IN (SELECT type FROM event_types) - ORDER BY 1 - ON CONFLICT DO NOTHING; - - FOR _aggregate, _events IN SELECT row->0, row->1 FROM jsonb_array_elements(_aggregates_with_events) AS row - ORDER BY row->0->'aggregate_id', row->1->0->'event_json'->'sequence_number' - LOOP - _aggregate_id = _aggregate->>'aggregate_id'; - _snapshot_threshold = NULLIF(_aggregate->'snapshot_threshold', 'null'::jsonb); - _provided_events_partition_key = _aggregate->>'events_partition_key'; - - SELECT events_partition_key INTO _existing_events_partition_key FROM aggregates WHERE aggregate_id = _aggregate_id FOR UPDATE; - _events_partition_key = COALESCE(_provided_events_partition_key, _existing_events_partition_key, ''); - - INSERT INTO aggregates (aggregate_id, created_at, aggregate_type_id, events_partition_key, snapshot_threshold) - VALUES ( - _aggregate_id, - (_events->0->>'created_at')::timestamptz, - (SELECT id FROM aggregate_types WHERE type = _aggregate->>'aggregate_type'), - _events_partition_key, - _snapshot_threshold - ) ON CONFLICT (aggregate_id) - DO UPDATE SET events_partition_key = EXCLUDED.events_partition_key, - snapshot_threshold = EXCLUDED.snapshot_threshold - WHERE aggregates.events_partition_key IS DISTINCT FROM EXCLUDED.events_partition_key - OR aggregates.snapshot_threshold IS DISTINCT FROM EXCLUDED.snapshot_threshold; - - INSERT INTO events (partition_key, aggregate_id, sequence_number, created_at, command_id, event_type_id, event_json) - SELECT _events_partition_key, - _aggregate_id, - (event->'event_json'->'sequence_number')::integer, - (event->>'created_at')::timestamptz, - _command_id, - (SELECT id FROM event_types WHERE type = event->>'event_type'), - (event->'event_json') - '{aggregate_id,created_at,event_type,sequence_number}'::text[] - FROM jsonb_array_elements(_events) AS event; - END LOOP; -END; -$$; - -CREATE OR REPLACE PROCEDURE store_snapshots(_snapshots jsonb) -LANGUAGE plpgsql AS $$ -DECLARE - _aggregate_id uuid; - _events_partition_key text; - _snapshot jsonb; -BEGIN - FOR _snapshot IN SELECT * FROM jsonb_array_elements(_snapshots) LOOP - _aggregate_id = _snapshot->>'aggregate_id'; - - INSERT INTO snapshot_records (aggregate_id, sequence_number, created_at, snapshot_type, snapshot_json) - VALUES ( - _aggregate_id, - (_snapshot->'sequence_number')::integer, - (_snapshot->>'created_at')::timestamptz, - _snapshot->>'snapshot_type', - _snapshot->'snapshot_json' - ); - END LOOP; -END; -$$; - -CREATE OR REPLACE FUNCTION load_latest_snapshot(_aggregate_id uuid) RETURNS aggregate_event_type -LANGUAGE SQL AS $$ - SELECT (SELECT type FROM aggregate_types WHERE id = a.aggregate_type_id), - a.aggregate_id, - a.events_partition_key, - a.snapshot_threshold, - s.snapshot_type, - s.snapshot_json - FROM aggregates a JOIN snapshot_records s ON a.aggregate_id = s.aggregate_id - WHERE a.aggregate_id = _aggregate_id - ORDER BY s.sequence_number DESC - LIMIT 1; -$$; - -CREATE OR REPLACE PROCEDURE delete_snapshots_before(_aggregate_id uuid, _sequence_number integer) -LANGUAGE plpgsql AS $$ -BEGIN - DELETE FROM snapshot_records - WHERE aggregate_id = _aggregate_id - AND sequence_number < _sequence_number; -END; -$$; - -CREATE OR REPLACE FUNCTION aggregates_that_need_snapshots(_last_aggregate_id uuid, _limit integer) - RETURNS TABLE (aggregate_id uuid) -LANGUAGE plpgsql AS $$ -BEGIN - RETURN QUERY SELECT stream.aggregate_id - FROM stream_records stream - WHERE (_last_aggregate_id IS NULL OR stream.aggregate_id > _last_aggregate_id) - AND snapshot_threshold IS NOT NULL - AND snapshot_threshold <= ( - (SELECT MAX(events.sequence_number) FROM event_records events WHERE stream.aggregate_id = events.aggregate_id) - - COALESCE((SELECT MAX(snapshots.sequence_number) FROM snapshot_records snapshots WHERE stream.aggregate_id = snapshots.aggregate_id), 0)) - ORDER BY 1 - LIMIT _limit; -END; -$$; - -CREATE OR REPLACE PROCEDURE permanently_delete_commands_without_events(_aggregate_id uuid, _organization_id uuid) -LANGUAGE plpgsql AS $$ -BEGIN - IF _aggregate_id IS NULL AND _organization_id IS NULL THEN - RAISE EXCEPTION 'aggregate_id or organization_id must be specified to delete commands'; - END IF; - - DELETE FROM commands - WHERE (_aggregate_id IS NULL OR aggregate_id = _aggregate_id) - AND NOT EXISTS (SELECT 1 FROM events WHERE command_id = commands.id); -END; -$$; - -CREATE OR REPLACE PROCEDURE permanently_delete_event_streams(_aggregate_ids jsonb) -LANGUAGE plpgsql AS $$ -BEGIN - DELETE FROM events - USING jsonb_array_elements_text(_aggregate_ids) AS ids (id) - JOIN aggregates ON ids.id::uuid = aggregates.aggregate_id - WHERE events.partition_key = aggregates.events_partition_key - AND events.aggregate_id = aggregates.aggregate_id; - DELETE FROM aggregates - USING jsonb_array_elements_text(_aggregate_ids) AS ids (id) - WHERE aggregates.aggregate_id = ids.id::uuid; -END; -$$; - -CREATE VIEW command_records (id, user_id, aggregate_id, command_type, command_json, created_at, event_aggregate_id, event_sequence_number) AS - SELECT id, - user_id, - aggregate_id, - (SELECT type FROM command_types WHERE command_types.id = command.command_type_id), - enrich_command_json(command), - created_at, - event_aggregate_id, - event_sequence_number - FROM commands command; - -CREATE VIEW event_records (aggregate_id, partition_key, sequence_number, created_at, event_type, event_json, command_record_id, xact_id) AS - SELECT aggregate.aggregate_id, - event.partition_key, - event.sequence_number, - event.created_at, - type.type, - enrich_event_json(event) AS event_json, - command_id, - event.xact_id - FROM aggregates aggregate - JOIN events event ON aggregate.aggregate_id = event.aggregate_id AND aggregate.events_partition_key = event.partition_key - JOIN event_types type ON event.event_type_id = type.id; - -CREATE VIEW stream_records (aggregate_id, events_partition_key, aggregate_type, snapshot_threshold, created_at) AS - SELECT aggregates.aggregate_id, - aggregates.events_partition_key, - aggregate_types.type, - aggregates.snapshot_threshold, - aggregates.created_at - FROM aggregates JOIN aggregate_types ON aggregates.aggregate_type_id = aggregate_types.id; +CREATE TABLE snapshot_records ( + aggregate_id uuid NOT NULL, + sequence_number integer NOT NULL, + created_at timestamptz NOT NULL DEFAULT NOW(), + snapshot_type text NOT NULL, + snapshot_json jsonb NOT NULL, + PRIMARY KEY (aggregate_id, sequence_number), + FOREIGN KEY (aggregate_id) REFERENCES aggregates (aggregate_id) + ON UPDATE CASCADE ON DELETE CASCADE +); From f5f9e3cf75150338cfc5d961b492d94ea902e7cc Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Thu, 13 Jun 2024 15:39:56 +0200 Subject: [PATCH 073/128] Update template project with new schema --- .../template_project/db/sequent_pgsql.sql | 318 ++++++++++++++++ .../template_project/db/sequent_schema.rb | 14 +- .../template_project/db/sequent_schema.sql | 356 +----------------- 3 files changed, 333 insertions(+), 355 deletions(-) create mode 100644 lib/sequent/generator/template_project/db/sequent_pgsql.sql diff --git a/lib/sequent/generator/template_project/db/sequent_pgsql.sql b/lib/sequent/generator/template_project/db/sequent_pgsql.sql new file mode 100644 index 00000000..b0ac83a2 --- /dev/null +++ b/lib/sequent/generator/template_project/db/sequent_pgsql.sql @@ -0,0 +1,318 @@ +DROP TYPE IF EXISTS aggregate_event_type CASCADE; +CREATE TYPE aggregate_event_type AS ( + aggregate_type text, + aggregate_id uuid, + events_partition_key text, + snapshot_threshold integer, + event_type text, + event_json jsonb +); + +CREATE OR REPLACE FUNCTION enrich_command_json(command commands) RETURNS jsonb +LANGUAGE plpgsql AS $$ +BEGIN + RETURN jsonb_build_object( + 'command_type', (SELECT type FROM command_types WHERE command_types.id = command.command_type_id), + 'created_at', command.created_at, + 'user_id', command.user_id, + 'aggregate_id', command.aggregate_id, + 'event_aggregate_id', command.event_aggregate_id, + 'event_sequence_number', command.event_sequence_number + ) + || command.command_json; +END +$$; + +CREATE OR REPLACE FUNCTION enrich_event_json(event events) RETURNS jsonb +LANGUAGE plpgsql AS $$ +BEGIN + RETURN jsonb_build_object( + 'aggregate_id', event.aggregate_id, + 'sequence_number', event.sequence_number, + 'created_at', event.created_at + ) + || event.event_json; +END +$$; + +CREATE OR REPLACE FUNCTION load_event( + _aggregate_id uuid, + _sequence_number integer +) RETURNS SETOF aggregate_event_type +LANGUAGE plpgsql AS $$ +BEGIN + RETURN QUERY SELECT aggregate_types.type, + a.aggregate_id, + a.events_partition_key, + a.snapshot_threshold, + event_types.type, + enrich_event_json(e) + FROM aggregates a + INNER JOIN events e ON (a.events_partition_key, a.aggregate_id) = (e.partition_key, e.aggregate_id) + INNER JOIN aggregate_types ON a.aggregate_type_id = aggregate_types.id + INNER JOIN event_types ON e.event_type_id = event_types.id + WHERE a.aggregate_id = _aggregate_id + AND e.sequence_number = _sequence_number; +END; +$$; + +CREATE OR REPLACE FUNCTION load_events( + _aggregate_ids jsonb, + _use_snapshots boolean DEFAULT TRUE, + _until timestamptz DEFAULT NULL +) RETURNS SETOF aggregate_event_type +LANGUAGE plpgsql AS $$ +DECLARE + _aggregate_id aggregates.aggregate_id%TYPE; +BEGIN + FOR _aggregate_id IN SELECT * FROM jsonb_array_elements_text(_aggregate_ids) LOOP + -- Use a single query to avoid race condition with UPDATEs to the events partition key + -- in case transaction isolation level is lower than repeatable read (the default of + -- PostgreSQL is read committed). + RETURN QUERY WITH + aggregate AS ( + SELECT aggregate_types.type, aggregate_id, events_partition_key, snapshot_threshold + FROM aggregates + JOIN aggregate_types ON aggregate_type_id = aggregate_types.id + WHERE aggregate_id = _aggregate_id + ), + snapshot AS ( + SELECT * + FROM snapshot_records + WHERE _use_snapshots + AND aggregate_id = _aggregate_id + AND (_until IS NULL OR created_at < _until) + ORDER BY sequence_number DESC LIMIT 1 + ) + (SELECT a.*, s.snapshot_type, s.snapshot_json FROM aggregate a, snapshot s) + UNION ALL + (SELECT a.*, event_types.type, enrich_event_json(e) + FROM aggregate a + JOIN events e ON (a.events_partition_key, a.aggregate_id) = (e.partition_key, e.aggregate_id) + JOIN event_types ON e.event_type_id = event_types.id + WHERE e.sequence_number >= COALESCE((SELECT sequence_number FROM snapshot), 0) + AND (_until IS NULL OR e.created_at < _until) + ORDER BY e.sequence_number ASC); + END LOOP; +END; +$$; + +CREATE OR REPLACE FUNCTION store_command(_command jsonb) RETURNS bigint +LANGUAGE plpgsql AS $$ +DECLARE + _id commands.id%TYPE; + _command_json jsonb = _command->'command_json'; +BEGIN + IF NOT EXISTS (SELECT 1 FROM command_types t WHERE t.type = _command->>'command_type') THEN + -- Only try inserting if it doesn't exist to avoid exhausting the id sequence + INSERT INTO command_types (type) + VALUES (_command->>'command_type') + ON CONFLICT DO NOTHING; + END IF; + + INSERT INTO commands ( + created_at, user_id, aggregate_id, command_type_id, command_json, + event_aggregate_id, event_sequence_number + ) VALUES ( + (_command->>'created_at')::timestamptz, + (_command_json->>'user_id')::uuid, + (_command_json->>'aggregate_id')::uuid, + (SELECT id FROM command_types WHERE type = _command->>'command_type'), + (_command->'command_json') - '{command_type,created_at,organization_id,user_id,aggregate_id,event_aggregate_id,event_sequence_number}'::text[], + (_command_json->>'event_aggregate_id')::uuid, + NULLIF(_command_json->'event_sequence_number', 'null'::jsonb)::integer + ) RETURNING id INTO STRICT _id; + RETURN _id; +END; +$$; + +CREATE OR REPLACE PROCEDURE store_events(_command jsonb, _aggregates_with_events jsonb) +LANGUAGE plpgsql AS $$ +DECLARE + _command_id commands.id%TYPE; + _aggregate jsonb; + _events jsonb; + _aggregate_id aggregates.aggregate_id%TYPE; + _provided_events_partition_key aggregates.events_partition_key%TYPE; + _existing_events_partition_key aggregates.events_partition_key%TYPE; + _events_partition_key aggregates.events_partition_key%TYPE; + _snapshot_threshold aggregates.snapshot_threshold%TYPE; +BEGIN + _command_id = store_command(_command); + + WITH types AS ( + SELECT DISTINCT row->0->>'aggregate_type' AS type + FROM jsonb_array_elements(_aggregates_with_events) AS row + ) + INSERT INTO aggregate_types (type) + SELECT type FROM types + WHERE type NOT IN (SELECT type FROM aggregate_types) + ORDER BY 1 + ON CONFLICT DO NOTHING; + + WITH types AS ( + SELECT DISTINCT events->>'event_type' AS type + FROM jsonb_array_elements(_aggregates_with_events) AS row + CROSS JOIN LATERAL jsonb_array_elements(row->1) AS events + ) + INSERT INTO event_types (type) + SELECT type FROM types + WHERE type NOT IN (SELECT type FROM event_types) + ORDER BY 1 + ON CONFLICT DO NOTHING; + + FOR _aggregate, _events IN SELECT row->0, row->1 FROM jsonb_array_elements(_aggregates_with_events) AS row + ORDER BY row->0->'aggregate_id', row->1->0->'event_json'->'sequence_number' + LOOP + _aggregate_id = _aggregate->>'aggregate_id'; + _snapshot_threshold = NULLIF(_aggregate->'snapshot_threshold', 'null'::jsonb); + _provided_events_partition_key = _aggregate->>'events_partition_key'; + + SELECT events_partition_key INTO _existing_events_partition_key FROM aggregates WHERE aggregate_id = _aggregate_id FOR UPDATE; + _events_partition_key = COALESCE(_provided_events_partition_key, _existing_events_partition_key, ''); + + INSERT INTO aggregates (aggregate_id, created_at, aggregate_type_id, events_partition_key, snapshot_threshold) + VALUES ( + _aggregate_id, + (_events->0->>'created_at')::timestamptz, + (SELECT id FROM aggregate_types WHERE type = _aggregate->>'aggregate_type'), + _events_partition_key, + _snapshot_threshold + ) ON CONFLICT (aggregate_id) + DO UPDATE SET events_partition_key = EXCLUDED.events_partition_key, + snapshot_threshold = EXCLUDED.snapshot_threshold + WHERE aggregates.events_partition_key IS DISTINCT FROM EXCLUDED.events_partition_key + OR aggregates.snapshot_threshold IS DISTINCT FROM EXCLUDED.snapshot_threshold; + + INSERT INTO events (partition_key, aggregate_id, sequence_number, created_at, command_id, event_type_id, event_json) + SELECT _events_partition_key, + _aggregate_id, + (event->'event_json'->'sequence_number')::integer, + (event->>'created_at')::timestamptz, + _command_id, + (SELECT id FROM event_types WHERE type = event->>'event_type'), + (event->'event_json') - '{aggregate_id,created_at,event_type,sequence_number}'::text[] + FROM jsonb_array_elements(_events) AS event; + END LOOP; +END; +$$; + +CREATE OR REPLACE PROCEDURE store_snapshots(_snapshots jsonb) +LANGUAGE plpgsql AS $$ +DECLARE + _aggregate_id uuid; + _events_partition_key text; + _snapshot jsonb; +BEGIN + FOR _snapshot IN SELECT * FROM jsonb_array_elements(_snapshots) LOOP + _aggregate_id = _snapshot->>'aggregate_id'; + + INSERT INTO snapshot_records (aggregate_id, sequence_number, created_at, snapshot_type, snapshot_json) + VALUES ( + _aggregate_id, + (_snapshot->'sequence_number')::integer, + (_snapshot->>'created_at')::timestamptz, + _snapshot->>'snapshot_type', + _snapshot->'snapshot_json' + ); + END LOOP; +END; +$$; + +CREATE OR REPLACE FUNCTION load_latest_snapshot(_aggregate_id uuid) RETURNS aggregate_event_type +LANGUAGE SQL AS $$ + SELECT (SELECT type FROM aggregate_types WHERE id = a.aggregate_type_id), + a.aggregate_id, + a.events_partition_key, + a.snapshot_threshold, + s.snapshot_type, + s.snapshot_json + FROM aggregates a JOIN snapshot_records s ON a.aggregate_id = s.aggregate_id + WHERE a.aggregate_id = _aggregate_id + ORDER BY s.sequence_number DESC + LIMIT 1; +$$; + +CREATE OR REPLACE PROCEDURE delete_snapshots_before(_aggregate_id uuid, _sequence_number integer) +LANGUAGE plpgsql AS $$ +BEGIN + DELETE FROM snapshot_records + WHERE aggregate_id = _aggregate_id + AND sequence_number < _sequence_number; +END; +$$; + +CREATE OR REPLACE FUNCTION aggregates_that_need_snapshots(_last_aggregate_id uuid, _limit integer) + RETURNS TABLE (aggregate_id uuid) +LANGUAGE plpgsql AS $$ +BEGIN + RETURN QUERY SELECT stream.aggregate_id + FROM stream_records stream + WHERE (_last_aggregate_id IS NULL OR stream.aggregate_id > _last_aggregate_id) + AND snapshot_threshold IS NOT NULL + AND snapshot_threshold <= ( + (SELECT MAX(events.sequence_number) FROM event_records events WHERE stream.aggregate_id = events.aggregate_id) - + COALESCE((SELECT MAX(snapshots.sequence_number) FROM snapshot_records snapshots WHERE stream.aggregate_id = snapshots.aggregate_id), 0)) + ORDER BY 1 + LIMIT _limit; +END; +$$; + +CREATE OR REPLACE PROCEDURE permanently_delete_commands_without_events(_aggregate_id uuid, _organization_id uuid) +LANGUAGE plpgsql AS $$ +BEGIN + IF _aggregate_id IS NULL AND _organization_id IS NULL THEN + RAISE EXCEPTION 'aggregate_id or organization_id must be specified to delete commands'; + END IF; + + DELETE FROM commands + WHERE (_aggregate_id IS NULL OR aggregate_id = _aggregate_id) + AND NOT EXISTS (SELECT 1 FROM events WHERE command_id = commands.id); +END; +$$; + +CREATE OR REPLACE PROCEDURE permanently_delete_event_streams(_aggregate_ids jsonb) +LANGUAGE plpgsql AS $$ +BEGIN + DELETE FROM events + USING jsonb_array_elements_text(_aggregate_ids) AS ids (id) + JOIN aggregates ON ids.id::uuid = aggregates.aggregate_id + WHERE events.partition_key = aggregates.events_partition_key + AND events.aggregate_id = aggregates.aggregate_id; + DELETE FROM aggregates + USING jsonb_array_elements_text(_aggregate_ids) AS ids (id) + WHERE aggregates.aggregate_id = ids.id::uuid; +END; +$$; + +CREATE OR REPLACE VIEW command_records (id, user_id, aggregate_id, command_type, command_json, created_at, event_aggregate_id, event_sequence_number) AS + SELECT id, + user_id, + aggregate_id, + (SELECT type FROM command_types WHERE command_types.id = command.command_type_id), + enrich_command_json(command), + created_at, + event_aggregate_id, + event_sequence_number + FROM commands command; + +CREATE OR REPLACE VIEW event_records (aggregate_id, partition_key, sequence_number, created_at, event_type, event_json, command_record_id, xact_id) AS + SELECT aggregate.aggregate_id, + event.partition_key, + event.sequence_number, + event.created_at, + type.type, + enrich_event_json(event) AS event_json, + command_id, + event.xact_id + FROM aggregates aggregate + JOIN events event ON aggregate.aggregate_id = event.aggregate_id AND aggregate.events_partition_key = event.partition_key + JOIN event_types type ON event.event_type_id = type.id; + +CREATE OR REPLACE VIEW stream_records (aggregate_id, events_partition_key, aggregate_type, snapshot_threshold, created_at) AS + SELECT aggregates.aggregate_id, + aggregates.events_partition_key, + aggregate_types.type, + aggregates.snapshot_threshold, + aggregates.created_at + FROM aggregates JOIN aggregate_types ON aggregates.aggregate_type_id = aggregate_types.id; diff --git a/lib/sequent/generator/template_project/db/sequent_schema.rb b/lib/sequent/generator/template_project/db/sequent_schema.rb index 97364112..4a4ed00a 100644 --- a/lib/sequent/generator/template_project/db/sequent_schema.rb +++ b/lib/sequent/generator/template_project/db/sequent_schema.rb @@ -1,14 +1,10 @@ # frozen_string_literal: true ActiveRecord::Schema.define do - create_table 'snapshot_records', primary_key: %w[aggregate_id sequence_number], force: true do |t| - t.uuid 'aggregate_id', null: false - t.integer 'sequence_number', null: false - t.datetime 'created_at', null: false - t.text 'snapshot_type', null: false - t.jsonb 'snapshot_json', null: false + say_with_time 'Installing Sequent schema' do + say 'Creating tables and indexes', true + suppress_messages { execute File.read("#{File.dirname(__FILE__)}/sequent_schema.sql") } + say 'Creating stored procedures and views', true + suppress_messages { execute File.read("#{File.dirname(__FILE__)}/sequent_pgsql.sql") } end - - schema = File.read("#{File.dirname(__FILE__)}/sequent_schema.sql") - execute schema end diff --git a/lib/sequent/generator/template_project/db/sequent_schema.sql b/lib/sequent/generator/template_project/db/sequent_schema.sql index e3813b38..faaebbe6 100644 --- a/lib/sequent/generator/template_project/db/sequent_schema.sql +++ b/lib/sequent/generator/template_project/db/sequent_schema.sql @@ -1,13 +1,3 @@ -DROP TYPE IF EXISTS aggregate_event_type CASCADE; -CREATE TYPE aggregate_event_type AS ( - aggregate_type text, - aggregate_id uuid, - events_partition_key text, - snapshot_threshold integer, - event_type text, - event_json jsonb -); - CREATE TABLE command_types (id SMALLINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, type text UNIQUE NOT NULL); CREATE TABLE aggregate_types (id SMALLINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, type text UNIQUE NOT NULL); CREATE TABLE event_types (id SMALLINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, type text UNIQUE NOT NULL); @@ -77,339 +67,13 @@ ALTER TABLE events_2025_and_later CLUSTER ON events_2025_and_later_pkey; CREATE TABLE events_aggregate PARTITION OF events FOR VALUES FROM ('A') TO ('Ag'); ALTER TABLE events_aggregate CLUSTER ON events_aggregate_pkey; -TRUNCATE TABLE snapshot_records; -ALTER TABLE snapshot_records - ADD CONSTRAINT aggregate_fkey FOREIGN KEY (aggregate_id) REFERENCES aggregates (aggregate_id) - ON UPDATE CASCADE ON DELETE CASCADE; - -CREATE OR REPLACE FUNCTION enrich_command_json(command commands) RETURNS jsonb -LANGUAGE plpgsql AS $$ -BEGIN - RETURN jsonb_build_object( - 'command_type', (SELECT type FROM command_types WHERE command_types.id = command.command_type_id), - 'created_at', command.created_at, - 'user_id', command.user_id, - 'aggregate_id', command.aggregate_id, - 'event_aggregate_id', command.event_aggregate_id, - 'event_sequence_number', command.event_sequence_number - ) - || command.command_json; -END -$$; - -CREATE OR REPLACE FUNCTION enrich_event_json(event events) RETURNS jsonb -LANGUAGE plpgsql AS $$ -BEGIN - RETURN jsonb_build_object( - 'aggregate_id', event.aggregate_id, - 'sequence_number', event.sequence_number, - 'created_at', event.created_at - ) - || event.event_json; -END -$$; - -CREATE OR REPLACE FUNCTION load_event( - _aggregate_id uuid, - _sequence_number integer -) RETURNS SETOF aggregate_event_type -LANGUAGE plpgsql AS $$ -DECLARE - _aggregate aggregates; - _aggregate_type text; -BEGIN - SELECT * INTO _aggregate - FROM aggregates - WHERE aggregate_id = _aggregate_id; - IF NOT FOUND THEN - RETURN; - END IF; - - SELECT type INTO STRICT _aggregate_type - FROM aggregate_types - WHERE id = _aggregate.aggregate_type_id; - - RETURN QUERY SELECT _aggregate_type, - _aggregate_id, - _aggregate.events_partition_key, - _aggregate.snapshot_threshold, - event_types.type, - enrich_event_json(events) - FROM events - INNER JOIN event_types ON events.event_type_id = event_types.id - WHERE events.partition_key = _aggregate.events_partition_key - AND events.aggregate_id = _aggregate_id - AND events.sequence_number = _sequence_number; -END; -$$; - -CREATE OR REPLACE FUNCTION load_events( - _aggregate_ids jsonb, - _use_snapshots boolean DEFAULT TRUE, - _until timestamptz DEFAULT NULL -) RETURNS SETOF aggregate_event_type -LANGUAGE plpgsql AS $$ -DECLARE - _aggregate_type text; - _aggregate_id aggregates.aggregate_id%TYPE; - _aggregate aggregates; - _snapshot snapshot_records; - _start_sequence_number events.sequence_number%TYPE; -BEGIN - FOR _aggregate_id IN SELECT * FROM jsonb_array_elements_text(_aggregate_ids) LOOP - SELECT * INTO _aggregate FROM aggregates WHERE aggregates.aggregate_id = _aggregate_id; - IF NOT FOUND THEN - CONTINUE; - END IF; - - SELECT type INTO STRICT _aggregate_type - FROM aggregate_types - WHERE id = _aggregate.aggregate_type_id; - - _start_sequence_number = 0; - IF _use_snapshots THEN - SELECT * INTO _snapshot FROM snapshot_records snapshots WHERE snapshots.aggregate_id = _aggregate.aggregate_id ORDER BY sequence_number DESC LIMIT 1; - IF FOUND THEN - _start_sequence_number := _snapshot.sequence_number; - RETURN NEXT (_aggregate_type, - _aggregate.aggregate_id, - _aggregate.events_partition_key, - _aggregate.snapshot_threshold, - _snapshot.snapshot_type, - _snapshot.snapshot_json); - END IF; - END IF; - RETURN QUERY SELECT _aggregate_type, - _aggregate.aggregate_id, - _aggregate.events_partition_key, - _aggregate.snapshot_threshold, - event_types.type, - enrich_event_json(events) - FROM events - INNER JOIN event_types ON events.event_type_id = event_types.id - WHERE events.partition_key = _aggregate.events_partition_key - AND events.aggregate_id = _aggregate.aggregate_id - AND events.sequence_number >= _start_sequence_number - AND (_until IS NULL OR events.created_at < _until) - ORDER BY events.sequence_number; - END LOOP; -END; -$$; - -CREATE OR REPLACE FUNCTION store_command(_command jsonb) RETURNS bigint -LANGUAGE plpgsql AS $$ -DECLARE - _id commands.id%TYPE; - _command_json jsonb = _command->'command_json'; -BEGIN - IF NOT EXISTS (SELECT 1 FROM command_types t WHERE t.type = _command->>'command_type') THEN - -- Only try inserting if it doesn't exist to avoid exhausting the id sequence - INSERT INTO command_types (type) - VALUES (_command->>'command_type') - ON CONFLICT DO NOTHING; - END IF; - - INSERT INTO commands ( - created_at, user_id, aggregate_id, command_type_id, command_json, - event_aggregate_id, event_sequence_number - ) VALUES ( - (_command->>'created_at')::timestamptz, - (_command_json->>'user_id')::uuid, - (_command_json->>'aggregate_id')::uuid, - (SELECT id FROM command_types WHERE type = _command->>'command_type'), - (_command->'command_json') - '{command_type,created_at,organization_id,user_id,aggregate_id,event_aggregate_id,event_sequence_number}'::text[], - (_command_json->>'event_aggregate_id')::uuid, - NULLIF(_command_json->'event_sequence_number', 'null'::jsonb)::integer - ) RETURNING id INTO STRICT _id; - RETURN _id; -END; -$$; - -CREATE OR REPLACE PROCEDURE store_events(_command jsonb, _aggregates_with_events jsonb) -LANGUAGE plpgsql AS $$ -DECLARE - _command_id commands.id%TYPE; - _aggregate jsonb; - _events jsonb; - _aggregate_id aggregates.aggregate_id%TYPE; - _provided_events_partition_key aggregates.events_partition_key%TYPE; - _existing_events_partition_key aggregates.events_partition_key%TYPE; - _events_partition_key aggregates.events_partition_key%TYPE; - _snapshot_threshold aggregates.snapshot_threshold%TYPE; -BEGIN - _command_id = store_command(_command); - - WITH types AS ( - SELECT DISTINCT row->0->>'aggregate_type' AS type - FROM jsonb_array_elements(_aggregates_with_events) AS row - ) - INSERT INTO aggregate_types (type) - SELECT type FROM types - WHERE type NOT IN (SELECT type FROM aggregate_types) - ORDER BY 1 - ON CONFLICT DO NOTHING; - - WITH types AS ( - SELECT DISTINCT events->>'event_type' AS type - FROM jsonb_array_elements(_aggregates_with_events) AS row - CROSS JOIN LATERAL jsonb_array_elements(row->1) AS events - ) - INSERT INTO event_types (type) - SELECT type FROM types - WHERE type NOT IN (SELECT type FROM event_types) - ORDER BY 1 - ON CONFLICT DO NOTHING; - - FOR _aggregate, _events IN SELECT row->0, row->1 FROM jsonb_array_elements(_aggregates_with_events) AS row LOOP - _aggregate_id = _aggregate->>'aggregate_id'; - _snapshot_threshold = NULLIF(_aggregate->'snapshot_threshold', 'null'::jsonb); - _provided_events_partition_key = _aggregate->>'events_partition_key'; - - SELECT events_partition_key INTO _existing_events_partition_key FROM aggregates WHERE aggregate_id = _aggregate_id; - _events_partition_key = COALESCE(_provided_events_partition_key, _existing_events_partition_key, ''); - - INSERT INTO aggregates (aggregate_id, created_at, aggregate_type_id, events_partition_key, snapshot_threshold) - VALUES ( - _aggregate_id, - (_events->0->>'created_at')::timestamptz, - (SELECT id FROM aggregate_types WHERE type = _aggregate->>'aggregate_type'), - _events_partition_key, - _snapshot_threshold - ) ON CONFLICT (aggregate_id) - DO UPDATE SET events_partition_key = EXCLUDED.events_partition_key, - snapshot_threshold = EXCLUDED.snapshot_threshold - WHERE aggregates.events_partition_key <> EXCLUDED.events_partition_key - OR aggregates.snapshot_threshold <> EXCLUDED.snapshot_threshold; - - INSERT INTO events (partition_key, aggregate_id, sequence_number, created_at, command_id, event_type_id, event_json) - SELECT _events_partition_key, - _aggregate_id, - (event->'event_json'->'sequence_number')::integer, - (event->>'created_at')::timestamptz, - _command_id, - (SELECT id FROM event_types WHERE type = event->>'event_type'), - (event->'event_json') - '{aggregate_id,created_at,event_type,sequence_number}'::text[] - FROM jsonb_array_elements(_events) AS event; - END LOOP; -END; -$$; - -CREATE OR REPLACE PROCEDURE store_snapshots(_snapshots jsonb) -LANGUAGE plpgsql AS $$ -DECLARE - _aggregate_id uuid; - _events_partition_key text; - _snapshot jsonb; -BEGIN - FOR _snapshot IN SELECT * FROM jsonb_array_elements(_snapshots) LOOP - _aggregate_id = _snapshot->>'aggregate_id'; - - INSERT INTO snapshot_records (aggregate_id, sequence_number, created_at, snapshot_type, snapshot_json) - VALUES ( - _aggregate_id, - (_snapshot->'sequence_number')::integer, - (_snapshot->>'created_at')::timestamptz, - _snapshot->>'snapshot_type', - _snapshot->'snapshot_json' - ); - END LOOP; -END; -$$; - -CREATE OR REPLACE FUNCTION load_latest_snapshot(_aggregate_id uuid) RETURNS aggregate_event_type -LANGUAGE SQL AS $$ - SELECT (SELECT type FROM aggregate_types WHERE id = a.aggregate_type_id), - a.aggregate_id, - a.events_partition_key, - a.snapshot_threshold, - s.snapshot_type, - s.snapshot_json - FROM aggregates a JOIN snapshot_records s ON a.aggregate_id = s.aggregate_id - WHERE a.aggregate_id = _aggregate_id - ORDER BY s.sequence_number DESC - LIMIT 1; -$$; - -CREATE OR REPLACE PROCEDURE delete_snapshots_before(_aggregate_id uuid, _sequence_number integer) -LANGUAGE plpgsql AS $$ -BEGIN - DELETE FROM snapshot_records - WHERE aggregate_id = _aggregate_id - AND sequence_number < _sequence_number; -END; -$$; - -CREATE OR REPLACE FUNCTION aggregates_that_need_snapshots(_last_aggregate_id uuid, _limit integer) - RETURNS TABLE (aggregate_id uuid) -LANGUAGE plpgsql AS $$ -BEGIN - RETURN QUERY SELECT stream.aggregate_id - FROM stream_records stream - WHERE (_last_aggregate_id IS NULL OR stream.aggregate_id > _last_aggregate_id) - AND snapshot_threshold IS NOT NULL - AND snapshot_threshold <= ( - (SELECT MAX(events.sequence_number) FROM event_records events WHERE stream.aggregate_id = events.aggregate_id) - - COALESCE((SELECT MAX(snapshots.sequence_number) FROM snapshot_records snapshots WHERE stream.aggregate_id = snapshots.aggregate_id), 0)) - ORDER BY 1 - LIMIT _limit; -END; -$$; - -CREATE OR REPLACE PROCEDURE permanently_delete_commands_without_events(_aggregate_id uuid, _organization_id uuid) -LANGUAGE plpgsql AS $$ -BEGIN - IF _aggregate_id IS NULL AND _organization_id IS NULL THEN - RAISE EXCEPTION 'aggregate_id or organization_id must be specified to delete commands'; - END IF; - - DELETE FROM commands - WHERE (_aggregate_id IS NULL OR aggregate_id = _aggregate_id) - AND NOT EXISTS (SELECT 1 FROM events WHERE command_id = commands.id); -END; -$$; - -CREATE OR REPLACE PROCEDURE permanently_delete_event_streams(_aggregate_ids jsonb) -LANGUAGE plpgsql AS $$ -BEGIN - DELETE FROM events - USING jsonb_array_elements_text(_aggregate_ids) AS ids (id) - JOIN aggregates ON ids.id::uuid = aggregates.aggregate_id - WHERE events.partition_key = aggregates.events_partition_key - AND events.aggregate_id = aggregates.aggregate_id; - DELETE FROM aggregates - USING jsonb_array_elements_text(_aggregate_ids) AS ids (id) - WHERE aggregates.aggregate_id = ids.id::uuid; -END; -$$; - -CREATE VIEW command_records (id, user_id, aggregate_id, command_type, command_json, created_at, event_aggregate_id, event_sequence_number) AS - SELECT id, - user_id, - aggregate_id, - (SELECT type FROM command_types WHERE command_types.id = command.command_type_id), - enrich_command_json(command), - created_at, - event_aggregate_id, - event_sequence_number - FROM commands command; - -CREATE VIEW event_records (aggregate_id, partition_key, sequence_number, created_at, event_type, event_json, command_record_id, xact_id) AS - SELECT aggregate.aggregate_id, - event.partition_key, - event.sequence_number, - event.created_at, - type.type, - enrich_event_json(event) AS event_json, - command_id, - event.xact_id - FROM aggregates aggregate - JOIN events event ON aggregate.aggregate_id = event.aggregate_id AND aggregate.events_partition_key = event.partition_key - JOIN event_types type ON event.event_type_id = type.id; - -CREATE VIEW stream_records (aggregate_id, events_partition_key, aggregate_type, snapshot_threshold, created_at) AS - SELECT aggregates.aggregate_id, - aggregates.events_partition_key, - aggregate_types.type, - aggregates.snapshot_threshold, - aggregates.created_at - FROM aggregates JOIN aggregate_types ON aggregates.aggregate_type_id = aggregate_types.id; +CREATE TABLE snapshot_records ( + aggregate_id uuid NOT NULL, + sequence_number integer NOT NULL, + created_at timestamptz NOT NULL DEFAULT NOW(), + snapshot_type text NOT NULL, + snapshot_json jsonb NOT NULL, + PRIMARY KEY (aggregate_id, sequence_number), + FOREIGN KEY (aggregate_id) REFERENCES aggregates (aggregate_id) + ON UPDATE CASCADE ON DELETE CASCADE +); From aa0c3264a73ef784c1f0af846d90854c5759033a Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Fri, 14 Jun 2024 11:25:56 +0200 Subject: [PATCH 074/128] Save events that are updated or deleted to separate table Uses database triggers to automatically save updated or deleted events. Updating events should normally not happen (only to fix corrupted data). The user of sequent is responsible for limiting the size of the `saved_event_records` table if updates or deletes are frequent. --- db/sequent_pgsql.sql | 48 +++++++++++++++++ db/sequent_schema.sql | 15 ++++++ spec/lib/sequent/core/event_store_spec.rb | 63 ++++++++++++++++++----- spec/spec_helper.rb | 2 +- 4 files changed, 115 insertions(+), 13 deletions(-) diff --git a/db/sequent_pgsql.sql b/db/sequent_pgsql.sql index b0ac83a2..0bba257a 100644 --- a/db/sequent_pgsql.sql +++ b/db/sequent_pgsql.sql @@ -316,3 +316,51 @@ CREATE OR REPLACE VIEW stream_records (aggregate_id, events_partition_key, aggre aggregates.snapshot_threshold, aggregates.created_at FROM aggregates JOIN aggregate_types ON aggregates.aggregate_type_id = aggregate_types.id; + +CREATE OR REPLACE FUNCTION save_events_on_delete_trigger() RETURNS TRIGGER AS $$ +BEGIN + INSERT INTO saved_event_records (operation, timestamp, "user", aggregate_id, partition_key, sequence_number, created_at, event_type, event_json, command_id, xact_id) + SELECT 'D', statement_timestamp(), user, + o.aggregate_id, + o.partition_key, + o.sequence_number, + o.created_at, + (SELECT type FROM event_types WHERE event_types.id = o.event_type_id), + o.event_json, + o.command_id, + o.xact_id + FROM old_table o; + RETURN NULL; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION save_events_on_update_trigger() RETURNS TRIGGER AS $$ +BEGIN + INSERT INTO saved_event_records (operation, timestamp, "user", aggregate_id, partition_key, sequence_number, created_at, event_type, event_json, command_id, xact_id) + SELECT 'U', statement_timestamp(), user, + o.aggregate_id, + o.partition_key, + o.sequence_number, + o.created_at, + (SELECT type FROM event_types WHERE event_types.id = o.event_type_id), + o.event_json, + o.command_id, + o.xact_id + FROM old_table o LEFT JOIN new_table n ON o.aggregate_id = n.aggregate_id AND o.sequence_number = n.sequence_number + WHERE n IS NULL + -- Only save when event related information changes + OR o.created_at <> n.created_at + OR o.event_type_id <> n.event_type_id + OR o.event_json <> n.event_json; + RETURN NULL; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE TRIGGER save_events_on_delete_trigger + AFTER DELETE ON events + REFERENCING OLD TABLE AS old_table + FOR EACH STATEMENT EXECUTE FUNCTION save_events_on_delete_trigger(); +CREATE OR REPLACE TRIGGER save_events_on_update_trigger + AFTER UPDATE ON events + REFERENCING OLD TABLE AS old_table NEW TABLE AS new_table + FOR EACH STATEMENT EXECUTE FUNCTION save_events_on_update_trigger(); diff --git a/db/sequent_schema.sql b/db/sequent_schema.sql index faaebbe6..54d72199 100644 --- a/db/sequent_schema.sql +++ b/db/sequent_schema.sql @@ -77,3 +77,18 @@ CREATE TABLE snapshot_records ( FOREIGN KEY (aggregate_id) REFERENCES aggregates (aggregate_id) ON UPDATE CASCADE ON DELETE CASCADE ); + +CREATE TABLE saved_event_records ( + operation varchar(1) NOT NULL CHECK (operation IN ('U', 'D')), + timestamp timestamptz NOT NULL, + "user" text NOT NULL, + aggregate_id uuid NOT NULL, + partition_key text DEFAULT '', + sequence_number integer NOT NULL, + created_at timestamp with time zone NOT NULL, + command_id bigint NOT NULL, + event_type text NOT NULL, + event_json jsonb NOT NULL, + xact_id bigint, + PRIMARY KEY (aggregate_id, sequence_number, timestamp) +); diff --git a/spec/lib/sequent/core/event_store_spec.rb b/spec/lib/sequent/core/event_store_spec.rb index ff0aed0b..bafe6360 100644 --- a/spec/lib/sequent/core/event_store_spec.rb +++ b/spec/lib/sequent/core/event_store_spec.rb @@ -138,34 +138,73 @@ class MyAggregate < Sequent::Core::AggregateRoot end end - describe '#events_exists?' do + describe '#permanently_delete_events' do before do event_store.commit_events( - Sequent::Core::Command.new(aggregate_id: aggregate_id), + Sequent::Core::Command.new(aggregate_id:), [ [ Sequent::Core::EventStream.new( aggregate_type: 'MyAggregate', - aggregate_id: aggregate_id, + aggregate_id:, snapshot_threshold: 13, ), - [MyEvent.new(aggregate_id: aggregate_id, sequence_number: 1)], + [MyEvent.new(aggregate_id:, sequence_number: 1)], ], ], ) end - it 'gets true for an existing aggregate' do - expect(event_store.events_exists?(aggregate_id)).to eq(true) - end + context 'should save deleted and updated events' do + it 'saves updated events into separate table' do + ActiveRecord::Base.connection.exec_update( + "UPDATE events SET event_json = '{}' WHERE aggregate_id = $1", + 'update event', + [aggregate_id], + ) - it 'gets false for an non-existing aggregate' do - expect(event_store.events_exists?(Sequent.new_uuid)).to eq(false) + saved_events = ActiveRecord::Base.connection.exec_query( + 'SELECT * FROM saved_event_records WHERE aggregate_id = $1', + 'saved_events', + [aggregate_id], + ).to_a + + expect(saved_events.size).to eq(1) + expect(saved_events[0]['operation']).to eq('U') + expect(saved_events[0]['event_type']).to eq('MyEvent') + expect(saved_events[0]['sequence_number']).to eq(1) + expect(saved_events[0]['event_json']).to eq('{"data": null}') + end + it 'saves deleted events into separate table' do + event_store.permanently_delete_event_stream(aggregate_id) + + saved_events = ActiveRecord::Base.connection.exec_query( + 'SELECT * FROM saved_event_records WHERE aggregate_id = $1', + 'saved_events', + [aggregate_id], + ).to_a + + expect(saved_events.size).to eq(1) + expect(saved_events[0]['operation']).to eq('D') + expect(saved_events[0]['event_type']).to eq('MyEvent') + expect(saved_events[0]['sequence_number']).to eq(1) + expect(saved_events[0]['event_json']).to eq('{"data": null}') + end end - it 'gets false after deletion' do - event_store.permanently_delete_event_stream(aggregate_id) - expect(event_store.events_exists?(aggregate_id)).to eq(false) + context '#events_exists?' do + it 'gets true for an existing aggregate' do + expect(event_store.events_exists?(aggregate_id)).to eq(true) + end + + it 'gets false for an non-existing aggregate' do + expect(event_store.events_exists?(Sequent.new_uuid)).to eq(false) + end + + it 'gets false after deletion' do + event_store.permanently_delete_event_stream(aggregate_id) + expect(event_store.events_exists?(aggregate_id)).to eq(false) + end end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index b8c822aa..c8cd9b93 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -23,7 +23,7 @@ RSpec.configure do |c| c.before do Database.establish_connection - Sequent::ApplicationRecord.connection.execute('TRUNCATE commands, aggregates CASCADE') + Sequent::ApplicationRecord.connection.execute('TRUNCATE commands, aggregates, saved_event_records CASCADE') Sequent::Configuration.reset Sequent.configuration.database_config_directory = 'tmp' end From 89adf865a7aabe772be2a94621c9825ef81948fa Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Mon, 17 Jun 2024 13:15:03 +0200 Subject: [PATCH 075/128] Formatting --- db/sequent_pgsql.sql | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/db/sequent_pgsql.sql b/db/sequent_pgsql.sql index 0bba257a..c476ecf1 100644 --- a/db/sequent_pgsql.sql +++ b/db/sequent_pgsql.sql @@ -320,7 +320,9 @@ CREATE OR REPLACE VIEW stream_records (aggregate_id, events_partition_key, aggre CREATE OR REPLACE FUNCTION save_events_on_delete_trigger() RETURNS TRIGGER AS $$ BEGIN INSERT INTO saved_event_records (operation, timestamp, "user", aggregate_id, partition_key, sequence_number, created_at, event_type, event_json, command_id, xact_id) - SELECT 'D', statement_timestamp(), user, + SELECT 'D', + statement_timestamp(), + user, o.aggregate_id, o.partition_key, o.sequence_number, @@ -337,7 +339,9 @@ $$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION save_events_on_update_trigger() RETURNS TRIGGER AS $$ BEGIN INSERT INTO saved_event_records (operation, timestamp, "user", aggregate_id, partition_key, sequence_number, created_at, event_type, event_json, command_id, xact_id) - SELECT 'U', statement_timestamp(), user, + SELECT 'U', + statement_timestamp(), + user, o.aggregate_id, o.partition_key, o.sequence_number, From d6c56ddf06fbe5d17ac1f1c90a46f641cc002574 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Wed, 26 Jun 2024 15:46:28 +0200 Subject: [PATCH 076/128] Update project template SQL schema --- .../template_project/db/sequent_pgsql.sql | 52 +++++++++++++++++++ .../template_project/db/sequent_schema.sql | 15 ++++++ 2 files changed, 67 insertions(+) diff --git a/lib/sequent/generator/template_project/db/sequent_pgsql.sql b/lib/sequent/generator/template_project/db/sequent_pgsql.sql index b0ac83a2..c476ecf1 100644 --- a/lib/sequent/generator/template_project/db/sequent_pgsql.sql +++ b/lib/sequent/generator/template_project/db/sequent_pgsql.sql @@ -316,3 +316,55 @@ CREATE OR REPLACE VIEW stream_records (aggregate_id, events_partition_key, aggre aggregates.snapshot_threshold, aggregates.created_at FROM aggregates JOIN aggregate_types ON aggregates.aggregate_type_id = aggregate_types.id; + +CREATE OR REPLACE FUNCTION save_events_on_delete_trigger() RETURNS TRIGGER AS $$ +BEGIN + INSERT INTO saved_event_records (operation, timestamp, "user", aggregate_id, partition_key, sequence_number, created_at, event_type, event_json, command_id, xact_id) + SELECT 'D', + statement_timestamp(), + user, + o.aggregate_id, + o.partition_key, + o.sequence_number, + o.created_at, + (SELECT type FROM event_types WHERE event_types.id = o.event_type_id), + o.event_json, + o.command_id, + o.xact_id + FROM old_table o; + RETURN NULL; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION save_events_on_update_trigger() RETURNS TRIGGER AS $$ +BEGIN + INSERT INTO saved_event_records (operation, timestamp, "user", aggregate_id, partition_key, sequence_number, created_at, event_type, event_json, command_id, xact_id) + SELECT 'U', + statement_timestamp(), + user, + o.aggregate_id, + o.partition_key, + o.sequence_number, + o.created_at, + (SELECT type FROM event_types WHERE event_types.id = o.event_type_id), + o.event_json, + o.command_id, + o.xact_id + FROM old_table o LEFT JOIN new_table n ON o.aggregate_id = n.aggregate_id AND o.sequence_number = n.sequence_number + WHERE n IS NULL + -- Only save when event related information changes + OR o.created_at <> n.created_at + OR o.event_type_id <> n.event_type_id + OR o.event_json <> n.event_json; + RETURN NULL; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE TRIGGER save_events_on_delete_trigger + AFTER DELETE ON events + REFERENCING OLD TABLE AS old_table + FOR EACH STATEMENT EXECUTE FUNCTION save_events_on_delete_trigger(); +CREATE OR REPLACE TRIGGER save_events_on_update_trigger + AFTER UPDATE ON events + REFERENCING OLD TABLE AS old_table NEW TABLE AS new_table + FOR EACH STATEMENT EXECUTE FUNCTION save_events_on_update_trigger(); diff --git a/lib/sequent/generator/template_project/db/sequent_schema.sql b/lib/sequent/generator/template_project/db/sequent_schema.sql index faaebbe6..54d72199 100644 --- a/lib/sequent/generator/template_project/db/sequent_schema.sql +++ b/lib/sequent/generator/template_project/db/sequent_schema.sql @@ -77,3 +77,18 @@ CREATE TABLE snapshot_records ( FOREIGN KEY (aggregate_id) REFERENCES aggregates (aggregate_id) ON UPDATE CASCADE ON DELETE CASCADE ); + +CREATE TABLE saved_event_records ( + operation varchar(1) NOT NULL CHECK (operation IN ('U', 'D')), + timestamp timestamptz NOT NULL, + "user" text NOT NULL, + aggregate_id uuid NOT NULL, + partition_key text DEFAULT '', + sequence_number integer NOT NULL, + created_at timestamp with time zone NOT NULL, + command_id bigint NOT NULL, + event_type text NOT NULL, + event_json jsonb NOT NULL, + xact_id bigint, + PRIMARY KEY (aggregate_id, sequence_number, timestamp) +); From cba6524b66587e588d5e4883cbb0b6023a270afe Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Fri, 7 Jun 2024 11:56:51 +0200 Subject: [PATCH 077/128] Track snapshot high watermark so that snapshotting can be prioritized After deleting all snapshots the first aggregates that we want to restore are the ones with the highest number of events. We track the high-water mark of the stored snapshots per aggregate to know which aggregates need to be snapshotted first. In normal operation the xact_id is used to prioritize snapshotting ordering. --- db/sequent_pgsql.sql | 90 ++++++++++++++++++++--- db/sequent_schema.sql | 13 ++++ lib/sequent/core/event_store.rb | 17 +++++ spec/lib/sequent/core/event_store_spec.rb | 27 ++++++- 4 files changed, 135 insertions(+), 12 deletions(-) diff --git a/db/sequent_pgsql.sql b/db/sequent_pgsql.sql index c476ecf1..d92ead3e 100644 --- a/db/sequent_pgsql.sql +++ b/db/sequent_pgsql.sql @@ -133,10 +133,11 @@ DECLARE _aggregate jsonb; _events jsonb; _aggregate_id aggregates.aggregate_id%TYPE; + _aggregate_row aggregates%ROWTYPE; _provided_events_partition_key aggregates.events_partition_key%TYPE; - _existing_events_partition_key aggregates.events_partition_key%TYPE; _events_partition_key aggregates.events_partition_key%TYPE; _snapshot_threshold aggregates.snapshot_threshold%TYPE; + _snapshot_outdated boolean; BEGIN _command_id = store_command(_command); @@ -168,8 +169,8 @@ BEGIN _snapshot_threshold = NULLIF(_aggregate->'snapshot_threshold', 'null'::jsonb); _provided_events_partition_key = _aggregate->>'events_partition_key'; - SELECT events_partition_key INTO _existing_events_partition_key FROM aggregates WHERE aggregate_id = _aggregate_id FOR UPDATE; - _events_partition_key = COALESCE(_provided_events_partition_key, _existing_events_partition_key, ''); + SELECT * INTO _aggregate_row FROM aggregates WHERE aggregate_id = _aggregate_id; + _events_partition_key = COALESCE(_provided_events_partition_key, _aggregate_row.events_partition_key, ''); INSERT INTO aggregates (aggregate_id, created_at, aggregate_type_id, events_partition_key, snapshot_threshold) VALUES ( @@ -193,6 +194,17 @@ BEGIN (SELECT id FROM event_types WHERE type = event->>'event_type'), (event->'event_json') - '{aggregate_id,created_at,event_type,sequence_number}'::text[] FROM jsonb_array_elements(_events) AS event; + + -- Require a new snapshot when an event is stored with a sequence number that is a multiple of snapshot_threshold + _snapshot_outdated = EXISTS (SELECT * FROM jsonb_array_elements(_events) AS event + WHERE (event->'event_json'->'sequence_number')::integer % _snapshot_threshold = 0); + IF _snapshot_outdated THEN + INSERT INTO aggregates_that_need_snapshots AS target + VALUES (_aggregate_id, pg_current_xact_id()::text::bigint, NULL) + ON CONFLICT (aggregate_id) DO UPDATE + SET snapshot_outdated_xact_id = EXCLUDED.snapshot_outdated_xact_id + WHERE target.snapshot_outdated_xact_id IS NULL; + END IF; END LOOP; END; $$; @@ -201,7 +213,6 @@ CREATE OR REPLACE PROCEDURE store_snapshots(_snapshots jsonb) LANGUAGE plpgsql AS $$ DECLARE _aggregate_id uuid; - _events_partition_key text; _snapshot jsonb; BEGIN FOR _snapshot IN SELECT * FROM jsonb_array_elements(_snapshots) LOOP @@ -215,6 +226,8 @@ BEGIN _snapshot->>'snapshot_type', _snapshot->'snapshot_json' ); + + CALL update_snapshot_status(_aggregate_id); END LOOP; END; $$; @@ -233,12 +246,58 @@ LANGUAGE SQL AS $$ LIMIT 1; $$; +CREATE OR REPLACE PROCEDURE delete_all_snapshots() +LANGUAGE plpgsql AS $$ +BEGIN + UPDATE aggregates_that_need_snapshots + SET snapshot_outdated_xact_id = pg_current_xact_id()::text::bigint + WHERE snapshot_outdated_xact_id IS NULL; + DELETE FROM snapshot_records; +END; +$$; + CREATE OR REPLACE PROCEDURE delete_snapshots_before(_aggregate_id uuid, _sequence_number integer) LANGUAGE plpgsql AS $$ BEGIN DELETE FROM snapshot_records WHERE aggregate_id = _aggregate_id AND sequence_number < _sequence_number; + + CALL update_snapshot_status(_aggregate_id); +END; +$$; + +CREATE OR REPLACE PROCEDURE update_snapshot_status(_aggregate_id uuid) +LANGUAGE plpgsql AS $$ +DECLARE + _snapshot_threshold aggregates.snapshot_threshold%TYPE; + _last_event_sequence_number events.sequence_number%TYPE; + _last_snapshot_sequence_number events.sequence_number%TYPE; + _snapshot_outdated_xact_id bigint = NULL; +BEGIN + SELECT a.snapshot_threshold, e.sequence_number INTO STRICT _snapshot_threshold, _last_event_sequence_number + FROM aggregates a JOIN events e ON a.events_partition_key = e.partition_key AND a.aggregate_id = e.aggregate_id + WHERE a.aggregate_id = _aggregate_id + ORDER BY 2 DESC LIMIT 1; + + SELECT sequence_number INTO _last_snapshot_sequence_number + FROM snapshot_records + WHERE aggregate_id = _aggregate_id + ORDER BY 1 DESC LIMIT 1; + + IF _last_event_sequence_number - COALESCE(_last_snapshot_sequence_number, 0) >= _snapshot_threshold THEN + _snapshot_outdated_xact_id = pg_current_xact_id()::text::bigint; + END IF; + + INSERT INTO aggregates_that_need_snapshots AS target + VALUES (_aggregate_id, _snapshot_outdated_xact_id, _last_snapshot_sequence_number) + ON CONFLICT (aggregate_id) DO UPDATE + SET snapshot_outdated_xact_id = (CASE + WHEN EXCLUDED.snapshot_outdated_xact_id IS NULL THEN NULL + ELSE LEAST(target.snapshot_outdated_xact_id, EXCLUDED.snapshot_outdated_xact_id) + END), + snapshot_sequence_number_high_water_mark = + GREATEST(target.snapshot_sequence_number_high_water_mark, EXCLUDED.snapshot_sequence_number_high_water_mark); END; $$; @@ -246,18 +305,27 @@ CREATE OR REPLACE FUNCTION aggregates_that_need_snapshots(_last_aggregate_id uui RETURNS TABLE (aggregate_id uuid) LANGUAGE plpgsql AS $$ BEGIN - RETURN QUERY SELECT stream.aggregate_id - FROM stream_records stream - WHERE (_last_aggregate_id IS NULL OR stream.aggregate_id > _last_aggregate_id) - AND snapshot_threshold IS NOT NULL - AND snapshot_threshold <= ( - (SELECT MAX(events.sequence_number) FROM event_records events WHERE stream.aggregate_id = events.aggregate_id) - - COALESCE((SELECT MAX(snapshots.sequence_number) FROM snapshot_records snapshots WHERE stream.aggregate_id = snapshots.aggregate_id), 0)) + RETURN QUERY SELECT a.aggregate_id + FROM aggregates_that_need_snapshots a + WHERE a.snapshot_outdated_xact_id IS NOT NULL + AND (_last_aggregate_id IS NULL OR a.aggregate_id > _last_aggregate_id) ORDER BY 1 LIMIT _limit; END; $$; +CREATE OR REPLACE FUNCTION aggregates_that_need_snapshots_ordered_by_priority(_limit integer) + RETURNS TABLE (aggregate_id uuid) +LANGUAGE plpgsql AS $$ +BEGIN + RETURN QUERY SELECT a.aggregate_id + FROM aggregates_that_need_snapshots a + WHERE snapshot_outdated_xact_id IS NOT NULL + ORDER BY snapshot_outdated_xact_id ASC, snapshot_sequence_number_high_water_mark DESC, aggregate_id ASC + LIMIT _limit; +END; +$$; + CREATE OR REPLACE PROCEDURE permanently_delete_commands_without_events(_aggregate_id uuid, _organization_id uuid) LANGUAGE plpgsql AS $$ BEGIN diff --git a/db/sequent_schema.sql b/db/sequent_schema.sql index 54d72199..36b79598 100644 --- a/db/sequent_schema.sql +++ b/db/sequent_schema.sql @@ -92,3 +92,16 @@ CREATE TABLE saved_event_records ( xact_id bigint, PRIMARY KEY (aggregate_id, sequence_number, timestamp) ); + +CREATE TABLE aggregates_that_need_snapshots ( + aggregate_id uuid NOT NULL PRIMARY KEY REFERENCES aggregates (aggregate_id) ON UPDATE CASCADE ON DELETE CASCADE, + snapshot_outdated_xact_id bigint, + snapshot_sequence_number_high_water_mark integer +); +CREATE INDEX aggregates_that_need_snapshots_outdated_idx + ON aggregates_that_need_snapshots (snapshot_outdated_xact_id ASC, snapshot_sequence_number_high_water_mark DESC, aggregate_id ASC) + WHERE snapshot_outdated_xact_id IS NOT NULL; +COMMENT ON TABLE aggregates_that_need_snapshots IS 'Contains a row for every aggregate with more events than its snapshot threshold.'; +COMMENT ON COLUMN aggregates_that_need_snapshots.snapshot_outdated_xact_id IS 'Not NULL indicates a snapshot is needed since the stored transaction id'; +COMMENT ON COLUMN aggregates_that_need_snapshots.snapshot_sequence_number_high_water_mark + IS 'The highest sequence number of the stored snapshot. Kept when snapshot are deleted to more easily query aggregates that need snapshotting the most'; diff --git a/lib/sequent/core/event_store.rb b/lib/sequent/core/event_store.rb index 625fef7c..3af03910 100644 --- a/lib/sequent/core/event_store.rb +++ b/lib/sequent/core/event_store.rb @@ -154,6 +154,15 @@ def load_latest_snapshot(aggregate_id) deserialize_event(snapshot_hash) unless snapshot_hash['aggregate_id'].nil? end + # Deletes all snapshots for all aggregates + def delete_all_snapshots + connection.exec_update( + 'CALL delete_all_snapshots()', + 'delete_all_snapshots', + [], + ) + end + # Deletes all snapshots for aggregate_id with a sequence_number lower than the specified sequence number. def delete_snapshots_before(aggregate_id, sequence_number) connection.exec_update( @@ -228,6 +237,14 @@ def aggregates_that_need_snapshots(last_aggregate_id, limit = 10) ).map { |x| x['aggregate_id'] } end + def aggregates_that_need_snapshots_ordered_by_priority(limit = 10) + connection.exec_query( + 'SELECT aggregate_id FROM aggregates_that_need_snapshots_ordered_by_priority($1)', + 'aggregates_that_need_snapshots', + [limit], + ).map { |x| x['aggregate_id'] } + end + def find_event_stream(aggregate_id) record = Sequent.configuration.stream_record_class.where(aggregate_id: aggregate_id).first record&.event_stream diff --git a/spec/lib/sequent/core/event_store_spec.rb b/spec/lib/sequent/core/event_store_spec.rb index bafe6360..d133634a 100644 --- a/spec/lib/sequent/core/event_store_spec.rb +++ b/spec/lib/sequent/core/event_store_spec.rb @@ -100,16 +100,41 @@ class MyAggregate < Sequent::Core::AggregateRoot end it 'can store and delete snapshots' do - aggregate = MyAggregate.new(aggregate_id) + stream, events = event_store.load_events(aggregate_id) + aggregate = Sequent::Core::AggregateRoot.load_from_history(stream, events) snapshot = aggregate.take_snapshot snapshot.created_at = Time.parse('2024-02-28T04:12:33Z') event_store.store_snapshots([snapshot]) + expect(event_store.aggregates_that_need_snapshots(nil)).to be_empty + expect(event_store.aggregates_that_need_snapshots_ordered_by_priority(nil)).to be_empty expect(event_store.load_latest_snapshot(aggregate_id)).to eq(snapshot) event_store.delete_snapshots_before(aggregate_id, snapshot.sequence_number + 1) + + expect(event_store.load_latest_snapshot(aggregate_id)).to eq(nil) + expect(event_store.aggregates_that_need_snapshots(nil)).to include(aggregate_id) + expect(event_store.aggregates_that_need_snapshots_ordered_by_priority(nil)).to include(aggregate_id) + end + + it 'can delete all snapshots' do + stream, events = event_store.load_events(aggregate_id) + aggregate = Sequent::Core::AggregateRoot.load_from_history(stream, events) + snapshot = aggregate.take_snapshot + snapshot.created_at = Time.parse('2024-02-28T04:12:33Z') + + event_store.store_snapshots([snapshot]) + + expect(event_store.aggregates_that_need_snapshots(nil)).to be_empty + expect(event_store.load_latest_snapshot(aggregate_id)).to eq(snapshot) + expect(event_store.aggregates_that_need_snapshots_ordered_by_priority(nil)).to be_empty + + event_store.delete_all_snapshots + expect(event_store.load_latest_snapshot(aggregate_id)).to eq(nil) + expect(event_store.aggregates_that_need_snapshots(nil)).to include(aggregate_id) + expect(event_store.aggregates_that_need_snapshots_ordered_by_priority(nil)).to include(aggregate_id) end end From fb81667c6d6c47a15a437d48013a8ff50ce69568 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Fri, 7 Jun 2024 16:55:11 +0200 Subject: [PATCH 078/128] Request new snapshot whenever new event exceeds snapshot threshold --- db/sequent_pgsql.sql | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/db/sequent_pgsql.sql b/db/sequent_pgsql.sql index d92ead3e..ccc2d13e 100644 --- a/db/sequent_pgsql.sql +++ b/db/sequent_pgsql.sql @@ -137,6 +137,7 @@ DECLARE _provided_events_partition_key aggregates.events_partition_key%TYPE; _events_partition_key aggregates.events_partition_key%TYPE; _snapshot_threshold aggregates.snapshot_threshold%TYPE; + _last_event_sequence_number events.sequence_number%TYPE; _snapshot_outdated boolean; BEGIN _command_id = store_command(_command); @@ -195,15 +196,19 @@ BEGIN (event->'event_json') - '{aggregate_id,created_at,event_type,sequence_number}'::text[] FROM jsonb_array_elements(_events) AS event; - -- Require a new snapshot when an event is stored with a sequence number that is a multiple of snapshot_threshold - _snapshot_outdated = EXISTS (SELECT * FROM jsonb_array_elements(_events) AS event - WHERE (event->'event_json'->'sequence_number')::integer % _snapshot_threshold = 0); - IF _snapshot_outdated THEN - INSERT INTO aggregates_that_need_snapshots AS target - VALUES (_aggregate_id, pg_current_xact_id()::text::bigint, NULL) - ON CONFLICT (aggregate_id) DO UPDATE - SET snapshot_outdated_xact_id = EXCLUDED.snapshot_outdated_xact_id - WHERE target.snapshot_outdated_xact_id IS NULL; + _last_event_sequence_number = (SELECT MAX((event->'event_json'->'sequence_number')::integer) FROM jsonb_array_elements(_events) AS event); + IF _last_event_sequence_number >= _snapshot_threshold THEN + _snapshot_outdated = _last_event_sequence_number - COALESCE( + (SELECT sequence_number FROM snapshot_records WHERE aggregate_id = _aggregate_id ORDER BY 1 DESC LIMIT 1), + 0 + ) >= _snapshot_threshold; + IF _snapshot_outdated THEN + INSERT INTO aggregates_that_need_snapshots AS target + VALUES (_aggregate_id, pg_current_xact_id()::text::bigint, NULL) + ON CONFLICT (aggregate_id) DO UPDATE + SET snapshot_outdated_xact_id = EXCLUDED.snapshot_outdated_xact_id + WHERE target.snapshot_outdated_xact_id IS NULL; + END IF; END IF; END LOOP; END; From ee68939ffb53f7568f616a840f2f0a08b7e3104e Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Mon, 24 Jun 2024 12:52:39 +0200 Subject: [PATCH 079/128] Add API to ignore aggregates that no longer need snapshotting Aggregates that have not been modified in a long time can be ignored for snapshotting, until new events get added. --- lib/sequent/core/event_store.rb | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/sequent/core/event_store.rb b/lib/sequent/core/event_store.rb index 3af03910..7ef87bf4 100644 --- a/lib/sequent/core/event_store.rb +++ b/lib/sequent/core/event_store.rb @@ -163,6 +163,15 @@ def delete_all_snapshots ) end + def ignore_aggregates_for_snapshotting_with_last_event_before(timestamp) + connection.exec_update(<<~EOS, 'ignore_aggregates_for_snapshotting_with_last_event_before', [timestamp]) + DELETE FROM aggregates_that_need_snapshotting s + WHERE NOT EXISTS (SELECT * + FROM aggregates a JOIN events e ON (a.aggregate_id, a.partition_key) = (e.aggregate_id, e.partition_key) + WHERE a.aggregate_id = s.aggregate_id AND e.created_at >= $1) + EOS + end + # Deletes all snapshots for aggregate_id with a sequence_number lower than the specified sequence number. def delete_snapshots_before(aggregate_id, sequence_number) connection.exec_update( From 0a64d236eba58d0ab0f054bf0ad5424f0e245f66 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Wed, 26 Jun 2024 14:23:45 +0200 Subject: [PATCH 080/128] Move snapshot threshold logic from SQL to Ruby --- db/sequent_pgsql.sql | 104 ++++++------------ db/sequent_schema.sql | 9 +- lib/sequent/core/aggregate_root.rb | 21 +++- lib/sequent/core/event_store.rb | 1 - lib/sequent/core/stream_record.rb | 15 ++- .../sequent/core/aggregate_repository_spec.rb | 41 +++---- spec/lib/sequent/core/aggregate_root_spec.rb | 1 + .../core/aggregate_snapshotter_spec.rb | 4 - spec/lib/sequent/core/event_store_spec.rb | 95 +++++++++------- 9 files changed, 136 insertions(+), 155 deletions(-) diff --git a/db/sequent_pgsql.sql b/db/sequent_pgsql.sql index ccc2d13e..4d521102 100644 --- a/db/sequent_pgsql.sql +++ b/db/sequent_pgsql.sql @@ -3,7 +3,6 @@ CREATE TYPE aggregate_event_type AS ( aggregate_type text, aggregate_id uuid, events_partition_key text, - snapshot_threshold integer, event_type text, event_json jsonb ); @@ -44,7 +43,6 @@ BEGIN RETURN QUERY SELECT aggregate_types.type, a.aggregate_id, a.events_partition_key, - a.snapshot_threshold, event_types.type, enrich_event_json(e) FROM aggregates a @@ -71,7 +69,7 @@ BEGIN -- PostgreSQL is read committed). RETURN QUERY WITH aggregate AS ( - SELECT aggregate_types.type, aggregate_id, events_partition_key, snapshot_threshold + SELECT aggregate_types.type, aggregate_id, events_partition_key FROM aggregates JOIN aggregate_types ON aggregate_type_id = aggregate_types.id WHERE aggregate_id = _aggregate_id @@ -136,9 +134,7 @@ DECLARE _aggregate_row aggregates%ROWTYPE; _provided_events_partition_key aggregates.events_partition_key%TYPE; _events_partition_key aggregates.events_partition_key%TYPE; - _snapshot_threshold aggregates.snapshot_threshold%TYPE; - _last_event_sequence_number events.sequence_number%TYPE; - _snapshot_outdated boolean; + _snapshot_outdated_at aggregates_that_need_snapshots.snapshot_outdated_at%TYPE; BEGIN _command_id = store_command(_command); @@ -167,24 +163,21 @@ BEGIN ORDER BY row->0->'aggregate_id', row->1->0->'event_json'->'sequence_number' LOOP _aggregate_id = _aggregate->>'aggregate_id'; - _snapshot_threshold = NULLIF(_aggregate->'snapshot_threshold', 'null'::jsonb); _provided_events_partition_key = _aggregate->>'events_partition_key'; + _snapshot_outdated_at = _aggregate->>'snapshot_outdated_at'; SELECT * INTO _aggregate_row FROM aggregates WHERE aggregate_id = _aggregate_id; _events_partition_key = COALESCE(_provided_events_partition_key, _aggregate_row.events_partition_key, ''); - INSERT INTO aggregates (aggregate_id, created_at, aggregate_type_id, events_partition_key, snapshot_threshold) + INSERT INTO aggregates (aggregate_id, created_at, aggregate_type_id, events_partition_key) VALUES ( _aggregate_id, (_events->0->>'created_at')::timestamptz, (SELECT id FROM aggregate_types WHERE type = _aggregate->>'aggregate_type'), - _events_partition_key, - _snapshot_threshold + _events_partition_key ) ON CONFLICT (aggregate_id) - DO UPDATE SET events_partition_key = EXCLUDED.events_partition_key, - snapshot_threshold = EXCLUDED.snapshot_threshold - WHERE aggregates.events_partition_key IS DISTINCT FROM EXCLUDED.events_partition_key - OR aggregates.snapshot_threshold IS DISTINCT FROM EXCLUDED.snapshot_threshold; + DO UPDATE SET events_partition_key = EXCLUDED.events_partition_key + WHERE aggregates.events_partition_key IS DISTINCT FROM EXCLUDED.events_partition_key; INSERT INTO events (partition_key, aggregate_id, sequence_number, created_at, command_id, event_type_id, event_json) SELECT _events_partition_key, @@ -196,19 +189,12 @@ BEGIN (event->'event_json') - '{aggregate_id,created_at,event_type,sequence_number}'::text[] FROM jsonb_array_elements(_events) AS event; - _last_event_sequence_number = (SELECT MAX((event->'event_json'->'sequence_number')::integer) FROM jsonb_array_elements(_events) AS event); - IF _last_event_sequence_number >= _snapshot_threshold THEN - _snapshot_outdated = _last_event_sequence_number - COALESCE( - (SELECT sequence_number FROM snapshot_records WHERE aggregate_id = _aggregate_id ORDER BY 1 DESC LIMIT 1), - 0 - ) >= _snapshot_threshold; - IF _snapshot_outdated THEN - INSERT INTO aggregates_that_need_snapshots AS target - VALUES (_aggregate_id, pg_current_xact_id()::text::bigint, NULL) - ON CONFLICT (aggregate_id) DO UPDATE - SET snapshot_outdated_xact_id = EXCLUDED.snapshot_outdated_xact_id - WHERE target.snapshot_outdated_xact_id IS NULL; - END IF; + IF _snapshot_outdated_at IS NOT NULL THEN + INSERT INTO aggregates_that_need_snapshots AS row (aggregate_id, snapshot_outdated_at) + VALUES (_aggregate_id, _snapshot_outdated_at) + ON CONFLICT (aggregate_id) DO UPDATE + SET snapshot_outdated_at = LEAST(row.snapshot_outdated_at, EXCLUDED.snapshot_outdated_at) + WHERE row.snapshot_outdated_at IS DISTINCT FROM EXCLUDED.snapshot_outdated_at; END IF; END LOOP; END; @@ -219,20 +205,28 @@ LANGUAGE plpgsql AS $$ DECLARE _aggregate_id uuid; _snapshot jsonb; + _sequence_number snapshot_records.sequence_number%TYPE; BEGIN FOR _snapshot IN SELECT * FROM jsonb_array_elements(_snapshots) LOOP _aggregate_id = _snapshot->>'aggregate_id'; + _sequence_number = _snapshot->'sequence_number'; INSERT INTO snapshot_records (aggregate_id, sequence_number, created_at, snapshot_type, snapshot_json) VALUES ( _aggregate_id, - (_snapshot->'sequence_number')::integer, + _sequence_number, (_snapshot->>'created_at')::timestamptz, _snapshot->>'snapshot_type', _snapshot->'snapshot_json' ); - CALL update_snapshot_status(_aggregate_id); + INSERT INTO aggregates_that_need_snapshots AS row (aggregate_id, snapshot_outdated_at, snapshot_sequence_number_high_water_mark) + VALUES (_aggregate_id, NULL, _sequence_number) + ON CONFLICT (aggregate_id) DO UPDATE + SET snapshot_outdated_at = NULL, + snapshot_sequence_number_high_water_mark = EXCLUDED.snapshot_sequence_number_high_water_mark + WHERE row.snapshot_sequence_number_high_water_mark IS NULL + OR row.snapshot_sequence_number_high_water_mark < EXCLUDED.snapshot_sequence_number_high_water_mark; END LOOP; END; $$; @@ -242,7 +236,6 @@ LANGUAGE SQL AS $$ SELECT (SELECT type FROM aggregate_types WHERE id = a.aggregate_type_id), a.aggregate_id, a.events_partition_key, - a.snapshot_threshold, s.snapshot_type, s.snapshot_json FROM aggregates a JOIN snapshot_records s ON a.aggregate_id = s.aggregate_id @@ -255,8 +248,8 @@ CREATE OR REPLACE PROCEDURE delete_all_snapshots() LANGUAGE plpgsql AS $$ BEGIN UPDATE aggregates_that_need_snapshots - SET snapshot_outdated_xact_id = pg_current_xact_id()::text::bigint - WHERE snapshot_outdated_xact_id IS NULL; + SET snapshot_outdated_at = NOW() + WHERE snapshot_outdated_at IS NULL; DELETE FROM snapshot_records; END; $$; @@ -268,41 +261,11 @@ BEGIN WHERE aggregate_id = _aggregate_id AND sequence_number < _sequence_number; - CALL update_snapshot_status(_aggregate_id); -END; -$$; - -CREATE OR REPLACE PROCEDURE update_snapshot_status(_aggregate_id uuid) -LANGUAGE plpgsql AS $$ -DECLARE - _snapshot_threshold aggregates.snapshot_threshold%TYPE; - _last_event_sequence_number events.sequence_number%TYPE; - _last_snapshot_sequence_number events.sequence_number%TYPE; - _snapshot_outdated_xact_id bigint = NULL; -BEGIN - SELECT a.snapshot_threshold, e.sequence_number INTO STRICT _snapshot_threshold, _last_event_sequence_number - FROM aggregates a JOIN events e ON a.events_partition_key = e.partition_key AND a.aggregate_id = e.aggregate_id - WHERE a.aggregate_id = _aggregate_id - ORDER BY 2 DESC LIMIT 1; - - SELECT sequence_number INTO _last_snapshot_sequence_number - FROM snapshot_records + UPDATE aggregates_that_need_snapshots + SET snapshot_outdated_at = NOW() WHERE aggregate_id = _aggregate_id - ORDER BY 1 DESC LIMIT 1; - - IF _last_event_sequence_number - COALESCE(_last_snapshot_sequence_number, 0) >= _snapshot_threshold THEN - _snapshot_outdated_xact_id = pg_current_xact_id()::text::bigint; - END IF; - - INSERT INTO aggregates_that_need_snapshots AS target - VALUES (_aggregate_id, _snapshot_outdated_xact_id, _last_snapshot_sequence_number) - ON CONFLICT (aggregate_id) DO UPDATE - SET snapshot_outdated_xact_id = (CASE - WHEN EXCLUDED.snapshot_outdated_xact_id IS NULL THEN NULL - ELSE LEAST(target.snapshot_outdated_xact_id, EXCLUDED.snapshot_outdated_xact_id) - END), - snapshot_sequence_number_high_water_mark = - GREATEST(target.snapshot_sequence_number_high_water_mark, EXCLUDED.snapshot_sequence_number_high_water_mark); + AND snapshot_outdated_at IS NULL + AND NOT EXISTS (SELECT 1 FROM snapshot_records WHERE aggregate_id = _aggregate_id); END; $$; @@ -312,7 +275,7 @@ LANGUAGE plpgsql AS $$ BEGIN RETURN QUERY SELECT a.aggregate_id FROM aggregates_that_need_snapshots a - WHERE a.snapshot_outdated_xact_id IS NOT NULL + WHERE a.snapshot_outdated_at IS NOT NULL AND (_last_aggregate_id IS NULL OR a.aggregate_id > _last_aggregate_id) ORDER BY 1 LIMIT _limit; @@ -325,8 +288,8 @@ LANGUAGE plpgsql AS $$ BEGIN RETURN QUERY SELECT a.aggregate_id FROM aggregates_that_need_snapshots a - WHERE snapshot_outdated_xact_id IS NOT NULL - ORDER BY snapshot_outdated_xact_id ASC, snapshot_sequence_number_high_water_mark DESC, aggregate_id ASC + WHERE snapshot_outdated_at IS NOT NULL + ORDER BY snapshot_outdated_at ASC, snapshot_sequence_number_high_water_mark DESC, aggregate_id ASC LIMIT _limit; END; $$; @@ -382,11 +345,10 @@ CREATE OR REPLACE VIEW event_records (aggregate_id, partition_key, sequence_numb JOIN events event ON aggregate.aggregate_id = event.aggregate_id AND aggregate.events_partition_key = event.partition_key JOIN event_types type ON event.event_type_id = type.id; -CREATE OR REPLACE VIEW stream_records (aggregate_id, events_partition_key, aggregate_type, snapshot_threshold, created_at) AS +CREATE OR REPLACE VIEW stream_records (aggregate_id, events_partition_key, aggregate_type, created_at) AS SELECT aggregates.aggregate_id, aggregates.events_partition_key, aggregate_types.type, - aggregates.snapshot_threshold, aggregates.created_at FROM aggregates JOIN aggregate_types ON aggregates.aggregate_type_id = aggregate_types.id; diff --git a/db/sequent_schema.sql b/db/sequent_schema.sql index 36b79598..140c7edc 100644 --- a/db/sequent_schema.sql +++ b/db/sequent_schema.sql @@ -22,7 +22,6 @@ CREATE TABLE aggregates ( aggregate_id uuid PRIMARY KEY, events_partition_key text NOT NULL DEFAULT '', aggregate_type_id SMALLINT NOT NULL REFERENCES aggregate_types (id), - snapshot_threshold integer, created_at timestamp with time zone NOT NULL DEFAULT NOW(), UNIQUE (events_partition_key, aggregate_id) ) PARTITION BY RANGE (aggregate_id); @@ -95,13 +94,13 @@ CREATE TABLE saved_event_records ( CREATE TABLE aggregates_that_need_snapshots ( aggregate_id uuid NOT NULL PRIMARY KEY REFERENCES aggregates (aggregate_id) ON UPDATE CASCADE ON DELETE CASCADE, - snapshot_outdated_xact_id bigint, + snapshot_outdated_at timestamp with time zone, snapshot_sequence_number_high_water_mark integer ); CREATE INDEX aggregates_that_need_snapshots_outdated_idx - ON aggregates_that_need_snapshots (snapshot_outdated_xact_id ASC, snapshot_sequence_number_high_water_mark DESC, aggregate_id ASC) - WHERE snapshot_outdated_xact_id IS NOT NULL; + ON aggregates_that_need_snapshots (snapshot_outdated_at ASC, snapshot_sequence_number_high_water_mark DESC, aggregate_id ASC) + WHERE snapshot_outdated_at IS NOT NULL; COMMENT ON TABLE aggregates_that_need_snapshots IS 'Contains a row for every aggregate with more events than its snapshot threshold.'; -COMMENT ON COLUMN aggregates_that_need_snapshots.snapshot_outdated_xact_id IS 'Not NULL indicates a snapshot is needed since the stored transaction id'; +COMMENT ON COLUMN aggregates_that_need_snapshots.snapshot_outdated_at IS 'Not NULL indicates a snapshot is needed since the stored timestamp'; COMMENT ON COLUMN aggregates_that_need_snapshots.snapshot_sequence_number_high_water_mark IS 'The highest sequence number of the stored snapshot. Kept when snapshot are deleted to more easily query aggregates that need snapshotting the most'; diff --git a/lib/sequent/core/aggregate_root.rb b/lib/sequent/core/aggregate_root.rb index f0d0e8e6..d3efc068 100644 --- a/lib/sequent/core/aggregate_root.rb +++ b/lib/sequent/core/aggregate_root.rb @@ -12,8 +12,7 @@ module SnapshotConfiguration module ClassMethods ## # Enable snapshots for this aggregate. The aggregate instance - # must define the *load_from_snapshot* and *save_to_snapshot* - # methods. + # must define the *take_snapshot* methods. # def enable_snapshots(default_threshold: 20) @snapshot_default_threshold = default_threshold @@ -42,6 +41,7 @@ class AggregateRoot extend ActiveSupport::DescendantsTracker attr_reader :id, :uncommitted_events, :sequence_number + attr_accessor :latest_snapshot_sequence_number def self.load_from_history(stream, events) first, *rest = events @@ -49,6 +49,7 @@ def self.load_from_history(stream, events) # rubocop:disable Security/MarshalLoad aggregate_root = Marshal.load(Base64.decode64(first.data)) # rubocop:enable Security/MarshalLoad + aggregate_root.latest_snapshot_sequence_number = first.sequence_number rest.each { |x| aggregate_root.apply_event(x) } else aggregate_root = allocate # allocate without calling new @@ -98,10 +99,12 @@ def to_s end def event_stream - EventStream.new aggregate_type: self.class.name, - aggregate_id: id, - events_partition_key: events_partition_key, - snapshot_threshold: self.class.snapshot_default_threshold + EventStream.new( + aggregate_type: self.class.name, + aggregate_id: id, + events_partition_key: events_partition_key, + snapshot_outdated_at: snapshot_outdated? ? Time.now : nil, + ) end # Provide the partitioning key for storing events. This value @@ -119,6 +122,12 @@ def clear_events @uncommitted_events = [] end + def snapshot_outdated? + snapshot_threshold = self.class.snapshot_default_threshold + events_since_latest_snapshot = @sequence_number - (latest_snapshot_sequence_number || 1) + snapshot_threshold.present? && events_since_latest_snapshot >= snapshot_threshold + end + def take_snapshot build_event SnapshotEvent, data: Base64.encode64(Marshal.dump(self)) end diff --git a/lib/sequent/core/event_store.rb b/lib/sequent/core/event_store.rb index 7ef87bf4..7065e561 100644 --- a/lib/sequent/core/event_store.rb +++ b/lib/sequent/core/event_store.rb @@ -119,7 +119,6 @@ def load_events_for_aggregates(aggregate_ids) aggregate_type: rows.first['aggregate_type'], aggregate_id: rows.first['aggregate_id'], events_partition_key: rows.first['events_partition_key'], - snapshot_threshold: rows.first['snapshot_threshold'], ), rows.map { |row| deserialize_event(row) }, ] diff --git a/lib/sequent/core/stream_record.rb b/lib/sequent/core/stream_record.rb index a3127e9f..4809ddf4 100644 --- a/lib/sequent/core/stream_record.rb +++ b/lib/sequent/core/stream_record.rb @@ -4,9 +4,9 @@ module Sequent module Core - EventStream = Data.define(:aggregate_type, :aggregate_id, :events_partition_key, :snapshot_threshold) do - def initialize(aggregate_type:, aggregate_id:, events_partition_key: '', snapshot_threshold: nil) - super(aggregate_type:, aggregate_id:, events_partition_key:, snapshot_threshold:) + EventStream = Data.define(:aggregate_type, :aggregate_id, :events_partition_key, :snapshot_outdated_at) do + def initialize(aggregate_type:, aggregate_id:, events_partition_key: '', snapshot_outdated_at: nil) + super(aggregate_type:, aggregate_id:, events_partition_key:, snapshot_outdated_at:) end end @@ -15,22 +15,21 @@ class StreamRecord < Sequent::ApplicationRecord self.table_name = 'stream_records' validates_presence_of :aggregate_type, :aggregate_id - validates_numericality_of :snapshot_threshold, only_integer: true, greater_than: 0, allow_nil: true has_many :event_records, foreign_key: :aggregate_id, primary_key: :aggregate_id def event_stream EventStream.new( - aggregate_type: aggregate_type, - aggregate_id: aggregate_id, - snapshot_threshold: snapshot_threshold, + aggregate_type:, + aggregate_id:, + events_partition_key:, ) end def event_stream=(data) self.aggregate_type = data.aggregate_type self.aggregate_id = data.aggregate_id - self.snapshot_threshold = data.snapshot_threshold + self.events_partition_key = data.events_partition_key end end end diff --git a/spec/lib/sequent/core/aggregate_repository_spec.rb b/spec/lib/sequent/core/aggregate_repository_spec.rb index 6b7e13e4..08df2df8 100644 --- a/spec/lib/sequent/core/aggregate_repository_spec.rb +++ b/spec/lib/sequent/core/aggregate_repository_spec.rb @@ -6,27 +6,25 @@ describe Sequent::Core::AggregateRepository do context 'Aggregate repository unit tests' do - class DummyAggregate2 < Sequent::Core::AggregateRoot + class DummyAggregate < Sequent::Core::AggregateRoot attr_reader :loaded_events attr_writer :uncommitted_events def load_from_history(stream, events) - @id = stream&.aggregate_id + super @event_stream = stream @loaded_events = events - @uncommitted_events = [] end end - class DummyAggregate < Sequent::Core::AggregateRoot + class DummyAggregate2 < Sequent::Core::AggregateRoot attr_reader :loaded_events attr_writer :uncommitted_events def load_from_history(stream, events) - @id = stream&.aggregate_id + super @event_stream = stream @loaded_events = events - @uncommitted_events = [] end end @@ -41,6 +39,9 @@ def load_from_history(stream, events) let(:event_store) { double } let(:repository) { Sequent.configuration.aggregate_repository } let(:aggregate) { DummyAggregate.new(Sequent.new_uuid) } + let(:events) do + [Sequent::Core::Event.new(aggregate_id: aggregate.id, sequence_number: 1)] + end it 'should track added aggregates by id' do allow(event_store).to receive(:load_events_for_aggregates).with([]).and_return([]).once @@ -54,7 +55,7 @@ def load_from_history(stream, events) [ [ aggregate.event_stream, - [:events], + events, ], ], ) @@ -62,7 +63,7 @@ def load_from_history(stream, events) loaded = repository.load_aggregate(:id, DummyAggregate) expect(loaded.event_stream).to eq(aggregate.event_stream) - expect(loaded.loaded_events).to eq([:events]) + expect(loaded.loaded_events).to eq(events) end it 'should not require expected aggregate class' do @@ -70,7 +71,7 @@ def load_from_history(stream, events) [ [ aggregate.event_stream, - [:events], + events, ], ], ) @@ -83,7 +84,7 @@ def load_from_history(stream, events) [ [ aggregate.event_stream, - [:events], + events, ], ], ) @@ -96,7 +97,7 @@ def load_from_history(stream, events) [ [ aggregate.event_stream, - [:events], + events, ], ], ) @@ -134,7 +135,7 @@ def load_from_history(stream, events) allow(event_store).to receive(:load_events_for_aggregates).with([aggregate.id]).and_return( [ [ - aggregate.event_stream, [:events] + aggregate.event_stream, events ], ], ).once @@ -233,13 +234,15 @@ class MyEvent < Sequent::Core::Event end context 'with aggregates in the event store' do - let(:aggregate_stream_with_events) { [aggregate.event_stream, [:events]] } + let(:aggregate_stream_with_events) { [aggregate.event_stream, events] } let(:aggregate_2) { DummyAggregate.new(Sequent.new_uuid) } - let(:aggregate_2_stream_with_events) { [aggregate_2.event_stream, [:events_2]] } + let(:events_2) { [Sequent::Core::Event.new(aggregate_id: aggregate_2.id, sequence_number: 1)] } + let(:aggregate_2_stream_with_events) { [aggregate_2.event_stream, events_2] } let(:aggregate_3) { DummyAggregate2.new(Sequent.new_uuid) } - let(:aggregate_3_stream_with_events) { [aggregate_3.event_stream, [:events_3]] } + let(:events_3) { [Sequent::Core::Event.new(aggregate_id: aggregate_3.id, sequence_number: 1)] } + let(:aggregate_3_stream_with_events) { [aggregate_3.event_stream, events_3] } it 'returns all the aggregates found' do allow(event_store) @@ -254,10 +257,10 @@ class MyEvent < Sequent::Core::Event expect(aggregates).to have(2).items expect(aggregates[0].event_stream).to eq aggregate.event_stream - expect(aggregates[0].loaded_events).to eq([:events]) + expect(aggregates[0].loaded_events).to eq(events) expect(aggregates[1].event_stream).to eq aggregate_2.event_stream - expect(aggregates[1].loaded_events).to eq([:events_2]) + expect(aggregates[1].loaded_events).to eq(events_2) end it 'raises error even if only one aggregate cannot be found' do @@ -330,11 +333,11 @@ class MyEvent < Sequent::Core::Event expect(aggregates).to have(2).items expect(aggregates[0].event_stream).to eq aggregate.event_stream - expect(aggregates[0].loaded_events).to eq([:events]) + expect(aggregates[0].loaded_events).to eq(events) expect(aggregates[1].class).to eq DummyAggregate2 expect(aggregates[1].event_stream).to eq aggregate_3.event_stream - expect(aggregates[1].loaded_events).to eq([:events_3]) + expect(aggregates[1].loaded_events).to eq(events_3) end context 'loaded in the identity map' do diff --git a/spec/lib/sequent/core/aggregate_root_spec.rb b/spec/lib/sequent/core/aggregate_root_spec.rb index 7db4fccc..f30bd745 100644 --- a/spec/lib/sequent/core/aggregate_root_spec.rb +++ b/spec/lib/sequent/core/aggregate_root_spec.rb @@ -95,6 +95,7 @@ def event_count snapshot_event = subject.take_snapshot restored = TestAggregateRoot.load_from_history :stream, [snapshot_event] expect(restored.event_count).to eq 1 + expect(restored.latest_snapshot_sequence_number).to eq(2) end end diff --git a/spec/lib/sequent/core/aggregate_snapshotter_spec.rb b/spec/lib/sequent/core/aggregate_snapshotter_spec.rb index f1ffaf70..aa4d1812 100644 --- a/spec/lib/sequent/core/aggregate_snapshotter_spec.rb +++ b/spec/lib/sequent/core/aggregate_snapshotter_spec.rb @@ -20,7 +20,6 @@ class MyAggregate < Sequent::Core::AggregateRoot; end Sequent.configuration.command_handlers = commands_handlers end end - let(:snapshot_threshold) { 1 } let(:events) { [MyEvent.new(aggregate_id: aggregate_id, sequence_number: 1)] } before :each do @@ -32,7 +31,6 @@ class MyAggregate < Sequent::Core::AggregateRoot; end Sequent::Core::EventStream.new( aggregate_type: 'MyAggregate', aggregate_id: aggregate_id, - snapshot_threshold: 1, ), events, ], @@ -47,7 +45,6 @@ class MyAggregate < Sequent::Core::AggregateRoot; end end context 'loads aggregates with snapshots' do - let(:snapshot_threshold) { 2 } let(:events) do [ MyEvent.new(aggregate_id: aggregate_id, sequence_number: 1), @@ -66,7 +63,6 @@ class MyAggregate < Sequent::Core::AggregateRoot; end Sequent::Core::EventStream.new( aggregate_type: 'MyAggregate', aggregate_id: aggregate_id_2, - snapshot_threshold: 10, ), [MyEvent.new(aggregate_id: aggregate_id_2, sequence_number: 1)], ], diff --git a/spec/lib/sequent/core/event_store_spec.rb b/spec/lib/sequent/core/event_store_spec.rb index d133634a..b3950175 100644 --- a/spec/lib/sequent/core/event_store_spec.rb +++ b/spec/lib/sequent/core/event_store_spec.rb @@ -44,7 +44,6 @@ class MyAggregate < Sequent::Core::AggregateRoot end context 'snapshotting' do - let(:snapshot_threshold) { 1 } before do event_store.commit_events( Sequent::Core::Command.new(aggregate_id: aggregate_id), @@ -53,7 +52,7 @@ class MyAggregate < Sequent::Core::AggregateRoot Sequent::Core::EventStream.new( aggregate_type: 'MyAggregate', aggregate_id: aggregate_id, - snapshot_threshold: snapshot_threshold, + snapshot_outdated_at: Time.now, ), [ MyEvent.new( @@ -68,33 +67,6 @@ class MyAggregate < Sequent::Core::AggregateRoot ) end - it 'stores the event as JSON object' do - # Test to ensure stored data is not accidentally doubly-encoded, - # so query database directly instead of using `load_event`. - row = ActiveRecord::Base.connection.exec_query( - "SELECT event_json, event_json::jsonb->>'data' AS data FROM event_records \ - WHERE aggregate_id = $1 and sequence_number = $2", - 'query_event', - [aggregate_id, 1], - ).first - - expect(row['data']).to eq("with ' unsafe SQL characters;\n") - json = Sequent::Core::Oj.strict_load(row['event_json']) - expect(json['aggregate_id']).to eq(aggregate_id) - expect(json['sequence_number']).to eq(1) - end - - it 'can store events' do - stream, events = event_store.load_events aggregate_id - - expect(stream.snapshot_threshold).to eq(1) - expect(stream.aggregate_type).to eq('MyAggregate') - expect(stream.aggregate_id).to eq(aggregate_id) - expect(events.first.aggregate_id).to eq(aggregate_id) - expect(events.first.sequence_number).to eq(1) - expect(events.first.data).to eq("with ' unsafe SQL characters;\n") - end - it 'can find streams that need snapshotting' do expect(event_store.aggregates_that_need_snapshots(nil)).to include(aggregate_id) end @@ -139,20 +111,68 @@ class MyAggregate < Sequent::Core::AggregateRoot end describe '#commit_events' do + before do + event_store.commit_events( + Sequent::Core::Command.new(aggregate_id: aggregate_id), + [ + [ + Sequent::Core::EventStream.new( + aggregate_type: 'MyAggregate', + aggregate_id: aggregate_id, + snapshot_outdated_at: Time.now, + ), + [ + MyEvent.new( + aggregate_id: aggregate_id, + sequence_number: 1, + created_at: Time.parse('2024-02-29T01:10:12Z'), + data: "with ' unsafe SQL characters;\n", + ), + ], + ], + ], + ) + end + + it 'can store events' do + stream, events = event_store.load_events aggregate_id + + expect(stream.aggregate_type).to eq('MyAggregate') + expect(stream.aggregate_id).to eq(aggregate_id) + expect(events.first.aggregate_id).to eq(aggregate_id) + expect(events.first.sequence_number).to eq(1) + expect(events.first.data).to eq("with ' unsafe SQL characters;\n") + end + + it 'stores the event as JSON object' do + # Test to ensure stored data is not accidentally doubly-encoded, + # so query database directly instead of using `load_event`. + row = ActiveRecord::Base.connection.exec_query( + "SELECT event_json, event_json->>'data' AS data FROM event_records \ + WHERE aggregate_id = $1 and sequence_number = $2", + 'query_event', + [aggregate_id, 1], + ).first + + expect(row['data']).to eq("with ' unsafe SQL characters;\n") + json = Sequent::Core::Oj.strict_load(row['event_json']) + expect(json['aggregate_id']).to eq(aggregate_id) + expect(json['sequence_number']).to eq(1) + end + it 'fails with OptimisticLockingError when RecordNotUnique' do expect do event_store.commit_events( - Sequent::Core::Command.new(aggregate_id: aggregate_id), + Sequent::Core::Command.new(aggregate_id:), [ [ Sequent::Core::EventStream.new( aggregate_type: 'MyAggregate', - aggregate_id: aggregate_id, - snapshot_threshold: 13, + aggregate_id:, ), [ - MyEvent.new(aggregate_id: aggregate_id, sequence_number: 1), - MyEvent.new(aggregate_id: aggregate_id, sequence_number: 1), + MyEvent.new(aggregate_id:, sequence_number: 2), + MyEvent.new(aggregate_id:, sequence_number: 2), ], ], ], @@ -172,7 +192,6 @@ class MyAggregate < Sequent::Core::AggregateRoot Sequent::Core::EventStream.new( aggregate_type: 'MyAggregate', aggregate_id:, - snapshot_threshold: 13, ), [MyEvent.new(aggregate_id:, sequence_number: 1)], ], @@ -242,7 +261,6 @@ class MyAggregate < Sequent::Core::AggregateRoot Sequent::Core::EventStream.new( aggregate_type: 'MyAggregate', aggregate_id: aggregate_id, - snapshot_threshold: 13, ), [MyEvent.new(aggregate_id: aggregate_id, sequence_number: 1)], ], @@ -549,7 +567,6 @@ class FailingHandler < Sequent::Core::Projector Sequent::Core::EventStream.new( aggregate_type: 'MyAggregate', aggregate_id: aggregate_id, - snapshot_threshold: 13, ), [my_event], ], @@ -569,7 +586,6 @@ class FailingHandler < Sequent::Core::Projector Sequent::Core::EventStream.new( aggregate_type: 'MyAggregate', aggregate_id: aggregate_id, - snapshot_threshold: 13, ), [my_event], ], @@ -591,7 +607,6 @@ class FailingHandler < Sequent::Core::Projector Sequent::Core::EventStream.new( aggregate_type: 'MyAggregate', aggregate_id: aggregate_id, - snapshot_threshold: 13, ), [my_event], ], @@ -686,7 +701,6 @@ class FailingHandler < Sequent::Core::Projector Sequent::Core::EventStream.new( aggregate_type: 'MyAggregate', aggregate_id: aggregate_id, - snapshot_threshold: 13, ), [MyEvent.new(aggregate_id: aggregate_id, sequence_number: 1)], ], @@ -729,7 +743,6 @@ def initialize Sequent::Core::EventStream.new( aggregate_type: 'MyAggregate', aggregate_id:, - snapshot_threshold: 13, ), [MyEvent.new(aggregate_id:, sequence_number: 1)], ], From 7f196c28fee27d7f877c1e2d20c2747db0016c52 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Wed, 26 Jun 2024 15:47:55 +0200 Subject: [PATCH 081/128] Update project template SQL schema --- .../template_project/db/sequent_pgsql.sql | 89 +++++++++++++------ .../template_project/db/sequent_schema.sql | 14 ++- 2 files changed, 75 insertions(+), 28 deletions(-) diff --git a/lib/sequent/generator/template_project/db/sequent_pgsql.sql b/lib/sequent/generator/template_project/db/sequent_pgsql.sql index c476ecf1..4d521102 100644 --- a/lib/sequent/generator/template_project/db/sequent_pgsql.sql +++ b/lib/sequent/generator/template_project/db/sequent_pgsql.sql @@ -3,7 +3,6 @@ CREATE TYPE aggregate_event_type AS ( aggregate_type text, aggregate_id uuid, events_partition_key text, - snapshot_threshold integer, event_type text, event_json jsonb ); @@ -44,7 +43,6 @@ BEGIN RETURN QUERY SELECT aggregate_types.type, a.aggregate_id, a.events_partition_key, - a.snapshot_threshold, event_types.type, enrich_event_json(e) FROM aggregates a @@ -71,7 +69,7 @@ BEGIN -- PostgreSQL is read committed). RETURN QUERY WITH aggregate AS ( - SELECT aggregate_types.type, aggregate_id, events_partition_key, snapshot_threshold + SELECT aggregate_types.type, aggregate_id, events_partition_key FROM aggregates JOIN aggregate_types ON aggregate_type_id = aggregate_types.id WHERE aggregate_id = _aggregate_id @@ -133,10 +131,10 @@ DECLARE _aggregate jsonb; _events jsonb; _aggregate_id aggregates.aggregate_id%TYPE; + _aggregate_row aggregates%ROWTYPE; _provided_events_partition_key aggregates.events_partition_key%TYPE; - _existing_events_partition_key aggregates.events_partition_key%TYPE; _events_partition_key aggregates.events_partition_key%TYPE; - _snapshot_threshold aggregates.snapshot_threshold%TYPE; + _snapshot_outdated_at aggregates_that_need_snapshots.snapshot_outdated_at%TYPE; BEGIN _command_id = store_command(_command); @@ -165,24 +163,21 @@ BEGIN ORDER BY row->0->'aggregate_id', row->1->0->'event_json'->'sequence_number' LOOP _aggregate_id = _aggregate->>'aggregate_id'; - _snapshot_threshold = NULLIF(_aggregate->'snapshot_threshold', 'null'::jsonb); _provided_events_partition_key = _aggregate->>'events_partition_key'; + _snapshot_outdated_at = _aggregate->>'snapshot_outdated_at'; - SELECT events_partition_key INTO _existing_events_partition_key FROM aggregates WHERE aggregate_id = _aggregate_id FOR UPDATE; - _events_partition_key = COALESCE(_provided_events_partition_key, _existing_events_partition_key, ''); + SELECT * INTO _aggregate_row FROM aggregates WHERE aggregate_id = _aggregate_id; + _events_partition_key = COALESCE(_provided_events_partition_key, _aggregate_row.events_partition_key, ''); - INSERT INTO aggregates (aggregate_id, created_at, aggregate_type_id, events_partition_key, snapshot_threshold) + INSERT INTO aggregates (aggregate_id, created_at, aggregate_type_id, events_partition_key) VALUES ( _aggregate_id, (_events->0->>'created_at')::timestamptz, (SELECT id FROM aggregate_types WHERE type = _aggregate->>'aggregate_type'), - _events_partition_key, - _snapshot_threshold + _events_partition_key ) ON CONFLICT (aggregate_id) - DO UPDATE SET events_partition_key = EXCLUDED.events_partition_key, - snapshot_threshold = EXCLUDED.snapshot_threshold - WHERE aggregates.events_partition_key IS DISTINCT FROM EXCLUDED.events_partition_key - OR aggregates.snapshot_threshold IS DISTINCT FROM EXCLUDED.snapshot_threshold; + DO UPDATE SET events_partition_key = EXCLUDED.events_partition_key + WHERE aggregates.events_partition_key IS DISTINCT FROM EXCLUDED.events_partition_key; INSERT INTO events (partition_key, aggregate_id, sequence_number, created_at, command_id, event_type_id, event_json) SELECT _events_partition_key, @@ -193,6 +188,14 @@ BEGIN (SELECT id FROM event_types WHERE type = event->>'event_type'), (event->'event_json') - '{aggregate_id,created_at,event_type,sequence_number}'::text[] FROM jsonb_array_elements(_events) AS event; + + IF _snapshot_outdated_at IS NOT NULL THEN + INSERT INTO aggregates_that_need_snapshots AS row (aggregate_id, snapshot_outdated_at) + VALUES (_aggregate_id, _snapshot_outdated_at) + ON CONFLICT (aggregate_id) DO UPDATE + SET snapshot_outdated_at = LEAST(row.snapshot_outdated_at, EXCLUDED.snapshot_outdated_at) + WHERE row.snapshot_outdated_at IS DISTINCT FROM EXCLUDED.snapshot_outdated_at; + END IF; END LOOP; END; $$; @@ -201,20 +204,29 @@ CREATE OR REPLACE PROCEDURE store_snapshots(_snapshots jsonb) LANGUAGE plpgsql AS $$ DECLARE _aggregate_id uuid; - _events_partition_key text; _snapshot jsonb; + _sequence_number snapshot_records.sequence_number%TYPE; BEGIN FOR _snapshot IN SELECT * FROM jsonb_array_elements(_snapshots) LOOP _aggregate_id = _snapshot->>'aggregate_id'; + _sequence_number = _snapshot->'sequence_number'; INSERT INTO snapshot_records (aggregate_id, sequence_number, created_at, snapshot_type, snapshot_json) VALUES ( _aggregate_id, - (_snapshot->'sequence_number')::integer, + _sequence_number, (_snapshot->>'created_at')::timestamptz, _snapshot->>'snapshot_type', _snapshot->'snapshot_json' ); + + INSERT INTO aggregates_that_need_snapshots AS row (aggregate_id, snapshot_outdated_at, snapshot_sequence_number_high_water_mark) + VALUES (_aggregate_id, NULL, _sequence_number) + ON CONFLICT (aggregate_id) DO UPDATE + SET snapshot_outdated_at = NULL, + snapshot_sequence_number_high_water_mark = EXCLUDED.snapshot_sequence_number_high_water_mark + WHERE row.snapshot_sequence_number_high_water_mark IS NULL + OR row.snapshot_sequence_number_high_water_mark < EXCLUDED.snapshot_sequence_number_high_water_mark; END LOOP; END; $$; @@ -224,7 +236,6 @@ LANGUAGE SQL AS $$ SELECT (SELECT type FROM aggregate_types WHERE id = a.aggregate_type_id), a.aggregate_id, a.events_partition_key, - a.snapshot_threshold, s.snapshot_type, s.snapshot_json FROM aggregates a JOIN snapshot_records s ON a.aggregate_id = s.aggregate_id @@ -233,12 +244,28 @@ LANGUAGE SQL AS $$ LIMIT 1; $$; +CREATE OR REPLACE PROCEDURE delete_all_snapshots() +LANGUAGE plpgsql AS $$ +BEGIN + UPDATE aggregates_that_need_snapshots + SET snapshot_outdated_at = NOW() + WHERE snapshot_outdated_at IS NULL; + DELETE FROM snapshot_records; +END; +$$; + CREATE OR REPLACE PROCEDURE delete_snapshots_before(_aggregate_id uuid, _sequence_number integer) LANGUAGE plpgsql AS $$ BEGIN DELETE FROM snapshot_records WHERE aggregate_id = _aggregate_id AND sequence_number < _sequence_number; + + UPDATE aggregates_that_need_snapshots + SET snapshot_outdated_at = NOW() + WHERE aggregate_id = _aggregate_id + AND snapshot_outdated_at IS NULL + AND NOT EXISTS (SELECT 1 FROM snapshot_records WHERE aggregate_id = _aggregate_id); END; $$; @@ -246,18 +273,27 @@ CREATE OR REPLACE FUNCTION aggregates_that_need_snapshots(_last_aggregate_id uui RETURNS TABLE (aggregate_id uuid) LANGUAGE plpgsql AS $$ BEGIN - RETURN QUERY SELECT stream.aggregate_id - FROM stream_records stream - WHERE (_last_aggregate_id IS NULL OR stream.aggregate_id > _last_aggregate_id) - AND snapshot_threshold IS NOT NULL - AND snapshot_threshold <= ( - (SELECT MAX(events.sequence_number) FROM event_records events WHERE stream.aggregate_id = events.aggregate_id) - - COALESCE((SELECT MAX(snapshots.sequence_number) FROM snapshot_records snapshots WHERE stream.aggregate_id = snapshots.aggregate_id), 0)) + RETURN QUERY SELECT a.aggregate_id + FROM aggregates_that_need_snapshots a + WHERE a.snapshot_outdated_at IS NOT NULL + AND (_last_aggregate_id IS NULL OR a.aggregate_id > _last_aggregate_id) ORDER BY 1 LIMIT _limit; END; $$; +CREATE OR REPLACE FUNCTION aggregates_that_need_snapshots_ordered_by_priority(_limit integer) + RETURNS TABLE (aggregate_id uuid) +LANGUAGE plpgsql AS $$ +BEGIN + RETURN QUERY SELECT a.aggregate_id + FROM aggregates_that_need_snapshots a + WHERE snapshot_outdated_at IS NOT NULL + ORDER BY snapshot_outdated_at ASC, snapshot_sequence_number_high_water_mark DESC, aggregate_id ASC + LIMIT _limit; +END; +$$; + CREATE OR REPLACE PROCEDURE permanently_delete_commands_without_events(_aggregate_id uuid, _organization_id uuid) LANGUAGE plpgsql AS $$ BEGIN @@ -309,11 +345,10 @@ CREATE OR REPLACE VIEW event_records (aggregate_id, partition_key, sequence_numb JOIN events event ON aggregate.aggregate_id = event.aggregate_id AND aggregate.events_partition_key = event.partition_key JOIN event_types type ON event.event_type_id = type.id; -CREATE OR REPLACE VIEW stream_records (aggregate_id, events_partition_key, aggregate_type, snapshot_threshold, created_at) AS +CREATE OR REPLACE VIEW stream_records (aggregate_id, events_partition_key, aggregate_type, created_at) AS SELECT aggregates.aggregate_id, aggregates.events_partition_key, aggregate_types.type, - aggregates.snapshot_threshold, aggregates.created_at FROM aggregates JOIN aggregate_types ON aggregates.aggregate_type_id = aggregate_types.id; diff --git a/lib/sequent/generator/template_project/db/sequent_schema.sql b/lib/sequent/generator/template_project/db/sequent_schema.sql index 54d72199..140c7edc 100644 --- a/lib/sequent/generator/template_project/db/sequent_schema.sql +++ b/lib/sequent/generator/template_project/db/sequent_schema.sql @@ -22,7 +22,6 @@ CREATE TABLE aggregates ( aggregate_id uuid PRIMARY KEY, events_partition_key text NOT NULL DEFAULT '', aggregate_type_id SMALLINT NOT NULL REFERENCES aggregate_types (id), - snapshot_threshold integer, created_at timestamp with time zone NOT NULL DEFAULT NOW(), UNIQUE (events_partition_key, aggregate_id) ) PARTITION BY RANGE (aggregate_id); @@ -92,3 +91,16 @@ CREATE TABLE saved_event_records ( xact_id bigint, PRIMARY KEY (aggregate_id, sequence_number, timestamp) ); + +CREATE TABLE aggregates_that_need_snapshots ( + aggregate_id uuid NOT NULL PRIMARY KEY REFERENCES aggregates (aggregate_id) ON UPDATE CASCADE ON DELETE CASCADE, + snapshot_outdated_at timestamp with time zone, + snapshot_sequence_number_high_water_mark integer +); +CREATE INDEX aggregates_that_need_snapshots_outdated_idx + ON aggregates_that_need_snapshots (snapshot_outdated_at ASC, snapshot_sequence_number_high_water_mark DESC, aggregate_id ASC) + WHERE snapshot_outdated_at IS NOT NULL; +COMMENT ON TABLE aggregates_that_need_snapshots IS 'Contains a row for every aggregate with more events than its snapshot threshold.'; +COMMENT ON COLUMN aggregates_that_need_snapshots.snapshot_outdated_at IS 'Not NULL indicates a snapshot is needed since the stored timestamp'; +COMMENT ON COLUMN aggregates_that_need_snapshots.snapshot_sequence_number_high_water_mark + IS 'The highest sequence number of the stored snapshot. Kept when snapshot are deleted to more easily query aggregates that need snapshotting the most'; From 77b1b2d2bfe059071ed94cdb163364f4db1c0249 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Wed, 26 Jun 2024 17:31:08 +0200 Subject: [PATCH 082/128] Add API to mark or clear aggregates for snapshotting Automatically delete snapshots for an aggregate when it is no longer marked for snapshotting. --- db/sequent_pgsql.sql | 25 ++++---- db/sequent_schema.sql | 28 ++++---- lib/sequent/core/event_store.rb | 48 +++++++++++--- .../template_project/db/sequent_pgsql.sql | 25 ++++---- .../template_project/db/sequent_schema.sql | 28 ++++---- spec/lib/sequent/core/event_store_spec.rb | 64 +++++++++++++++---- 6 files changed, 144 insertions(+), 74 deletions(-) diff --git a/db/sequent_pgsql.sql b/db/sequent_pgsql.sql index 4d521102..f481beb4 100644 --- a/db/sequent_pgsql.sql +++ b/db/sequent_pgsql.sql @@ -211,22 +211,21 @@ BEGIN _aggregate_id = _snapshot->>'aggregate_id'; _sequence_number = _snapshot->'sequence_number'; - INSERT INTO snapshot_records (aggregate_id, sequence_number, created_at, snapshot_type, snapshot_json) - VALUES ( - _aggregate_id, - _sequence_number, - (_snapshot->>'created_at')::timestamptz, - _snapshot->>'snapshot_type', - _snapshot->'snapshot_json' - ); - INSERT INTO aggregates_that_need_snapshots AS row (aggregate_id, snapshot_outdated_at, snapshot_sequence_number_high_water_mark) VALUES (_aggregate_id, NULL, _sequence_number) ON CONFLICT (aggregate_id) DO UPDATE - SET snapshot_outdated_at = NULL, - snapshot_sequence_number_high_water_mark = EXCLUDED.snapshot_sequence_number_high_water_mark - WHERE row.snapshot_sequence_number_high_water_mark IS NULL - OR row.snapshot_sequence_number_high_water_mark < EXCLUDED.snapshot_sequence_number_high_water_mark; + SET snapshot_outdated_at = EXCLUDED.snapshot_outdated_at, + snapshot_sequence_number_high_water_mark = + LEAST(row.snapshot_sequence_number_high_water_mark, EXCLUDED.snapshot_sequence_number_high_water_mark); + + INSERT INTO snapshot_records (aggregate_id, sequence_number, created_at, snapshot_type, snapshot_json) + VALUES ( + _aggregate_id, + _sequence_number, + (_snapshot->>'created_at')::timestamptz, + _snapshot->>'snapshot_type', + _snapshot->'snapshot_json' + ); END LOOP; END; $$; diff --git a/db/sequent_schema.sql b/db/sequent_schema.sql index 140c7edc..a7164edd 100644 --- a/db/sequent_schema.sql +++ b/db/sequent_schema.sql @@ -66,6 +66,19 @@ ALTER TABLE events_2025_and_later CLUSTER ON events_2025_and_later_pkey; CREATE TABLE events_aggregate PARTITION OF events FOR VALUES FROM ('A') TO ('Ag'); ALTER TABLE events_aggregate CLUSTER ON events_aggregate_pkey; +CREATE TABLE aggregates_that_need_snapshots ( + aggregate_id uuid NOT NULL PRIMARY KEY REFERENCES aggregates (aggregate_id) ON UPDATE CASCADE ON DELETE CASCADE, + snapshot_outdated_at timestamp with time zone, + snapshot_sequence_number_high_water_mark integer +); +CREATE INDEX aggregates_that_need_snapshots_outdated_idx + ON aggregates_that_need_snapshots (snapshot_outdated_at ASC, snapshot_sequence_number_high_water_mark DESC, aggregate_id ASC) + WHERE snapshot_outdated_at IS NOT NULL; +COMMENT ON TABLE aggregates_that_need_snapshots IS 'Contains a row for every aggregate with more events than its snapshot threshold.'; +COMMENT ON COLUMN aggregates_that_need_snapshots.snapshot_outdated_at IS 'Not NULL indicates a snapshot is needed since the stored timestamp'; +COMMENT ON COLUMN aggregates_that_need_snapshots.snapshot_sequence_number_high_water_mark + IS 'The highest sequence number of the stored snapshot. Kept when snapshot are deleted to more easily query aggregates that need snapshotting the most'; + CREATE TABLE snapshot_records ( aggregate_id uuid NOT NULL, sequence_number integer NOT NULL, @@ -73,7 +86,7 @@ CREATE TABLE snapshot_records ( snapshot_type text NOT NULL, snapshot_json jsonb NOT NULL, PRIMARY KEY (aggregate_id, sequence_number), - FOREIGN KEY (aggregate_id) REFERENCES aggregates (aggregate_id) + FOREIGN KEY (aggregate_id) REFERENCES aggregates_that_need_snapshots (aggregate_id) ON UPDATE CASCADE ON DELETE CASCADE ); @@ -91,16 +104,3 @@ CREATE TABLE saved_event_records ( xact_id bigint, PRIMARY KEY (aggregate_id, sequence_number, timestamp) ); - -CREATE TABLE aggregates_that_need_snapshots ( - aggregate_id uuid NOT NULL PRIMARY KEY REFERENCES aggregates (aggregate_id) ON UPDATE CASCADE ON DELETE CASCADE, - snapshot_outdated_at timestamp with time zone, - snapshot_sequence_number_high_water_mark integer -); -CREATE INDEX aggregates_that_need_snapshots_outdated_idx - ON aggregates_that_need_snapshots (snapshot_outdated_at ASC, snapshot_sequence_number_high_water_mark DESC, aggregate_id ASC) - WHERE snapshot_outdated_at IS NOT NULL; -COMMENT ON TABLE aggregates_that_need_snapshots IS 'Contains a row for every aggregate with more events than its snapshot threshold.'; -COMMENT ON COLUMN aggregates_that_need_snapshots.snapshot_outdated_at IS 'Not NULL indicates a snapshot is needed since the stored timestamp'; -COMMENT ON COLUMN aggregates_that_need_snapshots.snapshot_sequence_number_high_water_mark - IS 'The highest sequence number of the stored snapshot. Kept when snapshot are deleted to more easily query aggregates that need snapshotting the most'; diff --git a/lib/sequent/core/event_store.rb b/lib/sequent/core/event_store.rb index 7065e561..cf98a5ba 100644 --- a/lib/sequent/core/event_store.rb +++ b/lib/sequent/core/event_store.rb @@ -153,24 +153,54 @@ def load_latest_snapshot(aggregate_id) deserialize_event(snapshot_hash) unless snapshot_hash['aggregate_id'].nil? end - # Deletes all snapshots for all aggregates - def delete_all_snapshots + # Marks an aggregate for snapshotting. Marked aggregates will be + # picked up by the background snapshotting task. Another way to + # mark aggregates for snapshotting is to pass the + # *EventStream#snapshot_outdated_at* property to the + # *#store_events* method as is done automatically by the + # *AggregateRepository* based on the aggregate's + # *snapshot_threshold*. + def mark_aggregate_for_snapshotting(aggregate_id, snapshot_outdated_at = Time.now) + connection.exec_update(<<~EOS, 'mark_aggregate_for_snapshotting', [aggregate_id, snapshot_outdated_at]) + INSERT INTO aggregates_that_need_snapshots AS row (aggregate_id, snapshot_outdated_at) + VALUES ($1, $2) + ON CONFLICT (aggregate_id) DO UPDATE + SET snapshot_outdated_at = LEAST(row.snapshot_outdated_at, EXCLUDED.snapshot_outdated_at) + EOS + end + + # Stops snapshotting the specified aggregate. Any existing + # snapshots for this aggregate are also deleted. + def clear_aggregate_for_snapshotting(aggregate_id) connection.exec_update( - 'CALL delete_all_snapshots()', - 'delete_all_snapshots', - [], + 'DELETE FROM aggregates_that_need_snapshots WHERE aggregate_id = $1', + 'clear_aggregate_for_snapshotting', + [aggregate_id], ) end - def ignore_aggregates_for_snapshotting_with_last_event_before(timestamp) - connection.exec_update(<<~EOS, 'ignore_aggregates_for_snapshotting_with_last_event_before', [timestamp]) - DELETE FROM aggregates_that_need_snapshotting s + # Stops snapshotting all aggregates where the last event + # occurred before the indicated timestamp. Any existing + # snapshots for this aggregate are also deleted. + def clear_aggregates_for_snapshotting_with_last_event_before(timestamp) + connection.exec_update(<<~EOS, 'clear_aggregates_for_snapshotting_with_last_event_before', [timestamp]) + DELETE FROM aggregates_that_need_snapshots s WHERE NOT EXISTS (SELECT * - FROM aggregates a JOIN events e ON (a.aggregate_id, a.partition_key) = (e.aggregate_id, e.partition_key) + FROM aggregates a + JOIN events e ON (a.aggregate_id, a.events_partition_key) = (e.aggregate_id, e.partition_key) WHERE a.aggregate_id = s.aggregate_id AND e.created_at >= $1) EOS end + # Deletes all snapshots for all aggregates + def delete_all_snapshots + connection.exec_update( + 'CALL delete_all_snapshots()', + 'delete_all_snapshots', + [], + ) + end + # Deletes all snapshots for aggregate_id with a sequence_number lower than the specified sequence number. def delete_snapshots_before(aggregate_id, sequence_number) connection.exec_update( diff --git a/lib/sequent/generator/template_project/db/sequent_pgsql.sql b/lib/sequent/generator/template_project/db/sequent_pgsql.sql index 4d521102..f481beb4 100644 --- a/lib/sequent/generator/template_project/db/sequent_pgsql.sql +++ b/lib/sequent/generator/template_project/db/sequent_pgsql.sql @@ -211,22 +211,21 @@ BEGIN _aggregate_id = _snapshot->>'aggregate_id'; _sequence_number = _snapshot->'sequence_number'; - INSERT INTO snapshot_records (aggregate_id, sequence_number, created_at, snapshot_type, snapshot_json) - VALUES ( - _aggregate_id, - _sequence_number, - (_snapshot->>'created_at')::timestamptz, - _snapshot->>'snapshot_type', - _snapshot->'snapshot_json' - ); - INSERT INTO aggregates_that_need_snapshots AS row (aggregate_id, snapshot_outdated_at, snapshot_sequence_number_high_water_mark) VALUES (_aggregate_id, NULL, _sequence_number) ON CONFLICT (aggregate_id) DO UPDATE - SET snapshot_outdated_at = NULL, - snapshot_sequence_number_high_water_mark = EXCLUDED.snapshot_sequence_number_high_water_mark - WHERE row.snapshot_sequence_number_high_water_mark IS NULL - OR row.snapshot_sequence_number_high_water_mark < EXCLUDED.snapshot_sequence_number_high_water_mark; + SET snapshot_outdated_at = EXCLUDED.snapshot_outdated_at, + snapshot_sequence_number_high_water_mark = + LEAST(row.snapshot_sequence_number_high_water_mark, EXCLUDED.snapshot_sequence_number_high_water_mark); + + INSERT INTO snapshot_records (aggregate_id, sequence_number, created_at, snapshot_type, snapshot_json) + VALUES ( + _aggregate_id, + _sequence_number, + (_snapshot->>'created_at')::timestamptz, + _snapshot->>'snapshot_type', + _snapshot->'snapshot_json' + ); END LOOP; END; $$; diff --git a/lib/sequent/generator/template_project/db/sequent_schema.sql b/lib/sequent/generator/template_project/db/sequent_schema.sql index 140c7edc..a7164edd 100644 --- a/lib/sequent/generator/template_project/db/sequent_schema.sql +++ b/lib/sequent/generator/template_project/db/sequent_schema.sql @@ -66,6 +66,19 @@ ALTER TABLE events_2025_and_later CLUSTER ON events_2025_and_later_pkey; CREATE TABLE events_aggregate PARTITION OF events FOR VALUES FROM ('A') TO ('Ag'); ALTER TABLE events_aggregate CLUSTER ON events_aggregate_pkey; +CREATE TABLE aggregates_that_need_snapshots ( + aggregate_id uuid NOT NULL PRIMARY KEY REFERENCES aggregates (aggregate_id) ON UPDATE CASCADE ON DELETE CASCADE, + snapshot_outdated_at timestamp with time zone, + snapshot_sequence_number_high_water_mark integer +); +CREATE INDEX aggregates_that_need_snapshots_outdated_idx + ON aggregates_that_need_snapshots (snapshot_outdated_at ASC, snapshot_sequence_number_high_water_mark DESC, aggregate_id ASC) + WHERE snapshot_outdated_at IS NOT NULL; +COMMENT ON TABLE aggregates_that_need_snapshots IS 'Contains a row for every aggregate with more events than its snapshot threshold.'; +COMMENT ON COLUMN aggregates_that_need_snapshots.snapshot_outdated_at IS 'Not NULL indicates a snapshot is needed since the stored timestamp'; +COMMENT ON COLUMN aggregates_that_need_snapshots.snapshot_sequence_number_high_water_mark + IS 'The highest sequence number of the stored snapshot. Kept when snapshot are deleted to more easily query aggregates that need snapshotting the most'; + CREATE TABLE snapshot_records ( aggregate_id uuid NOT NULL, sequence_number integer NOT NULL, @@ -73,7 +86,7 @@ CREATE TABLE snapshot_records ( snapshot_type text NOT NULL, snapshot_json jsonb NOT NULL, PRIMARY KEY (aggregate_id, sequence_number), - FOREIGN KEY (aggregate_id) REFERENCES aggregates (aggregate_id) + FOREIGN KEY (aggregate_id) REFERENCES aggregates_that_need_snapshots (aggregate_id) ON UPDATE CASCADE ON DELETE CASCADE ); @@ -91,16 +104,3 @@ CREATE TABLE saved_event_records ( xact_id bigint, PRIMARY KEY (aggregate_id, sequence_number, timestamp) ); - -CREATE TABLE aggregates_that_need_snapshots ( - aggregate_id uuid NOT NULL PRIMARY KEY REFERENCES aggregates (aggregate_id) ON UPDATE CASCADE ON DELETE CASCADE, - snapshot_outdated_at timestamp with time zone, - snapshot_sequence_number_high_water_mark integer -); -CREATE INDEX aggregates_that_need_snapshots_outdated_idx - ON aggregates_that_need_snapshots (snapshot_outdated_at ASC, snapshot_sequence_number_high_water_mark DESC, aggregate_id ASC) - WHERE snapshot_outdated_at IS NOT NULL; -COMMENT ON TABLE aggregates_that_need_snapshots IS 'Contains a row for every aggregate with more events than its snapshot threshold.'; -COMMENT ON COLUMN aggregates_that_need_snapshots.snapshot_outdated_at IS 'Not NULL indicates a snapshot is needed since the stored timestamp'; -COMMENT ON COLUMN aggregates_that_need_snapshots.snapshot_sequence_number_high_water_mark - IS 'The highest sequence number of the stored snapshot. Kept when snapshot are deleted to more easily query aggregates that need snapshotting the most'; diff --git a/spec/lib/sequent/core/event_store_spec.rb b/spec/lib/sequent/core/event_store_spec.rb index b3950175..ca49a82d 100644 --- a/spec/lib/sequent/core/event_store_spec.rb +++ b/spec/lib/sequent/core/event_store_spec.rb @@ -52,7 +52,6 @@ class MyAggregate < Sequent::Core::AggregateRoot Sequent::Core::EventStream.new( aggregate_type: 'MyAggregate', aggregate_id: aggregate_id, - snapshot_outdated_at: Time.now, ), [ MyEvent.new( @@ -67,16 +66,64 @@ class MyAggregate < Sequent::Core::AggregateRoot ) end - it 'can find streams that need snapshotting' do - expect(event_store.aggregates_that_need_snapshots(nil)).to include(aggregate_id) + let(:aggregate) do + stream, events = event_store.load_events(aggregate_id) + Sequent::Core::AggregateRoot.load_from_history(stream, events) end - it 'can store and delete snapshots' do - stream, events = event_store.load_events(aggregate_id) - aggregate = Sequent::Core::AggregateRoot.load_from_history(stream, events) + let(:snapshot) do snapshot = aggregate.take_snapshot snapshot.created_at = Time.parse('2024-02-28T04:12:33Z') + snapshot + end + + it 'can mark aggregates for snapshotting when storing new events' do + event_store.commit_events( + Sequent::Core::Command.new(aggregate_id: aggregate_id), + [ + [ + Sequent::Core::EventStream.new( + aggregate_type: 'MyAggregate', + aggregate_id: aggregate_id, + snapshot_outdated_at: Time.now, + ), + [ + MyEvent.new( + aggregate_id: aggregate_id, + sequence_number: 2, + created_at: Time.parse('2024-02-30T01:10:12Z'), + data: "another event\n", + ), + ], + ], + ], + ) + expect(event_store.aggregates_that_need_snapshots(nil)).to include(aggregate_id) + + event_store.store_snapshots([snapshot]) + expect(event_store.aggregates_that_need_snapshots(nil)).to be_empty + end + + it 'can no longer find the aggregates that are cleared for snapshotting' do + event_store.store_snapshots([snapshot]) + + event_store.clear_aggregate_for_snapshotting(aggregate_id) + expect(event_store.aggregates_that_need_snapshots(nil)).to be_empty + expect(event_store.load_latest_snapshot(aggregate_id)).to eq(nil) + + event_store.mark_aggregate_for_snapshotting(aggregate_id) + expect(event_store.aggregates_that_need_snapshots(nil)).to include(aggregate_id) + end + + it 'can no longer find aggregates what are clear for snapshotting based on latest event timestamp' do + event_store.store_snapshots([snapshot]) + + event_store.clear_aggregates_for_snapshotting_with_last_event_before(Time.now) + expect(event_store.aggregates_that_need_snapshots(nil)).to be_empty + expect(event_store.load_latest_snapshot(aggregate_id)).to eq(nil) + end + it 'can store and delete snapshots' do event_store.store_snapshots([snapshot]) expect(event_store.aggregates_that_need_snapshots(nil)).to be_empty @@ -91,11 +138,6 @@ class MyAggregate < Sequent::Core::AggregateRoot end it 'can delete all snapshots' do - stream, events = event_store.load_events(aggregate_id) - aggregate = Sequent::Core::AggregateRoot.load_from_history(stream, events) - snapshot = aggregate.take_snapshot - snapshot.created_at = Time.parse('2024-02-28T04:12:33Z') - event_store.store_snapshots([snapshot]) expect(event_store.aggregates_that_need_snapshots(nil)).to be_empty From aab692a4ed41bb19462c23ab756f671b52287935 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Thu, 27 Jun 2024 08:59:12 +0200 Subject: [PATCH 083/128] Move snapshot related functions into separate included module --- lib/sequent/core/event_store.rb | 211 ++++++++++++++++---------------- 1 file changed, 107 insertions(+), 104 deletions(-) diff --git a/lib/sequent/core/event_store.rb b/lib/sequent/core/event_store.rb index cf98a5ba..355d8582 100644 --- a/lib/sequent/core/event_store.rb +++ b/lib/sequent/core/event_store.rb @@ -7,7 +7,114 @@ module Sequent module Core + module SnapshotStore + def store_snapshots(snapshots) + json = Sequent::Core::Oj.dump( + snapshots.map do |snapshot| + { + aggregate_id: snapshot.aggregate_id, + sequence_number: snapshot.sequence_number, + created_at: snapshot.created_at, + snapshot_type: snapshot.class.name, + snapshot_json: snapshot, + } + end, + ) + connection.exec_update( + 'CALL store_snapshots($1)', + 'store_snapshots', + [json], + ) + end + + def load_latest_snapshot(aggregate_id) + snapshot_hash = connection.exec_query( + 'SELECT * FROM load_latest_snapshot($1)', + 'load_latest_snapshot', + [aggregate_id], + ).first + deserialize_event(snapshot_hash) unless snapshot_hash['aggregate_id'].nil? + end + + # Deletes all snapshots for all aggregates + def delete_all_snapshots + connection.exec_update( + 'CALL delete_all_snapshots()', + 'delete_all_snapshots', + [], + ) + end + + # Deletes all snapshots for aggregate_id with a sequence_number lower than the specified sequence number. + def delete_snapshots_before(aggregate_id, sequence_number) + connection.exec_update( + 'CALL delete_snapshots_before($1, $2)', + 'delete_snapshots_before', + [aggregate_id, sequence_number], + ) + end + + # Marks an aggregate for snapshotting. Marked aggregates will be + # picked up by the background snapshotting task. Another way to + # mark aggregates for snapshotting is to pass the + # *EventStream#snapshot_outdated_at* property to the + # *#store_events* method as is done automatically by the + # *AggregateRepository* based on the aggregate's + # *snapshot_threshold*. + def mark_aggregate_for_snapshotting(aggregate_id, snapshot_outdated_at = Time.now) + connection.exec_update(<<~EOS, 'mark_aggregate_for_snapshotting', [aggregate_id, snapshot_outdated_at]) + INSERT INTO aggregates_that_need_snapshots AS row (aggregate_id, snapshot_outdated_at) + VALUES ($1, $2) + ON CONFLICT (aggregate_id) DO UPDATE + SET snapshot_outdated_at = LEAST(row.snapshot_outdated_at, EXCLUDED.snapshot_outdated_at) + EOS + end + + # Stops snapshotting the specified aggregate. Any existing + # snapshots for this aggregate are also deleted. + def clear_aggregate_for_snapshotting(aggregate_id) + connection.exec_update( + 'DELETE FROM aggregates_that_need_snapshots WHERE aggregate_id = $1', + 'clear_aggregate_for_snapshotting', + [aggregate_id], + ) + end + + # Stops snapshotting all aggregates where the last event + # occurred before the indicated timestamp. Any existing + # snapshots for this aggregate are also deleted. + def clear_aggregates_for_snapshotting_with_last_event_before(timestamp) + connection.exec_update(<<~EOS, 'clear_aggregates_for_snapshotting_with_last_event_before', [timestamp]) + DELETE FROM aggregates_that_need_snapshots s + WHERE NOT EXISTS (SELECT * + FROM aggregates a + JOIN events e ON (a.aggregate_id, a.events_partition_key) = (e.aggregate_id, e.partition_key) + WHERE a.aggregate_id = s.aggregate_id AND e.created_at >= $1) + EOS + end + + ## + # Returns the ids of aggregates that need a new snapshot. + # + def aggregates_that_need_snapshots(last_aggregate_id, limit = 10) + connection.exec_query( + 'SELECT aggregate_id FROM aggregates_that_need_snapshots($1, $2)', + 'aggregates_that_need_snapshots', + [last_aggregate_id, limit], + ).map { |x| x['aggregate_id'] } + end + + def aggregates_that_need_snapshots_ordered_by_priority(limit = 10) + connection.exec_query( + 'SELECT aggregate_id FROM aggregates_that_need_snapshots_ordered_by_priority($1)', + 'aggregates_that_need_snapshots', + [limit], + ).map { |x| x['aggregate_id'] } + end + end + class EventStore + include SnapshotStore include ActiveRecord::ConnectionAdapters::Quoting extend Forwardable @@ -125,91 +232,6 @@ def load_events_for_aggregates(aggregate_ids) end end - def store_snapshots(snapshots) - json = Sequent::Core::Oj.dump( - snapshots.map do |snapshot| - { - aggregate_id: snapshot.aggregate_id, - sequence_number: snapshot.sequence_number, - created_at: snapshot.created_at, - snapshot_type: snapshot.class.name, - snapshot_json: snapshot, - } - end, - ) - connection.exec_update( - 'CALL store_snapshots($1)', - 'store_snapshots', - [json], - ) - end - - def load_latest_snapshot(aggregate_id) - snapshot_hash = connection.exec_query( - 'SELECT * FROM load_latest_snapshot($1)', - 'load_latest_snapshot', - [aggregate_id], - ).first - deserialize_event(snapshot_hash) unless snapshot_hash['aggregate_id'].nil? - end - - # Marks an aggregate for snapshotting. Marked aggregates will be - # picked up by the background snapshotting task. Another way to - # mark aggregates for snapshotting is to pass the - # *EventStream#snapshot_outdated_at* property to the - # *#store_events* method as is done automatically by the - # *AggregateRepository* based on the aggregate's - # *snapshot_threshold*. - def mark_aggregate_for_snapshotting(aggregate_id, snapshot_outdated_at = Time.now) - connection.exec_update(<<~EOS, 'mark_aggregate_for_snapshotting', [aggregate_id, snapshot_outdated_at]) - INSERT INTO aggregates_that_need_snapshots AS row (aggregate_id, snapshot_outdated_at) - VALUES ($1, $2) - ON CONFLICT (aggregate_id) DO UPDATE - SET snapshot_outdated_at = LEAST(row.snapshot_outdated_at, EXCLUDED.snapshot_outdated_at) - EOS - end - - # Stops snapshotting the specified aggregate. Any existing - # snapshots for this aggregate are also deleted. - def clear_aggregate_for_snapshotting(aggregate_id) - connection.exec_update( - 'DELETE FROM aggregates_that_need_snapshots WHERE aggregate_id = $1', - 'clear_aggregate_for_snapshotting', - [aggregate_id], - ) - end - - # Stops snapshotting all aggregates where the last event - # occurred before the indicated timestamp. Any existing - # snapshots for this aggregate are also deleted. - def clear_aggregates_for_snapshotting_with_last_event_before(timestamp) - connection.exec_update(<<~EOS, 'clear_aggregates_for_snapshotting_with_last_event_before', [timestamp]) - DELETE FROM aggregates_that_need_snapshots s - WHERE NOT EXISTS (SELECT * - FROM aggregates a - JOIN events e ON (a.aggregate_id, a.events_partition_key) = (e.aggregate_id, e.partition_key) - WHERE a.aggregate_id = s.aggregate_id AND e.created_at >= $1) - EOS - end - - # Deletes all snapshots for all aggregates - def delete_all_snapshots - connection.exec_update( - 'CALL delete_all_snapshots()', - 'delete_all_snapshots', - [], - ) - end - - # Deletes all snapshots for aggregate_id with a sequence_number lower than the specified sequence number. - def delete_snapshots_before(aggregate_id, sequence_number) - connection.exec_update( - 'CALL delete_snapshots_before($1, $2)', - 'delete_snapshots_before', - [aggregate_id, sequence_number], - ) - end - def stream_exists?(aggregate_id) Sequent.configuration.stream_record_class.exists?(aggregate_id: aggregate_id) end @@ -264,25 +286,6 @@ def replay_events_from_cursor(get_events:, block_size: 2000, end end - ## - # Returns the ids of aggregates that need a new snapshot. - # - def aggregates_that_need_snapshots(last_aggregate_id, limit = 10) - connection.exec_query( - 'SELECT aggregate_id FROM aggregates_that_need_snapshots($1, $2)', - 'aggregates_that_need_snapshots', - [last_aggregate_id, limit], - ).map { |x| x['aggregate_id'] } - end - - def aggregates_that_need_snapshots_ordered_by_priority(limit = 10) - connection.exec_query( - 'SELECT aggregate_id FROM aggregates_that_need_snapshots_ordered_by_priority($1)', - 'aggregates_that_need_snapshots', - [limit], - ).map { |x| x['aggregate_id'] } - end - def find_event_stream(aggregate_id) record = Sequent.configuration.stream_record_class.where(aggregate_id: aggregate_id).first record&.event_stream From 6170c1b78c6d61b329daf3678d220701705eed25 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Thu, 27 Jun 2024 09:01:20 +0200 Subject: [PATCH 084/128] Remove unused private methods --- lib/sequent/core/event_store.rb | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/lib/sequent/core/event_store.rb b/lib/sequent/core/event_store.rb index 355d8582..cc05079d 100644 --- a/lib/sequent/core/event_store.rb +++ b/lib/sequent/core/event_store.rb @@ -317,14 +317,6 @@ def permanently_delete_commands_without_events(aggregate_id: nil, organization_i private - def event_types - @event_types = if Sequent.configuration.event_store_cache_event_types - ThreadSafe::Cache.new - else - NoEventTypesCache.new - end - end - def connection Sequent.configuration.event_record_class.connection end @@ -337,18 +329,6 @@ def query_events(aggregate_ids, use_snapshots = true, load_until = nil) ) end - def column_names - @column_names ||= Sequent - .configuration - .event_record_class - .column_names - .reject { |c| c == primary_key_event_records } - end - - def primary_key_event_records - @primary_key_event_records ||= Sequent.configuration.event_record_class.primary_key - end - def deserialize_event(event_hash) should_serialize_json = Sequent.configuration.event_record_class.serialize_json? record = Sequent.configuration.event_record_class.new @@ -367,10 +347,6 @@ def deserialize_event(event_hash) raise DeserializeEventError, event_hash end - def resolve_event_type(event_type) - event_types.fetch_or_store(event_type) { |k| Class.const_get(k) } - end - def publish_events(events) Sequent.configuration.event_publisher.publish_events(events) end From 1911da1a3c5dd23cbdfb0d865e8754fdb951372a Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Thu, 27 Jun 2024 09:55:42 +0200 Subject: [PATCH 085/128] Remove event type caching support `Module#const_get` is almost as fast as a Ruby hash, so simplify code and setup by removing the event type cache and configuration support. --- docs/docs/concepts/configuration.md | 1 - docs/docs/rails-sequent.md | 52 +++++++++---------- .../config/initializers/zz_sequent.rb | 1 - lib/sequent/configuration.rb | 2 - lib/sequent/core/event_store.rb | 9 ---- spec/lib/sequent/core/event_store_spec.rb | 40 -------------- 6 files changed, 25 insertions(+), 80 deletions(-) diff --git a/docs/docs/concepts/configuration.md b/docs/docs/concepts/configuration.md index 925321bc..a12c5891 100644 --- a/docs/docs/concepts/configuration.md +++ b/docs/docs/concepts/configuration.md @@ -130,7 +130,6 @@ For the latest configuration possibilities please check the `Sequent::Configurat | event_record_hooks_class | The class with EventRecord life cycle hooks | `Sequent::Core::EventRecordHooks` | | event_store | The [EventStore](event_store.html) | `Sequent::Core::EventStore.new` | | event_store_schema_name | The name of the db schema in which the [EventStore](event_store.html) is installed | `'sequent_schema'` | -| event_store_cache_event_types | If the EventStore should cache event types. Set this to false when running Rails in development mode | true | | migration_sql_files_directory | The location of the sql files for [Migrations](migrations.html) | `'db/tables'` | | number_of_replay_processes | The [number of process](#number_of_replay_processes) used while offline migration | `4` | | offline_replay_persistor_class | The class used to persist the `Projector`s during the offline migration part. | `Sequent::Core::Persistors::ActiveRecordPersistor` | diff --git a/docs/docs/rails-sequent.md b/docs/docs/rails-sequent.md index 346f38c7..e9be08de 100644 --- a/docs/docs/rails-sequent.md +++ b/docs/docs/rails-sequent.md @@ -28,16 +28,16 @@ See the [Rails autoloading and reloading guide](https://guides.rubyonrails.org/a 3) Copy the `sequent_schema.rb` file from [https://raw.githubusercontent.com/zilverline/sequent/master/db/sequent_schema.rb](https://raw.githubusercontent.com/zilverline/sequent/master/db/sequent_schema.rb) and put it in your `./db` directory. -4) Create `./db/sequent_migrations.rb`. This will contain your `view_schema` migrations. - +4) Create `./db/sequent_migrations.rb`. This will contain your `view_schema` migrations. + ```ruby VIEW_SCHEMA_VERSION = 1 - + class SequentMigrations < Sequent::Migrations::Projectors def self.version VIEW_SCHEMA_VERSION end - + def self.versions { '1' => [ @@ -46,36 +46,36 @@ See the [Rails autoloading and reloading guide](https://guides.rubyonrails.org/a } end end - + ``` For a complete overview on how Migrations work in Sequent, check out the [Migrations Guide](/docs/concepts/migrations.html) - - + + 5) Add the following snippet to your `Rakefile` ```ruby # Sequent requires a `SEQUENT_ENV` environment to be set - # next to a `RAILS_ENV` + # next to a `RAILS_ENV` ENV['SEQUENT_ENV'] = ENV['RAILS_ENV'] ||= 'development' - + require 'sequent/rake/migration_tasks' - + Sequent::Rake::MigrationTasks.new.register_tasks! - + # The dependency of sequent:init on :environment ensures the Rails app is loaded # when running the sequent migrations. This is needed otherwise # the sequent initializer - which is required to run these rake tasks - # doesn't run task 'sequent:init' => [:environment] task 'sequent:migrate:init' => [:sequent_db_connect] - + task 'sequent_db_connect' do Sequent::Support::Database.connect!(ENV['SEQUENT_ENV']) end - + # Create custom rake task setting the SEQUENT_MIGRATION_SCHEMAS for - # running the Rails migrations + # running the Rails migrations task :migrate_public_schema do ENV['SEQUENT_MIGRATION_SCHEMAS'] = 'public' Rake::Task['db:migrate'].invoke @@ -87,7 +87,7 @@ See the [Rails autoloading and reloading guide](https://guides.rubyonrails.org/a ``` - **You can't use rails db:migrate directly** anymore since + **You can't use rails db:migrate directly** anymore since that will add all the tables of the `view_schema` and `sequent_schema` to the `schema.rb` file after running a Rails migration. To fix this the `rails db:migrate` must be wrapped in your own task setting the @@ -97,7 +97,7 @@ See the [Rails autoloading and reloading guide](https://guides.rubyonrails.org/a so running it without `SEQUENT_MIGRATION_SCHEMAS` set will fail. {: .notice--warning} -6) Ensure your `database.yml` contains the schema_search_path: +6) Ensure your `database.yml` contains the schema_search_path: ```yaml default: @@ -109,22 +109,21 @@ See the [Rails autoloading and reloading guide](https://guides.rubyonrails.org/a Sequent internally relies on registries of classes of certain types. For instance it keeps track of all `AggregateRoot` classes by adding them to a registry when `Sequent::Core::AggregateRoot` is extended. For this to work properly, all classes must be eager loaded otherwise code depending on this fact might - produce unpredictable results. Set the `config.eager_load` to `true` for all environments + produce unpredictable results. Set the `config.eager_load` to `true` for all environments (in production the Rails default is already `true`). 8) Add `./config/initializers/sequent.rb` containing at least: ```ruby require_relative '../../db/sequent_migrations' - + Rails.application.reloader.to_prepare do Sequent.configure do |config| config.migrations_class_name = 'SequentMigrations' config.enable_autoregistration = true - config.event_store_cache_event_types = !Rails.env.development? config.database_config_directory = 'config' - + # this is the location of your sql files for your view_schema config.migration_sql_files_directory = 'db/sequent' end @@ -134,21 +133,21 @@ See the [Rails autoloading and reloading guide](https://guides.rubyonrails.org/a **You must** wrap the sequent initializer code in `Rails.application.reloader.to_prepare` because during initialization, the autoloading hasn't run yet. -9) Run the following commands to create the `sequent_schema` and `view_schema` +9) Run the following commands to create the `sequent_schema` and `view_schema` ```bash bundle exec rake sequent:db:create_event_store bundle exec rake sequent:db:create_view_schema - + # only run this when you add or change projectors in SequentMigrations bundle exec rake sequent:migrate:online - bundle exec rake sequent:migrate:offline + bundle exec rake sequent:migrate:offline ``` 10) Add the following to application.rb This step is actually only necessary if you load Aggregates outside the scope - of the Unit Of Work which is automatically started and committed via the `execute_commands` call. + of the Unit Of Work which is automatically started and committed via the `execute_commands` call. If you for instance load Aggregates inside Controllers or or ActiveJob you have to clear Sequent's Unit Of Work (stored in the Thread.current) yourself. For the web you can add the following Rack middleware: @@ -203,7 +202,7 @@ module Banking end ``` -The "downside" here is that you need to introduce an extra layer of naming to be able to group your events into a single file. +The "downside" here is that you need to introduce an extra layer of naming to be able to group your events into a single file. ### Rails Engines @@ -211,8 +210,7 @@ Sequent in [Rails Engines](https://guides.rubyonrails.org/engines.html) work bas Some things to remember when working with Rails Engines: 1. The Sequent config must be set in the main application `config/initializers` -2. The main application is the maintainer of the `sequent_schema` and `view_schema`. +2. The main application is the maintainer of the `sequent_schema` and `view_schema`. So copy over the migration sql files to the main application directory like you would when an Engine provides active record migrations. Please checkout the Rails & Sequent example app in our [sequent-examples](https://github.com/zilverline/sequent-examples) Github repository. - diff --git a/integration-specs/rails-app/config/initializers/zz_sequent.rb b/integration-specs/rails-app/config/initializers/zz_sequent.rb index 4a4dbefc..12b2ebf1 100644 --- a/integration-specs/rails-app/config/initializers/zz_sequent.rb +++ b/integration-specs/rails-app/config/initializers/zz_sequent.rb @@ -7,7 +7,6 @@ Rails.application.reloader.to_prepare do Sequent.configure do |config| config.enable_autoregistration = true - config.event_store_cache_event_types = !Rails.env.development? config.migrations_class_name = 'SequentMigrations' diff --git a/lib/sequent/configuration.rb b/lib/sequent/configuration.rb index 04df322b..79e778d2 100644 --- a/lib/sequent/configuration.rb +++ b/lib/sequent/configuration.rb @@ -36,7 +36,6 @@ class Configuration attr_accessor :aggregate_repository, :event_store, - :event_store_cache_event_types, :command_service, :event_record_class, :snapshot_record_class, @@ -93,7 +92,6 @@ def initialize self.command_middleware = Sequent::Core::Middleware::Chain.new self.aggregate_repository = Sequent::Core::AggregateRepository.new - self.event_store_cache_event_types = true self.event_store = Sequent::Core::EventStore.new self.command_service = Sequent::Core::CommandService.new self.event_record_class = Sequent::Core::EventRecord diff --git a/lib/sequent/core/event_store.rb b/lib/sequent/core/event_store.rb index cc05079d..e0c1d226 100644 --- a/lib/sequent/core/event_store.rb +++ b/lib/sequent/core/event_store.rb @@ -134,15 +134,6 @@ def message end end - ## - # Disables event type caching (ie. for in development). - # - class NoEventTypesCache - def fetch_or_store(event_type) - yield(event_type) - end - end - ## # Stores the events in the EventStore and publishes the events # to the registered event_handlers. diff --git a/spec/lib/sequent/core/event_store_spec.rb b/spec/lib/sequent/core/event_store_spec.rb index ca49a82d..da3cb9b0 100644 --- a/spec/lib/sequent/core/event_store_spec.rb +++ b/spec/lib/sequent/core/event_store_spec.rb @@ -418,46 +418,6 @@ def stop EOS end end - - context 'and event type caching disabled' do - around do |example| - current = Sequent.configuration.event_store_cache_event_types - - Sequent.configuration.event_store_cache_event_types = false - - example.run - ensure - Sequent.configuration.event_store_cache_event_types = current - end - let(:event_store) { Sequent::Core::EventStore.new } - - it 'returns the stream and events for existing aggregates' do - TestEventForCaching = Class.new(Sequent::Core::Event) - - event_store.commit_events( - Sequent::Core::Command.new(aggregate_id: aggregate_id), - [ - [ - Sequent::Core::EventStream.new(aggregate_type: 'MyAggregate', aggregate_id: aggregate_id), - [TestEventForCaching.new(aggregate_id: aggregate_id, sequence_number: 1)], - ], - ], - ) - stream, events = event_store.load_events(aggregate_id) - expect(stream).to be - expect(events.first).to be_kind_of(TestEventForCaching) - - # redefine TestEventForCaching class (ie. simulate Rails auto-loading) - OldTestEventForCaching = TestEventForCaching - TestEventForCaching = Class.new(Sequent::Core::Event) - - stream, events = event_store.load_events(aggregate_id) - expect(stream).to be - expect(events.first).to be_kind_of(TestEventForCaching) - - expect(event_store.load_event(aggregate_id, events.first.sequence_number)).to eq(events.first) - end - end end describe '#load_events_for_aggregates' do From c02c7f9bf7e6fc25a7e49cdacb4b6d2a512ac449 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Fri, 28 Jun 2024 09:37:06 +0200 Subject: [PATCH 086/128] Update view SQL for consistency --- db/sequent_pgsql.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db/sequent_pgsql.sql b/db/sequent_pgsql.sql index f481beb4..49b66b4b 100644 --- a/db/sequent_pgsql.sql +++ b/db/sequent_pgsql.sql @@ -340,8 +340,8 @@ CREATE OR REPLACE VIEW event_records (aggregate_id, partition_key, sequence_numb enrich_event_json(event) AS event_json, command_id, event.xact_id - FROM aggregates aggregate - JOIN events event ON aggregate.aggregate_id = event.aggregate_id AND aggregate.events_partition_key = event.partition_key + FROM events event + JOIN aggregates aggregate ON aggregate.aggregate_id = event.aggregate_id AND aggregate.events_partition_key = event.partition_key JOIN event_types type ON event.event_type_id = type.id; CREATE OR REPLACE VIEW stream_records (aggregate_id, events_partition_key, aggregate_type, created_at) AS From c10fed0b3f2ebe4291cef362970fb866a4c51a6e Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Fri, 28 Jun 2024 11:02:43 +0200 Subject: [PATCH 087/128] Drop views and re-create so that column definitions can change Ignore the obsolete `snapshot_threshold` column for stream records so that the column can be dropped in the database without affecting running applications. --- db/sequent_pgsql.sql | 9 ++++++--- lib/sequent/core/stream_record.rb | 1 + 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/db/sequent_pgsql.sql b/db/sequent_pgsql.sql index 49b66b4b..1a897bd0 100644 --- a/db/sequent_pgsql.sql +++ b/db/sequent_pgsql.sql @@ -320,7 +320,8 @@ BEGIN END; $$; -CREATE OR REPLACE VIEW command_records (id, user_id, aggregate_id, command_type, command_json, created_at, event_aggregate_id, event_sequence_number) AS +DROP VIEW IF EXISTS command_records; +CREATE VIEW command_records (id, user_id, aggregate_id, command_type, command_json, created_at, event_aggregate_id, event_sequence_number) AS SELECT id, user_id, aggregate_id, @@ -331,7 +332,8 @@ CREATE OR REPLACE VIEW command_records (id, user_id, aggregate_id, command_type, event_sequence_number FROM commands command; -CREATE OR REPLACE VIEW event_records (aggregate_id, partition_key, sequence_number, created_at, event_type, event_json, command_record_id, xact_id) AS +DROP VIEW IF EXISTS event_records; +CREATE VIEW event_records (aggregate_id, partition_key, sequence_number, created_at, event_type, event_json, command_record_id, xact_id) AS SELECT aggregate.aggregate_id, event.partition_key, event.sequence_number, @@ -344,7 +346,8 @@ CREATE OR REPLACE VIEW event_records (aggregate_id, partition_key, sequence_numb JOIN aggregates aggregate ON aggregate.aggregate_id = event.aggregate_id AND aggregate.events_partition_key = event.partition_key JOIN event_types type ON event.event_type_id = type.id; -CREATE OR REPLACE VIEW stream_records (aggregate_id, events_partition_key, aggregate_type, created_at) AS +DROP VIEW IF EXISTS stream_records; +CREATE VIEW stream_records (aggregate_id, events_partition_key, aggregate_type, created_at) AS SELECT aggregates.aggregate_id, aggregates.events_partition_key, aggregate_types.type, diff --git a/lib/sequent/core/stream_record.rb b/lib/sequent/core/stream_record.rb index 4809ddf4..e40b7d42 100644 --- a/lib/sequent/core/stream_record.rb +++ b/lib/sequent/core/stream_record.rb @@ -13,6 +13,7 @@ def initialize(aggregate_type:, aggregate_id:, events_partition_key: '', snapsho class StreamRecord < Sequent::ApplicationRecord self.primary_key = %i[aggregate_id] self.table_name = 'stream_records' + self.ignored_columns = %w[snapshot_threshold] validates_presence_of :aggregate_type, :aggregate_id From 3f3712b6bf24688eefcb2396cc212398de50a3f0 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Tue, 2 Jul 2024 17:03:19 +0200 Subject: [PATCH 088/128] Limit the number of aggregates that are scheduled for snapshotting When selecting aggregates for snapshotting (aggregates with an outdated snapshot) there can never be more than the specified limit aggregates returned that are currently not yet scheduled for snapshotting (snapshot_scheduled_at is null). This allows background tasks to finish snapshotting aggregates before new (or the same) aggregates are scheduled for snapshotting again. --- db/sequent_pgsql.sql | 37 +++++++++++------- db/sequent_schema.sql | 6 ++- lib/sequent/core/event_store.rb | 21 +++++----- spec/lib/sequent/core/event_store_spec.rb | 47 +++++++++++++++++++++-- spec/spec_helper.rb | 1 + 5 files changed, 83 insertions(+), 29 deletions(-) diff --git a/db/sequent_pgsql.sql b/db/sequent_pgsql.sql index 1a897bd0..fd4bf700 100644 --- a/db/sequent_pgsql.sql +++ b/db/sequent_pgsql.sql @@ -211,12 +211,13 @@ BEGIN _aggregate_id = _snapshot->>'aggregate_id'; _sequence_number = _snapshot->'sequence_number'; - INSERT INTO aggregates_that_need_snapshots AS row (aggregate_id, snapshot_outdated_at, snapshot_sequence_number_high_water_mark) - VALUES (_aggregate_id, NULL, _sequence_number) + INSERT INTO aggregates_that_need_snapshots AS row (aggregate_id, snapshot_sequence_number_high_water_mark) + VALUES (_aggregate_id, _sequence_number) ON CONFLICT (aggregate_id) DO UPDATE - SET snapshot_outdated_at = EXCLUDED.snapshot_outdated_at, - snapshot_sequence_number_high_water_mark = - LEAST(row.snapshot_sequence_number_high_water_mark, EXCLUDED.snapshot_sequence_number_high_water_mark); + SET snapshot_sequence_number_high_water_mark = + LEAST(row.snapshot_sequence_number_high_water_mark, EXCLUDED.snapshot_sequence_number_high_water_mark), + snapshot_outdated_at = NULL, + snapshot_scheduled_at = NULL; INSERT INTO snapshot_records (aggregate_id, sequence_number, created_at, snapshot_type, snapshot_json) VALUES ( @@ -247,7 +248,8 @@ CREATE OR REPLACE PROCEDURE delete_all_snapshots() LANGUAGE plpgsql AS $$ BEGIN UPDATE aggregates_that_need_snapshots - SET snapshot_outdated_at = NOW() + SET snapshot_outdated_at = NOW(), + snapshot_scheduled_at = NULL WHERE snapshot_outdated_at IS NULL; DELETE FROM snapshot_records; END; @@ -261,7 +263,8 @@ BEGIN AND sequence_number < _sequence_number; UPDATE aggregates_that_need_snapshots - SET snapshot_outdated_at = NOW() + SET snapshot_outdated_at = NOW(), + snapshot_scheduled_at = NULL WHERE aggregate_id = _aggregate_id AND snapshot_outdated_at IS NULL AND NOT EXISTS (SELECT 1 FROM snapshot_records WHERE aggregate_id = _aggregate_id); @@ -281,15 +284,23 @@ BEGIN END; $$; -CREATE OR REPLACE FUNCTION aggregates_that_need_snapshots_ordered_by_priority(_limit integer) +CREATE OR REPLACE FUNCTION select_aggregates_for_snapshotting(_limit integer, _reschedule_snapshot_scheduled_before timestamp with time zone) RETURNS TABLE (aggregate_id uuid) LANGUAGE plpgsql AS $$ BEGIN - RETURN QUERY SELECT a.aggregate_id - FROM aggregates_that_need_snapshots a - WHERE snapshot_outdated_at IS NOT NULL - ORDER BY snapshot_outdated_at ASC, snapshot_sequence_number_high_water_mark DESC, aggregate_id ASC - LIMIT _limit; + RETURN QUERY WITH scheduled AS MATERIALIZED ( + SELECT a.aggregate_id + FROM aggregates_that_need_snapshots AS a + WHERE snapshot_outdated_at IS NOT NULL + ORDER BY snapshot_outdated_at ASC, snapshot_sequence_number_high_water_mark DESC, aggregate_id ASC + LIMIT _limit + FOR UPDATE + ) UPDATE aggregates_that_need_snapshots AS row + SET snapshot_scheduled_at = NOW() + FROM scheduled + WHERE row.aggregate_id = scheduled.aggregate_id + AND (row.snapshot_scheduled_at IS NULL OR row.snapshot_scheduled_at < _reschedule_snapshot_scheduled_before) + RETURNING row.aggregate_id; END; $$; diff --git a/db/sequent_schema.sql b/db/sequent_schema.sql index a7164edd..3bc3b7a0 100644 --- a/db/sequent_schema.sql +++ b/db/sequent_schema.sql @@ -68,16 +68,18 @@ ALTER TABLE events_aggregate CLUSTER ON events_aggregate_pkey; CREATE TABLE aggregates_that_need_snapshots ( aggregate_id uuid NOT NULL PRIMARY KEY REFERENCES aggregates (aggregate_id) ON UPDATE CASCADE ON DELETE CASCADE, + snapshot_sequence_number_high_water_mark integer, snapshot_outdated_at timestamp with time zone, - snapshot_sequence_number_high_water_mark integer + snapshot_scheduled_at timestamp with time zone ); CREATE INDEX aggregates_that_need_snapshots_outdated_idx ON aggregates_that_need_snapshots (snapshot_outdated_at ASC, snapshot_sequence_number_high_water_mark DESC, aggregate_id ASC) WHERE snapshot_outdated_at IS NOT NULL; COMMENT ON TABLE aggregates_that_need_snapshots IS 'Contains a row for every aggregate with more events than its snapshot threshold.'; -COMMENT ON COLUMN aggregates_that_need_snapshots.snapshot_outdated_at IS 'Not NULL indicates a snapshot is needed since the stored timestamp'; COMMENT ON COLUMN aggregates_that_need_snapshots.snapshot_sequence_number_high_water_mark IS 'The highest sequence number of the stored snapshot. Kept when snapshot are deleted to more easily query aggregates that need snapshotting the most'; +COMMENT ON COLUMN aggregates_that_need_snapshots.snapshot_outdated_at IS 'Not NULL indicates a snapshot is needed since the stored timestamp'; +COMMENT ON COLUMN aggregates_that_need_snapshots.snapshot_scheduled_at IS 'Not NULL indicates a snapshot is in the process of being taken'; CREATE TABLE snapshot_records ( aggregate_id uuid NOT NULL, diff --git a/lib/sequent/core/event_store.rb b/lib/sequent/core/event_store.rb index e0c1d226..cd5ca691 100644 --- a/lib/sequent/core/event_store.rb +++ b/lib/sequent/core/event_store.rb @@ -57,16 +57,17 @@ def delete_snapshots_before(aggregate_id, sequence_number) # Marks an aggregate for snapshotting. Marked aggregates will be # picked up by the background snapshotting task. Another way to # mark aggregates for snapshotting is to pass the - # *EventStream#snapshot_outdated_at* property to the - # *#store_events* method as is done automatically by the - # *AggregateRepository* based on the aggregate's - # *snapshot_threshold*. - def mark_aggregate_for_snapshotting(aggregate_id, snapshot_outdated_at = Time.now) + # +EventStream#snapshot_outdated_at+ property to the + # +#store_events+ method as is done automatically by the + # +AggregateRepository+ based on the aggregate's + # +snapshot_threshold+. + def mark_aggregate_for_snapshotting(aggregate_id, snapshot_outdated_at: Time.now) connection.exec_update(<<~EOS, 'mark_aggregate_for_snapshotting', [aggregate_id, snapshot_outdated_at]) INSERT INTO aggregates_that_need_snapshots AS row (aggregate_id, snapshot_outdated_at) VALUES ($1, $2) ON CONFLICT (aggregate_id) DO UPDATE - SET snapshot_outdated_at = LEAST(row.snapshot_outdated_at, EXCLUDED.snapshot_outdated_at) + SET snapshot_outdated_at = LEAST(row.snapshot_outdated_at, EXCLUDED.snapshot_outdated_at), + snapshot_scheduled_at = NULL EOS end @@ -104,11 +105,11 @@ def aggregates_that_need_snapshots(last_aggregate_id, limit = 10) ).map { |x| x['aggregate_id'] } end - def aggregates_that_need_snapshots_ordered_by_priority(limit = 10) + def select_aggregates_for_snapshotting(limit:, reschedule_snapshots_scheduled_before: nil) connection.exec_query( - 'SELECT aggregate_id FROM aggregates_that_need_snapshots_ordered_by_priority($1)', - 'aggregates_that_need_snapshots', - [limit], + 'SELECT aggregate_id FROM select_aggregates_for_snapshotting($1, $2)', + 'select_aggregates_for_snapshotting', + [limit, reschedule_snapshots_scheduled_before], ).map { |x| x['aggregate_id'] } end end diff --git a/spec/lib/sequent/core/event_store_spec.rb b/spec/lib/sequent/core/event_store_spec.rb index da3cb9b0..cbec8896 100644 --- a/spec/lib/sequent/core/event_store_spec.rb +++ b/spec/lib/sequent/core/event_store_spec.rb @@ -104,6 +104,45 @@ class MyAggregate < Sequent::Core::AggregateRoot expect(event_store.aggregates_that_need_snapshots(nil)).to be_empty end + it 'limits the number of concurrent aggregates scheduled for snapshotting' do + event_store.mark_aggregate_for_snapshotting(aggregate_id, snapshot_outdated_at: 1.hour.ago) + + aggregate_id_2 = Sequent.new_uuid + event_store.commit_events( + Sequent::Core::Command.new(aggregate_id: aggregate_id_2), + [ + [ + Sequent::Core::EventStream.new( + aggregate_type: 'MyAggregate', + aggregate_id: aggregate_id_2, + snapshot_outdated_at: 2.minutes.ago, + ), + [ + MyEvent.new( + aggregate_id: aggregate_id_2, + sequence_number: 1, + created_at: Time.parse('2024-02-30T01:10:12Z'), + data: "another event\n", + ), + ], + ], + ], + ) + + expect(event_store.select_aggregates_for_snapshotting(limit: 1)).to include(aggregate_id) + expect(event_store.select_aggregates_for_snapshotting(limit: 1)).to be_empty + + event_store.store_snapshots([snapshot]) + + expect(event_store.select_aggregates_for_snapshotting(limit: 1)).to include(aggregate_id_2) + expect(event_store.select_aggregates_for_snapshotting(limit: 1)).to be_empty + + event_store.mark_aggregate_for_snapshotting(aggregate_id, snapshot_outdated_at: 1.minute.ago) + + expect(event_store.select_aggregates_for_snapshotting(limit: 10, reschedule_snapshots_scheduled_before: Time.now)) + .to include(aggregate_id, aggregate_id_2) + end + it 'can no longer find the aggregates that are cleared for snapshotting' do event_store.store_snapshots([snapshot]) @@ -127,14 +166,14 @@ class MyAggregate < Sequent::Core::AggregateRoot event_store.store_snapshots([snapshot]) expect(event_store.aggregates_that_need_snapshots(nil)).to be_empty - expect(event_store.aggregates_that_need_snapshots_ordered_by_priority(nil)).to be_empty + expect(event_store.select_aggregates_for_snapshotting(limit: 1)).to be_empty expect(event_store.load_latest_snapshot(aggregate_id)).to eq(snapshot) event_store.delete_snapshots_before(aggregate_id, snapshot.sequence_number + 1) expect(event_store.load_latest_snapshot(aggregate_id)).to eq(nil) expect(event_store.aggregates_that_need_snapshots(nil)).to include(aggregate_id) - expect(event_store.aggregates_that_need_snapshots_ordered_by_priority(nil)).to include(aggregate_id) + expect(event_store.select_aggregates_for_snapshotting(limit: 1)).to include(aggregate_id) end it 'can delete all snapshots' do @@ -142,13 +181,13 @@ class MyAggregate < Sequent::Core::AggregateRoot expect(event_store.aggregates_that_need_snapshots(nil)).to be_empty expect(event_store.load_latest_snapshot(aggregate_id)).to eq(snapshot) - expect(event_store.aggregates_that_need_snapshots_ordered_by_priority(nil)).to be_empty + expect(event_store.select_aggregates_for_snapshotting(limit: 1)).to be_empty event_store.delete_all_snapshots expect(event_store.load_latest_snapshot(aggregate_id)).to eq(nil) expect(event_store.aggregates_that_need_snapshots(nil)).to include(aggregate_id) - expect(event_store.aggregates_that_need_snapshots_ordered_by_priority(nil)).to include(aggregate_id) + expect(event_store.select_aggregates_for_snapshotting(limit: 1)).to include(aggregate_id) end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index c8cd9b93..1a2db7a0 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -22,6 +22,7 @@ RSpec.configure do |c| c.before do + Timecop.return Database.establish_connection Sequent::ApplicationRecord.connection.execute('TRUNCATE commands, aggregates, saved_event_records CASCADE') Sequent::Configuration.reset From 385c9069a4e245379785b839baf62b0be7bb98b0 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Wed, 3 Jul 2024 09:07:35 +0200 Subject: [PATCH 089/128] Pass current time to SQL procedures instead of using NOW() This allows proper mocking of the time in specs using (for example) Timecop. --- db/sequent_pgsql.sql | 14 ++++++-------- lib/sequent/core/event_store.rb | 12 ++++++------ 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/db/sequent_pgsql.sql b/db/sequent_pgsql.sql index fd4bf700..ef05ea78 100644 --- a/db/sequent_pgsql.sql +++ b/db/sequent_pgsql.sql @@ -244,18 +244,17 @@ LANGUAGE SQL AS $$ LIMIT 1; $$; -CREATE OR REPLACE PROCEDURE delete_all_snapshots() +CREATE OR REPLACE PROCEDURE delete_all_snapshots(_now timestamp with time zone DEFAULT NOW()) LANGUAGE plpgsql AS $$ BEGIN UPDATE aggregates_that_need_snapshots - SET snapshot_outdated_at = NOW(), - snapshot_scheduled_at = NULL + SET snapshot_outdated_at = _now WHERE snapshot_outdated_at IS NULL; DELETE FROM snapshot_records; END; $$; -CREATE OR REPLACE PROCEDURE delete_snapshots_before(_aggregate_id uuid, _sequence_number integer) +CREATE OR REPLACE PROCEDURE delete_snapshots_before(_aggregate_id uuid, _sequence_number integer, _now timestamp with time zone DEFAULT NOW()) LANGUAGE plpgsql AS $$ BEGIN DELETE FROM snapshot_records @@ -263,8 +262,7 @@ BEGIN AND sequence_number < _sequence_number; UPDATE aggregates_that_need_snapshots - SET snapshot_outdated_at = NOW(), - snapshot_scheduled_at = NULL + SET snapshot_outdated_at = _now WHERE aggregate_id = _aggregate_id AND snapshot_outdated_at IS NULL AND NOT EXISTS (SELECT 1 FROM snapshot_records WHERE aggregate_id = _aggregate_id); @@ -284,7 +282,7 @@ BEGIN END; $$; -CREATE OR REPLACE FUNCTION select_aggregates_for_snapshotting(_limit integer, _reschedule_snapshot_scheduled_before timestamp with time zone) +CREATE OR REPLACE FUNCTION select_aggregates_for_snapshotting(_limit integer, _reschedule_snapshot_scheduled_before timestamp with time zone, _now timestamp with time zone DEFAULT NOW()) RETURNS TABLE (aggregate_id uuid) LANGUAGE plpgsql AS $$ BEGIN @@ -296,7 +294,7 @@ BEGIN LIMIT _limit FOR UPDATE ) UPDATE aggregates_that_need_snapshots AS row - SET snapshot_scheduled_at = NOW() + SET snapshot_scheduled_at = _now FROM scheduled WHERE row.aggregate_id = scheduled.aggregate_id AND (row.snapshot_scheduled_at IS NULL OR row.snapshot_scheduled_at < _reschedule_snapshot_scheduled_before) diff --git a/lib/sequent/core/event_store.rb b/lib/sequent/core/event_store.rb index cd5ca691..6eae9979 100644 --- a/lib/sequent/core/event_store.rb +++ b/lib/sequent/core/event_store.rb @@ -39,18 +39,18 @@ def load_latest_snapshot(aggregate_id) # Deletes all snapshots for all aggregates def delete_all_snapshots connection.exec_update( - 'CALL delete_all_snapshots()', + 'CALL delete_all_snapshots($1)', 'delete_all_snapshots', - [], + [Time.now], ) end # Deletes all snapshots for aggregate_id with a sequence_number lower than the specified sequence number. def delete_snapshots_before(aggregate_id, sequence_number) connection.exec_update( - 'CALL delete_snapshots_before($1, $2)', + 'CALL delete_snapshots_before($1, $2, $3)', 'delete_snapshots_before', - [aggregate_id, sequence_number], + [aggregate_id, sequence_number, Time.now], ) end @@ -107,9 +107,9 @@ def aggregates_that_need_snapshots(last_aggregate_id, limit = 10) def select_aggregates_for_snapshotting(limit:, reschedule_snapshots_scheduled_before: nil) connection.exec_query( - 'SELECT aggregate_id FROM select_aggregates_for_snapshotting($1, $2)', + 'SELECT aggregate_id FROM select_aggregates_for_snapshotting($1, $2, $3)', 'select_aggregates_for_snapshotting', - [limit, reschedule_snapshots_scheduled_before], + [limit, reschedule_snapshots_scheduled_before, Time.now], ).map { |x| x['aggregate_id'] } end end From f704689683b3d115c05d64f7de8c284116ce1527 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Wed, 3 Jul 2024 10:05:10 +0200 Subject: [PATCH 090/128] Properly track the high-water mark --- db/sequent_pgsql.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/sequent_pgsql.sql b/db/sequent_pgsql.sql index ef05ea78..0cfe3996 100644 --- a/db/sequent_pgsql.sql +++ b/db/sequent_pgsql.sql @@ -215,7 +215,7 @@ BEGIN VALUES (_aggregate_id, _sequence_number) ON CONFLICT (aggregate_id) DO UPDATE SET snapshot_sequence_number_high_water_mark = - LEAST(row.snapshot_sequence_number_high_water_mark, EXCLUDED.snapshot_sequence_number_high_water_mark), + GREATEST(row.snapshot_sequence_number_high_water_mark, EXCLUDED.snapshot_sequence_number_high_water_mark), snapshot_outdated_at = NULL, snapshot_scheduled_at = NULL; From eb8d9f89033a99b041dd149c564cda4ea9cf0685 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Wed, 3 Jul 2024 10:06:25 +0200 Subject: [PATCH 091/128] Update template project schema --- .../template_project/db/sequent_pgsql.sql | 52 ++++++++++++------- .../template_project/db/sequent_schema.sql | 6 ++- 2 files changed, 36 insertions(+), 22 deletions(-) diff --git a/lib/sequent/generator/template_project/db/sequent_pgsql.sql b/lib/sequent/generator/template_project/db/sequent_pgsql.sql index f481beb4..0cfe3996 100644 --- a/lib/sequent/generator/template_project/db/sequent_pgsql.sql +++ b/lib/sequent/generator/template_project/db/sequent_pgsql.sql @@ -211,12 +211,13 @@ BEGIN _aggregate_id = _snapshot->>'aggregate_id'; _sequence_number = _snapshot->'sequence_number'; - INSERT INTO aggregates_that_need_snapshots AS row (aggregate_id, snapshot_outdated_at, snapshot_sequence_number_high_water_mark) - VALUES (_aggregate_id, NULL, _sequence_number) + INSERT INTO aggregates_that_need_snapshots AS row (aggregate_id, snapshot_sequence_number_high_water_mark) + VALUES (_aggregate_id, _sequence_number) ON CONFLICT (aggregate_id) DO UPDATE - SET snapshot_outdated_at = EXCLUDED.snapshot_outdated_at, - snapshot_sequence_number_high_water_mark = - LEAST(row.snapshot_sequence_number_high_water_mark, EXCLUDED.snapshot_sequence_number_high_water_mark); + SET snapshot_sequence_number_high_water_mark = + GREATEST(row.snapshot_sequence_number_high_water_mark, EXCLUDED.snapshot_sequence_number_high_water_mark), + snapshot_outdated_at = NULL, + snapshot_scheduled_at = NULL; INSERT INTO snapshot_records (aggregate_id, sequence_number, created_at, snapshot_type, snapshot_json) VALUES ( @@ -243,17 +244,17 @@ LANGUAGE SQL AS $$ LIMIT 1; $$; -CREATE OR REPLACE PROCEDURE delete_all_snapshots() +CREATE OR REPLACE PROCEDURE delete_all_snapshots(_now timestamp with time zone DEFAULT NOW()) LANGUAGE plpgsql AS $$ BEGIN UPDATE aggregates_that_need_snapshots - SET snapshot_outdated_at = NOW() + SET snapshot_outdated_at = _now WHERE snapshot_outdated_at IS NULL; DELETE FROM snapshot_records; END; $$; -CREATE OR REPLACE PROCEDURE delete_snapshots_before(_aggregate_id uuid, _sequence_number integer) +CREATE OR REPLACE PROCEDURE delete_snapshots_before(_aggregate_id uuid, _sequence_number integer, _now timestamp with time zone DEFAULT NOW()) LANGUAGE plpgsql AS $$ BEGIN DELETE FROM snapshot_records @@ -261,7 +262,7 @@ BEGIN AND sequence_number < _sequence_number; UPDATE aggregates_that_need_snapshots - SET snapshot_outdated_at = NOW() + SET snapshot_outdated_at = _now WHERE aggregate_id = _aggregate_id AND snapshot_outdated_at IS NULL AND NOT EXISTS (SELECT 1 FROM snapshot_records WHERE aggregate_id = _aggregate_id); @@ -281,15 +282,23 @@ BEGIN END; $$; -CREATE OR REPLACE FUNCTION aggregates_that_need_snapshots_ordered_by_priority(_limit integer) +CREATE OR REPLACE FUNCTION select_aggregates_for_snapshotting(_limit integer, _reschedule_snapshot_scheduled_before timestamp with time zone, _now timestamp with time zone DEFAULT NOW()) RETURNS TABLE (aggregate_id uuid) LANGUAGE plpgsql AS $$ BEGIN - RETURN QUERY SELECT a.aggregate_id - FROM aggregates_that_need_snapshots a - WHERE snapshot_outdated_at IS NOT NULL - ORDER BY snapshot_outdated_at ASC, snapshot_sequence_number_high_water_mark DESC, aggregate_id ASC - LIMIT _limit; + RETURN QUERY WITH scheduled AS MATERIALIZED ( + SELECT a.aggregate_id + FROM aggregates_that_need_snapshots AS a + WHERE snapshot_outdated_at IS NOT NULL + ORDER BY snapshot_outdated_at ASC, snapshot_sequence_number_high_water_mark DESC, aggregate_id ASC + LIMIT _limit + FOR UPDATE + ) UPDATE aggregates_that_need_snapshots AS row + SET snapshot_scheduled_at = _now + FROM scheduled + WHERE row.aggregate_id = scheduled.aggregate_id + AND (row.snapshot_scheduled_at IS NULL OR row.snapshot_scheduled_at < _reschedule_snapshot_scheduled_before) + RETURNING row.aggregate_id; END; $$; @@ -320,7 +329,8 @@ BEGIN END; $$; -CREATE OR REPLACE VIEW command_records (id, user_id, aggregate_id, command_type, command_json, created_at, event_aggregate_id, event_sequence_number) AS +DROP VIEW IF EXISTS command_records; +CREATE VIEW command_records (id, user_id, aggregate_id, command_type, command_json, created_at, event_aggregate_id, event_sequence_number) AS SELECT id, user_id, aggregate_id, @@ -331,7 +341,8 @@ CREATE OR REPLACE VIEW command_records (id, user_id, aggregate_id, command_type, event_sequence_number FROM commands command; -CREATE OR REPLACE VIEW event_records (aggregate_id, partition_key, sequence_number, created_at, event_type, event_json, command_record_id, xact_id) AS +DROP VIEW IF EXISTS event_records; +CREATE VIEW event_records (aggregate_id, partition_key, sequence_number, created_at, event_type, event_json, command_record_id, xact_id) AS SELECT aggregate.aggregate_id, event.partition_key, event.sequence_number, @@ -340,11 +351,12 @@ CREATE OR REPLACE VIEW event_records (aggregate_id, partition_key, sequence_numb enrich_event_json(event) AS event_json, command_id, event.xact_id - FROM aggregates aggregate - JOIN events event ON aggregate.aggregate_id = event.aggregate_id AND aggregate.events_partition_key = event.partition_key + FROM events event + JOIN aggregates aggregate ON aggregate.aggregate_id = event.aggregate_id AND aggregate.events_partition_key = event.partition_key JOIN event_types type ON event.event_type_id = type.id; -CREATE OR REPLACE VIEW stream_records (aggregate_id, events_partition_key, aggregate_type, created_at) AS +DROP VIEW IF EXISTS stream_records; +CREATE VIEW stream_records (aggregate_id, events_partition_key, aggregate_type, created_at) AS SELECT aggregates.aggregate_id, aggregates.events_partition_key, aggregate_types.type, diff --git a/lib/sequent/generator/template_project/db/sequent_schema.sql b/lib/sequent/generator/template_project/db/sequent_schema.sql index a7164edd..3bc3b7a0 100644 --- a/lib/sequent/generator/template_project/db/sequent_schema.sql +++ b/lib/sequent/generator/template_project/db/sequent_schema.sql @@ -68,16 +68,18 @@ ALTER TABLE events_aggregate CLUSTER ON events_aggregate_pkey; CREATE TABLE aggregates_that_need_snapshots ( aggregate_id uuid NOT NULL PRIMARY KEY REFERENCES aggregates (aggregate_id) ON UPDATE CASCADE ON DELETE CASCADE, + snapshot_sequence_number_high_water_mark integer, snapshot_outdated_at timestamp with time zone, - snapshot_sequence_number_high_water_mark integer + snapshot_scheduled_at timestamp with time zone ); CREATE INDEX aggregates_that_need_snapshots_outdated_idx ON aggregates_that_need_snapshots (snapshot_outdated_at ASC, snapshot_sequence_number_high_water_mark DESC, aggregate_id ASC) WHERE snapshot_outdated_at IS NOT NULL; COMMENT ON TABLE aggregates_that_need_snapshots IS 'Contains a row for every aggregate with more events than its snapshot threshold.'; -COMMENT ON COLUMN aggregates_that_need_snapshots.snapshot_outdated_at IS 'Not NULL indicates a snapshot is needed since the stored timestamp'; COMMENT ON COLUMN aggregates_that_need_snapshots.snapshot_sequence_number_high_water_mark IS 'The highest sequence number of the stored snapshot. Kept when snapshot are deleted to more easily query aggregates that need snapshotting the most'; +COMMENT ON COLUMN aggregates_that_need_snapshots.snapshot_outdated_at IS 'Not NULL indicates a snapshot is needed since the stored timestamp'; +COMMENT ON COLUMN aggregates_that_need_snapshots.snapshot_scheduled_at IS 'Not NULL indicates a snapshot is in the process of being taken'; CREATE TABLE snapshot_records ( aggregate_id uuid NOT NULL, From c931d9153cb5d0b12a300cf9cb80cbe3424ab51b Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Wed, 10 Jul 2024 11:19:18 +0200 Subject: [PATCH 092/128] Move SnapshotStore module into separate file --- lib/sequent/core/event_store.rb | 108 +-------------------------- lib/sequent/core/snapshot_store.rb | 114 +++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+), 107 deletions(-) create mode 100644 lib/sequent/core/snapshot_store.rb diff --git a/lib/sequent/core/event_store.rb b/lib/sequent/core/event_store.rb index 6eae9979..91a31ade 100644 --- a/lib/sequent/core/event_store.rb +++ b/lib/sequent/core/event_store.rb @@ -4,116 +4,10 @@ require_relative 'event_record' require_relative 'sequent_oj' require_relative 'snapshot_record' +require_relative 'snapshot_store' module Sequent module Core - module SnapshotStore - def store_snapshots(snapshots) - json = Sequent::Core::Oj.dump( - snapshots.map do |snapshot| - { - aggregate_id: snapshot.aggregate_id, - sequence_number: snapshot.sequence_number, - created_at: snapshot.created_at, - snapshot_type: snapshot.class.name, - snapshot_json: snapshot, - } - end, - ) - connection.exec_update( - 'CALL store_snapshots($1)', - 'store_snapshots', - [json], - ) - end - - def load_latest_snapshot(aggregate_id) - snapshot_hash = connection.exec_query( - 'SELECT * FROM load_latest_snapshot($1)', - 'load_latest_snapshot', - [aggregate_id], - ).first - deserialize_event(snapshot_hash) unless snapshot_hash['aggregate_id'].nil? - end - - # Deletes all snapshots for all aggregates - def delete_all_snapshots - connection.exec_update( - 'CALL delete_all_snapshots($1)', - 'delete_all_snapshots', - [Time.now], - ) - end - - # Deletes all snapshots for aggregate_id with a sequence_number lower than the specified sequence number. - def delete_snapshots_before(aggregate_id, sequence_number) - connection.exec_update( - 'CALL delete_snapshots_before($1, $2, $3)', - 'delete_snapshots_before', - [aggregate_id, sequence_number, Time.now], - ) - end - - # Marks an aggregate for snapshotting. Marked aggregates will be - # picked up by the background snapshotting task. Another way to - # mark aggregates for snapshotting is to pass the - # +EventStream#snapshot_outdated_at+ property to the - # +#store_events+ method as is done automatically by the - # +AggregateRepository+ based on the aggregate's - # +snapshot_threshold+. - def mark_aggregate_for_snapshotting(aggregate_id, snapshot_outdated_at: Time.now) - connection.exec_update(<<~EOS, 'mark_aggregate_for_snapshotting', [aggregate_id, snapshot_outdated_at]) - INSERT INTO aggregates_that_need_snapshots AS row (aggregate_id, snapshot_outdated_at) - VALUES ($1, $2) - ON CONFLICT (aggregate_id) DO UPDATE - SET snapshot_outdated_at = LEAST(row.snapshot_outdated_at, EXCLUDED.snapshot_outdated_at), - snapshot_scheduled_at = NULL - EOS - end - - # Stops snapshotting the specified aggregate. Any existing - # snapshots for this aggregate are also deleted. - def clear_aggregate_for_snapshotting(aggregate_id) - connection.exec_update( - 'DELETE FROM aggregates_that_need_snapshots WHERE aggregate_id = $1', - 'clear_aggregate_for_snapshotting', - [aggregate_id], - ) - end - - # Stops snapshotting all aggregates where the last event - # occurred before the indicated timestamp. Any existing - # snapshots for this aggregate are also deleted. - def clear_aggregates_for_snapshotting_with_last_event_before(timestamp) - connection.exec_update(<<~EOS, 'clear_aggregates_for_snapshotting_with_last_event_before', [timestamp]) - DELETE FROM aggregates_that_need_snapshots s - WHERE NOT EXISTS (SELECT * - FROM aggregates a - JOIN events e ON (a.aggregate_id, a.events_partition_key) = (e.aggregate_id, e.partition_key) - WHERE a.aggregate_id = s.aggregate_id AND e.created_at >= $1) - EOS - end - - ## - # Returns the ids of aggregates that need a new snapshot. - # - def aggregates_that_need_snapshots(last_aggregate_id, limit = 10) - connection.exec_query( - 'SELECT aggregate_id FROM aggregates_that_need_snapshots($1, $2)', - 'aggregates_that_need_snapshots', - [last_aggregate_id, limit], - ).map { |x| x['aggregate_id'] } - end - - def select_aggregates_for_snapshotting(limit:, reschedule_snapshots_scheduled_before: nil) - connection.exec_query( - 'SELECT aggregate_id FROM select_aggregates_for_snapshotting($1, $2, $3)', - 'select_aggregates_for_snapshotting', - [limit, reschedule_snapshots_scheduled_before, Time.now], - ).map { |x| x['aggregate_id'] } - end - end - class EventStore include SnapshotStore include ActiveRecord::ConnectionAdapters::Quoting diff --git a/lib/sequent/core/snapshot_store.rb b/lib/sequent/core/snapshot_store.rb new file mode 100644 index 00000000..0d96faae --- /dev/null +++ b/lib/sequent/core/snapshot_store.rb @@ -0,0 +1,114 @@ +# frozen_string_literal: true + +require_relative 'sequent_oj' + +module Sequent + module Core + module SnapshotStore + def store_snapshots(snapshots) + json = Sequent::Core::Oj.dump( + snapshots.map do |snapshot| + { + aggregate_id: snapshot.aggregate_id, + sequence_number: snapshot.sequence_number, + created_at: snapshot.created_at, + snapshot_type: snapshot.class.name, + snapshot_json: snapshot, + } + end, + ) + connection.exec_update( + 'CALL store_snapshots($1)', + 'store_snapshots', + [json], + ) + end + + def load_latest_snapshot(aggregate_id) + snapshot_hash = connection.exec_query( + 'SELECT * FROM load_latest_snapshot($1)', + 'load_latest_snapshot', + [aggregate_id], + ).first + deserialize_event(snapshot_hash) unless snapshot_hash['aggregate_id'].nil? + end + + # Deletes all snapshots for all aggregates + def delete_all_snapshots + connection.exec_update( + 'CALL delete_all_snapshots($1)', + 'delete_all_snapshots', + [Time.now], + ) + end + + # Deletes all snapshots for aggregate_id with a sequence_number lower than the specified sequence number. + def delete_snapshots_before(aggregate_id, sequence_number) + connection.exec_update( + 'CALL delete_snapshots_before($1, $2, $3)', + 'delete_snapshots_before', + [aggregate_id, sequence_number, Time.now], + ) + end + + # Marks an aggregate for snapshotting. Marked aggregates will be + # picked up by the background snapshotting task. Another way to + # mark aggregates for snapshotting is to pass the + # +EventStream#snapshot_outdated_at+ property to the + # +#store_events+ method as is done automatically by the + # +AggregateRepository+ based on the aggregate's + # +snapshot_threshold+. + def mark_aggregate_for_snapshotting(aggregate_id, snapshot_outdated_at: Time.now) + connection.exec_update(<<~EOS, 'mark_aggregate_for_snapshotting', [aggregate_id, snapshot_outdated_at]) + INSERT INTO aggregates_that_need_snapshots AS row (aggregate_id, snapshot_outdated_at) + VALUES ($1, $2) + ON CONFLICT (aggregate_id) DO UPDATE + SET snapshot_outdated_at = LEAST(row.snapshot_outdated_at, EXCLUDED.snapshot_outdated_at), + snapshot_scheduled_at = NULL + EOS + end + + # Stops snapshotting the specified aggregate. Any existing + # snapshots for this aggregate are also deleted. + def clear_aggregate_for_snapshotting(aggregate_id) + connection.exec_update( + 'DELETE FROM aggregates_that_need_snapshots WHERE aggregate_id = $1', + 'clear_aggregate_for_snapshotting', + [aggregate_id], + ) + end + + # Stops snapshotting all aggregates where the last event + # occurred before the indicated timestamp. Any existing + # snapshots for this aggregate are also deleted. + def clear_aggregates_for_snapshotting_with_last_event_before(timestamp) + connection.exec_update(<<~EOS, 'clear_aggregates_for_snapshotting_with_last_event_before', [timestamp]) + DELETE FROM aggregates_that_need_snapshots s + WHERE NOT EXISTS (SELECT * + FROM aggregates a + JOIN events e ON (a.aggregate_id, a.events_partition_key) = (e.aggregate_id, e.partition_key) + WHERE a.aggregate_id = s.aggregate_id AND e.created_at >= $1) + EOS + end + + ## + # Returns the ids of aggregates that need a new snapshot. + # + def aggregates_that_need_snapshots(last_aggregate_id, limit = 10) + connection.exec_query( + 'SELECT aggregate_id FROM aggregates_that_need_snapshots($1, $2)', + 'aggregates_that_need_snapshots', + [last_aggregate_id, limit], + ).map { |x| x['aggregate_id'] } + end + + def select_aggregates_for_snapshotting(limit:, reschedule_snapshots_scheduled_before: nil) + connection.exec_query( + 'SELECT aggregate_id FROM select_aggregates_for_snapshotting($1, $2, $3)', + 'select_aggregates_for_snapshotting', + [limit, reschedule_snapshots_scheduled_before, Time.now], + ).map { |x| x['aggregate_id'] } + end + end + end +end From 104c59d43b9446e1e3f1fa1fb1e2914ac0179fb7 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Wed, 10 Jul 2024 12:07:55 +0200 Subject: [PATCH 093/128] Use helper functions to call Pg/SQL functions and procedures --- lib/sequent/core/event_store.rb | 29 ++++------------ lib/sequent/core/helpers/pgsql_helpers.rb | 25 ++++++++++++++ lib/sequent/core/snapshot_store.rb | 41 +++++++---------------- 3 files changed, 45 insertions(+), 50 deletions(-) create mode 100644 lib/sequent/core/helpers/pgsql_helpers.rb diff --git a/lib/sequent/core/event_store.rb b/lib/sequent/core/event_store.rb index 91a31ade..c7a9c6fa 100644 --- a/lib/sequent/core/event_store.rb +++ b/lib/sequent/core/event_store.rb @@ -2,6 +2,7 @@ require 'forwardable' require_relative 'event_record' +require_relative 'helpers/pgsql_helpers' require_relative 'sequent_oj' require_relative 'snapshot_record' require_relative 'snapshot_store' @@ -9,6 +10,7 @@ module Sequent module Core class EventStore + include Helpers::PgsqlHelpers include SnapshotStore include ActiveRecord::ConnectionAdapters::Quoting extend Forwardable @@ -84,11 +86,7 @@ def stream_events_for_aggregate(aggregate_id, load_until: nil, &block) end def load_event(aggregate_id, sequence_number) - event_hash = connection.exec_query( - 'SELECT * FROM load_event($1, $2)', - 'load_event', - [aggregate_id, sequence_number], - ).first + event_hash = query_function('load_event', [aggregate_id, sequence_number]).first deserialize_event(event_hash) if event_hash end @@ -182,11 +180,7 @@ def permanently_delete_event_stream(aggregate_id) end def permanently_delete_event_streams(aggregate_ids) - connection.exec_update( - 'CALL permanently_delete_event_streams($1)', - 'permanently_delete_event_streams', - [aggregate_ids.to_json], - ) + call_procedure('permanently_delete_event_streams', [aggregate_ids.to_json]) end def permanently_delete_commands_without_events(aggregate_id: nil, organization_id: nil) @@ -194,11 +188,7 @@ def permanently_delete_commands_without_events(aggregate_id: nil, organization_i fail ArgumentError, 'aggregate_id and/or organization_id must be specified' end - connection.exec_update( - 'CALL permanently_delete_commands_without_events($1, $2)', - 'permanently_delete_commands_without_events', - [aggregate_id, organization_id], - ) + call_procedure('permanently_delete_commands_without_events', [aggregate_id, organization_id]) end private @@ -208,11 +198,7 @@ def connection end def query_events(aggregate_ids, use_snapshots = true, load_until = nil) - connection.exec_query( - 'SELECT * FROM load_events($1::JSONB, $2, $3)', - 'load_events', - [aggregate_ids.to_json, use_snapshots, load_until], - ) + query_function('load_events', [aggregate_ids.to_json, use_snapshots, load_until]) end def deserialize_event(event_hash) @@ -256,8 +242,7 @@ def store_events(command, streams_with_events = []) end, ] end - connection.exec_update( - 'CALL store_events($1, $2)', + call_procedure( 'store_events', [ Sequent::Core::Oj.dump(command_record), diff --git a/lib/sequent/core/helpers/pgsql_helpers.rb b/lib/sequent/core/helpers/pgsql_helpers.rb new file mode 100644 index 00000000..bf65f0c9 --- /dev/null +++ b/lib/sequent/core/helpers/pgsql_helpers.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Sequent + module Core + module Helpers + module PgsqlHelpers + def call_procedure(procedure, params) + statement = "CALL #{procedure}(#{bind_placeholders(params)})" + connection.exec_update(statement, procedure, params) + end + + def query_function(function, params, columns = ['*']) + query = "SELECT #{columns.join(', ')} FROM #{function}(#{bind_placeholders(params)})" + connection.exec_query(query, function, params) + end + + private + + def bind_placeholders(params) + (1..params.size).map { |n| "$#{n}" }.join(', ') + end + end + end + end +end diff --git a/lib/sequent/core/snapshot_store.rb b/lib/sequent/core/snapshot_store.rb index 0d96faae..cd5edd15 100644 --- a/lib/sequent/core/snapshot_store.rb +++ b/lib/sequent/core/snapshot_store.rb @@ -1,10 +1,13 @@ # frozen_string_literal: true require_relative 'sequent_oj' +require_relative 'helpers/pgsql_helpers' module Sequent module Core module SnapshotStore + include Helpers::PgsqlHelpers + def store_snapshots(snapshots) json = Sequent::Core::Oj.dump( snapshots.map do |snapshot| @@ -17,38 +20,23 @@ def store_snapshots(snapshots) } end, ) - connection.exec_update( - 'CALL store_snapshots($1)', - 'store_snapshots', - [json], - ) + + call_procedure('store_snapshots', [json]) end def load_latest_snapshot(aggregate_id) - snapshot_hash = connection.exec_query( - 'SELECT * FROM load_latest_snapshot($1)', - 'load_latest_snapshot', - [aggregate_id], - ).first + snapshot_hash = query_function('load_latest_snapshot', [aggregate_id]).first deserialize_event(snapshot_hash) unless snapshot_hash['aggregate_id'].nil? end # Deletes all snapshots for all aggregates def delete_all_snapshots - connection.exec_update( - 'CALL delete_all_snapshots($1)', - 'delete_all_snapshots', - [Time.now], - ) + call_procedure('delete_all_snapshots', [Time.now]) end # Deletes all snapshots for aggregate_id with a sequence_number lower than the specified sequence number. def delete_snapshots_before(aggregate_id, sequence_number) - connection.exec_update( - 'CALL delete_snapshots_before($1, $2, $3)', - 'delete_snapshots_before', - [aggregate_id, sequence_number, Time.now], - ) + call_procedure('delete_snapshots_before', [aggregate_id, sequence_number, Time.now]) end # Marks an aggregate for snapshotting. Marked aggregates will be @@ -95,19 +83,16 @@ def clear_aggregates_for_snapshotting_with_last_event_before(timestamp) # Returns the ids of aggregates that need a new snapshot. # def aggregates_that_need_snapshots(last_aggregate_id, limit = 10) - connection.exec_query( - 'SELECT aggregate_id FROM aggregates_that_need_snapshots($1, $2)', - 'aggregates_that_need_snapshots', - [last_aggregate_id, limit], - ).map { |x| x['aggregate_id'] } + query_function('aggregates_that_need_snapshots', [last_aggregate_id, limit], ['aggregate_id']) + .pluck('aggregate_id') end def select_aggregates_for_snapshotting(limit:, reschedule_snapshots_scheduled_before: nil) - connection.exec_query( - 'SELECT aggregate_id FROM select_aggregates_for_snapshotting($1, $2, $3)', + query_function( 'select_aggregates_for_snapshotting', [limit, reschedule_snapshots_scheduled_before, Time.now], - ).map { |x| x['aggregate_id'] } + ['aggregate_id'], + ).pluck('aggregate_id') end end end From 25c046a395f432f283bf9dd1e0ad30488e70f061 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Wed, 10 Jul 2024 14:31:43 +0200 Subject: [PATCH 094/128] Updated rake tasks to take and delete snapshots --- docs/docs/concepts/snapshotting.md | 24 +++++--------- lib/sequent/rake/migration_tasks.rb | 49 +++++++++++++++++++---------- 2 files changed, 39 insertions(+), 34 deletions(-) diff --git a/docs/docs/concepts/snapshotting.md b/docs/docs/concepts/snapshotting.md index f2fb0887..0da17788 100644 --- a/docs/docs/concepts/snapshotting.md +++ b/docs/docs/concepts/snapshotting.md @@ -13,26 +13,16 @@ class UserNames < Sequent::AggregateRoot end ``` +Whenever more events are stored for an aggregate than its snapshot +threshold a record is stored in the `aggregates_that_need_snapshots` +table. You can use the rake `sequent:snapshots:take_snapshots[limit]` +task to snapshot up to `limit` highest priority aggregates. -You then also need to update the existing [StreamRecords](event_store.html#stream_records) in the database to ensure they are also eligible for snapshotting. -This can be done via `bundle exec rake sequent:snapshotting:set_snapshot_threshold[Usernames,100]`. - -After this, snapshots can be taken with a `Sequent::Core::SnapshotCommand` for example by a Rake task. - -```ruby -namespace :snapshot do - task :take_all do - catch (:done) do - while true - Sequent.command_service.execute_commands Sequent::Core::SnapshotCommand.new(limit: 10) - end - end - end -end -``` +You can schedule this task to run in the background regularly as it +will simply do nothing if there are no aggregates that need a new +snapshot. **Important:** When you enable snapshotting you **must** delete all snapshots after each deploy. The AggregateRoot root state is dumped in the database. If there is a new version of an AggregateRoot class definition, the snapshotted state can not be loaded. {: .notice--danger} To delete all snapshots, you can execute `bundle exec rake sequent:snapshotting:delete_all`. - diff --git a/lib/sequent/rake/migration_tasks.rb b/lib/sequent/rake/migration_tasks.rb index e4dc8481..88834690 100644 --- a/lib/sequent/rake/migration_tasks.rb +++ b/lib/sequent/rake/migration_tasks.rb @@ -208,33 +208,48 @@ def register_tasks! desc <<~EOS Rake task that runs before all snapshots rake tasks. Hook applications can use to for instance run other rake tasks. EOS - task :init + task :init, :set_env_var do + ensure_sequent_env_set! + + Sequent.configuration.command_handlers << Sequent::Core::AggregateSnapshotter.new + + db_config = Sequent::Support::Database.read_config(@env) + Sequent::Support::Database.establish_connection(db_config) + end - task :set_snapshot_threshold, %i[aggregate_type threshold] => ['sequent:init', :init] do |_t, args| - aggregate_type = args['aggregate_type'] - threshold = args['threshold'] + desc <<~EOS + Takes up-to `limit` snapshots, starting with the highest priority aggregates (based on snapshot outdated time and number of events) + EOS + task :take_snapshots, %i[limit] => ['sequent:init', :init] do |_t, args| + limit = args['limit']&.to_i - unless aggregate_type + unless limit fail ArgumentError, - 'usage rake sequent:snapshots:set_snapshot_threshold[AggregegateType,threshold]' + 'usage rake sequent:snapshots:take_snapshots[limit]' end - unless threshold + + aggregate_ids = Sequent.configuration.event_store.select_aggregates_for_snapshotting(limit:) + + Sequent.logger.info "Taking #{aggregate_ids.size} snapshots" + aggregate_ids.each do |aggregate_id| + Sequent.command_service.execute_commands(Sequent::Core::TakeSnapshot.new(aggregate_id:)) + end + end + + task :take_snapshot, %i[aggregate_id] => ['sequent:init', :init] do |_t, args| + aggregate_id = args['aggregate_id'] + + unless aggregate_id fail ArgumentError, - 'usage rake sequent:snapshots:set_snapshot_threshold[AggregegateType,threshold]' + 'usage rake sequent:snapshots:take_snapshot[aggregate_id]' end - execute <<~EOS - UPDATE #{Sequent.configuration.stream_record_class} SET snapshot_threshold = #{threshold.to_i} WHERE aggregate_type = '#{aggregate_type}' - EOS + Sequent.command_service.execute_commands(Sequent::Core::TakeSnapshot.new(aggregate_id:)) end task delete_all: ['sequent:init', :init] do - result = Sequent::ApplicationRecord - .connection - .execute(<<~EOS) - DELETE FROM #{Sequent.configuration.snapshot_record_class.table_name}' - EOS - Sequent.logger.info "Deleted #{result.cmd_tuples} aggregate snapshots from the event store" + Sequent.configuration.event_store.delete_all_snapshots + Sequent.logger.info 'Deleted all aggregate snapshots from the event store' end end end From b211b62ab239b93d6a3aa3a87a689c9ebdba4ef7 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Wed, 11 Sep 2024 15:27:39 +0200 Subject: [PATCH 095/128] Restore overridable `init` task and document new tasks --- lib/sequent/rake/migration_tasks.rb | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/lib/sequent/rake/migration_tasks.rb b/lib/sequent/rake/migration_tasks.rb index 88834690..0545b5f3 100644 --- a/lib/sequent/rake/migration_tasks.rb +++ b/lib/sequent/rake/migration_tasks.rb @@ -208,7 +208,9 @@ def register_tasks! desc <<~EOS Rake task that runs before all snapshots rake tasks. Hook applications can use to for instance run other rake tasks. EOS - task :init, :set_env_var do + task :init + + task :connect, ['sequent:init', :init, :set_env_var] do ensure_sequent_env_set! Sequent.configuration.command_handlers << Sequent::Core::AggregateSnapshotter.new @@ -220,7 +222,7 @@ def register_tasks! desc <<~EOS Takes up-to `limit` snapshots, starting with the highest priority aggregates (based on snapshot outdated time and number of events) EOS - task :take_snapshots, %i[limit] => ['sequent:init', :init] do |_t, args| + task :take_snapshots, %i[limit] => :connect do |_t, args| limit = args['limit']&.to_i unless limit @@ -236,7 +238,10 @@ def register_tasks! end end - task :take_snapshot, %i[aggregate_id] => ['sequent:init', :init] do |_t, args| + desc <<~EOS + Takes a new snapshot for the aggregate specified by `aggregate_id` + EOS + task :take_snapshot, %i[aggregate_id] => :connect do |_t, args| aggregate_id = args['aggregate_id'] unless aggregate_id @@ -247,7 +252,10 @@ def register_tasks! Sequent.command_service.execute_commands(Sequent::Core::TakeSnapshot.new(aggregate_id:)) end - task delete_all: ['sequent:init', :init] do + desc <<~EOS + Delete all aggregate snapshots, which can negatively impact performance of a running system. + EOS + task delete_all: :connect do Sequent.configuration.event_store.delete_all_snapshots Sequent.logger.info 'Deleted all aggregate snapshots from the event store' end From 7e895983deef3c9148288e30558f667c90ee4ecf Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Thu, 12 Sep 2024 08:57:38 +0200 Subject: [PATCH 096/128] Updated changelog for version 8 changes (partitioning, etc) --- CHANGELOG.md | 56 ++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 41 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cd8eca93..2e9470d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,25 +1,51 @@ # Changelog 8.0.x (changes since 7.0.1) -- Sequent now requires at least Ruby 3.1 and ActiveRecord 7.1. +- Sequent now requires at least Ruby 3.2, ActiveRecord 7.1, and + PostgreSQL 14. +- The internal database schema has been re-organized to support table + partitioning so the events of related aggregates can be stored + together. Partitioning is optional. You can override the + `AggregateRoot#events_partition_key` method to return the partition + key of an aggregate that will be used by PostgreSQL to store the + events in the correct partition. +- Storage logic of Sequent is now done by PostgreSQL stored + procedures. This reduces the number of round-trips necessary between + the Ruby process and the PostgreSQL server and optimizes storage + requirements. - `AggregateRoot#take_snapshot!` has been replaced with `AggregateRoot#take_snapshot` which returns the snapshot instead of adding it to the uncommitted events. Snapshots can be stored using - `EventStore#store_snapshots`. -- Events, commands, and snapshot can now be serialized directly to a - `JSON` or `JSONB` PostgreSQL column. Support for the `TEXT` column - type will be removed in a future release. + `EventStore#store_snapshots` method. +- The `stream_records`, `event_records`, and `command_records` + relations are no longer tables but views on the underlying + `aggregates`, `events`, and `commands` tables. These views take care + of joining related tables using the correct partitioning key and + type ids. The views are mainly for backwards compatibility with the + `*Record` Ruby classes and can only be used for querying. For + updates the `EventStore` (and higher-level `AggregateRepository`) + classes should be used. New APIs have been added so there is no + longer a need to use ActiveRecord methods directly to manage the + event store. +- Events, commands, and snapshot are now serialized directly to a + `JSON` or `JSONB` column. The `TEXT` column type is no longer + supported. JSON size is minimized by extracting various repetitive + fields (such as `aggregate_id`) into database columns. - Snapshots are now stored into a separate `snapshot_records` table - instead of being mixed in with events in the `event_records` table. - This also makes it possible to store snapshots without an associated - command record. -- The `id` column `event_streams` has been removed and the primary key - is now the `aggregate_id` column. -- The `id` column of `event_records` has been removed and the primary + instead of being mixed in with events in the `events` table. This + also makes it possible to store snapshots without an associated + command record. Aggregates that need snapshots are tracked in a + separate `aggregates_that_need_snapshots` so that snapshots can be + created more quickly by a background snapshotting process. +- The `parent`, `origin` and `children` methods of `CommandRecord` and + `EventRecord` have been replaced by more specific methods + (`origin_command`, `parent_event`, etc) and will always return a + record of the appropriate type. +- Events that are updated or deleted (a rare occurrence) are archived + into the `saved_event_records` table. +- The `id` column of `aggregates` has been removed and the primary + key is now the `aggregate_id` column. +- The `id` column of `events` has been removed and the primary key are now the `aggregate_id` and `sequence_number` columns. -- New APIs have been added to the event store to prepare for more - internal storage changes. Events streams and commands can now be - permanently deleted, snapshots can be managed, and events can be - loaded individually. # Changelog 7.0.1 (changes since 7.0.0) From f65e189174c2b7478ba7dc1d25f20505dda465bb Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Thu, 12 Sep 2024 09:44:16 +0200 Subject: [PATCH 097/128] Update template project generator to require less maintenance --- lib/sequent/generator/project.rb | 10 +- .../template_project/db/sequent_pgsql.sql | 416 ------------------ .../template_project/db/sequent_schema.rb | 10 - .../template_project/db/sequent_schema.sql | 108 ----- .../generator/template_project/ruby-version | 1 - 5 files changed, 3 insertions(+), 542 deletions(-) delete mode 100644 lib/sequent/generator/template_project/db/sequent_pgsql.sql delete mode 100644 lib/sequent/generator/template_project/db/sequent_schema.rb delete mode 100644 lib/sequent/generator/template_project/db/sequent_schema.sql delete mode 100644 lib/sequent/generator/template_project/ruby-version diff --git a/lib/sequent/generator/project.rb b/lib/sequent/generator/project.rb index 4c0fc227..dfcdbd27 100644 --- a/lib/sequent/generator/project.rb +++ b/lib/sequent/generator/project.rb @@ -14,7 +14,6 @@ def initialize(path_or_name) def execute make_directory copy_files - rename_ruby_version rename_app_file replace_app_name end @@ -27,12 +26,9 @@ def make_directory def copy_files FileUtils.copy_entry(File.expand_path('template_project', __dir__), path) - end - - # Hidden files are by default excluded from gem build. - # Therefor we need to rename the ruby-version to .ruby-version. - def rename_ruby_version - FileUtils.mv("#{path}/ruby-version", "#{path}/.ruby-version") + ['.ruby-version', 'db/sequent_schema.rb', 'db/sequent_schema.sql', 'db/sequent_pgsql.sql'].each do |file| + FileUtils.copy_entry(File.expand_path("../../../#{file}", __dir__), "#{path}/#{file}") + end end def rename_app_file diff --git a/lib/sequent/generator/template_project/db/sequent_pgsql.sql b/lib/sequent/generator/template_project/db/sequent_pgsql.sql deleted file mode 100644 index 0cfe3996..00000000 --- a/lib/sequent/generator/template_project/db/sequent_pgsql.sql +++ /dev/null @@ -1,416 +0,0 @@ -DROP TYPE IF EXISTS aggregate_event_type CASCADE; -CREATE TYPE aggregate_event_type AS ( - aggregate_type text, - aggregate_id uuid, - events_partition_key text, - event_type text, - event_json jsonb -); - -CREATE OR REPLACE FUNCTION enrich_command_json(command commands) RETURNS jsonb -LANGUAGE plpgsql AS $$ -BEGIN - RETURN jsonb_build_object( - 'command_type', (SELECT type FROM command_types WHERE command_types.id = command.command_type_id), - 'created_at', command.created_at, - 'user_id', command.user_id, - 'aggregate_id', command.aggregate_id, - 'event_aggregate_id', command.event_aggregate_id, - 'event_sequence_number', command.event_sequence_number - ) - || command.command_json; -END -$$; - -CREATE OR REPLACE FUNCTION enrich_event_json(event events) RETURNS jsonb -LANGUAGE plpgsql AS $$ -BEGIN - RETURN jsonb_build_object( - 'aggregate_id', event.aggregate_id, - 'sequence_number', event.sequence_number, - 'created_at', event.created_at - ) - || event.event_json; -END -$$; - -CREATE OR REPLACE FUNCTION load_event( - _aggregate_id uuid, - _sequence_number integer -) RETURNS SETOF aggregate_event_type -LANGUAGE plpgsql AS $$ -BEGIN - RETURN QUERY SELECT aggregate_types.type, - a.aggregate_id, - a.events_partition_key, - event_types.type, - enrich_event_json(e) - FROM aggregates a - INNER JOIN events e ON (a.events_partition_key, a.aggregate_id) = (e.partition_key, e.aggregate_id) - INNER JOIN aggregate_types ON a.aggregate_type_id = aggregate_types.id - INNER JOIN event_types ON e.event_type_id = event_types.id - WHERE a.aggregate_id = _aggregate_id - AND e.sequence_number = _sequence_number; -END; -$$; - -CREATE OR REPLACE FUNCTION load_events( - _aggregate_ids jsonb, - _use_snapshots boolean DEFAULT TRUE, - _until timestamptz DEFAULT NULL -) RETURNS SETOF aggregate_event_type -LANGUAGE plpgsql AS $$ -DECLARE - _aggregate_id aggregates.aggregate_id%TYPE; -BEGIN - FOR _aggregate_id IN SELECT * FROM jsonb_array_elements_text(_aggregate_ids) LOOP - -- Use a single query to avoid race condition with UPDATEs to the events partition key - -- in case transaction isolation level is lower than repeatable read (the default of - -- PostgreSQL is read committed). - RETURN QUERY WITH - aggregate AS ( - SELECT aggregate_types.type, aggregate_id, events_partition_key - FROM aggregates - JOIN aggregate_types ON aggregate_type_id = aggregate_types.id - WHERE aggregate_id = _aggregate_id - ), - snapshot AS ( - SELECT * - FROM snapshot_records - WHERE _use_snapshots - AND aggregate_id = _aggregate_id - AND (_until IS NULL OR created_at < _until) - ORDER BY sequence_number DESC LIMIT 1 - ) - (SELECT a.*, s.snapshot_type, s.snapshot_json FROM aggregate a, snapshot s) - UNION ALL - (SELECT a.*, event_types.type, enrich_event_json(e) - FROM aggregate a - JOIN events e ON (a.events_partition_key, a.aggregate_id) = (e.partition_key, e.aggregate_id) - JOIN event_types ON e.event_type_id = event_types.id - WHERE e.sequence_number >= COALESCE((SELECT sequence_number FROM snapshot), 0) - AND (_until IS NULL OR e.created_at < _until) - ORDER BY e.sequence_number ASC); - END LOOP; -END; -$$; - -CREATE OR REPLACE FUNCTION store_command(_command jsonb) RETURNS bigint -LANGUAGE plpgsql AS $$ -DECLARE - _id commands.id%TYPE; - _command_json jsonb = _command->'command_json'; -BEGIN - IF NOT EXISTS (SELECT 1 FROM command_types t WHERE t.type = _command->>'command_type') THEN - -- Only try inserting if it doesn't exist to avoid exhausting the id sequence - INSERT INTO command_types (type) - VALUES (_command->>'command_type') - ON CONFLICT DO NOTHING; - END IF; - - INSERT INTO commands ( - created_at, user_id, aggregate_id, command_type_id, command_json, - event_aggregate_id, event_sequence_number - ) VALUES ( - (_command->>'created_at')::timestamptz, - (_command_json->>'user_id')::uuid, - (_command_json->>'aggregate_id')::uuid, - (SELECT id FROM command_types WHERE type = _command->>'command_type'), - (_command->'command_json') - '{command_type,created_at,organization_id,user_id,aggregate_id,event_aggregate_id,event_sequence_number}'::text[], - (_command_json->>'event_aggregate_id')::uuid, - NULLIF(_command_json->'event_sequence_number', 'null'::jsonb)::integer - ) RETURNING id INTO STRICT _id; - RETURN _id; -END; -$$; - -CREATE OR REPLACE PROCEDURE store_events(_command jsonb, _aggregates_with_events jsonb) -LANGUAGE plpgsql AS $$ -DECLARE - _command_id commands.id%TYPE; - _aggregate jsonb; - _events jsonb; - _aggregate_id aggregates.aggregate_id%TYPE; - _aggregate_row aggregates%ROWTYPE; - _provided_events_partition_key aggregates.events_partition_key%TYPE; - _events_partition_key aggregates.events_partition_key%TYPE; - _snapshot_outdated_at aggregates_that_need_snapshots.snapshot_outdated_at%TYPE; -BEGIN - _command_id = store_command(_command); - - WITH types AS ( - SELECT DISTINCT row->0->>'aggregate_type' AS type - FROM jsonb_array_elements(_aggregates_with_events) AS row - ) - INSERT INTO aggregate_types (type) - SELECT type FROM types - WHERE type NOT IN (SELECT type FROM aggregate_types) - ORDER BY 1 - ON CONFLICT DO NOTHING; - - WITH types AS ( - SELECT DISTINCT events->>'event_type' AS type - FROM jsonb_array_elements(_aggregates_with_events) AS row - CROSS JOIN LATERAL jsonb_array_elements(row->1) AS events - ) - INSERT INTO event_types (type) - SELECT type FROM types - WHERE type NOT IN (SELECT type FROM event_types) - ORDER BY 1 - ON CONFLICT DO NOTHING; - - FOR _aggregate, _events IN SELECT row->0, row->1 FROM jsonb_array_elements(_aggregates_with_events) AS row - ORDER BY row->0->'aggregate_id', row->1->0->'event_json'->'sequence_number' - LOOP - _aggregate_id = _aggregate->>'aggregate_id'; - _provided_events_partition_key = _aggregate->>'events_partition_key'; - _snapshot_outdated_at = _aggregate->>'snapshot_outdated_at'; - - SELECT * INTO _aggregate_row FROM aggregates WHERE aggregate_id = _aggregate_id; - _events_partition_key = COALESCE(_provided_events_partition_key, _aggregate_row.events_partition_key, ''); - - INSERT INTO aggregates (aggregate_id, created_at, aggregate_type_id, events_partition_key) - VALUES ( - _aggregate_id, - (_events->0->>'created_at')::timestamptz, - (SELECT id FROM aggregate_types WHERE type = _aggregate->>'aggregate_type'), - _events_partition_key - ) ON CONFLICT (aggregate_id) - DO UPDATE SET events_partition_key = EXCLUDED.events_partition_key - WHERE aggregates.events_partition_key IS DISTINCT FROM EXCLUDED.events_partition_key; - - INSERT INTO events (partition_key, aggregate_id, sequence_number, created_at, command_id, event_type_id, event_json) - SELECT _events_partition_key, - _aggregate_id, - (event->'event_json'->'sequence_number')::integer, - (event->>'created_at')::timestamptz, - _command_id, - (SELECT id FROM event_types WHERE type = event->>'event_type'), - (event->'event_json') - '{aggregate_id,created_at,event_type,sequence_number}'::text[] - FROM jsonb_array_elements(_events) AS event; - - IF _snapshot_outdated_at IS NOT NULL THEN - INSERT INTO aggregates_that_need_snapshots AS row (aggregate_id, snapshot_outdated_at) - VALUES (_aggregate_id, _snapshot_outdated_at) - ON CONFLICT (aggregate_id) DO UPDATE - SET snapshot_outdated_at = LEAST(row.snapshot_outdated_at, EXCLUDED.snapshot_outdated_at) - WHERE row.snapshot_outdated_at IS DISTINCT FROM EXCLUDED.snapshot_outdated_at; - END IF; - END LOOP; -END; -$$; - -CREATE OR REPLACE PROCEDURE store_snapshots(_snapshots jsonb) -LANGUAGE plpgsql AS $$ -DECLARE - _aggregate_id uuid; - _snapshot jsonb; - _sequence_number snapshot_records.sequence_number%TYPE; -BEGIN - FOR _snapshot IN SELECT * FROM jsonb_array_elements(_snapshots) LOOP - _aggregate_id = _snapshot->>'aggregate_id'; - _sequence_number = _snapshot->'sequence_number'; - - INSERT INTO aggregates_that_need_snapshots AS row (aggregate_id, snapshot_sequence_number_high_water_mark) - VALUES (_aggregate_id, _sequence_number) - ON CONFLICT (aggregate_id) DO UPDATE - SET snapshot_sequence_number_high_water_mark = - GREATEST(row.snapshot_sequence_number_high_water_mark, EXCLUDED.snapshot_sequence_number_high_water_mark), - snapshot_outdated_at = NULL, - snapshot_scheduled_at = NULL; - - INSERT INTO snapshot_records (aggregate_id, sequence_number, created_at, snapshot_type, snapshot_json) - VALUES ( - _aggregate_id, - _sequence_number, - (_snapshot->>'created_at')::timestamptz, - _snapshot->>'snapshot_type', - _snapshot->'snapshot_json' - ); - END LOOP; -END; -$$; - -CREATE OR REPLACE FUNCTION load_latest_snapshot(_aggregate_id uuid) RETURNS aggregate_event_type -LANGUAGE SQL AS $$ - SELECT (SELECT type FROM aggregate_types WHERE id = a.aggregate_type_id), - a.aggregate_id, - a.events_partition_key, - s.snapshot_type, - s.snapshot_json - FROM aggregates a JOIN snapshot_records s ON a.aggregate_id = s.aggregate_id - WHERE a.aggregate_id = _aggregate_id - ORDER BY s.sequence_number DESC - LIMIT 1; -$$; - -CREATE OR REPLACE PROCEDURE delete_all_snapshots(_now timestamp with time zone DEFAULT NOW()) -LANGUAGE plpgsql AS $$ -BEGIN - UPDATE aggregates_that_need_snapshots - SET snapshot_outdated_at = _now - WHERE snapshot_outdated_at IS NULL; - DELETE FROM snapshot_records; -END; -$$; - -CREATE OR REPLACE PROCEDURE delete_snapshots_before(_aggregate_id uuid, _sequence_number integer, _now timestamp with time zone DEFAULT NOW()) -LANGUAGE plpgsql AS $$ -BEGIN - DELETE FROM snapshot_records - WHERE aggregate_id = _aggregate_id - AND sequence_number < _sequence_number; - - UPDATE aggregates_that_need_snapshots - SET snapshot_outdated_at = _now - WHERE aggregate_id = _aggregate_id - AND snapshot_outdated_at IS NULL - AND NOT EXISTS (SELECT 1 FROM snapshot_records WHERE aggregate_id = _aggregate_id); -END; -$$; - -CREATE OR REPLACE FUNCTION aggregates_that_need_snapshots(_last_aggregate_id uuid, _limit integer) - RETURNS TABLE (aggregate_id uuid) -LANGUAGE plpgsql AS $$ -BEGIN - RETURN QUERY SELECT a.aggregate_id - FROM aggregates_that_need_snapshots a - WHERE a.snapshot_outdated_at IS NOT NULL - AND (_last_aggregate_id IS NULL OR a.aggregate_id > _last_aggregate_id) - ORDER BY 1 - LIMIT _limit; -END; -$$; - -CREATE OR REPLACE FUNCTION select_aggregates_for_snapshotting(_limit integer, _reschedule_snapshot_scheduled_before timestamp with time zone, _now timestamp with time zone DEFAULT NOW()) - RETURNS TABLE (aggregate_id uuid) -LANGUAGE plpgsql AS $$ -BEGIN - RETURN QUERY WITH scheduled AS MATERIALIZED ( - SELECT a.aggregate_id - FROM aggregates_that_need_snapshots AS a - WHERE snapshot_outdated_at IS NOT NULL - ORDER BY snapshot_outdated_at ASC, snapshot_sequence_number_high_water_mark DESC, aggregate_id ASC - LIMIT _limit - FOR UPDATE - ) UPDATE aggregates_that_need_snapshots AS row - SET snapshot_scheduled_at = _now - FROM scheduled - WHERE row.aggregate_id = scheduled.aggregate_id - AND (row.snapshot_scheduled_at IS NULL OR row.snapshot_scheduled_at < _reschedule_snapshot_scheduled_before) - RETURNING row.aggregate_id; -END; -$$; - -CREATE OR REPLACE PROCEDURE permanently_delete_commands_without_events(_aggregate_id uuid, _organization_id uuid) -LANGUAGE plpgsql AS $$ -BEGIN - IF _aggregate_id IS NULL AND _organization_id IS NULL THEN - RAISE EXCEPTION 'aggregate_id or organization_id must be specified to delete commands'; - END IF; - - DELETE FROM commands - WHERE (_aggregate_id IS NULL OR aggregate_id = _aggregate_id) - AND NOT EXISTS (SELECT 1 FROM events WHERE command_id = commands.id); -END; -$$; - -CREATE OR REPLACE PROCEDURE permanently_delete_event_streams(_aggregate_ids jsonb) -LANGUAGE plpgsql AS $$ -BEGIN - DELETE FROM events - USING jsonb_array_elements_text(_aggregate_ids) AS ids (id) - JOIN aggregates ON ids.id::uuid = aggregates.aggregate_id - WHERE events.partition_key = aggregates.events_partition_key - AND events.aggregate_id = aggregates.aggregate_id; - DELETE FROM aggregates - USING jsonb_array_elements_text(_aggregate_ids) AS ids (id) - WHERE aggregates.aggregate_id = ids.id::uuid; -END; -$$; - -DROP VIEW IF EXISTS command_records; -CREATE VIEW command_records (id, user_id, aggregate_id, command_type, command_json, created_at, event_aggregate_id, event_sequence_number) AS - SELECT id, - user_id, - aggregate_id, - (SELECT type FROM command_types WHERE command_types.id = command.command_type_id), - enrich_command_json(command), - created_at, - event_aggregate_id, - event_sequence_number - FROM commands command; - -DROP VIEW IF EXISTS event_records; -CREATE VIEW event_records (aggregate_id, partition_key, sequence_number, created_at, event_type, event_json, command_record_id, xact_id) AS - SELECT aggregate.aggregate_id, - event.partition_key, - event.sequence_number, - event.created_at, - type.type, - enrich_event_json(event) AS event_json, - command_id, - event.xact_id - FROM events event - JOIN aggregates aggregate ON aggregate.aggregate_id = event.aggregate_id AND aggregate.events_partition_key = event.partition_key - JOIN event_types type ON event.event_type_id = type.id; - -DROP VIEW IF EXISTS stream_records; -CREATE VIEW stream_records (aggregate_id, events_partition_key, aggregate_type, created_at) AS - SELECT aggregates.aggregate_id, - aggregates.events_partition_key, - aggregate_types.type, - aggregates.created_at - FROM aggregates JOIN aggregate_types ON aggregates.aggregate_type_id = aggregate_types.id; - -CREATE OR REPLACE FUNCTION save_events_on_delete_trigger() RETURNS TRIGGER AS $$ -BEGIN - INSERT INTO saved_event_records (operation, timestamp, "user", aggregate_id, partition_key, sequence_number, created_at, event_type, event_json, command_id, xact_id) - SELECT 'D', - statement_timestamp(), - user, - o.aggregate_id, - o.partition_key, - o.sequence_number, - o.created_at, - (SELECT type FROM event_types WHERE event_types.id = o.event_type_id), - o.event_json, - o.command_id, - o.xact_id - FROM old_table o; - RETURN NULL; -END; -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION save_events_on_update_trigger() RETURNS TRIGGER AS $$ -BEGIN - INSERT INTO saved_event_records (operation, timestamp, "user", aggregate_id, partition_key, sequence_number, created_at, event_type, event_json, command_id, xact_id) - SELECT 'U', - statement_timestamp(), - user, - o.aggregate_id, - o.partition_key, - o.sequence_number, - o.created_at, - (SELECT type FROM event_types WHERE event_types.id = o.event_type_id), - o.event_json, - o.command_id, - o.xact_id - FROM old_table o LEFT JOIN new_table n ON o.aggregate_id = n.aggregate_id AND o.sequence_number = n.sequence_number - WHERE n IS NULL - -- Only save when event related information changes - OR o.created_at <> n.created_at - OR o.event_type_id <> n.event_type_id - OR o.event_json <> n.event_json; - RETURN NULL; -END; -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE TRIGGER save_events_on_delete_trigger - AFTER DELETE ON events - REFERENCING OLD TABLE AS old_table - FOR EACH STATEMENT EXECUTE FUNCTION save_events_on_delete_trigger(); -CREATE OR REPLACE TRIGGER save_events_on_update_trigger - AFTER UPDATE ON events - REFERENCING OLD TABLE AS old_table NEW TABLE AS new_table - FOR EACH STATEMENT EXECUTE FUNCTION save_events_on_update_trigger(); diff --git a/lib/sequent/generator/template_project/db/sequent_schema.rb b/lib/sequent/generator/template_project/db/sequent_schema.rb deleted file mode 100644 index 4a4ed00a..00000000 --- a/lib/sequent/generator/template_project/db/sequent_schema.rb +++ /dev/null @@ -1,10 +0,0 @@ -# frozen_string_literal: true - -ActiveRecord::Schema.define do - say_with_time 'Installing Sequent schema' do - say 'Creating tables and indexes', true - suppress_messages { execute File.read("#{File.dirname(__FILE__)}/sequent_schema.sql") } - say 'Creating stored procedures and views', true - suppress_messages { execute File.read("#{File.dirname(__FILE__)}/sequent_pgsql.sql") } - end -end diff --git a/lib/sequent/generator/template_project/db/sequent_schema.sql b/lib/sequent/generator/template_project/db/sequent_schema.sql deleted file mode 100644 index 3bc3b7a0..00000000 --- a/lib/sequent/generator/template_project/db/sequent_schema.sql +++ /dev/null @@ -1,108 +0,0 @@ -CREATE TABLE command_types (id SMALLINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, type text UNIQUE NOT NULL); -CREATE TABLE aggregate_types (id SMALLINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, type text UNIQUE NOT NULL); -CREATE TABLE event_types (id SMALLINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, type text UNIQUE NOT NULL); - -CREATE TABLE commands ( - id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - created_at timestamp with time zone NOT NULL, - user_id uuid, - aggregate_id uuid, - command_type_id SMALLINT NOT NULL REFERENCES command_types (id), - command_json jsonb NOT NULL, - event_aggregate_id uuid, - event_sequence_number integer -) PARTITION BY RANGE (id); -CREATE INDEX commands_command_type_id_idx ON commands (command_type_id); -CREATE INDEX commands_aggregate_id_idx ON commands (aggregate_id); -CREATE INDEX commands_event_idx ON commands (event_aggregate_id, event_sequence_number); - -CREATE TABLE commands_default PARTITION OF commands DEFAULT; - -CREATE TABLE aggregates ( - aggregate_id uuid PRIMARY KEY, - events_partition_key text NOT NULL DEFAULT '', - aggregate_type_id SMALLINT NOT NULL REFERENCES aggregate_types (id), - created_at timestamp with time zone NOT NULL DEFAULT NOW(), - UNIQUE (events_partition_key, aggregate_id) -) PARTITION BY RANGE (aggregate_id); -CREATE INDEX aggregates_aggregate_type_id_idx ON aggregates (aggregate_type_id); - -CREATE TABLE aggregates_0 PARTITION OF aggregates FOR VALUES FROM (MINVALUE) TO ('40000000-0000-0000-0000-000000000000'); -ALTER TABLE aggregates_0 CLUSTER ON aggregates_0_events_partition_key_aggregate_id_key; -CREATE TABLE aggregates_4 PARTITION OF aggregates FOR VALUES FROM ('40000000-0000-0000-0000-000000000000') TO ('80000000-0000-0000-0000-000000000000'); -ALTER TABLE aggregates_4 CLUSTER ON aggregates_4_events_partition_key_aggregate_id_key; -CREATE TABLE aggregates_8 PARTITION OF aggregates FOR VALUES FROM ('80000000-0000-0000-0000-000000000000') TO ('c0000000-0000-0000-0000-000000000000'); -ALTER TABLE aggregates_8 CLUSTER ON aggregates_8_events_partition_key_aggregate_id_key; -CREATE TABLE aggregates_c PARTITION OF aggregates FOR VALUES FROM ('c0000000-0000-0000-0000-000000000000') TO (MAXVALUE); -ALTER TABLE aggregates_c CLUSTER ON aggregates_c_events_partition_key_aggregate_id_key; - -CREATE TABLE events ( - aggregate_id uuid NOT NULL, - partition_key text DEFAULT '', - sequence_number integer NOT NULL, - created_at timestamp with time zone NOT NULL, - command_id bigint NOT NULL, - event_type_id SMALLINT NOT NULL REFERENCES event_types (id), - event_json jsonb NOT NULL, - xact_id bigint DEFAULT pg_current_xact_id()::text::bigint, - PRIMARY KEY (partition_key, aggregate_id, sequence_number), - FOREIGN KEY (partition_key, aggregate_id) - REFERENCES aggregates (events_partition_key, aggregate_id) - ON UPDATE CASCADE ON DELETE RESTRICT, - FOREIGN KEY (command_id) REFERENCES commands (id) -) PARTITION BY RANGE (partition_key); -CREATE INDEX events_command_id_idx ON events (command_id); -CREATE INDEX events_event_type_id_idx ON events (event_type_id); -CREATE INDEX events_xact_id_idx ON events (xact_id) WHERE xact_id IS NOT NULL; - -CREATE TABLE events_default PARTITION OF events DEFAULT; -ALTER TABLE events_default CLUSTER ON events_default_pkey; -CREATE TABLE events_2023_and_earlier PARTITION OF events FOR VALUES FROM ('Y00') TO ('Y24'); -ALTER TABLE events_2023_and_earlier CLUSTER ON events_2023_and_earlier_pkey; -CREATE TABLE events_2024 PARTITION OF events FOR VALUES FROM ('Y24') TO ('Y25'); -ALTER TABLE events_2024 CLUSTER ON events_2024_pkey; -CREATE TABLE events_2025_and_later PARTITION OF events FOR VALUES FROM ('Y25') TO ('Y99'); -ALTER TABLE events_2025_and_later CLUSTER ON events_2025_and_later_pkey; -CREATE TABLE events_aggregate PARTITION OF events FOR VALUES FROM ('A') TO ('Ag'); -ALTER TABLE events_aggregate CLUSTER ON events_aggregate_pkey; - -CREATE TABLE aggregates_that_need_snapshots ( - aggregate_id uuid NOT NULL PRIMARY KEY REFERENCES aggregates (aggregate_id) ON UPDATE CASCADE ON DELETE CASCADE, - snapshot_sequence_number_high_water_mark integer, - snapshot_outdated_at timestamp with time zone, - snapshot_scheduled_at timestamp with time zone -); -CREATE INDEX aggregates_that_need_snapshots_outdated_idx - ON aggregates_that_need_snapshots (snapshot_outdated_at ASC, snapshot_sequence_number_high_water_mark DESC, aggregate_id ASC) - WHERE snapshot_outdated_at IS NOT NULL; -COMMENT ON TABLE aggregates_that_need_snapshots IS 'Contains a row for every aggregate with more events than its snapshot threshold.'; -COMMENT ON COLUMN aggregates_that_need_snapshots.snapshot_sequence_number_high_water_mark - IS 'The highest sequence number of the stored snapshot. Kept when snapshot are deleted to more easily query aggregates that need snapshotting the most'; -COMMENT ON COLUMN aggregates_that_need_snapshots.snapshot_outdated_at IS 'Not NULL indicates a snapshot is needed since the stored timestamp'; -COMMENT ON COLUMN aggregates_that_need_snapshots.snapshot_scheduled_at IS 'Not NULL indicates a snapshot is in the process of being taken'; - -CREATE TABLE snapshot_records ( - aggregate_id uuid NOT NULL, - sequence_number integer NOT NULL, - created_at timestamptz NOT NULL DEFAULT NOW(), - snapshot_type text NOT NULL, - snapshot_json jsonb NOT NULL, - PRIMARY KEY (aggregate_id, sequence_number), - FOREIGN KEY (aggregate_id) REFERENCES aggregates_that_need_snapshots (aggregate_id) - ON UPDATE CASCADE ON DELETE CASCADE -); - -CREATE TABLE saved_event_records ( - operation varchar(1) NOT NULL CHECK (operation IN ('U', 'D')), - timestamp timestamptz NOT NULL, - "user" text NOT NULL, - aggregate_id uuid NOT NULL, - partition_key text DEFAULT '', - sequence_number integer NOT NULL, - created_at timestamp with time zone NOT NULL, - command_id bigint NOT NULL, - event_type text NOT NULL, - event_json jsonb NOT NULL, - xact_id bigint, - PRIMARY KEY (aggregate_id, sequence_number, timestamp) -); diff --git a/lib/sequent/generator/template_project/ruby-version b/lib/sequent/generator/template_project/ruby-version deleted file mode 100644 index 4a36342f..00000000 --- a/lib/sequent/generator/template_project/ruby-version +++ /dev/null @@ -1 +0,0 @@ -3.0.0 From 31649f6fac5151c25688ace2a5e60d44f1009d91 Mon Sep 17 00:00:00 2001 From: Lars Vonk Date: Thu, 12 Sep 2024 14:29:37 +0200 Subject: [PATCH 098/128] Add support for ruby3 and ar72 --- .github/workflows/rspec.yml | 2 + .ruby-version | 2 +- gemfiles/ar_7_2.gemfile | 10 ++ gemfiles/ar_7_2.gemfile.lock | 137 ++++++++++++++++++ lib/sequent/internal/partitioned_aggregate.rb | 5 +- lib/sequent/internal/partitioned_command.rb | 5 +- lib/sequent/internal/partitioned_event.rb | 13 +- .../internal/partitioned_storage_spec.rb | 8 +- spec/lib/sequent/support/database_spec.rb | 3 + 9 files changed, 168 insertions(+), 17 deletions(-) create mode 100644 gemfiles/ar_7_2.gemfile create mode 100644 gemfiles/ar_7_2.gemfile.lock diff --git a/.github/workflows/rspec.yml b/.github/workflows/rspec.yml index 298207bd..8f18055c 100644 --- a/.github/workflows/rspec.yml +++ b/.github/workflows/rspec.yml @@ -21,6 +21,8 @@ jobs: strategy: matrix: include: + - ruby-version: '3.3' + gemfile: 'ar_7_2' - ruby-version: '3.3' gemfile: 'ar_7_1' - ruby-version: '3.2' diff --git a/.ruby-version b/.ruby-version index be94e6f5..fa7adc7a 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -3.2.2 +3.3.5 diff --git a/gemfiles/ar_7_2.gemfile b/gemfiles/ar_7_2.gemfile new file mode 100644 index 00000000..1b5c5720 --- /dev/null +++ b/gemfiles/ar_7_2.gemfile @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +source 'https://rubygems.org' + +active_star_version = '= 7.2.1' + +gem 'activemodel', active_star_version +gem 'activerecord', active_star_version + +gemspec path: '../' diff --git a/gemfiles/ar_7_2.gemfile.lock b/gemfiles/ar_7_2.gemfile.lock new file mode 100644 index 00000000..e30d4b6f --- /dev/null +++ b/gemfiles/ar_7_2.gemfile.lock @@ -0,0 +1,137 @@ +PATH + remote: .. + specs: + sequent (8.0.0.pre.dev.1) + activemodel (>= 7.1) + activerecord (>= 7.1) + bcrypt (~> 3.1) + i18n + oj (~> 3) + parallel (~> 1.20) + parser (>= 2.6.5, < 3.3) + pg (~> 1.2) + postgresql_cursor (~> 0.6) + thread_safe (~> 0.3.6) + tzinfo (>= 1.1) + +GEM + remote: https://rubygems.org/ + specs: + activemodel (7.2.1) + activesupport (= 7.2.1) + activerecord (7.2.1) + activemodel (= 7.2.1) + activesupport (= 7.2.1) + timeout (>= 0.4.0) + activesupport (7.2.1) + base64 + bigdecimal + concurrent-ruby (~> 1.0, >= 1.3.1) + connection_pool (>= 2.2.5) + drb + i18n (>= 1.6, < 2) + logger (>= 1.4.2) + minitest (>= 5.1) + securerandom (>= 0.3) + tzinfo (~> 2.0, >= 2.0.5) + ast (2.4.2) + base64 (0.2.0) + bcrypt (3.1.20) + bigdecimal (3.1.8) + coderay (1.1.3) + concurrent-ruby (1.3.4) + connection_pool (2.4.1) + diff-lcs (1.5.1) + docile (1.4.1) + drb (2.2.1) + i18n (1.14.5) + concurrent-ruby (~> 1.0) + json (2.7.2) + language_server-protocol (3.17.0.3) + logger (1.6.1) + method_source (1.1.0) + minitest (5.25.1) + oj (3.16.6) + bigdecimal (>= 3.0) + ostruct (>= 0.2) + ostruct (0.6.0) + parallel (1.26.3) + parser (3.2.2.4) + ast (~> 2.4.1) + racc + pg (1.5.8) + postgresql_cursor (0.6.9) + activerecord (>= 6.0) + prop_check (1.0.0) + pry (0.14.2) + coderay (~> 1.1) + method_source (~> 1.0) + racc (1.8.1) + rainbow (3.1.1) + rake (13.2.1) + regexp_parser (2.9.2) + rexml (3.3.7) + rspec (3.13.0) + rspec-core (~> 3.13.0) + rspec-expectations (~> 3.13.0) + rspec-mocks (~> 3.13.0) + rspec-collection_matchers (1.2.1) + rspec-expectations (>= 2.99.0.beta1) + rspec-core (3.13.1) + rspec-support (~> 3.13.0) + rspec-expectations (3.13.3) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-mocks (3.13.1) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-support (3.13.1) + rubocop (1.59.0) + json (~> 2.3) + language_server-protocol (>= 3.17.0) + parallel (~> 1.10) + parser (>= 3.2.2.4) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 1.8, < 3.0) + rexml (>= 3.2.5, < 4.0) + rubocop-ast (>= 1.30.0, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 3.0) + rubocop-ast (1.30.0) + parser (>= 3.2.1.0) + ruby-progressbar (1.13.0) + securerandom (0.3.1) + simplecov (0.22.0) + docile (~> 1.1) + simplecov-html (~> 0.11) + simplecov_json_formatter (~> 0.1) + simplecov-html (0.13.1) + simplecov_json_formatter (0.1.4) + thread_safe (0.3.6) + timecop (0.9.10) + timeout (0.4.1) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + unicode-display_width (2.5.0) + +PLATFORMS + arm64-darwin-23 + ruby + x86_64-linux + +DEPENDENCIES + activemodel (= 7.2.1) + activerecord (= 7.2.1) + prop_check (~> 1.0) + pry (~> 0.13) + rake (~> 13) + rspec (~> 3.10) + rspec-collection_matchers (~> 1.2) + rspec-mocks (~> 3.10) + rubocop (~> 1.56, >= 1.56.3) + sequent! + simplecov (~> 0.21) + timecop (~> 0.9) + +BUNDLED WITH + 2.5.16 diff --git a/lib/sequent/internal/partitioned_aggregate.rb b/lib/sequent/internal/partitioned_aggregate.rb index 8cf6f279..0dfa8643 100644 --- a/lib/sequent/internal/partitioned_aggregate.rb +++ b/lib/sequent/internal/partitioned_aggregate.rb @@ -10,9 +10,8 @@ class PartitionedAggregate < Sequent::ApplicationRecord self.primary_key = %i[aggregate_id] belongs_to :aggregate_type - has_many :events, - inverse_of: :aggregate, - class_name: :PartitionedEvent, + has_many :partitioned_events, + inverse_of: :partitioned_aggregate, primary_key: %i[events_partition_key aggregate_id], query_constraints: %i[partition_key aggregate_id] end diff --git a/lib/sequent/internal/partitioned_command.rb b/lib/sequent/internal/partitioned_command.rb index eeb0740e..765fb676 100644 --- a/lib/sequent/internal/partitioned_command.rb +++ b/lib/sequent/internal/partitioned_command.rb @@ -9,9 +9,8 @@ class PartitionedCommand < Sequent::ApplicationRecord self.table_name = :commands belongs_to :command_type - has_many :events, - inverse_of: :command, - class_name: :PartitionedEvent + has_many :partitioned_events, + inverse_of: :partitioned_command end end end diff --git a/lib/sequent/internal/partitioned_event.rb b/lib/sequent/internal/partitioned_event.rb index f40c79cd..69f6d550 100644 --- a/lib/sequent/internal/partitioned_event.rb +++ b/lib/sequent/internal/partitioned_event.rb @@ -10,12 +10,13 @@ class PartitionedEvent < Sequent::ApplicationRecord self.primary_key = %i[partition_key aggregate_id sequence_number] belongs_to :event_type - belongs_to :command, - inverse_of: :events, - class_name: :PartitionedCommand - belongs_to :aggregate, - inverse_of: :events, - class_name: :PartitionedAggregate + belongs_to :partitioned_command, + inverse_of: :partitioned_events, + foreign_key: :command_id + belongs_to :partitioned_aggregate, + inverse_of: :partitioned_events, + primary_key: %w[partition_key aggregate_id], + foreign_key: %w[events_partition_key aggregate_id] end end end diff --git a/spec/lib/sequent/internal/partitioned_storage_spec.rb b/spec/lib/sequent/internal/partitioned_storage_spec.rb index 5728278a..2ef5e309 100644 --- a/spec/lib/sequent/internal/partitioned_storage_spec.rb +++ b/spec/lib/sequent/internal/partitioned_storage_spec.rb @@ -31,19 +31,19 @@ module Internal expect(aggregate.events_partition_key).to eq(events_partition_key) expect(aggregate.aggregate_type.type).to eq('Aggregate') - events = aggregate.events.to_a + events = aggregate.partitioned_events.to_a expect(events.size).to eq(1) event = events[0] - expect(event.aggregate).to be(aggregate) + expect(event.partitioned_aggregate).to be(aggregate) expect(event.aggregate_id).to eq(aggregate_id) expect(event.partition_key).to eq(events_partition_key) expect(event.event_type.type).to eq('Sequent::Core::Event') - command = event.command + command = event.partitioned_command expect(command).to be_present expect(command.command_type.type).to eq('Sequent::Core::Command') - expect(command.events).to eq(events) + expect(command.partitioned_events).to eq(events) end end end diff --git a/spec/lib/sequent/support/database_spec.rb b/spec/lib/sequent/support/database_spec.rb index 00b3373e..8c539991 100644 --- a/spec/lib/sequent/support/database_spec.rb +++ b/spec/lib/sequent/support/database_spec.rb @@ -47,6 +47,7 @@ it 'connects the Sequent::ApplicationRecord pool' do Sequent::Support::Database.establish_connection(db_config) + Sequent::ApplicationRecord.connection.reconnect! expect(Sequent::ApplicationRecord.connection).to be_active end end @@ -60,6 +61,7 @@ it 'connects the Sequent::ApplicationRecord pool' do Sequent::Support::Database.with_schema_search_path('test2', db_config) do + Sequent::ApplicationRecord.connection.reconnect! expect(Sequent::ApplicationRecord.connection).to be_active end end @@ -176,6 +178,7 @@ it 'connects using an url option' do Sequent::Support::Database.establish_connection(db_config) + Sequent::ApplicationRecord.connection.reconnect! expect(Sequent::ApplicationRecord.connection).to be_active end end From ba342a926a6b714f3ac461dfac981859949bb6c4 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Mon, 16 Sep 2024 17:07:36 +0200 Subject: [PATCH 099/128] Update required versions --- docs/docs/getting-started.md | 8 ++++---- lib/sequent/generator/template_project/Gemfile | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/docs/getting-started.md b/docs/docs/getting-started.md index 9957544c..70231ede 100644 --- a/docs/docs/getting-started.md +++ b/docs/docs/getting-started.md @@ -39,10 +39,10 @@ Verify that you have a current version of Ruby installed: ```bash $ ruby -v -ruby 3.0.0p0 (2020-12-25 revision 95aff21468) [x86_64-darwin20] +ruby 3.2.4 (2024-04-23 revision af471c0e01) [arm64-darwin23] ``` -Sequent requires Ruby version 2.7.0 or later. If the version number returned is lower, you'll need to upgrade your Ruby version. For managing Ruby versions we recommend [rbenv](https://github.com/rbenv/rbenv). +Sequent requires Ruby version 3.2.0 or later. If the version number returned is lower, you'll need to upgrade your Ruby version. For managing Ruby versions we recommend [rbenv](https://github.com/rbenv/rbenv). #### Postgres @@ -50,10 +50,10 @@ You will also need to have the PostgreSQL database server installed. Verify that ```bash $ pg_config --version -PostgreSQL 11.2 +PostgreSQL 16.3 ``` -Sequent works with PostgreSQL version 9.4 or later, but we recommend you install the lastest version. For installation instructions refer to your OS or see [postgresql.org](https://www.postgresql.org). +Sequent works with PostgreSQL version 14.x or later, but we recommend you install the lastest version. For installation instructions refer to your OS or see [postgresql.org](https://www.postgresql.org). #### Sequent diff --git a/lib/sequent/generator/template_project/Gemfile b/lib/sequent/generator/template_project/Gemfile index 2139a3e7..2ee81b4f 100644 --- a/lib/sequent/generator/template_project/Gemfile +++ b/lib/sequent/generator/template_project/Gemfile @@ -1,5 +1,5 @@ source "https://rubygems.org" -ruby "3.0.0" +ruby file: ".ruby-version" gem 'rake' # let's use the latest and greatest From 70d8c24baddffb12ffd637c2ae4ea80a126381ac Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Mon, 16 Sep 2024 17:30:24 +0200 Subject: [PATCH 100/128] Rubocop fixes in template project --- .rubocop.yml | 2 -- .../template_aggregate/template_aggregate.rb | 2 ++ .../template_aggregate/template_aggregate/commands.rb | 2 ++ .../template_aggregate/template_aggregate/events.rb | 2 ++ .../template_aggregate/template_aggregate.rb | 2 ++ .../template_aggregate_command_handler.rb | 2 ++ lib/sequent/generator/template_project/Gemfile | 8 +++++--- lib/sequent/generator/template_project/Rakefile | 6 ++++-- .../template_project/app/projectors/post_projector.rb | 2 ++ .../template_project/app/records/post_record.rb | 2 ++ .../template_project/config/initializers/sequent.rb | 6 ++++-- .../generator/template_project/db/migrations.rb | 6 +++--- lib/sequent/generator/template_project/lib/post.rb | 2 ++ .../generator/template_project/lib/post/commands.rb | 2 ++ .../generator/template_project/lib/post/events.rb | 2 ++ .../generator/template_project/lib/post/post.rb | 2 ++ .../template_project/lib/post/post_command_handler.rb | 2 ++ lib/sequent/generator/template_project/my_app.rb | 3 ++- .../spec/app/projectors/post_projector_spec.rb | 2 ++ .../spec/lib/post/post_command_handler_spec.rb | 11 +++++++++-- .../generator/template_project/spec/spec_helper.rb | 4 +++- 21 files changed, 56 insertions(+), 16 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index 038f4ecf..b1bd6cee 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -19,8 +19,6 @@ AllCops: - '**/vendor/**/*' - '**/.git/**/*' # sequent - - '**/sequent/generator/template_aggregate/**/*' - - '**/sequent/generator/template_project/**/*' - 'db/sequent_schema.rb' - 'docs/**/*' - 'integration-specs/rails-app/bin/*' diff --git a/lib/sequent/generator/template_aggregate/template_aggregate.rb b/lib/sequent/generator/template_aggregate/template_aggregate.rb index 620372ff..acf09e2c 100644 --- a/lib/sequent/generator/template_aggregate/template_aggregate.rb +++ b/lib/sequent/generator/template_aggregate/template_aggregate.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require_relative 'template_aggregate/commands' require_relative 'template_aggregate/events' require_relative 'template_aggregate/template_aggregate' diff --git a/lib/sequent/generator/template_aggregate/template_aggregate/commands.rb b/lib/sequent/generator/template_aggregate/template_aggregate/commands.rb index b7b38359..ca30d55e 100644 --- a/lib/sequent/generator/template_aggregate/template_aggregate/commands.rb +++ b/lib/sequent/generator/template_aggregate/template_aggregate/commands.rb @@ -1,2 +1,4 @@ +# frozen_string_literal: true + class AddTemplateAggregate < Sequent::Command end diff --git a/lib/sequent/generator/template_aggregate/template_aggregate/events.rb b/lib/sequent/generator/template_aggregate/template_aggregate/events.rb index 51785450..a3aae44e 100644 --- a/lib/sequent/generator/template_aggregate/template_aggregate/events.rb +++ b/lib/sequent/generator/template_aggregate/template_aggregate/events.rb @@ -1,2 +1,4 @@ +# frozen_string_literal: true + class TemplateAggregateAdded < Sequent::Event end diff --git a/lib/sequent/generator/template_aggregate/template_aggregate/template_aggregate.rb b/lib/sequent/generator/template_aggregate/template_aggregate/template_aggregate.rb index c4b5a865..2699ecf7 100644 --- a/lib/sequent/generator/template_aggregate/template_aggregate/template_aggregate.rb +++ b/lib/sequent/generator/template_aggregate/template_aggregate/template_aggregate.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class TemplateAggregate < Sequent::AggregateRoot def initialize(command) super(command.aggregate_id) diff --git a/lib/sequent/generator/template_aggregate/template_aggregate/template_aggregate_command_handler.rb b/lib/sequent/generator/template_aggregate/template_aggregate/template_aggregate_command_handler.rb index 2324bde0..2c1e5049 100644 --- a/lib/sequent/generator/template_aggregate/template_aggregate/template_aggregate_command_handler.rb +++ b/lib/sequent/generator/template_aggregate/template_aggregate/template_aggregate_command_handler.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class TemplateAggregateCommandHandler < Sequent::CommandHandler on AddTemplateAggregate do |command| repository.add_aggregate TemplateAggregate.new(command) diff --git a/lib/sequent/generator/template_project/Gemfile b/lib/sequent/generator/template_project/Gemfile index 2ee81b4f..9e836243 100644 --- a/lib/sequent/generator/template_project/Gemfile +++ b/lib/sequent/generator/template_project/Gemfile @@ -1,11 +1,13 @@ -source "https://rubygems.org" -ruby file: ".ruby-version" +# frozen_string_literal: true + +source 'https://rubygems.org' +ruby file: '.ruby-version' gem 'rake' # let's use the latest and greatest gem 'sequent', git: 'https://github.com/zilverline/sequent' group :test do - gem 'rspec' gem 'database_cleaner' + gem 'rspec' end diff --git a/lib/sequent/generator/template_project/Rakefile b/lib/sequent/generator/template_project/Rakefile index c018702a..feeedfac 100644 --- a/lib/sequent/generator/template_project/Rakefile +++ b/lib/sequent/generator/template_project/Rakefile @@ -1,3 +1,5 @@ +# frozen_string_literal: true + ENV['SEQUENT_ENV'] ||= 'development' require './my_app' @@ -5,8 +7,8 @@ require 'sequent/rake/migration_tasks' Sequent::Rake::MigrationTasks.new.register_tasks! -task "sequent:migrate:init" => [:db_connect] +task 'sequent:migrate:init' => [:db_connect] -task "db_connect" do +task 'db_connect' do Sequent::Support::Database.connect!(ENV['SEQUENT_ENV']) end diff --git a/lib/sequent/generator/template_project/app/projectors/post_projector.rb b/lib/sequent/generator/template_project/app/projectors/post_projector.rb index 1c5ed60f..2dd3fd19 100644 --- a/lib/sequent/generator/template_project/app/projectors/post_projector.rb +++ b/lib/sequent/generator/template_project/app/projectors/post_projector.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require_relative '../records/post_record' require_relative '../../lib/post/events' diff --git a/lib/sequent/generator/template_project/app/records/post_record.rb b/lib/sequent/generator/template_project/app/records/post_record.rb index 905668b8..6ffcff4e 100644 --- a/lib/sequent/generator/template_project/app/records/post_record.rb +++ b/lib/sequent/generator/template_project/app/records/post_record.rb @@ -1,2 +1,4 @@ +# frozen_string_literal: true + class PostRecord < Sequent::ApplicationRecord end diff --git a/lib/sequent/generator/template_project/config/initializers/sequent.rb b/lib/sequent/generator/template_project/config/initializers/sequent.rb index e0f09bc6..87b26ba7 100644 --- a/lib/sequent/generator/template_project/config/initializers/sequent.rb +++ b/lib/sequent/generator/template_project/config/initializers/sequent.rb @@ -1,13 +1,15 @@ +# frozen_string_literal: true + require './db/migrations' Sequent.configure do |config| config.migrations_class_name = 'Migrations' config.command_handlers = [ - PostCommandHandler.new + PostCommandHandler.new, ] config.event_handlers = [ - PostProjector.new + PostProjector.new, ] end diff --git a/lib/sequent/generator/template_project/db/migrations.rb b/lib/sequent/generator/template_project/db/migrations.rb index a27a7b73..00a0a2fa 100644 --- a/lib/sequent/generator/template_project/db/migrations.rb +++ b/lib/sequent/generator/template_project/db/migrations.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'sequent/migrations/projectors' VIEW_SCHEMA_VERSION = 1 @@ -9,9 +11,7 @@ def self.version def self.versions { - '1' => [ - PostProjector - ] + '1' => [PostProjector], } end end diff --git a/lib/sequent/generator/template_project/lib/post.rb b/lib/sequent/generator/template_project/lib/post.rb index aad19cb6..003731a9 100644 --- a/lib/sequent/generator/template_project/lib/post.rb +++ b/lib/sequent/generator/template_project/lib/post.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require_relative 'post/commands' require_relative 'post/events' require_relative 'post/post' diff --git a/lib/sequent/generator/template_project/lib/post/commands.rb b/lib/sequent/generator/template_project/lib/post/commands.rb index 6be6f587..f2892f70 100644 --- a/lib/sequent/generator/template_project/lib/post/commands.rb +++ b/lib/sequent/generator/template_project/lib/post/commands.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class AddPost < Sequent::Command attrs author: String, title: String, content: String validates_presence_of :author, :title, :content diff --git a/lib/sequent/generator/template_project/lib/post/events.rb b/lib/sequent/generator/template_project/lib/post/events.rb index e394278e..b10c0551 100644 --- a/lib/sequent/generator/template_project/lib/post/events.rb +++ b/lib/sequent/generator/template_project/lib/post/events.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class PostAdded < Sequent::Event end diff --git a/lib/sequent/generator/template_project/lib/post/post.rb b/lib/sequent/generator/template_project/lib/post/post.rb index 641c7f7d..57347f7f 100644 --- a/lib/sequent/generator/template_project/lib/post/post.rb +++ b/lib/sequent/generator/template_project/lib/post/post.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class Post < Sequent::AggregateRoot def initialize(command) super(command.aggregate_id) diff --git a/lib/sequent/generator/template_project/lib/post/post_command_handler.rb b/lib/sequent/generator/template_project/lib/post/post_command_handler.rb index bef1ec13..23ddcb57 100644 --- a/lib/sequent/generator/template_project/lib/post/post_command_handler.rb +++ b/lib/sequent/generator/template_project/lib/post/post_command_handler.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class PostCommandHandler < Sequent::CommandHandler on AddPost do |command| repository.add_aggregate Post.new(command) diff --git a/lib/sequent/generator/template_project/my_app.rb b/lib/sequent/generator/template_project/my_app.rb index 86e63628..1d8c4945 100644 --- a/lib/sequent/generator/template_project/my_app.rb +++ b/lib/sequent/generator/template_project/my_app.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'sequent' require 'sequent/support' require 'erb' @@ -7,5 +9,4 @@ require_relative 'config/initializers/sequent' module MyApp - end diff --git a/lib/sequent/generator/template_project/spec/app/projectors/post_projector_spec.rb b/lib/sequent/generator/template_project/spec/app/projectors/post_projector_spec.rb index c1ad2baf..59584035 100644 --- a/lib/sequent/generator/template_project/spec/app/projectors/post_projector_spec.rb +++ b/lib/sequent/generator/template_project/spec/app/projectors/post_projector_spec.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'spec_helper' require_relative '../../../app/projectors/post_projector' diff --git a/lib/sequent/generator/template_project/spec/lib/post/post_command_handler_spec.rb b/lib/sequent/generator/template_project/spec/lib/post/post_command_handler_spec.rb index d1f48131..bd131364 100644 --- a/lib/sequent/generator/template_project/spec/lib/post/post_command_handler_spec.rb +++ b/lib/sequent/generator/template_project/spec/lib/post/post_command_handler_spec.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'spec_helper' require_relative '../../../lib/post' @@ -9,12 +11,17 @@ end it 'creates a post' do - when_command AddPost.new(aggregate_id: aggregate_id, author: 'ben', title: 'My first blogpost', content: 'Hello World!') + when_command AddPost.new( + aggregate_id: aggregate_id, + author: 'ben', + title: 'My first blogpost', + content: 'Hello World!', + ) then_events( PostAdded.new(aggregate_id: aggregate_id, sequence_number: 1), PostAuthorChanged.new(aggregate_id: aggregate_id, sequence_number: 2, author: 'ben'), PostTitleChanged, - PostContentChanged + PostContentChanged, ) end end diff --git a/lib/sequent/generator/template_project/spec/spec_helper.rb b/lib/sequent/generator/template_project/spec/spec_helper.rb index 506320c3..ee4cbd07 100644 --- a/lib/sequent/generator/template_project/spec/spec_helper.rb +++ b/lib/sequent/generator/template_project/spec/spec_helper.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'bundler/setup' Bundler.setup @@ -23,7 +25,7 @@ def self.included(base) RSpec.configure do |config| config.include Sequent::Test::CommandHandlerHelpers - config.include DomainTests, file_path: /spec\/lib/ + config.include DomainTests, file_path: %r{/spec\/lib/} # Domain tests run with a clean sequent configuration and the in memory FakeEventStore config.around :each, :domain_tests do |example| From 41b05a78de64c127aae0d06292d66af3040efcf9 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Tue, 17 Sep 2024 11:39:15 +0200 Subject: [PATCH 101/128] Updated documentation --- docs/docs/modelling-the-domain.md | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/docs/docs/modelling-the-domain.md b/docs/docs/modelling-the-domain.md index eda76394..239a6ffa 100644 --- a/docs/docs/modelling-the-domain.md +++ b/docs/docs/modelling-the-domain.md @@ -111,6 +111,7 @@ Let's define how the domain should behave when receiving our new `PublishPost` c ```ruby def publish(publication_date) fail PostAlreadyPubishedError if @publication_date.any? + apply PostPublished, publication_date: publication_date end ``` @@ -139,7 +140,7 @@ _Learn all about events in the [Event](/docs/concepts/event.html) Reference Guid ## Adding an Author -We have now gone through the generated example files. +We have now gone through the generated example files. Before we can add a `Post` we need to add an `Author`. In this guide, we will 'upgrade' `Author` to its own Aggregate Root. This means we need to add new files defining the `Author` Aggregate Root, and make some changes to the `Post` Commands and Events. i.e. using the author `aggregate_id` instead of an author String. @@ -171,7 +172,7 @@ class Usernames < Sequent::AggregateRoot class UsernameAlreadyRegistered < StandardError; end # We can generate and hardcode the UUID since there is only one instance - ID = "85507d60-8645-4a8a-bdb8-3a9c86a0c635" + ID = '85507d60-8645-4a8a-bdb8-3a9c86a0c635' def self.instance(id = ID) Sequent.configuration.aggregate_repository.load_aggregate(id) @@ -267,7 +268,7 @@ Sequent.configure do |config| end ``` -When we run the tests in `spec/lib/author/author_command_handler_spec.rb`, all are marked as `Pending: Not yet implemented`. Before we can go any further, we need to think about what kind of Events we are interested in. What do we want to know in this case? When registering our very first `Author`, it will not only create the Author, but also create our `Usernames` Aggregate to ensure uniqueness of the usernames. +When we run the tests in `spec/lib/author/author_command_handler_spec.rb`, all are marked as `Pending: Not yet implemented`. Before we can go any further, we need to think about what kind of Events we are interested in. What do we want to know in this case? When registering our very first `Author`, it will not only create the Author, but also create our `Usernames` Aggregate to ensure uniqueness of the usernames. The test will read something like: @@ -285,21 +286,21 @@ context AddAuthor do let(:user_aggregate_id) { Sequent.new_uuid } let(:email) { 'ben@sequent.io' } - it "creates a user when valid input" do - when_command AddAuthor.new(aggregate_id: user_aggregate_id, name: "Ben", email: email) + it 'creates a user when valid input' do + when_command AddAuthor.new(aggregate_id: user_aggregate_id, name: 'Ben', email: email) then_events UsernamesCreated.new(aggregate_id: Usernames::ID, sequence_number: 1), UsernameAdded.new(aggregate_id: Usernames::ID, username: email, sequence_number: 2), AuthorCreated.new(aggregate_id: user_aggregate_id, sequence_number: 1), AuthorNameSet, AuthorEmailSet.new(aggregate_id: user_aggregate_id, email: email, sequence_number: 3) end - it "fails if the username already exists" - it "ignores case in usernames" + it 'fails if the username already exists' + it 'ignores case in usernames' end ``` In Sequent (or other event sourcing libraries) you test your code by checking the applied events, and which order they were run in. -In this case we modelled the `AuthorNameSet` and `AuthorEmailSet` as separate events, since they probably don't change together. +In this case we modelled the `AuthorNameSet` and `AuthorEmailSet` as separate events, since they probably don't change together. In more comprehensive cases we can imagine triggering other events, e.g. when the email changes, a confirmation is sent. You should take these considerations into account when modelling your domain and defining your Events. @@ -309,11 +310,9 @@ In `lib/usernames/events.rb` ```ruby class UsernamesCreated < Sequent::Event - end class UsernameAdded < Sequent::Event - end ``` @@ -326,7 +325,7 @@ class Usernames < Sequent::AggregateRoot class UsernameAlreadyRegistered < StandardError; end # We can generate and hardcode the UUID since there is only one instance - ID = "85507d60-8645-4a8a-bdb8-3a9c86a0c635" + ID = '85507d60-8645-4a8a-bdb8-3a9c86a0c635' def self.instance(id = ID) Sequent.aggregate_repository.load_aggregate(id) @@ -350,7 +349,6 @@ end In `lib/author/events.rb` ```ruby class AuthorCreated < Sequent::Event - end class AuthorNameSet < Sequent::Event @@ -562,7 +560,7 @@ class Post < Sequent::AggregateRoot end # omitted ... - + on PostAuthorChanged do |event| @author_aggregate_id = event.author_aggregate_id end From 73665d19ee18447d193550f74ffce19557cec450 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Tue, 17 Sep 2024 11:49:05 +0200 Subject: [PATCH 102/128] Add PostgreSQL migration script for Sequent 8 --- db/sequent_8_migration.sql | 249 +++++++++++++++++++++++++++++++++++++ 1 file changed, 249 insertions(+) create mode 100644 db/sequent_8_migration.sql diff --git a/db/sequent_8_migration.sql b/db/sequent_8_migration.sql new file mode 100644 index 00000000..42b28063 --- /dev/null +++ b/db/sequent_8_migration.sql @@ -0,0 +1,249 @@ +\set ECHO all +\set ON_ERROR_STOP +\timing on + +SELECT clock_timestamp() AS migration_started_at \gset + +\echo Migration started at :migration_started_at + +SET work_mem TO '8MB'; +SET max_parallel_workers = 8; +SET max_parallel_workers_per_gather = 8; +SET max_parallel_maintenance_workers = 8; + +BEGIN; + +SET temp_tablespaces = 'pg_default'; +SET search_path TO sequent_schema; + +CREATE TYPE aggregate_event_type AS ( + aggregate_type text, + aggregate_id uuid, + events_partition_key text, + snapshot_threshold integer, + event_type text, + event_json jsonb +); + +CREATE TABLE command_types (id SMALLINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, type text UNIQUE NOT NULL); +CREATE TABLE aggregate_types (id SMALLINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, type text UNIQUE NOT NULL); +CREATE TABLE event_types (id SMALLINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, type text UNIQUE NOT NULL); + + +CREATE TABLE commands ( + id bigint NOT NULL DEFAULT nextval('sequent_schema.command_records_id_seq'), + created_at timestamp with time zone NOT NULL, + user_id uuid, + aggregate_id uuid, + command_type_id SMALLINT NOT NULL, + command_json jsonb NOT NULL, + event_aggregate_id uuid, + event_sequence_number integer +) PARTITION BY RANGE (id); + +CREATE TABLE commands_default PARTITION OF commands DEFAULT; +-- ### Add additional partitions as needed +CREATE TABLE commands_0 PARTITION OF commands FOR VALUES FROM (1) TO (100e6); +-- CREATE TABLE commands_1 PARTITION OF commands FOR VALUES FROM (100e6) TO (200e6); +-- CREATE TABLE commands_2 PARTITION OF commands FOR VALUES FROM (200e6) TO (300e6); +-- CREATE TABLE commands_3 PARTITION OF commands FOR VALUES FROM (300e6) TO (400e6); + +CREATE TABLE aggregates ( + aggregate_id uuid NOT NULL, + events_partition_key text NOT NULL DEFAULT '', + aggregate_type_id SMALLINT NOT NULL, + snapshot_threshold integer, + created_at timestamp with time zone NOT NULL DEFAULT NOW() +) PARTITION BY RANGE (aggregate_id); + +CREATE TABLE aggregates_0 PARTITION OF aggregates FOR VALUES FROM (MINVALUE) TO ('40000000-0000-0000-0000-000000000000'); +CREATE TABLE aggregates_4 PARTITION OF aggregates FOR VALUES FROM ('40000000-0000-0000-0000-000000000000') TO ('80000000-0000-0000-0000-000000000000'); +CREATE TABLE aggregates_8 PARTITION OF aggregates FOR VALUES FROM ('80000000-0000-0000-0000-000000000000') TO ('c0000000-0000-0000-0000-000000000000'); +CREATE TABLE aggregates_c PARTITION OF aggregates FOR VALUES FROM ('c0000000-0000-0000-0000-000000000000') TO (MAXVALUE); + +CREATE TABLE events ( + aggregate_id uuid NOT NULL, + partition_key text NOT NULL DEFAULT '', + sequence_number integer NOT NULL, + created_at timestamp with time zone NOT NULL, + command_id bigint NOT NULL, + event_type_id SMALLINT NOT NULL, + event_json jsonb NOT NULL, + xact_id bigint +) PARTITION BY RANGE (partition_key); + +CREATE INDEX events_xact_id_idx ON events (xact_id) WHERE xact_id IS NOT NULL; + +CREATE TABLE events_default PARTITION OF events DEFAULT; +CREATE TABLE events_2023_and_earlier PARTITION OF events FOR VALUES FROM ('Y00') TO ('Y24'); +CREATE TABLE events_2024 PARTITION OF events FOR VALUES FROM ('Y24') TO ('Y25'); +CREATE TABLE events_2025_and_later PARTITION OF events FOR VALUES FROM ('Y25') TO ('Y99'); +CREATE TABLE events_aggregate PARTITION OF events FOR VALUES FROM ('A') TO ('Ag'); + +INSERT INTO aggregate_types (type) +SELECT DISTINCT aggregate_type + FROM sequent_schema.stream_records + ORDER BY 1; + +INSERT INTO event_types (type) +SELECT DISTINCT event_type + FROM sequent_schema.event_records + WHERE event_type <> 'Sequent::Core::SnapshotEvent' + ORDER BY 1; + +INSERT INTO command_types (type) +SELECT DISTINCT command_type + FROM sequent_schema.command_records + ORDER BY 1; + +ANALYZE aggregate_types, event_types, command_types; + +INSERT INTO aggregates +SELECT aggregate_id, '', (SELECT t.id FROM aggregate_types t WHERE aggregate_type = t.type), snapshot_threshold, created_at AT TIME ZONE 'Europe/Amsterdam' + FROM stream_records; + +WITH e AS MATERIALIZED ( + SELECT aggregate_id, + sequence_number, + command_record_id, + t.id AS event_type_id, + event_json::jsonb - '{aggregate_id,sequence_number}'::text[] AS event_json + FROM sequent_schema.event_records e + JOIN event_types t ON e.event_type = t.type +) +INSERT INTO events (aggregate_id, sequence_number, created_at, command_id, event_type_id, event_json) +SELECT aggregate_id, + sequence_number, + (event_json->>'created_at')::timestamptz AS created_at, + command_record_id, + event_type_id, + event_json - 'created_at' + FROM e; + + +WITH command AS MATERIALIZED ( + SELECT c.id, created_at, + t.id AS command_type_id, + command_json::jsonb AS json + FROM sequent_schema.command_records c + JOIN command_types t ON t.type = c.command_type +) +INSERT INTO commands ( + id, created_at, user_id, aggregate_id, command_type_id, command_json, + event_aggregate_id, event_sequence_number +) +SELECT id, + COALESCE((json->>'created_at')::timestamptz, created_at AT TIME ZONE 'Europe/Amsterdam'), + (json->>'user_id')::uuid, + (json->>'aggregate_id')::uuid, + command_type_id, + json - '{created_at,user_id,aggregate_id,event_aggregate_id,event_sequence_number}'::text[], + (json->>'event_aggregate_id')::uuid, + (json->>'event_sequence_number')::integer + FROM command; + +ALTER TABLE aggregates ADD UNIQUE (events_partition_key, aggregate_id); +ALTER TABLE aggregates_0 CLUSTER ON aggregates_0_events_partition_key_aggregate_id_key; +ALTER TABLE aggregates_4 CLUSTER ON aggregates_4_events_partition_key_aggregate_id_key; +ALTER TABLE aggregates_8 CLUSTER ON aggregates_8_events_partition_key_aggregate_id_key; +ALTER TABLE aggregates_c CLUSTER ON aggregates_c_events_partition_key_aggregate_id_key; + +ALTER TABLE events + ADD PRIMARY KEY (partition_key, aggregate_id, sequence_number); +ALTER TABLE events_default CLUSTER ON events_default_pkey; +ALTER TABLE events_2023_and_earlier CLUSTER ON events_2023_and_earlier_pkey; +ALTER TABLE events_2024 CLUSTER ON events_2024_pkey; +ALTER TABLE events_2025_and_later CLUSTER ON events_2025_and_later_pkey; +ALTER TABLE events_aggregate CLUSTER ON events_aggregate_pkey; + +ALTER TABLE commands ADD PRIMARY KEY (id); + +ALTER TABLE aggregates ADD PRIMARY KEY (aggregate_id); +CREATE INDEX aggregates_aggregate_type_id_idx ON aggregates (aggregate_type_id); +ALTER TABLE aggregates + ADD FOREIGN KEY (aggregate_type_id) REFERENCES aggregate_types (id) ON UPDATE CASCADE; + +CREATE INDEX events_command_id_idx ON events (command_id); +CREATE INDEX events_event_type_id_idx ON events (event_type_id); +ALTER TABLE events + ADD FOREIGN KEY (partition_key, aggregate_id) REFERENCES aggregates (events_partition_key, aggregate_id) + ON UPDATE CASCADE ON DELETE RESTRICT; +ALTER TABLE events + ADD FOREIGN KEY (command_id) REFERENCES commands (id) ON UPDATE RESTRICT ON DELETE RESTRICT; +ALTER TABLE events + ADD FOREIGN KEY (event_type_id) REFERENCES event_types (id) ON UPDATE CASCADE; +ALTER TABLE events ALTER COLUMN xact_id SET DEFAULT pg_current_xact_id()::text::bigint; + +CREATE INDEX commands_command_type_id_idx ON commands (command_type_id); +CREATE INDEX commands_aggregate_id_idx ON commands (aggregate_id); +CREATE INDEX commands_event_idx ON commands (event_aggregate_id, event_sequence_number); +ALTER TABLE commands + ADD FOREIGN KEY (command_type_id) REFERENCES command_types (id) ON UPDATE CASCADE; + +CREATE TABLE aggregates_that_need_snapshots ( + aggregate_id uuid NOT NULL PRIMARY KEY REFERENCES aggregates (aggregate_id) ON UPDATE CASCADE ON DELETE CASCADE, + snapshot_sequence_number_high_water_mark integer, + snapshot_outdated_at timestamp with time zone, + snapshot_scheduled_at timestamp with time zone +); +INSERT INTO aggregates_that_need_snapshots (aggregate_id, snapshot_sequence_number_high_water_mark, snapshot_outdated_at) +SELECT aggregate_id, MAX(sequence_number), NOW() + FROM event_records + WHERE event_type = 'Sequent::Core::SnapshotEvent' + GROUP BY 1 + ORDER BY 1; + +CREATE INDEX aggregates_that_need_snapshots_outdated_idx + ON aggregates_that_need_snapshots (snapshot_outdated_at ASC, snapshot_sequence_number_high_water_mark DESC, aggregate_id ASC) + WHERE snapshot_outdated_at IS NOT NULL; +COMMENT ON TABLE aggregates_that_need_snapshots IS 'Contains a row for every aggregate with more events than its snapshot threshold.'; +COMMENT ON COLUMN aggregates_that_need_snapshots.snapshot_sequence_number_high_water_mark + IS 'The highest sequence number of the stored snapshot. Kept when snapshot are deleted to more easily query aggregates that need snapshotting the most'; +COMMENT ON COLUMN aggregates_that_need_snapshots.snapshot_outdated_at IS 'Not NULL indicates a snapshot is needed since the stored timestamp'; +COMMENT ON COLUMN aggregates_that_need_snapshots.snapshot_scheduled_at IS 'Not NULL indicates a snapshot is in the process of being taken'; + +CREATE TABLE snapshot_records ( + aggregate_id uuid NOT NULL, + sequence_number integer NOT NULL, + created_at timestamptz NOT NULL, + snapshot_type text NOT NULL, + snapshot_json jsonb NOT NULL, + PRIMARY KEY (aggregate_id, sequence_number), + FOREIGN KEY (aggregate_id) REFERENCES aggregates (aggregate_id) ON UPDATE CASCADE ON DELETE CASCADE +); + +CREATE TABLE saved_event_records ( + operation varchar(1) NOT NULL CHECK (operation IN ('U', 'D')), + timestamp timestamptz NOT NULL, + "user" text NOT NULL, + aggregate_id uuid NOT NULL, + partition_key text DEFAULT '', + sequence_number integer NOT NULL, + created_at timestamp with time zone NOT NULL, + command_id bigint NOT NULL, + event_type text NOT NULL, + event_json jsonb NOT NULL, + xact_id bigint, + PRIMARY KEY (aggregate_id, sequence_number, timestamp) +); + +ALTER SEQUENCE command_records_id_seq OWNED BY NONE; +ALTER SEQUENCE command_records_id_seq OWNED BY commands.id; +ALTER SEQUENCE command_records_id_seq RENAME TO commands_id_seq; + +ALTER TABLE command_records RENAME TO old_command_records; +ALTER TABLE event_records RENAME TO old_event_records; +ALTER TABLE stream_records RENAME TO old_stream_records; + +\set ECHO none + +\ir ./sequent_pgsql.sql + +\set ECHO all + +SELECT clock_timestamp() AS migration_completed_at, + clock_timestamp() - :'migration_started_at'::timestamptz AS migration_duration \gset + +\echo Migration complated in :migration_duration (started at :migration_started_at, completed at :migration_completed_at) + +\echo execute COMMIT to commit, ROLLBACK to abort From 70b88516b28da2e1453b7b1e05850e3722b0ba4d Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Tue, 17 Sep 2024 15:49:27 +0200 Subject: [PATCH 103/128] Add Sequent 8 migration script --- db/sequent_8_migration.sql | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/db/sequent_8_migration.sql b/db/sequent_8_migration.sql index 42b28063..fa346013 100644 --- a/db/sequent_8_migration.sql +++ b/db/sequent_8_migration.sql @@ -1,3 +1,12 @@ +-- This script migrates a pre-sequent 8 database to the sequent 8 schema while preserving the data. +-- It runs in a single transaction and when completed you can COMMIT or ROLLBACK the results. +-- +-- Adjust this script to your needs (number of table partitions, etc). See comments marked with ### +-- for configuration sections. +-- +-- Ensure you test this on a copy of your production system to verify everything works and to +-- get an indication of the required downtime for your system. + \set ECHO all \set ON_ERROR_STOP \timing on @@ -41,8 +50,8 @@ CREATE TABLE commands ( event_sequence_number integer ) PARTITION BY RANGE (id); +-- ### Configure partitions as needed CREATE TABLE commands_default PARTITION OF commands DEFAULT; --- ### Add additional partitions as needed CREATE TABLE commands_0 PARTITION OF commands FOR VALUES FROM (1) TO (100e6); -- CREATE TABLE commands_1 PARTITION OF commands FOR VALUES FROM (100e6) TO (200e6); -- CREATE TABLE commands_2 PARTITION OF commands FOR VALUES FROM (200e6) TO (300e6); @@ -56,6 +65,7 @@ CREATE TABLE aggregates ( created_at timestamp with time zone NOT NULL DEFAULT NOW() ) PARTITION BY RANGE (aggregate_id); +-- ### Configure partitions as needed CREATE TABLE aggregates_0 PARTITION OF aggregates FOR VALUES FROM (MINVALUE) TO ('40000000-0000-0000-0000-000000000000'); CREATE TABLE aggregates_4 PARTITION OF aggregates FOR VALUES FROM ('40000000-0000-0000-0000-000000000000') TO ('80000000-0000-0000-0000-000000000000'); CREATE TABLE aggregates_8 PARTITION OF aggregates FOR VALUES FROM ('80000000-0000-0000-0000-000000000000') TO ('c0000000-0000-0000-0000-000000000000'); @@ -74,6 +84,7 @@ CREATE TABLE events ( CREATE INDEX events_xact_id_idx ON events (xact_id) WHERE xact_id IS NOT NULL; +-- ### Configure partitions as needed CREATE TABLE events_default PARTITION OF events DEFAULT; CREATE TABLE events_2023_and_earlier PARTITION OF events FOR VALUES FROM ('Y00') TO ('Y24'); CREATE TABLE events_2024 PARTITION OF events FOR VALUES FROM ('Y24') TO ('Y25'); @@ -143,6 +154,7 @@ SELECT id, FROM command; ALTER TABLE aggregates ADD UNIQUE (events_partition_key, aggregate_id); +-- ### Configure clustering as needed ALTER TABLE aggregates_0 CLUSTER ON aggregates_0_events_partition_key_aggregate_id_key; ALTER TABLE aggregates_4 CLUSTER ON aggregates_4_events_partition_key_aggregate_id_key; ALTER TABLE aggregates_8 CLUSTER ON aggregates_8_events_partition_key_aggregate_id_key; @@ -150,6 +162,7 @@ ALTER TABLE aggregates_c CLUSTER ON aggregates_c_events_partition_key_aggregate_ ALTER TABLE events ADD PRIMARY KEY (partition_key, aggregate_id, sequence_number); +-- ### Configure clustering as needed ALTER TABLE events_default CLUSTER ON events_default_pkey; ALTER TABLE events_2023_and_earlier CLUSTER ON events_2023_and_earlier_pkey; ALTER TABLE events_2024 CLUSTER ON events_2024_pkey; From 12a649b4d7cfe205998f1332a77b395b407e023b Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Tue, 17 Sep 2024 15:52:20 +0200 Subject: [PATCH 104/128] Updated link to Erik's blog series --- docs/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.md b/docs/index.md index 28a456ff..045067f9 100644 --- a/docs/index.md +++ b/docs/index.md @@ -51,7 +51,7 @@ There is a lot more information available on CQRS and event sourcing: - [Event sourcing](http://martinfowler.com/eaaDev/EventSourcing.html) - [Lars and Bob's presentation at GOTO Amsterdam](http://gotocon.com/dl/goto-amsterdam-2013/slides/BobForma_and_LarsVonk_EventSourcingInProductionSystems.pdf) -- [Erik's blog series](http://blog.zilverline.com/2011/02/10/towards-an-immutable-domain-model-monads-part-5/) +- [Erik's blog series](https://www.zilverline.com/blog/towards-an-immutable-domain-model-introduction-part-1) - [Simple CQRS example by Greg Young](https://github.com/gregoryyoung/m-r) - [Google](http://www.google.nl/search?ie=UTF-8&q=cqrs+event+sourcing) From d601719497c001d4c50f913a32c4b2398d365da7 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Tue, 17 Sep 2024 16:03:42 +0200 Subject: [PATCH 105/128] Add some documentation on upgrading --- docs/docs/upgrade-guide.md | 35 +++++++++++++++++++++++++++++++++++ docs/index.md | 6 ++++++ 2 files changed, 41 insertions(+) create mode 100644 docs/docs/upgrade-guide.md diff --git a/docs/docs/upgrade-guide.md b/docs/docs/upgrade-guide.md new file mode 100644 index 00000000..b9e8bc5c --- /dev/null +++ b/docs/docs/upgrade-guide.md @@ -0,0 +1,35 @@ +--- +title: Upgrade Guide +--- + +## Upgrade to Sequent 8.x from older versions + +Sequent 8 remodels the PostgreSQL event store to allow partitioning of +the aggregates, commands, and events tables. Furthermore it contains +various storage optimizations. To migrate your older Sequent database +an example script is provided in `db/sequent_8_migration.sql` that can +be run using `psql`. + +You will have to adjust this script to match your desired partitioning +setup, although the default configuration will work for many cases as +well. + +To make use of partitioning you will have to adjust your aggregates by +overriding the `events_partitio_key` method to indicate in which +partition the aggregate's events should be stored. For example, if you +wish to store your events in yearly partitions you might do something +like: + +```ruby +class MyAggregate < Sequent::Core::Aggregate + def events_partition_key + "Y#{@created_at.strftime('%y')}" + end +end +``` + +The partition key should be a string that: + +- is short (to optimize storage and indexing) +- put related aggregates together (e.g. based on user, time, tenant, + client, etc). diff --git a/docs/index.md b/docs/index.md index 045067f9..c3c2b5b5 100644 --- a/docs/index.md +++ b/docs/index.md @@ -45,6 +45,12 @@ The [Rails & Sequent](/docs/rails-sequent.html) guide shows how to use Sequent i Next to the tutorials there is the [Reference Guide](/docs/concepts.html) to provide and in-depth explanation of the several concepts (like `AggregateRoot`, `Event`, `Command` etc) used in Sequent. +## Upgrade Guide + +If you wish to upgrade your code and database to Sequent 8 from an +older version please reasd the [Upgrade +Guide](/docs/upgrade-guide.html). + ## Further reading There is a lot more information available on CQRS and event sourcing: From 054846b4b2661fbed86a4853052ea5395b6ffa23 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Wed, 18 Sep 2024 08:54:02 +0200 Subject: [PATCH 106/128] Add note to perform vacuum after committing Sequent 8 migration --- db/sequent_8_migration.sql | 2 +- docs/docs/upgrade-guide.md | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/db/sequent_8_migration.sql b/db/sequent_8_migration.sql index fa346013..0af5c7f5 100644 --- a/db/sequent_8_migration.sql +++ b/db/sequent_8_migration.sql @@ -259,4 +259,4 @@ SELECT clock_timestamp() AS migration_completed_at, \echo Migration complated in :migration_duration (started at :migration_started_at, completed at :migration_completed_at) -\echo execute COMMIT to commit, ROLLBACK to abort +\echo execute ROLLBACK to abort, COMMIT to commit followed by VACUUM VERBOSE ANALYZE to ensure good performance diff --git a/docs/docs/upgrade-guide.md b/docs/docs/upgrade-guide.md index b9e8bc5c..7ebae41e 100644 --- a/docs/docs/upgrade-guide.md +++ b/docs/docs/upgrade-guide.md @@ -14,6 +14,10 @@ You will have to adjust this script to match your desired partitioning setup, although the default configuration will work for many cases as well. +**IMPORTANT**: If the migration succeeds and you COMMIT the results +you must vacuum (e.g. using VACUUM VERBOSE ANALYZE) the new tables to +ensure good performance! + To make use of partitioning you will have to adjust your aggregates by overriding the `events_partitio_key` method to indicate in which partition the aggregate's events should be stored. For example, if you From bc3e61c2061fd4036515bbb746ce4d9b61bffa1b Mon Sep 17 00:00:00 2001 From: Lars Vonk Date: Tue, 8 Oct 2024 12:56:47 +0200 Subject: [PATCH 107/128] Update docs/index.md --- docs/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.md b/docs/index.md index c3c2b5b5..8adafd65 100644 --- a/docs/index.md +++ b/docs/index.md @@ -48,7 +48,7 @@ in-depth explanation of the several concepts (like `AggregateRoot`, `Event`, `Co ## Upgrade Guide If you wish to upgrade your code and database to Sequent 8 from an -older version please reasd the [Upgrade +older version please read the [Upgrade Guide](/docs/upgrade-guide.html). ## Further reading From 3c3c0e17948c4bbb568ee74b2714148e1d44808c Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Fri, 18 Oct 2024 16:27:17 +0200 Subject: [PATCH 108/128] Require sequent 8.x for new projects and fix warnings about csv gem --- lib/sequent/generator/template_project/Gemfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/sequent/generator/template_project/Gemfile b/lib/sequent/generator/template_project/Gemfile index d7d2622a..08a822b5 100644 --- a/lib/sequent/generator/template_project/Gemfile +++ b/lib/sequent/generator/template_project/Gemfile @@ -4,7 +4,8 @@ source 'https://rubygems.org' ruby file: '.ruby-version' gem 'rake' -gem 'sequent' +gem 'sequent', '>= 8' +gem 'csv' group :test do gem 'database_cleaner' From 139780fe4e27bac981d8b0a9a65402540afd8388 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Fri, 18 Oct 2024 16:27:34 +0200 Subject: [PATCH 109/128] Disable project build and spec test The generated project should be tested against the locally checkout out `sequent` gem but unfortunately bundler does not seem to find the correct version (even with the `config local.sequent` option). So disable for now. We could add it also as a separate github workflow test. --- spec/lib/sequent/generator/project_spec.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/spec/lib/sequent/generator/project_spec.rb b/spec/lib/sequent/generator/project_spec.rb index c247f049..d19c577b 100644 --- a/spec/lib/sequent/generator/project_spec.rb +++ b/spec/lib/sequent/generator/project_spec.rb @@ -58,7 +58,9 @@ expect(File.read('blog-with_special-symbols/Rakefile')).to include("require './blog_with_special_symbols'") end - it 'has working example with specs' do + # Ignore for now until we find a way to correctly test the template project + # against the locally checked out sequent code. + xit 'has working example with specs' do execute Bundler.with_unbundled_env do @@ -73,7 +75,10 @@ ruby_version=$(ruby -v | awk '{print $2}' | grep -o '^[0-9.]*') echo "$ruby_version" > .ruby-version + export BUNDLE_GEMFILE=./Gemfile gem install bundler + bundle config set local.sequent ../../.. + bundle config bundle install bundle exec rake sequent:db:drop bundle exec rake sequent:db:create From c2c89086bc3f5dc0377345ed53e8bd986442d391 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Mon, 21 Oct 2024 09:59:04 +0200 Subject: [PATCH 110/128] Rubocop updates and fixes --- .rubocop.yml | 36 +++++++++++++++++-- lib/sequent/core/aggregate_repository.rb | 4 +-- lib/sequent/core/command_service.rb | 2 +- .../core/helpers/string_to_value_parsers.rb | 2 +- lib/sequent/core/stream_record.rb | 2 +- .../generator/template_project/Gemfile | 2 +- sequent.gemspec | 2 +- .../sequent/core/aggregate_repository_spec.rb | 2 +- spec/lib/sequent/core/command_service_spec.rb | 2 +- spec/lib/sequent/core/event_publisher_spec.rb | 2 +- spec/lib/sequent/core/fixtures.rb | 2 +- .../sequent/fixtures/commands_and_events.rb | 2 +- 12 files changed, 46 insertions(+), 14 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index b1bd6cee..65e12ae3 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -10,6 +10,7 @@ # See https://github.com/rubocop-hq/rubocop/blob/master/manual/configuration.md AllCops: + TargetRubyVersion: 3.2 SuggestExtensions: rubocop-rake: false rubocop-rspec: false @@ -65,8 +66,10 @@ Layout/LineLength: Max: 120 AutoCorrect: true +Gemspec/AddRuntimeDependency: + Enabled: true Gemspec/RequiredRubyVersion: - Enabled: false + Enabled: true ## # Metrics (all disabled; we have our own opinion on this) @@ -142,9 +145,21 @@ Naming/MethodParameterName: ## # Lint ## +Lint/DuplicateSetElement: + Enabled: true + +Lint/ItWithoutArgumentsInBlock: + Enabled: true + +Lint/LiteralAssignmentInCondition: + Enabled: true + Lint/UriEscapeUnescape: Enabled: false # TODO enable +Lint/UselessNumericOperation: + Enabled: true + Lint/SuppressedException: AllowComments: true @@ -239,7 +254,24 @@ Layout/BlockAlignment: Enabled: true EnforcedStyleAlignWith: start_of_block -# TODO Ruby 3.0 + +Style/MapIntoArray: + Enabled: true + +Style/RedundantInterpolationUnfreeze: + Enabled: true + +Style/SendWithLiteralMethodName: + Enabled: true + +Style/SingleLineDoEndBlock: + Enabled: true + +Style/SuperArguments: + Enabled: true + +Style/SuperWithArgsParentheses: + Enabled: true Style/HashTransformKeys: Enabled: false diff --git a/lib/sequent/core/aggregate_repository.rb b/lib/sequent/core/aggregate_repository.rb index 037b33e8..f357ceed 100644 --- a/lib/sequent/core/aggregate_repository.rb +++ b/lib/sequent/core/aggregate_repository.rb @@ -20,13 +20,13 @@ class AggregateRepository class NonUniqueAggregateId < StandardError def initialize(existing, new) - super "Duplicate aggregate #{new} with same key as existing #{existing}" + super("Duplicate aggregate #{new} with same key as existing #{existing}") end end class AggregateNotFound < StandardError def initialize(id) - super "Aggregate with id #{id} not found" + super("Aggregate with id #{id} not found") end end diff --git a/lib/sequent/core/command_service.rb b/lib/sequent/core/command_service.rb index 39771344..e6ed5a32 100644 --- a/lib/sequent/core/command_service.rb +++ b/lib/sequent/core/command_service.rb @@ -118,7 +118,7 @@ class CommandNotValid < ArgumentError def initialize(command) @command = command msg = @command.respond_to?(:aggregate_id) ? " #{@command.aggregate_id}" : '' - super "Invalid command #{@command.class}#{msg}, errors: #{@command.validation_errors}" + super("Invalid command #{@command.class}#{msg}, errors: #{@command.validation_errors}") end def errors(prefix = nil) diff --git a/lib/sequent/core/helpers/string_to_value_parsers.rb b/lib/sequent/core/helpers/string_to_value_parsers.rb index e6ebae47..3f0f20c3 100644 --- a/lib/sequent/core/helpers/string_to_value_parsers.rb +++ b/lib/sequent/core/helpers/string_to_value_parsers.rb @@ -39,7 +39,7 @@ def self.parse_to_bool(value) if value.blank? && !(value.is_a?(TrueClass) || value.is_a?(FalseClass)) nil else - (value.is_a?(TrueClass) || value == 'true') + value.is_a?(TrueClass) || value == 'true' end end diff --git a/lib/sequent/core/stream_record.rb b/lib/sequent/core/stream_record.rb index e40b7d42..940691df 100644 --- a/lib/sequent/core/stream_record.rb +++ b/lib/sequent/core/stream_record.rb @@ -6,7 +6,7 @@ module Sequent module Core EventStream = Data.define(:aggregate_type, :aggregate_id, :events_partition_key, :snapshot_outdated_at) do def initialize(aggregate_type:, aggregate_id:, events_partition_key: '', snapshot_outdated_at: nil) - super(aggregate_type:, aggregate_id:, events_partition_key:, snapshot_outdated_at:) + super end end diff --git a/lib/sequent/generator/template_project/Gemfile b/lib/sequent/generator/template_project/Gemfile index 08a822b5..d835d923 100644 --- a/lib/sequent/generator/template_project/Gemfile +++ b/lib/sequent/generator/template_project/Gemfile @@ -3,9 +3,9 @@ source 'https://rubygems.org' ruby file: '.ruby-version' +gem 'csv' gem 'rake' gem 'sequent', '>= 8' -gem 'csv' group :test do gem 'database_cleaner' diff --git a/sequent.gemspec b/sequent.gemspec index cd663bcc..c9fa4597 100644 --- a/sequent.gemspec +++ b/sequent.gemspec @@ -4,7 +4,7 @@ require_relative 'lib/version' Gem::Specification.new do |s| s.platform = Gem::Platform::RUBY - s.required_ruby_version = '>= 3.0' + s.required_ruby_version = '>= 3.2' s.name = 'sequent' s.version = Sequent::VERSION s.summary = 'Event sourcing framework for Ruby' diff --git a/spec/lib/sequent/core/aggregate_repository_spec.rb b/spec/lib/sequent/core/aggregate_repository_spec.rb index 08df2df8..184dfcca 100644 --- a/spec/lib/sequent/core/aggregate_repository_spec.rb +++ b/spec/lib/sequent/core/aggregate_repository_spec.rb @@ -388,7 +388,7 @@ class DummyAggregate3 < Sequent::Core::AggregateRoot attr_reader :pinged def initialize(id) - super(id) + super apply DummyCreated end diff --git a/spec/lib/sequent/core/command_service_spec.rb b/spec/lib/sequent/core/command_service_spec.rb index d9bab4d8..fc3f6a1d 100644 --- a/spec/lib/sequent/core/command_service_spec.rb +++ b/spec/lib/sequent/core/command_service_spec.rb @@ -33,7 +33,7 @@ class CommandWithSecret < Sequent::Core::BaseCommand def initialize(*args) reset - super(*args) + super end def reset diff --git a/spec/lib/sequent/core/event_publisher_spec.rb b/spec/lib/sequent/core/event_publisher_spec.rb index 018be168..f3b7b910 100644 --- a/spec/lib/sequent/core/event_publisher_spec.rb +++ b/spec/lib/sequent/core/event_publisher_spec.rb @@ -50,7 +50,7 @@ class TestEventHandler < Sequent::Core::Projector def initialize(*args) @sequence_numbers = [] - super(*args) + super end attr_reader :sequence_numbers diff --git a/spec/lib/sequent/core/fixtures.rb b/spec/lib/sequent/core/fixtures.rb index 6e91bf5b..24491ea2 100644 --- a/spec/lib/sequent/core/fixtures.rb +++ b/spec/lib/sequent/core/fixtures.rb @@ -26,7 +26,7 @@ class PersonAggregate < Sequent::Core::AggregateRoot is_a(Statusable) def initialize(id) - super(id) + super apply TestEvent, field: 'value' end diff --git a/spec/lib/sequent/fixtures/commands_and_events.rb b/spec/lib/sequent/fixtures/commands_and_events.rb index d20258d4..10e7ac75 100644 --- a/spec/lib/sequent/fixtures/commands_and_events.rb +++ b/spec/lib/sequent/fixtures/commands_and_events.rb @@ -25,7 +25,7 @@ module Fixtures AggregateClass = Class.new(Sequent::Core::AggregateRoot) do def initialize(id) - super(id) + super apply Event1 end From 6ad07222587add5797997bbaf163ed907c10043a Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Mon, 21 Oct 2024 10:39:14 +0200 Subject: [PATCH 111/128] Workaround for deprecated `query_constraints` option Composite foreign keys require the use of `query_options` in AR 7.1.3 but cause a deprecation warning in AR 7.2. Conditionally configure the affected associations. --- gemfiles/ar_7_1.gemfile | 2 +- gemfiles/ar_7_1.gemfile.lock | 20 +++++++++---------- gemfiles/ar_7_2.gemfile.lock | 4 ++-- lib/sequent/core/event_record.rb | 15 ++++++++++---- lib/sequent/internal/partitioned_aggregate.rb | 15 ++++++++++---- lib/sequent/internal/partitioned_event.rb | 15 ++++++++++---- sequent.gemspec | 2 +- .../internal/partitioned_storage_spec.rb | 12 +++++++++++ 8 files changed, 59 insertions(+), 26 deletions(-) diff --git a/gemfiles/ar_7_1.gemfile b/gemfiles/ar_7_1.gemfile index 6ecbac58..edbfd29a 100644 --- a/gemfiles/ar_7_1.gemfile +++ b/gemfiles/ar_7_1.gemfile @@ -2,7 +2,7 @@ source 'https://rubygems.org' -active_star_version = '= 7.1.1' +active_star_version = '= 7.1.3' gem 'activemodel', active_star_version gem 'activerecord', active_star_version diff --git a/gemfiles/ar_7_1.gemfile.lock b/gemfiles/ar_7_1.gemfile.lock index 9a017944..01ecb78c 100644 --- a/gemfiles/ar_7_1.gemfile.lock +++ b/gemfiles/ar_7_1.gemfile.lock @@ -2,8 +2,8 @@ PATH remote: .. specs: sequent (8.0.0.pre.dev.1) - activemodel (>= 7.1) - activerecord (>= 7.1) + activemodel (>= 7.1.3) + activerecord (>= 7.1.3) bcrypt (~> 3.1) csv (~> 3.3) i18n @@ -19,13 +19,13 @@ PATH GEM remote: https://rubygems.org/ specs: - activemodel (7.1.1) - activesupport (= 7.1.1) - activerecord (7.1.1) - activemodel (= 7.1.1) - activesupport (= 7.1.1) + activemodel (7.1.3) + activesupport (= 7.1.3) + activerecord (7.1.3) + activemodel (= 7.1.3) + activesupport (= 7.1.3) timeout (>= 0.4.0) - activesupport (7.1.1) + activesupport (7.1.3) base64 bigdecimal concurrent-ruby (~> 1.0, >= 1.0.2) @@ -120,8 +120,8 @@ PLATFORMS x86_64-linux DEPENDENCIES - activemodel (= 7.1.1) - activerecord (= 7.1.1) + activemodel (= 7.1.3) + activerecord (= 7.1.3) prop_check (~> 1.0) pry (~> 0.13) rake (~> 13) diff --git a/gemfiles/ar_7_2.gemfile.lock b/gemfiles/ar_7_2.gemfile.lock index 554bc0a8..32cf7414 100644 --- a/gemfiles/ar_7_2.gemfile.lock +++ b/gemfiles/ar_7_2.gemfile.lock @@ -2,8 +2,8 @@ PATH remote: .. specs: sequent (8.0.0.pre.dev.1) - activemodel (>= 7.1) - activerecord (>= 7.1) + activemodel (>= 7.1.3) + activerecord (>= 7.1.3) bcrypt (~> 3.1) csv (~> 3.3) i18n diff --git a/lib/sequent/core/event_record.rb b/lib/sequent/core/event_record.rb index 89e2d3fe..ea768d3b 100644 --- a/lib/sequent/core/event_record.rb +++ b/lib/sequent/core/event_record.rb @@ -94,10 +94,17 @@ class EventRecord < Sequent::ApplicationRecord belongs_to :parent_command, class_name: :CommandRecord, foreign_key: :command_record_id - has_many :child_commands, - class_name: :CommandRecord, - query_constraints: %i[event_aggregate_id event_sequence_number], - primary_key: %i[aggregate_id sequence_number] + if Gem.loaded_specs['activerecord'].version < Gem::Version.create('7.2') + has_many :child_commands, + class_name: :CommandRecord, + primary_key: %i[aggregate_id sequence_number], + query_constraints: %i[event_aggregate_id event_sequence_number] + else + has_many :child_commands, + class_name: :CommandRecord, + primary_key: %i[aggregate_id sequence_number], + foreign_key: %i[event_aggregate_id event_sequence_number] + end validates_presence_of :aggregate_id, :sequence_number, :event_type, :event_json, :stream_record, :parent_command validates_numericality_of :sequence_number, only_integer: true, greater_than: 0 diff --git a/lib/sequent/internal/partitioned_aggregate.rb b/lib/sequent/internal/partitioned_aggregate.rb index 0dfa8643..72eba0fb 100644 --- a/lib/sequent/internal/partitioned_aggregate.rb +++ b/lib/sequent/internal/partitioned_aggregate.rb @@ -10,10 +10,17 @@ class PartitionedAggregate < Sequent::ApplicationRecord self.primary_key = %i[aggregate_id] belongs_to :aggregate_type - has_many :partitioned_events, - inverse_of: :partitioned_aggregate, - primary_key: %i[events_partition_key aggregate_id], - query_constraints: %i[partition_key aggregate_id] + if Gem.loaded_specs['activerecord'].version < Gem::Version.create('7.2') + has_many :partitioned_events, + inverse_of: :partitioned_aggregate, + primary_key: %w[events_partition_key aggregate_id], + query_constraints: %w[partition_key aggregate_id] + else + has_many :partitioned_events, + inverse_of: :partitioned_aggregate, + primary_key: %i[events_partition_key aggregate_id], + foreign_key: %i[partition_key aggregate_id] + end end end end diff --git a/lib/sequent/internal/partitioned_event.rb b/lib/sequent/internal/partitioned_event.rb index 69f6d550..e81dcab1 100644 --- a/lib/sequent/internal/partitioned_event.rb +++ b/lib/sequent/internal/partitioned_event.rb @@ -13,10 +13,17 @@ class PartitionedEvent < Sequent::ApplicationRecord belongs_to :partitioned_command, inverse_of: :partitioned_events, foreign_key: :command_id - belongs_to :partitioned_aggregate, - inverse_of: :partitioned_events, - primary_key: %w[partition_key aggregate_id], - foreign_key: %w[events_partition_key aggregate_id] + if Gem.loaded_specs['activerecord'].version < Gem::Version.create('7.2') + belongs_to :partitioned_aggregate, + inverse_of: :partitioned_events, + primary_key: %w[partition_key aggregate_id], + query_constraints: %w[events_partition_key aggregate_id] + else + belongs_to :partitioned_aggregate, + inverse_of: :partitioned_events, + primary_key: %w[partition_key aggregate_id], + foreign_key: %w[events_partition_key aggregate_id] + end end end end diff --git a/sequent.gemspec b/sequent.gemspec index c9fa4597..d8ca9279 100644 --- a/sequent.gemspec +++ b/sequent.gemspec @@ -23,7 +23,7 @@ Gem::Specification.new do |s| 'https://github.com/zilverline/sequent' s.license = 'MIT' - active_star_version = '>= 7.1' + active_star_version = '>= 7.1.3' rspec_version = '~> 3.10' diff --git a/spec/lib/sequent/internal/partitioned_storage_spec.rb b/spec/lib/sequent/internal/partitioned_storage_spec.rb index 2ef5e309..15e1dca1 100644 --- a/spec/lib/sequent/internal/partitioned_storage_spec.rb +++ b/spec/lib/sequent/internal/partitioned_storage_spec.rb @@ -25,6 +25,18 @@ module Internal end it 'persists to the partitioned tables' do + if Gem.loaded_specs['activerecord'].version < Gem::Version.create('7.2') + skip("AR 7.1.3 doesn't allow correct configuration of composite foreign key constraint") + # AR 7.1.3 fails with `Association Sequent::Internal::PartitionedEvent#partitioned_aggregate primary key + # ["partition_key", "aggregate_id"] doesn't match with foreign key ["events_partition_key", + # "aggregate_id"]. Please specify query_constraints, or primary_key and foreign_key values.`, however + # specifying the `foreign_key` with multiple columns results in the error: `Passing ["events_partition_key", + # "aggregate_id"] array to :foreign_key option on the + # Sequent::Internal::PartitionedEvent#partitioned_aggregate association is not supported. Use the + # query_constraints: ["events_partition_key", "aggregate_id"] option instead to represent a composite foreign + # key.` + end + aggregate = PartitionedAggregate.first expect(aggregate).to be_present expect(aggregate.aggregate_id).to eq(aggregate_id) From e3dc12a2c445d2d3e0c0b18508137b9e653e2330 Mon Sep 17 00:00:00 2001 From: Lars Vonk Date: Thu, 24 Oct 2024 13:19:40 +0200 Subject: [PATCH 112/128] Update docs/docs/upgrade-guide.md --- docs/docs/upgrade-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/upgrade-guide.md b/docs/docs/upgrade-guide.md index 7ebae41e..03b2cfdf 100644 --- a/docs/docs/upgrade-guide.md +++ b/docs/docs/upgrade-guide.md @@ -19,7 +19,7 @@ you must vacuum (e.g. using VACUUM VERBOSE ANALYZE) the new tables to ensure good performance! To make use of partitioning you will have to adjust your aggregates by -overriding the `events_partitio_key` method to indicate in which +overriding the `events_partition_key` method to indicate in which partition the aggregate's events should be stored. For example, if you wish to store your events in yearly partitions you might do something like: From 372e121e5d528b766cb844b257d52f038479cf3b Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Thu, 24 Oct 2024 16:55:35 +0200 Subject: [PATCH 113/128] Split SQL schema creation files to ease partitioning customization Schema parts are now used by both the Sequent 8 migration and the initial install. --- db/sequent_8_migration.sql | 159 ++----------------------------- db/sequent_schema.rb | 8 +- db/sequent_schema.sql | 108 --------------------- db/sequent_schema_indexes.sql | 37 +++++++ db/sequent_schema_partitions.sql | 19 ++++ db/sequent_schema_tables.sql | 74 ++++++++++++++ lib/sequent/generator/project.rb | 2 +- 7 files changed, 145 insertions(+), 262 deletions(-) delete mode 100644 db/sequent_schema.sql create mode 100644 db/sequent_schema_indexes.sql create mode 100644 db/sequent_schema_partitions.sql create mode 100644 db/sequent_schema_tables.sql diff --git a/db/sequent_8_migration.sql b/db/sequent_8_migration.sql index 0af5c7f5..523a53e3 100644 --- a/db/sequent_8_migration.sql +++ b/db/sequent_8_migration.sql @@ -25,71 +25,11 @@ BEGIN; SET temp_tablespaces = 'pg_default'; SET search_path TO sequent_schema; -CREATE TYPE aggregate_event_type AS ( - aggregate_type text, - aggregate_id uuid, - events_partition_key text, - snapshot_threshold integer, - event_type text, - event_json jsonb -); - -CREATE TABLE command_types (id SMALLINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, type text UNIQUE NOT NULL); -CREATE TABLE aggregate_types (id SMALLINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, type text UNIQUE NOT NULL); -CREATE TABLE event_types (id SMALLINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, type text UNIQUE NOT NULL); - - -CREATE TABLE commands ( - id bigint NOT NULL DEFAULT nextval('sequent_schema.command_records_id_seq'), - created_at timestamp with time zone NOT NULL, - user_id uuid, - aggregate_id uuid, - command_type_id SMALLINT NOT NULL, - command_json jsonb NOT NULL, - event_aggregate_id uuid, - event_sequence_number integer -) PARTITION BY RANGE (id); - --- ### Configure partitions as needed -CREATE TABLE commands_default PARTITION OF commands DEFAULT; -CREATE TABLE commands_0 PARTITION OF commands FOR VALUES FROM (1) TO (100e6); --- CREATE TABLE commands_1 PARTITION OF commands FOR VALUES FROM (100e6) TO (200e6); --- CREATE TABLE commands_2 PARTITION OF commands FOR VALUES FROM (200e6) TO (300e6); --- CREATE TABLE commands_3 PARTITION OF commands FOR VALUES FROM (300e6) TO (400e6); - -CREATE TABLE aggregates ( - aggregate_id uuid NOT NULL, - events_partition_key text NOT NULL DEFAULT '', - aggregate_type_id SMALLINT NOT NULL, - snapshot_threshold integer, - created_at timestamp with time zone NOT NULL DEFAULT NOW() -) PARTITION BY RANGE (aggregate_id); - --- ### Configure partitions as needed -CREATE TABLE aggregates_0 PARTITION OF aggregates FOR VALUES FROM (MINVALUE) TO ('40000000-0000-0000-0000-000000000000'); -CREATE TABLE aggregates_4 PARTITION OF aggregates FOR VALUES FROM ('40000000-0000-0000-0000-000000000000') TO ('80000000-0000-0000-0000-000000000000'); -CREATE TABLE aggregates_8 PARTITION OF aggregates FOR VALUES FROM ('80000000-0000-0000-0000-000000000000') TO ('c0000000-0000-0000-0000-000000000000'); -CREATE TABLE aggregates_c PARTITION OF aggregates FOR VALUES FROM ('c0000000-0000-0000-0000-000000000000') TO (MAXVALUE); - -CREATE TABLE events ( - aggregate_id uuid NOT NULL, - partition_key text NOT NULL DEFAULT '', - sequence_number integer NOT NULL, - created_at timestamp with time zone NOT NULL, - command_id bigint NOT NULL, - event_type_id SMALLINT NOT NULL, - event_json jsonb NOT NULL, - xact_id bigint -) PARTITION BY RANGE (partition_key); - -CREATE INDEX events_xact_id_idx ON events (xact_id) WHERE xact_id IS NOT NULL; +ALTER SEQUENCE command_records_id_seq OWNED BY NONE; +ALTER SEQUENCE command_records_id_seq RENAME TO commands_id_seq; --- ### Configure partitions as needed -CREATE TABLE events_default PARTITION OF events DEFAULT; -CREATE TABLE events_2023_and_earlier PARTITION OF events FOR VALUES FROM ('Y00') TO ('Y24'); -CREATE TABLE events_2024 PARTITION OF events FOR VALUES FROM ('Y24') TO ('Y25'); -CREATE TABLE events_2025_and_later PARTITION OF events FOR VALUES FROM ('Y25') TO ('Y99'); -CREATE TABLE events_aggregate PARTITION OF events FOR VALUES FROM ('A') TO ('Ag'); +\ir ./sequent_schema_tables.sql +\ir ./sequent_schema_partitions.sql INSERT INTO aggregate_types (type) SELECT DISTINCT aggregate_type @@ -109,8 +49,8 @@ SELECT DISTINCT command_type ANALYZE aggregate_types, event_types, command_types; -INSERT INTO aggregates -SELECT aggregate_id, '', (SELECT t.id FROM aggregate_types t WHERE aggregate_type = t.type), snapshot_threshold, created_at AT TIME ZONE 'Europe/Amsterdam' +INSERT INTO aggregates (aggregate_id, aggregate_type_id, snapshot_threshold, created_at) +SELECT aggregate_id, (SELECT t.id FROM aggregate_types t WHERE aggregate_type = t.type), snapshot_threshold, created_at AT TIME ZONE 'Europe/Amsterdam' FROM stream_records; WITH e AS MATERIALIZED ( @@ -131,7 +71,6 @@ SELECT aggregate_id, event_json - 'created_at' FROM e; - WITH command AS MATERIALIZED ( SELECT c.id, created_at, t.id AS command_type_id, @@ -153,52 +92,6 @@ SELECT id, (json->>'event_sequence_number')::integer FROM command; -ALTER TABLE aggregates ADD UNIQUE (events_partition_key, aggregate_id); --- ### Configure clustering as needed -ALTER TABLE aggregates_0 CLUSTER ON aggregates_0_events_partition_key_aggregate_id_key; -ALTER TABLE aggregates_4 CLUSTER ON aggregates_4_events_partition_key_aggregate_id_key; -ALTER TABLE aggregates_8 CLUSTER ON aggregates_8_events_partition_key_aggregate_id_key; -ALTER TABLE aggregates_c CLUSTER ON aggregates_c_events_partition_key_aggregate_id_key; - -ALTER TABLE events - ADD PRIMARY KEY (partition_key, aggregate_id, sequence_number); --- ### Configure clustering as needed -ALTER TABLE events_default CLUSTER ON events_default_pkey; -ALTER TABLE events_2023_and_earlier CLUSTER ON events_2023_and_earlier_pkey; -ALTER TABLE events_2024 CLUSTER ON events_2024_pkey; -ALTER TABLE events_2025_and_later CLUSTER ON events_2025_and_later_pkey; -ALTER TABLE events_aggregate CLUSTER ON events_aggregate_pkey; - -ALTER TABLE commands ADD PRIMARY KEY (id); - -ALTER TABLE aggregates ADD PRIMARY KEY (aggregate_id); -CREATE INDEX aggregates_aggregate_type_id_idx ON aggregates (aggregate_type_id); -ALTER TABLE aggregates - ADD FOREIGN KEY (aggregate_type_id) REFERENCES aggregate_types (id) ON UPDATE CASCADE; - -CREATE INDEX events_command_id_idx ON events (command_id); -CREATE INDEX events_event_type_id_idx ON events (event_type_id); -ALTER TABLE events - ADD FOREIGN KEY (partition_key, aggregate_id) REFERENCES aggregates (events_partition_key, aggregate_id) - ON UPDATE CASCADE ON DELETE RESTRICT; -ALTER TABLE events - ADD FOREIGN KEY (command_id) REFERENCES commands (id) ON UPDATE RESTRICT ON DELETE RESTRICT; -ALTER TABLE events - ADD FOREIGN KEY (event_type_id) REFERENCES event_types (id) ON UPDATE CASCADE; -ALTER TABLE events ALTER COLUMN xact_id SET DEFAULT pg_current_xact_id()::text::bigint; - -CREATE INDEX commands_command_type_id_idx ON commands (command_type_id); -CREATE INDEX commands_aggregate_id_idx ON commands (aggregate_id); -CREATE INDEX commands_event_idx ON commands (event_aggregate_id, event_sequence_number); -ALTER TABLE commands - ADD FOREIGN KEY (command_type_id) REFERENCES command_types (id) ON UPDATE CASCADE; - -CREATE TABLE aggregates_that_need_snapshots ( - aggregate_id uuid NOT NULL PRIMARY KEY REFERENCES aggregates (aggregate_id) ON UPDATE CASCADE ON DELETE CASCADE, - snapshot_sequence_number_high_water_mark integer, - snapshot_outdated_at timestamp with time zone, - snapshot_scheduled_at timestamp with time zone -); INSERT INTO aggregates_that_need_snapshots (aggregate_id, snapshot_sequence_number_high_water_mark, snapshot_outdated_at) SELECT aggregate_id, MAX(sequence_number), NOW() FROM event_records @@ -206,48 +99,12 @@ SELECT aggregate_id, MAX(sequence_number), NOW() GROUP BY 1 ORDER BY 1; -CREATE INDEX aggregates_that_need_snapshots_outdated_idx - ON aggregates_that_need_snapshots (snapshot_outdated_at ASC, snapshot_sequence_number_high_water_mark DESC, aggregate_id ASC) - WHERE snapshot_outdated_at IS NOT NULL; -COMMENT ON TABLE aggregates_that_need_snapshots IS 'Contains a row for every aggregate with more events than its snapshot threshold.'; -COMMENT ON COLUMN aggregates_that_need_snapshots.snapshot_sequence_number_high_water_mark - IS 'The highest sequence number of the stored snapshot. Kept when snapshot are deleted to more easily query aggregates that need snapshotting the most'; -COMMENT ON COLUMN aggregates_that_need_snapshots.snapshot_outdated_at IS 'Not NULL indicates a snapshot is needed since the stored timestamp'; -COMMENT ON COLUMN aggregates_that_need_snapshots.snapshot_scheduled_at IS 'Not NULL indicates a snapshot is in the process of being taken'; - -CREATE TABLE snapshot_records ( - aggregate_id uuid NOT NULL, - sequence_number integer NOT NULL, - created_at timestamptz NOT NULL, - snapshot_type text NOT NULL, - snapshot_json jsonb NOT NULL, - PRIMARY KEY (aggregate_id, sequence_number), - FOREIGN KEY (aggregate_id) REFERENCES aggregates (aggregate_id) ON UPDATE CASCADE ON DELETE CASCADE -); - -CREATE TABLE saved_event_records ( - operation varchar(1) NOT NULL CHECK (operation IN ('U', 'D')), - timestamp timestamptz NOT NULL, - "user" text NOT NULL, - aggregate_id uuid NOT NULL, - partition_key text DEFAULT '', - sequence_number integer NOT NULL, - created_at timestamp with time zone NOT NULL, - command_id bigint NOT NULL, - event_type text NOT NULL, - event_json jsonb NOT NULL, - xact_id bigint, - PRIMARY KEY (aggregate_id, sequence_number, timestamp) -); - -ALTER SEQUENCE command_records_id_seq OWNED BY NONE; -ALTER SEQUENCE command_records_id_seq OWNED BY commands.id; -ALTER SEQUENCE command_records_id_seq RENAME TO commands_id_seq; - ALTER TABLE command_records RENAME TO old_command_records; ALTER TABLE event_records RENAME TO old_event_records; ALTER TABLE stream_records RENAME TO old_stream_records; +\ir ./sequent_schema_indexes.sql + \set ECHO none \ir ./sequent_pgsql.sql diff --git a/db/sequent_schema.rb b/db/sequent_schema.rb index 4a4ed00a..b4e63dab 100644 --- a/db/sequent_schema.rb +++ b/db/sequent_schema.rb @@ -2,8 +2,12 @@ ActiveRecord::Schema.define do say_with_time 'Installing Sequent schema' do - say 'Creating tables and indexes', true - suppress_messages { execute File.read("#{File.dirname(__FILE__)}/sequent_schema.sql") } + say 'Creating tables', true + suppress_messages { execute File.read("#{File.dirname(__FILE__)}/sequent_schema_tables.sql") } + say 'Creating table partitions', true + suppress_messages { execute File.read("#{File.dirname(__FILE__)}/sequent_schema_partitions.sql") } + say 'Creating constraints and indexes', true + suppress_messages { execute File.read("#{File.dirname(__FILE__)}/sequent_schema_indexes.sql") } say 'Creating stored procedures and views', true suppress_messages { execute File.read("#{File.dirname(__FILE__)}/sequent_pgsql.sql") } end diff --git a/db/sequent_schema.sql b/db/sequent_schema.sql deleted file mode 100644 index 3bc3b7a0..00000000 --- a/db/sequent_schema.sql +++ /dev/null @@ -1,108 +0,0 @@ -CREATE TABLE command_types (id SMALLINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, type text UNIQUE NOT NULL); -CREATE TABLE aggregate_types (id SMALLINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, type text UNIQUE NOT NULL); -CREATE TABLE event_types (id SMALLINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, type text UNIQUE NOT NULL); - -CREATE TABLE commands ( - id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - created_at timestamp with time zone NOT NULL, - user_id uuid, - aggregate_id uuid, - command_type_id SMALLINT NOT NULL REFERENCES command_types (id), - command_json jsonb NOT NULL, - event_aggregate_id uuid, - event_sequence_number integer -) PARTITION BY RANGE (id); -CREATE INDEX commands_command_type_id_idx ON commands (command_type_id); -CREATE INDEX commands_aggregate_id_idx ON commands (aggregate_id); -CREATE INDEX commands_event_idx ON commands (event_aggregate_id, event_sequence_number); - -CREATE TABLE commands_default PARTITION OF commands DEFAULT; - -CREATE TABLE aggregates ( - aggregate_id uuid PRIMARY KEY, - events_partition_key text NOT NULL DEFAULT '', - aggregate_type_id SMALLINT NOT NULL REFERENCES aggregate_types (id), - created_at timestamp with time zone NOT NULL DEFAULT NOW(), - UNIQUE (events_partition_key, aggregate_id) -) PARTITION BY RANGE (aggregate_id); -CREATE INDEX aggregates_aggregate_type_id_idx ON aggregates (aggregate_type_id); - -CREATE TABLE aggregates_0 PARTITION OF aggregates FOR VALUES FROM (MINVALUE) TO ('40000000-0000-0000-0000-000000000000'); -ALTER TABLE aggregates_0 CLUSTER ON aggregates_0_events_partition_key_aggregate_id_key; -CREATE TABLE aggregates_4 PARTITION OF aggregates FOR VALUES FROM ('40000000-0000-0000-0000-000000000000') TO ('80000000-0000-0000-0000-000000000000'); -ALTER TABLE aggregates_4 CLUSTER ON aggregates_4_events_partition_key_aggregate_id_key; -CREATE TABLE aggregates_8 PARTITION OF aggregates FOR VALUES FROM ('80000000-0000-0000-0000-000000000000') TO ('c0000000-0000-0000-0000-000000000000'); -ALTER TABLE aggregates_8 CLUSTER ON aggregates_8_events_partition_key_aggregate_id_key; -CREATE TABLE aggregates_c PARTITION OF aggregates FOR VALUES FROM ('c0000000-0000-0000-0000-000000000000') TO (MAXVALUE); -ALTER TABLE aggregates_c CLUSTER ON aggregates_c_events_partition_key_aggregate_id_key; - -CREATE TABLE events ( - aggregate_id uuid NOT NULL, - partition_key text DEFAULT '', - sequence_number integer NOT NULL, - created_at timestamp with time zone NOT NULL, - command_id bigint NOT NULL, - event_type_id SMALLINT NOT NULL REFERENCES event_types (id), - event_json jsonb NOT NULL, - xact_id bigint DEFAULT pg_current_xact_id()::text::bigint, - PRIMARY KEY (partition_key, aggregate_id, sequence_number), - FOREIGN KEY (partition_key, aggregate_id) - REFERENCES aggregates (events_partition_key, aggregate_id) - ON UPDATE CASCADE ON DELETE RESTRICT, - FOREIGN KEY (command_id) REFERENCES commands (id) -) PARTITION BY RANGE (partition_key); -CREATE INDEX events_command_id_idx ON events (command_id); -CREATE INDEX events_event_type_id_idx ON events (event_type_id); -CREATE INDEX events_xact_id_idx ON events (xact_id) WHERE xact_id IS NOT NULL; - -CREATE TABLE events_default PARTITION OF events DEFAULT; -ALTER TABLE events_default CLUSTER ON events_default_pkey; -CREATE TABLE events_2023_and_earlier PARTITION OF events FOR VALUES FROM ('Y00') TO ('Y24'); -ALTER TABLE events_2023_and_earlier CLUSTER ON events_2023_and_earlier_pkey; -CREATE TABLE events_2024 PARTITION OF events FOR VALUES FROM ('Y24') TO ('Y25'); -ALTER TABLE events_2024 CLUSTER ON events_2024_pkey; -CREATE TABLE events_2025_and_later PARTITION OF events FOR VALUES FROM ('Y25') TO ('Y99'); -ALTER TABLE events_2025_and_later CLUSTER ON events_2025_and_later_pkey; -CREATE TABLE events_aggregate PARTITION OF events FOR VALUES FROM ('A') TO ('Ag'); -ALTER TABLE events_aggregate CLUSTER ON events_aggregate_pkey; - -CREATE TABLE aggregates_that_need_snapshots ( - aggregate_id uuid NOT NULL PRIMARY KEY REFERENCES aggregates (aggregate_id) ON UPDATE CASCADE ON DELETE CASCADE, - snapshot_sequence_number_high_water_mark integer, - snapshot_outdated_at timestamp with time zone, - snapshot_scheduled_at timestamp with time zone -); -CREATE INDEX aggregates_that_need_snapshots_outdated_idx - ON aggregates_that_need_snapshots (snapshot_outdated_at ASC, snapshot_sequence_number_high_water_mark DESC, aggregate_id ASC) - WHERE snapshot_outdated_at IS NOT NULL; -COMMENT ON TABLE aggregates_that_need_snapshots IS 'Contains a row for every aggregate with more events than its snapshot threshold.'; -COMMENT ON COLUMN aggregates_that_need_snapshots.snapshot_sequence_number_high_water_mark - IS 'The highest sequence number of the stored snapshot. Kept when snapshot are deleted to more easily query aggregates that need snapshotting the most'; -COMMENT ON COLUMN aggregates_that_need_snapshots.snapshot_outdated_at IS 'Not NULL indicates a snapshot is needed since the stored timestamp'; -COMMENT ON COLUMN aggregates_that_need_snapshots.snapshot_scheduled_at IS 'Not NULL indicates a snapshot is in the process of being taken'; - -CREATE TABLE snapshot_records ( - aggregate_id uuid NOT NULL, - sequence_number integer NOT NULL, - created_at timestamptz NOT NULL DEFAULT NOW(), - snapshot_type text NOT NULL, - snapshot_json jsonb NOT NULL, - PRIMARY KEY (aggregate_id, sequence_number), - FOREIGN KEY (aggregate_id) REFERENCES aggregates_that_need_snapshots (aggregate_id) - ON UPDATE CASCADE ON DELETE CASCADE -); - -CREATE TABLE saved_event_records ( - operation varchar(1) NOT NULL CHECK (operation IN ('U', 'D')), - timestamp timestamptz NOT NULL, - "user" text NOT NULL, - aggregate_id uuid NOT NULL, - partition_key text DEFAULT '', - sequence_number integer NOT NULL, - created_at timestamp with time zone NOT NULL, - command_id bigint NOT NULL, - event_type text NOT NULL, - event_json jsonb NOT NULL, - xact_id bigint, - PRIMARY KEY (aggregate_id, sequence_number, timestamp) -); diff --git a/db/sequent_schema_indexes.sql b/db/sequent_schema_indexes.sql new file mode 100644 index 00000000..fe0a1e95 --- /dev/null +++ b/db/sequent_schema_indexes.sql @@ -0,0 +1,37 @@ +ALTER TABLE aggregates ADD PRIMARY KEY (aggregate_id); +ALTER TABLE aggregates ADD UNIQUE (events_partition_key, aggregate_id); +CREATE INDEX aggregates_aggregate_type_id_idx ON aggregates (aggregate_type_id); + +ALTER TABLE commands ADD PRIMARY KEY (id); +CREATE INDEX commands_command_type_id_idx ON commands (command_type_id); +CREATE INDEX commands_aggregate_id_idx ON commands (aggregate_id); +CREATE INDEX commands_event_idx ON commands (event_aggregate_id, event_sequence_number); + +ALTER TABLE events ADD PRIMARY KEY (partition_key, aggregate_id, sequence_number); +CREATE INDEX events_command_id_idx ON events (command_id); +CREATE INDEX events_event_type_id_idx ON events (event_type_id); + +ALTER TABLE aggregates + ADD FOREIGN KEY (aggregate_type_id) REFERENCES aggregate_types (id) ON UPDATE CASCADE; + +ALTER TABLE events + ADD FOREIGN KEY (partition_key, aggregate_id) REFERENCES aggregates (events_partition_key, aggregate_id) + ON UPDATE CASCADE ON DELETE RESTRICT; +ALTER TABLE events + ADD FOREIGN KEY (command_id) REFERENCES commands (id) ON UPDATE RESTRICT ON DELETE RESTRICT; +ALTER TABLE events + ADD FOREIGN KEY (event_type_id) REFERENCES event_types (id) ON UPDATE CASCADE; +ALTER TABLE events ALTER COLUMN xact_id SET DEFAULT pg_current_xact_id()::text::bigint; + +ALTER TABLE commands + ADD FOREIGN KEY (command_type_id) REFERENCES command_types (id) ON UPDATE CASCADE; + +ALTER TABLE aggregates_that_need_snapshots + ADD FOREIGN KEY (aggregate_id) REFERENCES aggregates (aggregate_id) ON UPDATE CASCADE ON DELETE CASCADE; + +CREATE INDEX aggregates_that_need_snapshots_outdated_idx + ON aggregates_that_need_snapshots (snapshot_outdated_at ASC, snapshot_sequence_number_high_water_mark DESC, aggregate_id ASC) + WHERE snapshot_outdated_at IS NOT NULL; + +ALTER TABLE snapshot_records + ADD FOREIGN KEY (aggregate_id) REFERENCES aggregates_that_need_snapshots (aggregate_id) ON UPDATE CASCADE ON DELETE CASCADE; diff --git a/db/sequent_schema_partitions.sql b/db/sequent_schema_partitions.sql new file mode 100644 index 00000000..dcb5de0c --- /dev/null +++ b/db/sequent_schema_partitions.sql @@ -0,0 +1,19 @@ +-- ### Configure partitions as needed +CREATE TABLE commands_default PARTITION OF commands DEFAULT; +CREATE TABLE commands_0 PARTITION OF commands FOR VALUES FROM (1) TO (100e6); +-- CREATE TABLE commands_1 PARTITION OF commands FOR VALUES FROM (100e6) TO (200e6); +-- CREATE TABLE commands_2 PARTITION OF commands FOR VALUES FROM (200e6) TO (300e6); +-- CREATE TABLE commands_3 PARTITION OF commands FOR VALUES FROM (300e6) TO (400e6); + +-- ### Configure partitions as needed +CREATE TABLE aggregates_0 PARTITION OF aggregates FOR VALUES FROM (MINVALUE) TO ('40000000-0000-0000-0000-000000000000'); +CREATE TABLE aggregates_4 PARTITION OF aggregates FOR VALUES FROM ('40000000-0000-0000-0000-000000000000') TO ('80000000-0000-0000-0000-000000000000'); +CREATE TABLE aggregates_8 PARTITION OF aggregates FOR VALUES FROM ('80000000-0000-0000-0000-000000000000') TO ('c0000000-0000-0000-0000-000000000000'); +CREATE TABLE aggregates_c PARTITION OF aggregates FOR VALUES FROM ('c0000000-0000-0000-0000-000000000000') TO (MAXVALUE); + +-- ### Configure partitions as needed +CREATE TABLE events_default PARTITION OF events DEFAULT; +CREATE TABLE events_2023_and_earlier PARTITION OF events FOR VALUES FROM ('Y00') TO ('Y24'); +CREATE TABLE events_2024 PARTITION OF events FOR VALUES FROM ('Y24') TO ('Y25'); +CREATE TABLE events_2025_and_later PARTITION OF events FOR VALUES FROM ('Y25') TO ('Y99'); +CREATE TABLE events_aggregate PARTITION OF events FOR VALUES FROM ('A') TO ('Ag'); diff --git a/db/sequent_schema_tables.sql b/db/sequent_schema_tables.sql new file mode 100644 index 00000000..4a498f4b --- /dev/null +++ b/db/sequent_schema_tables.sql @@ -0,0 +1,74 @@ +CREATE TABLE command_types (id SMALLINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, type text UNIQUE NOT NULL); +CREATE TABLE aggregate_types (id SMALLINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, type text UNIQUE NOT NULL); +CREATE TABLE event_types (id SMALLINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, type text UNIQUE NOT NULL); + +CREATE SEQUENCE IF NOT EXISTS commands_id_seq; + +CREATE TABLE commands ( + id bigint NOT NULL DEFAULT nextval('commands_id_seq'), + created_at timestamp with time zone NOT NULL, + user_id uuid, + aggregate_id uuid, + command_type_id SMALLINT NOT NULL, + command_json jsonb NOT NULL, + event_aggregate_id uuid, + event_sequence_number integer +) PARTITION BY RANGE (id); + +ALTER SEQUENCE commands_id_seq OWNED BY commands.id; + +CREATE TABLE aggregates ( + aggregate_id uuid NOT NULL, + events_partition_key text NOT NULL DEFAULT '', + aggregate_type_id SMALLINT NOT NULL, + snapshot_threshold integer, + created_at timestamp with time zone NOT NULL DEFAULT NOW() +) PARTITION BY RANGE (aggregate_id); + +CREATE TABLE events ( + aggregate_id uuid NOT NULL, + partition_key text NOT NULL DEFAULT '', + sequence_number integer NOT NULL, + created_at timestamp with time zone NOT NULL, + command_id bigint NOT NULL, + event_type_id SMALLINT NOT NULL, + event_json jsonb NOT NULL, + xact_id bigint +) PARTITION BY RANGE (partition_key); + +CREATE TABLE aggregates_that_need_snapshots ( + aggregate_id uuid NOT NULL PRIMARY KEY, + snapshot_sequence_number_high_water_mark integer, + snapshot_outdated_at timestamp with time zone, + snapshot_scheduled_at timestamp with time zone +); + +COMMENT ON TABLE aggregates_that_need_snapshots IS 'Contains a row for every aggregate with more events than its snapshot threshold.'; +COMMENT ON COLUMN aggregates_that_need_snapshots.snapshot_sequence_number_high_water_mark + IS 'The highest sequence number of the stored snapshot. Kept when snapshot are deleted to more easily query aggregates that need snapshotting the most'; +COMMENT ON COLUMN aggregates_that_need_snapshots.snapshot_outdated_at IS 'Not NULL indicates a snapshot is needed since the stored timestamp'; +COMMENT ON COLUMN aggregates_that_need_snapshots.snapshot_scheduled_at IS 'Not NULL indicates a snapshot is in the process of being taken'; + +CREATE TABLE snapshot_records ( + aggregate_id uuid NOT NULL, + sequence_number integer NOT NULL, + created_at timestamptz NOT NULL, + snapshot_type text NOT NULL, + snapshot_json jsonb NOT NULL, + PRIMARY KEY (aggregate_id, sequence_number) +); + +CREATE TABLE saved_event_records ( + operation varchar(1) NOT NULL CHECK (operation IN ('U', 'D')), + timestamp timestamptz NOT NULL, + "user" text NOT NULL, + aggregate_id uuid NOT NULL, + partition_key text DEFAULT '', + sequence_number integer NOT NULL, + created_at timestamp with time zone NOT NULL, + command_id bigint NOT NULL, + event_type text NOT NULL, + event_json jsonb NOT NULL, + xact_id bigint, + PRIMARY KEY (aggregate_id, sequence_number, timestamp) +); diff --git a/lib/sequent/generator/project.rb b/lib/sequent/generator/project.rb index dfcdbd27..fcc988d3 100644 --- a/lib/sequent/generator/project.rb +++ b/lib/sequent/generator/project.rb @@ -26,7 +26,7 @@ def make_directory def copy_files FileUtils.copy_entry(File.expand_path('template_project', __dir__), path) - ['.ruby-version', 'db/sequent_schema.rb', 'db/sequent_schema.sql', 'db/sequent_pgsql.sql'].each do |file| + ['.ruby-version', 'db'].each do |file| FileUtils.copy_entry(File.expand_path("../../../#{file}", __dir__), "#{path}/#{file}") end end From 31b705286ec3220339b6b56b0e99d87c13f126ef Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Fri, 25 Oct 2024 11:06:42 +0200 Subject: [PATCH 114/128] Improve sequent script error handling --- bin/sequent | 82 ++++++++++++++++++-------------- lib/sequent/generator/project.rb | 2 + 2 files changed, 49 insertions(+), 35 deletions(-) diff --git a/bin/sequent b/bin/sequent index 414ec1cd..916dbff2 100755 --- a/bin/sequent +++ b/bin/sequent @@ -4,15 +4,23 @@ require_relative '../lib/sequent/generator' command = ARGV[0].to_s.strip -abort('Please specify a command. i.e. `sequent new myapp`') if command.empty? -abort('Please specify a command. i.e. `sequent new myapp`') if ARGV[1..-1].empty? +args = (ARGV[1..-1] || []).map(&:to_s).map(&:strip) -args = ARGV[1..-1].map(&:to_s).map(&:strip) +if command.empty? + abort <<~EOS + Usage: #{$PROGRAM_NAME} command arguments... -def new_project(args) - arguments = args.dup - name = arguments.shift + Please specify a command, for example `sequent new myapp`. Available commands: + + new appname Generate a new project from the Sequent template + project + generate type arguments... Generate a new aggregate, command, or event + EOS +end + +def new_project(name = nil, *args) abort('Please specify a directory name. i.e. `sequent new myapp`') if name.empty? + abort("Unknown arguments '#{args.join(' ')}', aborting") unless args.empty? Sequent::Generator::Project.new(name).execute puts <<~NEXTSTEPS @@ -43,69 +51,73 @@ def new_project(args) Happy coding! NEXTSTEPS +rescue TargetAlreadyExists + abort("Target '#{name}' already exists, aborting") end -def generate_aggregate(args) - arguments = args.dup - aggregate_name = arguments.shift +def generate_aggregate(aggregate_name = nil, *args) abort('Please specify an aggregate name. i.e. `sequent g aggregate user`') unless args_valid?(aggregate_name) + abort("Unknown arguments '#{args.join(' ')}', aborting") unless args.empty? Sequent::Generator::Aggregate.new(aggregate_name).execute puts "#{aggregate_name} aggregate has been generated" +rescue TargetAlreadyExists + abort("Target '#{aggregate_name}' already exists, aborting") end -def generate_command(args) - arguments = args.dup - aggregate_name = arguments.shift - command_name = arguments.shift - attrs = arguments - +def generate_command(aggregate_name = nil, command_name = nil, *attrs) unless args_valid?(aggregate_name, command_name) - abort('Please specify an aggregate name and command name. i.e. `sequent g command user AddUser`') + abort('Please specify an aggregate name and command name. i.e. `sequent g command User AddUser`') end Sequent::Generator::Command.new(aggregate_name, command_name, attrs).execute puts "#{command_name} command has been added to #{aggregate_name}" +rescue NoAggregateFound + abort("Aggregate '#{aggregate_name}' not found, aborting") end -def generate_event(args) - arguments = args.dup - aggregate_name = arguments.shift - event_name = arguments.shift - attrs = arguments - - abort('Please specify an aggregate name and event name. i.e. `sequent g event user AddUser`') unless args_valid?( +def generate_event(aggregate_name = nil, event_name = nil, *attrs) + abort('Please specify an aggregate name and event name. i.e. `sequent g event User UserAdded`') unless args_valid?( aggregate_name, event_name ) Sequent::Generator::Event.new(aggregate_name, event_name, attrs).execute puts "#{event_name} event has been added to #{aggregate_name}" +rescue NoAggregateFound + abort("Aggregate '#{aggregate_name}' not found, aborting") end -def generate(args) - arguments = args.dup - entity = arguments.shift - abort('Please specify a command. i.e. `sequent g aggregate user`') if entity.empty? - +def generate(entity = nil, *args) case entity when 'aggregate' - generate_aggregate(arguments) + generate_aggregate(*args) when 'command' - generate_command(arguments) + generate_command(*args) when 'event' - generate_event(arguments) + generate_event(*args) else - abort("Unknown argument #{entity} for `generate`. Try `sequent g aggregate user`") + abort <<~EOS + Unknown type for `generate`. Try `sequent g aggregate User`. Available options: + + generate aggregate Name + Generates the aggregate `Name` + + generate command Aggregate Command attributes... + Generates the command `Command` for aggregate `Aggregate` + + generate event Aggregate Event attributes... + Generates the event `Event` for aggregate `Aggregate` + EOS end end def args_valid?(*args) - args.all?(&:present?) + args.none?(&:empty?) end case command when 'new' - new_project(args) + new_project(*args) when 'generate', 'g' - generate(args) + generate(*args) else abort("Unknown command #{command}. Try `sequent new myapp`") end diff --git a/lib/sequent/generator/project.rb b/lib/sequent/generator/project.rb index fcc988d3..75a2d831 100644 --- a/lib/sequent/generator/project.rb +++ b/lib/sequent/generator/project.rb @@ -12,6 +12,8 @@ def initialize(path_or_name) end def execute + fail TargetAlreadyExists if File.exist?(path) + make_directory copy_files rename_app_file From 72121d6d2de6aa08399124ef9c2dccb3e710a125 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Fri, 25 Oct 2024 11:28:24 +0200 Subject: [PATCH 115/128] Only create a single partition by default Additional partitions can be defined by modifying the SQL file and depend on the application. --- db/sequent_schema_partitions.sql | 33 +++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/db/sequent_schema_partitions.sql b/db/sequent_schema_partitions.sql index dcb5de0c..ea93241d 100644 --- a/db/sequent_schema_partitions.sql +++ b/db/sequent_schema_partitions.sql @@ -1,19 +1,34 @@ -- ### Configure partitions as needed CREATE TABLE commands_default PARTITION OF commands DEFAULT; -CREATE TABLE commands_0 PARTITION OF commands FOR VALUES FROM (1) TO (100e6); +-- CREATE TABLE commands_0 PARTITION OF commands FOR VALUES FROM (1) TO (100e6); -- CREATE TABLE commands_1 PARTITION OF commands FOR VALUES FROM (100e6) TO (200e6); -- CREATE TABLE commands_2 PARTITION OF commands FOR VALUES FROM (200e6) TO (300e6); -- CREATE TABLE commands_3 PARTITION OF commands FOR VALUES FROM (300e6) TO (400e6); -- ### Configure partitions as needed -CREATE TABLE aggregates_0 PARTITION OF aggregates FOR VALUES FROM (MINVALUE) TO ('40000000-0000-0000-0000-000000000000'); -CREATE TABLE aggregates_4 PARTITION OF aggregates FOR VALUES FROM ('40000000-0000-0000-0000-000000000000') TO ('80000000-0000-0000-0000-000000000000'); -CREATE TABLE aggregates_8 PARTITION OF aggregates FOR VALUES FROM ('80000000-0000-0000-0000-000000000000') TO ('c0000000-0000-0000-0000-000000000000'); -CREATE TABLE aggregates_c PARTITION OF aggregates FOR VALUES FROM ('c0000000-0000-0000-0000-000000000000') TO (MAXVALUE); +CREATE TABLE aggregates_default PARTITION OF aggregates DEFAULT; +-- CREATE TABLE aggregates_0 PARTITION OF aggregates FOR VALUES FROM (MINVALUE) TO ('10000000-0000-0000-0000-000000000000'); +-- CREATE TABLE aggregates_1 PARTITION OF aggregates FOR VALUES FROM ('10000000-0000-0000-0000-000000000000') TO ('20000000-0000-0000-0000-000000000000'); +-- CREATE TABLE aggregates_2 PARTITION OF aggregates FOR VALUES FROM ('20000000-0000-0000-0000-000000000000') TO ('30000000-0000-0000-0000-000000000000'); +-- CREATE TABLE aggregates_3 PARTITION OF aggregates FOR VALUES FROM ('30000000-0000-0000-0000-000000000000') TO ('40000000-0000-0000-0000-000000000000'); +-- CREATE TABLE aggregates_4 PARTITION OF aggregates FOR VALUES FROM ('40000000-0000-0000-0000-000000000000') TO ('50000000-0000-0000-0000-000000000000'); +-- CREATE TABLE aggregates_5 PARTITION OF aggregates FOR VALUES FROM ('50000000-0000-0000-0000-000000000000') TO ('60000000-0000-0000-0000-000000000000'); +-- CREATE TABLE aggregates_6 PARTITION OF aggregates FOR VALUES FROM ('60000000-0000-0000-0000-000000000000') TO ('70000000-0000-0000-0000-000000000000'); +-- CREATE TABLE aggregates_7 PARTITION OF aggregates FOR VALUES FROM ('70000000-0000-0000-0000-000000000000') TO ('80000000-0000-0000-0000-000000000000'); +-- CREATE TABLE aggregates_8 PARTITION OF aggregates FOR VALUES FROM ('80000000-0000-0000-0000-000000000000') TO ('90000000-0000-0000-0000-000000000000'); +-- CREATE TABLE aggregates_9 PARTITION OF aggregates FOR VALUES FROM ('90000000-0000-0000-0000-000000000000') TO ('a0000000-0000-0000-0000-000000000000'); +-- CREATE TABLE aggregates_a PARTITION OF aggregates FOR VALUES FROM ('a0000000-0000-0000-0000-000000000000') TO ('b0000000-0000-0000-0000-000000000000'); +-- CREATE TABLE aggregates_b PARTITION OF aggregates FOR VALUES FROM ('b0000000-0000-0000-0000-000000000000') TO ('c0000000-0000-0000-0000-000000000000'); +-- CREATE TABLE aggregates_c PARTITION OF aggregates FOR VALUES FROM ('c0000000-0000-0000-0000-000000000000') TO ('d0000000-0000-0000-0000-000000000000'); +-- CREATE TABLE aggregates_d PARTITION OF aggregates FOR VALUES FROM ('d0000000-0000-0000-0000-000000000000') TO ('e0000000-0000-0000-0000-000000000000'); +-- CREATE TABLE aggregates_e PARTITION OF aggregates FOR VALUES FROM ('e0000000-0000-0000-0000-000000000000') TO ('f0000000-0000-0000-0000-000000000000'); +-- CREATE TABLE aggregates_f PARTITION OF aggregates FOR VALUES FROM ('f0000000-0000-0000-0000-000000000000') TO (MAXVALUE); -- ### Configure partitions as needed CREATE TABLE events_default PARTITION OF events DEFAULT; -CREATE TABLE events_2023_and_earlier PARTITION OF events FOR VALUES FROM ('Y00') TO ('Y24'); -CREATE TABLE events_2024 PARTITION OF events FOR VALUES FROM ('Y24') TO ('Y25'); -CREATE TABLE events_2025_and_later PARTITION OF events FOR VALUES FROM ('Y25') TO ('Y99'); -CREATE TABLE events_aggregate PARTITION OF events FOR VALUES FROM ('A') TO ('Ag'); +-- CREATE TABLE events_2023_and_earlier PARTITION OF events FOR VALUES FROM ('Y00') TO ('Y24'); +-- CREATE TABLE events_2024 PARTITION OF events FOR VALUES FROM ('Y24') TO ('Y25'); +-- CREATE TABLE events_2025 PARTITION OF events FOR VALUES FROM ('Y25') TO ('Y26'); +-- CREATE TABLE events_2026 PARTITION OF events FOR VALUES FROM ('Y26') TO ('Y27'); +-- CREATE TABLE events_2027_and_later PARTITION OF events FOR VALUES FROM ('Y27') TO ('Y99'); +-- CREATE TABLE events_aggregate PARTITION OF events FOR VALUES FROM ('A') TO ('Ag'); From 4158c242a4c33397aaa1b39da4097827d8f51ce6 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Fri, 25 Oct 2024 13:15:12 +0200 Subject: [PATCH 116/128] Add a migrate command to help with database migration --- bin/sequent | 102 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/bin/sequent b/bin/sequent index 916dbff2..da54d381 100755 --- a/bin/sequent +++ b/bin/sequent @@ -15,6 +15,8 @@ if command.empty? new appname Generate a new project from the Sequent template project generate type arguments... Generate a new aggregate, command, or event + + migrate Migrate a Sequent 7.1 database to Sequent 8 EOS end @@ -109,6 +111,104 @@ def generate(entity = nil, *args) end end +def confirm_or_abort + loop do + puts 'Enter `yes` to continue, `abort` to abort' + + line = $stdin.readline + puts + + abort('Aborted by user') if line.strip == 'abort' + + break if %w[y yes].include? line.strip.downcase + end +rescue EOFError + abort('Aborted by user') +end + +def migrate(*args) + abort("Unknown arguments '#{args.join(' ')}', aborting") unless args.empty? + + puts <<~EOS + The Sequent 8 database has been further optimized for disk usage and + performance. In addition it supports partitioning the tables for aggregates, + commands, and events, making it easier to manage the database (VACUUM, + CLUSTER) since these can work on the smaller partition tables. + + To migrate you first need to decide if you want to define partitions. This + is mainly useful when your database tables are larger than 10 gigabytes or + so. By default Sequent 8 uses a single "default" partition. + + To change this you can edit the `db/sequent_schema_partitions.sql` file to + define your own partition scheme for the `aggregates`, `commands`, and + `events` tables. + + After editing, shut down your application and open a `psql` session to your + existing database, for example: + + ``` + $ psql -U sequent sequent_dev + ``` + + Then run the migration script. The script starts a transaction but DOES NOT + commit the results: + + ``` + psql> \\i db/sequent_8_migration.sql + ``` + + If all goes well you can now COMMIT or ROLLBACK the result. If you COMMIT, + you must perform a VACUUM ANALYZE to ensure PostgreSQL can efficiently query + the new tables: + + ``` + psql> COMMIT; VACUUM VERBOSE ANALYZE; + ``` + + Now you can deploy your Sequent 8 based application and start it again. + + EOS + + step = 0 + + puts "#{step += 1}. Please shut down your existing application." + confirm_or_abort + + puts <<~EOS + #{step += 1}. Open a `psql` connection to the database you wish to migrate: + + ``` + psql -U myapp_user myapp_db + ``` + EOS + confirm_or_abort + + puts <<~EOS + #{step += 1}. Run the database migration. If you have a large database this + can take some time: + + ``` + psql> \\i db/sequent_8_migration.sql + ``` + EOS + confirm_or_abort + + puts <<~EOS + #{step += 1}. After checking everything went OK, COMMIT and optimize the + database: + + ``` + psql> COMMIT; VACUUM VERBOSE ANALYZE; + ``` + EOS + confirm_or_abort + + puts "#{step}. Deploy your Sequent 8 based application and start it." + confirm_or_abort + + puts 'Congratulations! You are now running your application on Sequent 8!' +end + def args_valid?(*args) args.none?(&:empty?) end @@ -118,6 +218,8 @@ when 'new' new_project(*args) when 'generate', 'g' generate(*args) +when 'migrate' + migrate(*args) else abort("Unknown command #{command}. Try `sequent new myapp`") end From 87631049e45eadb09e3f4cde15f96458783a1f9d Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Tue, 12 Nov 2024 16:08:57 +0100 Subject: [PATCH 117/128] Properly escape identifiers when invoking stored procedures Only constants were currently used when invoking stored procedures and functions, but it is better to be safe than sorry here. --- gemfiles/ar_7_1.gemfile.lock | 1 + gemfiles/ar_7_2.gemfile.lock | 1 + lib/sequent/core/event_store.rb | 9 +++++---- lib/sequent/core/helpers/pgsql_helpers.rb | 15 +++++++++++---- lib/sequent/core/snapshot_store.rb | 18 ++++++++++++------ 5 files changed, 30 insertions(+), 14 deletions(-) diff --git a/gemfiles/ar_7_1.gemfile.lock b/gemfiles/ar_7_1.gemfile.lock index 01ecb78c..98705402 100644 --- a/gemfiles/ar_7_1.gemfile.lock +++ b/gemfiles/ar_7_1.gemfile.lock @@ -117,6 +117,7 @@ GEM PLATFORMS arm64-darwin-22 arm64-darwin-23 + arm64-darwin-24 x86_64-linux DEPENDENCIES diff --git a/gemfiles/ar_7_2.gemfile.lock b/gemfiles/ar_7_2.gemfile.lock index 32cf7414..3a182559 100644 --- a/gemfiles/ar_7_2.gemfile.lock +++ b/gemfiles/ar_7_2.gemfile.lock @@ -117,6 +117,7 @@ GEM PLATFORMS arm64-darwin-23 + arm64-darwin-24 x86_64-linux DEPENDENCIES diff --git a/lib/sequent/core/event_store.rb b/lib/sequent/core/event_store.rb index c7a9c6fa..3521851b 100644 --- a/lib/sequent/core/event_store.rb +++ b/lib/sequent/core/event_store.rb @@ -86,7 +86,7 @@ def stream_events_for_aggregate(aggregate_id, load_until: nil, &block) end def load_event(aggregate_id, sequence_number) - event_hash = query_function('load_event', [aggregate_id, sequence_number]).first + event_hash = query_function(connection, 'load_event', [aggregate_id, sequence_number]).first deserialize_event(event_hash) if event_hash end @@ -180,7 +180,7 @@ def permanently_delete_event_stream(aggregate_id) end def permanently_delete_event_streams(aggregate_ids) - call_procedure('permanently_delete_event_streams', [aggregate_ids.to_json]) + call_procedure(connection, 'permanently_delete_event_streams', [aggregate_ids.to_json]) end def permanently_delete_commands_without_events(aggregate_id: nil, organization_id: nil) @@ -188,7 +188,7 @@ def permanently_delete_commands_without_events(aggregate_id: nil, organization_i fail ArgumentError, 'aggregate_id and/or organization_id must be specified' end - call_procedure('permanently_delete_commands_without_events', [aggregate_id, organization_id]) + call_procedure(connection, 'permanently_delete_commands_without_events', [aggregate_id, organization_id]) end private @@ -198,7 +198,7 @@ def connection end def query_events(aggregate_ids, use_snapshots = true, load_until = nil) - query_function('load_events', [aggregate_ids.to_json, use_snapshots, load_until]) + query_function(connection, 'load_events', [aggregate_ids.to_json, use_snapshots, load_until]) end def deserialize_event(event_hash) @@ -243,6 +243,7 @@ def store_events(command, streams_with_events = []) ] end call_procedure( + connection, 'store_events', [ Sequent::Core::Oj.dump(command_record), diff --git a/lib/sequent/core/helpers/pgsql_helpers.rb b/lib/sequent/core/helpers/pgsql_helpers.rb index bf65f0c9..fcf448c9 100644 --- a/lib/sequent/core/helpers/pgsql_helpers.rb +++ b/lib/sequent/core/helpers/pgsql_helpers.rb @@ -4,13 +4,18 @@ module Sequent module Core module Helpers module PgsqlHelpers - def call_procedure(procedure, params) - statement = "CALL #{procedure}(#{bind_placeholders(params)})" + def call_procedure(connection, procedure, params) + fail ArgumentError if procedure.blank? + + statement = "CALL #{quote_ident(procedure)}(#{bind_placeholders(params)})" connection.exec_update(statement, procedure, params) end - def query_function(function, params, columns = ['*']) - query = "SELECT #{columns.join(', ')} FROM #{function}(#{bind_placeholders(params)})" + def query_function(connection, function, params, columns: []) + fail ArgumentError if function.blank? + + cols = columns.blank? ? '*' : columns.map { |c| PG::Connection.quote_ident(c) }.join(', ') + query = "SELECT #{cols} FROM #{quote_ident(function)}(#{bind_placeholders(params)})" connection.exec_query(query, function, params) end @@ -19,6 +24,8 @@ def query_function(function, params, columns = ['*']) def bind_placeholders(params) (1..params.size).map { |n| "$#{n}" }.join(', ') end + + def quote_ident(...) = PG::Connection.quote_ident(...) end end end diff --git a/lib/sequent/core/snapshot_store.rb b/lib/sequent/core/snapshot_store.rb index cd5edd15..25c9137e 100644 --- a/lib/sequent/core/snapshot_store.rb +++ b/lib/sequent/core/snapshot_store.rb @@ -21,22 +21,22 @@ def store_snapshots(snapshots) end, ) - call_procedure('store_snapshots', [json]) + call_procedure(connection, 'store_snapshots', [json]) end def load_latest_snapshot(aggregate_id) - snapshot_hash = query_function('load_latest_snapshot', [aggregate_id]).first + snapshot_hash = query_function(connection, 'load_latest_snapshot', [aggregate_id]).first deserialize_event(snapshot_hash) unless snapshot_hash['aggregate_id'].nil? end # Deletes all snapshots for all aggregates def delete_all_snapshots - call_procedure('delete_all_snapshots', [Time.now]) + call_procedure(connection, 'delete_all_snapshots', [Time.now]) end # Deletes all snapshots for aggregate_id with a sequence_number lower than the specified sequence number. def delete_snapshots_before(aggregate_id, sequence_number) - call_procedure('delete_snapshots_before', [aggregate_id, sequence_number, Time.now]) + call_procedure(connection, 'delete_snapshots_before', [aggregate_id, sequence_number, Time.now]) end # Marks an aggregate for snapshotting. Marked aggregates will be @@ -83,15 +83,21 @@ def clear_aggregates_for_snapshotting_with_last_event_before(timestamp) # Returns the ids of aggregates that need a new snapshot. # def aggregates_that_need_snapshots(last_aggregate_id, limit = 10) - query_function('aggregates_that_need_snapshots', [last_aggregate_id, limit], ['aggregate_id']) + query_function( + connection, + 'aggregates_that_need_snapshots', + [last_aggregate_id, limit], + columns: ['aggregate_id'], + ) .pluck('aggregate_id') end def select_aggregates_for_snapshotting(limit:, reschedule_snapshots_scheduled_before: nil) query_function( + connection, 'select_aggregates_for_snapshotting', [limit, reschedule_snapshots_scheduled_before, Time.now], - ['aggregate_id'], + columns: ['aggregate_id'], ).pluck('aggregate_id') end end From 98e11942ab961081e9968f6b001ac6903be66250 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Tue, 12 Nov 2024 17:38:12 +0100 Subject: [PATCH 118/128] Copy Sequent 8 database schema and migrations script from gem dir Upgraded projects do not have a recent copy of the Sequent `db/` subdirectory, so copy the files from the gem to the project's `db/` directory. --- bin/sequent | 42 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/bin/sequent b/bin/sequent index da54d381..0e9437a0 100755 --- a/bin/sequent +++ b/bin/sequent @@ -129,19 +129,30 @@ end def migrate(*args) abort("Unknown arguments '#{args.join(' ')}', aborting") unless args.empty? + copy_schema, sequent_gem_dir = + begin + [true, Gem::Specification.find_by_name('sequent').gem_dir] + rescue Gem::MissingSpecError + [false, File.expand_path('..', __dir__)] + end + puts <<~EOS The Sequent 8 database has been further optimized for disk usage and performance. In addition it supports partitioning the tables for aggregates, commands, and events, making it easier to manage the database (VACUUM, CLUSTER) since these can work on the smaller partition tables. - To migrate you first need to decide if you want to define partitions. This - is mainly useful when your database tables are larger than 10 gigabytes or - so. By default Sequent 8 uses a single "default" partition. + In the first step this script will copy the Sequent 8 database schema and + migration files to your project's `db/` directory. When this step is completed you + can customize these files to your liking and commit the changes. + + One decision you need to make is whether you want to define partitions. This is + mainly useful when your database tables are larger than 10 gigabytes or so. By + default Sequent 8 uses a single "default" partition. - To change this you can edit the `db/sequent_schema_partitions.sql` file to - define your own partition scheme for the `aggregates`, `commands`, and - `events` tables. + The `db/sequent_schema_partitions.sql` file contains the database partitions for + the `aggregates`, `commands`, and `events` tables, you can customize your + partitions here. After editing, shut down your application and open a `psql` session to your existing database, for example: @@ -171,6 +182,25 @@ def migrate(*args) step = 0 + if copy_schema + puts <<~EOS + #{step += 1}. First a copy of the Sequent 8 database schema and migration scripts are + added to your db/ directory. + + WARNING: this may overwrite your existing scripts, please use your version control + system to commit or abort any of the changes! + EOS + confirm_or_abort + + FileUtils.copy_entry("#{sequent_gem_dir}/db", 'db') + + puts <<~EOS + WARNING: The schema files have been copied, please verify and adjust the contents + before committing and continuing. + EOS + confirm_or_abort + end + puts "#{step += 1}. Please shut down your existing application." confirm_or_abort From bb58e79f887d2f46eb3413c93212826c01cfd967 Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Wed, 13 Nov 2024 08:54:38 +0100 Subject: [PATCH 119/128] Upgrade documentation now suggests using `sequent migrate` --- docs/docs/upgrade-guide.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/docs/upgrade-guide.md b/docs/docs/upgrade-guide.md index 03b2cfdf..90abccff 100644 --- a/docs/docs/upgrade-guide.md +++ b/docs/docs/upgrade-guide.md @@ -7,12 +7,14 @@ title: Upgrade Guide Sequent 8 remodels the PostgreSQL event store to allow partitioning of the aggregates, commands, and events tables. Furthermore it contains various storage optimizations. To migrate your older Sequent database -an example script is provided in `db/sequent_8_migration.sql` that can -be run using `psql`. - -You will have to adjust this script to match your desired partitioning -setup, although the default configuration will work for many cases as -well. +you can use the `sequent migrate` command. Make sure to run this after +updating Sequent in your Gemfile, running `bundle update --source +sequent`, and from the root directory of your project. + +This command will help you perform the database upgrade by providing +you with a default schema and database upgrade script that you can +customize to match your desired partitioning setup, although the +default configuration will work for many cases as well. **IMPORTANT**: If the migration succeeds and you COMMIT the results you must vacuum (e.g. using VACUUM VERBOSE ANALYZE) the new tables to From 9c2780495a3abd185639378ce705032d9cdd260c Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Wed, 13 Nov 2024 08:58:28 +0100 Subject: [PATCH 120/128] Use filter_map instead of map and compact --- lib/sequent/core/aggregate_snapshotter.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/sequent/core/aggregate_snapshotter.rb b/lib/sequent/core/aggregate_snapshotter.rb index 847394bd..f296bbdd 100644 --- a/lib/sequent/core/aggregate_snapshotter.rb +++ b/lib/sequent/core/aggregate_snapshotter.rb @@ -24,9 +24,7 @@ class AggregateSnapshotter < BaseCommandHandler @last_aggregate_id, command.limit, ) - snapshots = aggregate_ids.map do |aggregate_id| - take_snapshot(aggregate_id) - end.compact + snapshots = aggregate_ids.filter_map { |aggregate_id| take_snapshot(aggregate_id) } Sequent.configuration.event_store.store_snapshots(snapshots) @last_aggregate_id = aggregate_ids.last From ae39d9a169e77e1d65893839fdd00efe740ac74f Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Wed, 13 Nov 2024 09:01:32 +0100 Subject: [PATCH 121/128] Update doc --- db/sequent_8_migration.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/db/sequent_8_migration.sql b/db/sequent_8_migration.sql index 523a53e3..2a8734c0 100644 --- a/db/sequent_8_migration.sql +++ b/db/sequent_8_migration.sql @@ -1,8 +1,9 @@ -- This script migrates a pre-sequent 8 database to the sequent 8 schema while preserving the data. -- It runs in a single transaction and when completed you can COMMIT or ROLLBACK the results. -- --- Adjust this script to your needs (number of table partitions, etc). See comments marked with ### --- for configuration sections. +-- To adjust the partitioning setup you can modify `./sequent_schema_partitions.sql`. By default +-- only a single partition is present for each partitioned table, which works well for smaller +-- (e.g. less than 10 Gigabytes) databases. -- -- Ensure you test this on a copy of your production system to verify everything works and to -- get an indication of the required downtime for your system. From f3d27651fd80df40ae318bb742c18c52172e0970 Mon Sep 17 00:00:00 2001 From: Lars Vonk Date: Wed, 13 Nov 2024 12:00:58 +0100 Subject: [PATCH 122/128] Improve text on upgrade command Clarify that the script will guide you through the steps. --- bin/sequent | 33 ++++++++++++--------------------- docs/docs/upgrade-guide.md | 7 ++++++- 2 files changed, 18 insertions(+), 22 deletions(-) diff --git a/bin/sequent b/bin/sequent index 0e9437a0..a5b06794 100755 --- a/bin/sequent +++ b/bin/sequent @@ -137,12 +137,18 @@ def migrate(*args) end puts <<~EOS + This script will guide you through upgrading your Sequent application to Sequent 8. + The Sequent 8 database has been further optimized for disk usage and performance. In addition it supports partitioning the tables for aggregates, commands, and events, making it easier to manage the database (VACUUM, CLUSTER) since these can work on the smaller partition tables. - In the first step this script will copy the Sequent 8 database schema and + It is highly recommended to test this upgrade on a copy of your production database first. + + This script will guide you through the following steps: + + Step 1: Copy the Sequent 8 database schema and migration files to your project's `db/` directory. When this step is completed you can customize these files to your liking and commit the changes. @@ -154,29 +160,14 @@ def migrate(*args) the `aggregates`, `commands`, and `events` tables, you can customize your partitions here. - After editing, shut down your application and open a `psql` session to your - existing database, for example: - - ``` - $ psql -U sequent sequent_dev - ``` + Step 2: Shutdown your application and the migration script. The script starts a transaction but DOES NOT + commit the results. - Then run the migration script. The script starts a transaction but DOES NOT - commit the results: - - ``` - psql> \\i db/sequent_8_migration.sql - ``` - - If all goes well you can now COMMIT or ROLLBACK the result. If you COMMIT, + Step 3: Check the results and COMMIT or ROLLBACK the result. If you COMMIT, you must perform a VACUUM ANALYZE to ensure PostgreSQL can efficiently query - the new tables: - - ``` - psql> COMMIT; VACUUM VERBOSE ANALYZE; - ``` + the new tables - Now you can deploy your Sequent 8 based application and start it again. + Step 4: Now you can deploy your Sequent 8 based application and start it again. EOS diff --git a/docs/docs/upgrade-guide.md b/docs/docs/upgrade-guide.md index 90abccff..3b5ccb6b 100644 --- a/docs/docs/upgrade-guide.md +++ b/docs/docs/upgrade-guide.md @@ -7,8 +7,8 @@ title: Upgrade Guide Sequent 8 remodels the PostgreSQL event store to allow partitioning of the aggregates, commands, and events tables. Furthermore it contains various storage optimizations. To migrate your older Sequent database -you can use the `sequent migrate` command. Make sure to run this after updating Sequent in your Gemfile, running `bundle update --source +you can use the `bundle exec sequent migrate` command. Make sure to run this after sequent`, and from the root directory of your project. This command will help you perform the database upgrade by providing @@ -16,6 +16,11 @@ you with a default schema and database upgrade script that you can customize to match your desired partitioning setup, although the default configuration will work for many cases as well. +**IMPORTANT**: Ensure you test your migration on a copy of your database first! This will give you +a good indication on how long the migration will take so you can schedule downtime appropriately. +Next to that it will ensure all data in your event store is compatible with Sequent 8. Normally this won't be a problem +unless you somehow have corrupted data in your event store. + **IMPORTANT**: If the migration succeeds and you COMMIT the results you must vacuum (e.g. using VACUUM VERBOSE ANALYZE) the new tables to ensure good performance! From 7ea18a6a65de4bf1fe36e6ae3ecefe520adfc154 Mon Sep 17 00:00:00 2001 From: Lars Vonk Date: Wed, 13 Nov 2024 12:01:21 +0100 Subject: [PATCH 123/128] Suggest `--conservative` instead of `--source`. --- docs/docs/upgrade-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/upgrade-guide.md b/docs/docs/upgrade-guide.md index 3b5ccb6b..381d01b1 100644 --- a/docs/docs/upgrade-guide.md +++ b/docs/docs/upgrade-guide.md @@ -7,8 +7,8 @@ title: Upgrade Guide Sequent 8 remodels the PostgreSQL event store to allow partitioning of the aggregates, commands, and events tables. Furthermore it contains various storage optimizations. To migrate your older Sequent database -updating Sequent in your Gemfile, running `bundle update --source you can use the `bundle exec sequent migrate` command. Make sure to run this after +updating Sequent in your Gemfile, running `bundle update --conservative sequent`, and from the root directory of your project. This command will help you perform the database upgrade by providing From 4cdd1bb71e4fea5b157e159698db05e03fbe52f1 Mon Sep 17 00:00:00 2001 From: Lars Vonk Date: Thu, 14 Nov 2024 09:15:19 +0100 Subject: [PATCH 124/128] Use gli and tty-prompt to improve sequent cli We can use colors and leverage the possibility of those gems. --- README.md | 3 - bin/sequent | 248 +------------------------ gemfiles/ar_7_1.gemfile.lock | 24 ++- gemfiles/ar_7_2.gemfile.lock | 24 ++- lib/sequent/cli/app.rb | 132 +++++++++++++ lib/sequent/cli/sequent_8_migration.rb | 180 ++++++++++++++++++ sequent.gemspec | 4 +- 7 files changed, 361 insertions(+), 254 deletions(-) create mode 100644 lib/sequent/cli/app.rb create mode 100644 lib/sequent/cli/sequent_8_migration.rb diff --git a/README.md b/README.md index 62fb106b..48ad82c4 100644 --- a/README.md +++ b/README.md @@ -45,9 +45,6 @@ Then run `rake release`. A git tag will be created and pushed, and the new versi Increase version to new working version, update the sequent version for all the `gemfiles`: ``` -BUNDLE_GEMFILE=gemfiles/ar_6_0.gemfile bundle update sequent --conservative -BUNDLE_GEMFILE=gemfiles/ar_6_1.gemfile bundle update sequent --conservative -BUNDLE_GEMFILE=gemfiles/ar_7_0.gemfile bundle update sequent --conservative BUNDLE_GEMFILE=gemfiles/ar_7_1.gemfile bundle update sequent --conservative BUNDLE_GEMFILE=gemfiles/ar_7_2.gemfile bundle update sequent --conservative ``` diff --git a/bin/sequent b/bin/sequent index a5b06794..bf48055f 100755 --- a/bin/sequent +++ b/bin/sequent @@ -1,246 +1,10 @@ #!/usr/bin/env ruby # frozen_string_literal: true -require_relative '../lib/sequent/generator' +require 'gli' +require 'tty-prompt' +require './lib/version' +require './lib/sequent/cli/app' -command = ARGV[0].to_s.strip -args = (ARGV[1..-1] || []).map(&:to_s).map(&:strip) - -if command.empty? - abort <<~EOS - Usage: #{$PROGRAM_NAME} command arguments... - - Please specify a command, for example `sequent new myapp`. Available commands: - - new appname Generate a new project from the Sequent template - project - generate type arguments... Generate a new aggregate, command, or event - - migrate Migrate a Sequent 7.1 database to Sequent 8 - EOS -end - -def new_project(name = nil, *args) - abort('Please specify a directory name. i.e. `sequent new myapp`') if name.empty? - abort("Unknown arguments '#{args.join(' ')}', aborting") unless args.empty? - - Sequent::Generator::Project.new(name).execute - puts <<~NEXTSTEPS - - Success! - - Your brand spanking new sequent app is waiting for you in: - #{File.expand_path(name, Dir.pwd)} - - To finish setting up your app: - cd #{name} - bundle install - bundle exec rake sequent:db:create - bundle exec rake sequent:db:create_view_schema - bundle exec rake sequent:migrate:online - bundle exec rake sequent:migrate:offline - - Run the example specs: - SEQUENT_ENV=test bundle exec rake sequent:db:create - bundle exec rspec spec - - To generate new aggregates use: - sequent generate . e.g. sequent generate address - - For more information see: - https://www.sequent.io - - Happy coding! - - NEXTSTEPS -rescue TargetAlreadyExists - abort("Target '#{name}' already exists, aborting") -end - -def generate_aggregate(aggregate_name = nil, *args) - abort('Please specify an aggregate name. i.e. `sequent g aggregate user`') unless args_valid?(aggregate_name) - abort("Unknown arguments '#{args.join(' ')}', aborting") unless args.empty? - - Sequent::Generator::Aggregate.new(aggregate_name).execute - puts "#{aggregate_name} aggregate has been generated" -rescue TargetAlreadyExists - abort("Target '#{aggregate_name}' already exists, aborting") -end - -def generate_command(aggregate_name = nil, command_name = nil, *attrs) - unless args_valid?(aggregate_name, command_name) - abort('Please specify an aggregate name and command name. i.e. `sequent g command User AddUser`') - end - Sequent::Generator::Command.new(aggregate_name, command_name, attrs).execute - puts "#{command_name} command has been added to #{aggregate_name}" -rescue NoAggregateFound - abort("Aggregate '#{aggregate_name}' not found, aborting") -end - -def generate_event(aggregate_name = nil, event_name = nil, *attrs) - abort('Please specify an aggregate name and event name. i.e. `sequent g event User UserAdded`') unless args_valid?( - aggregate_name, event_name - ) - Sequent::Generator::Event.new(aggregate_name, event_name, attrs).execute - puts "#{event_name} event has been added to #{aggregate_name}" -rescue NoAggregateFound - abort("Aggregate '#{aggregate_name}' not found, aborting") -end - -def generate(entity = nil, *args) - case entity - when 'aggregate' - generate_aggregate(*args) - when 'command' - generate_command(*args) - when 'event' - generate_event(*args) - else - abort <<~EOS - Unknown type for `generate`. Try `sequent g aggregate User`. Available options: - - generate aggregate Name - Generates the aggregate `Name` - - generate command Aggregate Command attributes... - Generates the command `Command` for aggregate `Aggregate` - - generate event Aggregate Event attributes... - Generates the event `Event` for aggregate `Aggregate` - EOS - end -end - -def confirm_or_abort - loop do - puts 'Enter `yes` to continue, `abort` to abort' - - line = $stdin.readline - puts - - abort('Aborted by user') if line.strip == 'abort' - - break if %w[y yes].include? line.strip.downcase - end -rescue EOFError - abort('Aborted by user') -end - -def migrate(*args) - abort("Unknown arguments '#{args.join(' ')}', aborting") unless args.empty? - - copy_schema, sequent_gem_dir = - begin - [true, Gem::Specification.find_by_name('sequent').gem_dir] - rescue Gem::MissingSpecError - [false, File.expand_path('..', __dir__)] - end - - puts <<~EOS - This script will guide you through upgrading your Sequent application to Sequent 8. - - The Sequent 8 database has been further optimized for disk usage and - performance. In addition it supports partitioning the tables for aggregates, - commands, and events, making it easier to manage the database (VACUUM, - CLUSTER) since these can work on the smaller partition tables. - - It is highly recommended to test this upgrade on a copy of your production database first. - - This script will guide you through the following steps: - - Step 1: Copy the Sequent 8 database schema and - migration files to your project's `db/` directory. When this step is completed you - can customize these files to your liking and commit the changes. - - One decision you need to make is whether you want to define partitions. This is - mainly useful when your database tables are larger than 10 gigabytes or so. By - default Sequent 8 uses a single "default" partition. - - The `db/sequent_schema_partitions.sql` file contains the database partitions for - the `aggregates`, `commands`, and `events` tables, you can customize your - partitions here. - - Step 2: Shutdown your application and the migration script. The script starts a transaction but DOES NOT - commit the results. - - Step 3: Check the results and COMMIT or ROLLBACK the result. If you COMMIT, - you must perform a VACUUM ANALYZE to ensure PostgreSQL can efficiently query - the new tables - - Step 4: Now you can deploy your Sequent 8 based application and start it again. - - EOS - - step = 0 - - if copy_schema - puts <<~EOS - #{step += 1}. First a copy of the Sequent 8 database schema and migration scripts are - added to your db/ directory. - - WARNING: this may overwrite your existing scripts, please use your version control - system to commit or abort any of the changes! - EOS - confirm_or_abort - - FileUtils.copy_entry("#{sequent_gem_dir}/db", 'db') - - puts <<~EOS - WARNING: The schema files have been copied, please verify and adjust the contents - before committing and continuing. - EOS - confirm_or_abort - end - - puts "#{step += 1}. Please shut down your existing application." - confirm_or_abort - - puts <<~EOS - #{step += 1}. Open a `psql` connection to the database you wish to migrate: - - ``` - psql -U myapp_user myapp_db - ``` - EOS - confirm_or_abort - - puts <<~EOS - #{step += 1}. Run the database migration. If you have a large database this - can take some time: - - ``` - psql> \\i db/sequent_8_migration.sql - ``` - EOS - confirm_or_abort - - puts <<~EOS - #{step += 1}. After checking everything went OK, COMMIT and optimize the - database: - - ``` - psql> COMMIT; VACUUM VERBOSE ANALYZE; - ``` - EOS - confirm_or_abort - - puts "#{step}. Deploy your Sequent 8 based application and start it." - confirm_or_abort - - puts 'Congratulations! You are now running your application on Sequent 8!' -end - -def args_valid?(*args) - args.none?(&:empty?) -end - -case command -when 'new' - new_project(*args) -when 'generate', 'g' - generate(*args) -when 'migrate' - migrate(*args) -else - abort("Unknown command #{command}. Try `sequent new myapp`") -end +exit_code = Sequent::Cli::App.run(ARGV) +exit(exit_code) diff --git a/gemfiles/ar_7_1.gemfile.lock b/gemfiles/ar_7_1.gemfile.lock index 98705402..ea9766a7 100644 --- a/gemfiles/ar_7_1.gemfile.lock +++ b/gemfiles/ar_7_1.gemfile.lock @@ -6,6 +6,7 @@ PATH activerecord (>= 7.1.3) bcrypt (~> 3.1) csv (~> 3.3) + gli (~> 2.22) i18n logger (~> 1.6) oj (~> 3.3) @@ -14,6 +15,7 @@ PATH pg (~> 1.2) postgresql_cursor (~> 0.6) thread_safe (~> 0.3.6) + tty-prompt (~> 0.23.1) tzinfo (>= 1.1) GEM @@ -46,9 +48,10 @@ GEM diff-lcs (1.5.1) docile (1.4.1) drb (2.2.1) + gli (2.22.0) i18n (1.14.6) concurrent-ruby (~> 1.0) - json (2.7.2) + json (2.8.1) language_server-protocol (3.17.0.3) logger (1.6.1) method_source (1.1.0) @@ -62,6 +65,8 @@ GEM parser (3.3.5.0) ast (~> 2.4.1) racc + pastel (0.8.0) + tty-color (~> 0.5) pg (1.5.8) postgresql_cursor (0.6.9) activerecord (>= 6.0) @@ -88,7 +93,7 @@ GEM diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.13.0) rspec-support (3.13.1) - rubocop (1.67.0) + rubocop (1.68.0) json (~> 2.3) language_server-protocol (>= 3.17.0) parallel (~> 1.10) @@ -98,7 +103,7 @@ GEM rubocop-ast (>= 1.32.2, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.32.3) + rubocop-ast (1.36.1) parser (>= 3.3.1.0) ruby-progressbar (1.13.0) simplecov (0.22.0) @@ -110,9 +115,20 @@ GEM thread_safe (0.3.6) timecop (0.9.10) timeout (0.4.1) + tty-color (0.6.0) + tty-cursor (0.7.1) + tty-prompt (0.23.1) + pastel (~> 0.8) + tty-reader (~> 0.8) + tty-reader (0.9.0) + tty-cursor (~> 0.7) + tty-screen (~> 0.8) + wisper (~> 2.0) + tty-screen (0.8.2) tzinfo (2.0.6) concurrent-ruby (~> 1.0) unicode-display_width (2.6.0) + wisper (2.0.1) PLATFORMS arm64-darwin-22 @@ -129,7 +145,7 @@ DEPENDENCIES rspec (~> 3.10) rspec-collection_matchers (~> 1.2) rspec-mocks (~> 3.10) - rubocop (~> 1.56, >= 1.56.3) + rubocop (~> 1.68.0) sequent! simplecov (~> 0.21) timecop (~> 0.9) diff --git a/gemfiles/ar_7_2.gemfile.lock b/gemfiles/ar_7_2.gemfile.lock index 3a182559..c24c784f 100644 --- a/gemfiles/ar_7_2.gemfile.lock +++ b/gemfiles/ar_7_2.gemfile.lock @@ -6,6 +6,7 @@ PATH activerecord (>= 7.1.3) bcrypt (~> 3.1) csv (~> 3.3) + gli (~> 2.22) i18n logger (~> 1.6) oj (~> 3.3) @@ -14,6 +15,7 @@ PATH pg (~> 1.2) postgresql_cursor (~> 0.6) thread_safe (~> 0.3.6) + tty-prompt (~> 0.23.1) tzinfo (>= 1.1) GEM @@ -47,9 +49,10 @@ GEM diff-lcs (1.5.1) docile (1.4.1) drb (2.2.1) + gli (2.22.0) i18n (1.14.6) concurrent-ruby (~> 1.0) - json (2.7.2) + json (2.8.1) language_server-protocol (3.17.0.3) logger (1.6.1) method_source (1.1.0) @@ -62,6 +65,8 @@ GEM parser (3.3.5.0) ast (~> 2.4.1) racc + pastel (0.8.0) + tty-color (~> 0.5) pg (1.5.8) postgresql_cursor (0.6.9) activerecord (>= 6.0) @@ -88,7 +93,7 @@ GEM diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.13.0) rspec-support (3.13.1) - rubocop (1.67.0) + rubocop (1.68.0) json (~> 2.3) language_server-protocol (>= 3.17.0) parallel (~> 1.10) @@ -98,7 +103,7 @@ GEM rubocop-ast (>= 1.32.2, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.32.3) + rubocop-ast (1.36.1) parser (>= 3.3.1.0) ruby-progressbar (1.13.0) securerandom (0.3.1) @@ -111,9 +116,20 @@ GEM thread_safe (0.3.6) timecop (0.9.10) timeout (0.4.1) + tty-color (0.6.0) + tty-cursor (0.7.1) + tty-prompt (0.23.1) + pastel (~> 0.8) + tty-reader (~> 0.8) + tty-reader (0.9.0) + tty-cursor (~> 0.7) + tty-screen (~> 0.8) + wisper (~> 2.0) + tty-screen (0.8.2) tzinfo (2.0.6) concurrent-ruby (~> 1.0) unicode-display_width (2.6.0) + wisper (2.0.1) PLATFORMS arm64-darwin-23 @@ -129,7 +145,7 @@ DEPENDENCIES rspec (~> 3.10) rspec-collection_matchers (~> 1.2) rspec-mocks (~> 3.10) - rubocop (~> 1.56, >= 1.56.3) + rubocop (~> 1.68.0) sequent! simplecov (~> 0.21) timecop (~> 0.9) diff --git a/lib/sequent/cli/app.rb b/lib/sequent/cli/app.rb new file mode 100644 index 00000000..205435f3 --- /dev/null +++ b/lib/sequent/cli/app.rb @@ -0,0 +1,132 @@ +# frozen_string_literal: true + +require_relative '../generator' +require_relative './sequent8_migration' +module Sequent + module Cli + class App + extend GLI::App + + program_desc 'Sequent Command Line Interface (CLI)' + + version Sequent::VERSION + on_error do |_error| + true + end + + desc 'Generate a directory structure for a Sequent project' + command :new do |c| + prompt = TTY::Prompt.new(interrupt: :exit) + + c.arg_name 'project_name' + c.action do |_global, _options, args| + help_now!('can only specify one single argument e.g. `sequent new project_name`') if args&.length != 1 + + project_name = args[0] + Sequent::Generator::Project.new(project_name).execute + prompt.say(<<~EOS) + Success! + + Your brand spanking new sequent app is waiting for you in: + #{File.expand_path(project_name, Dir.pwd)} + + To finish setting up your app: + cd #{project_name} + bundle install + bundle exec rake sequent:db:create + bundle exec rake sequent:db:create_view_schema + bundle exec rake sequent:migrate:online + bundle exec rake sequent:migrate:offline + + Run the example specs: + SEQUENT_ENV=test bundle exec rake sequent:db:create + bundle exec rspec spec + + To generate new aggregates use: + sequent generate . e.g. sequent generate address + + For more information see: + https://www.sequent.io + + Happy coding! + EOS + rescue TargetAlreadyExists + prompt.error("Target '#{project_name}' already exists, aborting") + end + end + + desc 'Generate a new aggregate, command, or event' + command [:generate, :g] do |c| + prompt = TTY::Prompt.new(interrupt: :exit) + + c.arg_name 'aggregate_name' + c.desc 'Generate an aggregate' + c.command :aggregate do |a| + a.action do |_global, _options, args| + if args&.length != 1 + help_now!('must specify one single argument e.g. `sequent generate aggregate Employee`') + end + + aggregate_name = args[0] + + Sequent::Generator::Aggregate.new(aggregate_name).execute + + prompt.say(<<~EOS) + #{aggregate_name} aggregate has been generated + EOS + end + end + + c.desc 'Generate a command' + c.arg_name 'aggregate_name command_name' + c.command :command do |command| + command.action do |_global, _options, args| + if args&.length&.< 2 + help_now!('must specify at least two arguments e.g. `sequent generate command Employee CreateEmployee`') + end + + aggregate_name, command_name, *attributes = args + + Sequent::Generator::Command.new(aggregate_name, command_name, attributes).execute + prompt.say(<<~EOS) + "#{command_name} command has been added to #{aggregate_name}" + EOS + rescue NoAggregateFound + prompt.error("Aggregate '#{aggregate_name}' not found, aborting") + end + end + + c.desc 'Generate an Event' + c.arg_name 'aggregate_name event_name' + c.command :event do |command| + command.action do |_global, _options, args| + if args&.length&.< 2 + help_now!('must specify at least two arguments e.g. `sequent generate event Employee EmployeeCreated`') + end + + aggregate_name, event_name, *attributes = args + + Sequent::Generator::Command.new(aggregate_name, event_name, attributes).execute + prompt.say(<<~EOS) + "#{event_name} event has been added to #{aggregate_name}" + EOS + rescue NoAggregateFound + prompt.error("Aggregate '#{aggregate_name}' not found, aborting") + end + end + end + + desc 'Migrates a Sequent 7 project to Sequent 8' + command :migrate do |c| + prompt = TTY::Prompt.new(interrupt: :exit) + c.action do |_global, _options, _args| + Sequent8Migration.new(prompt).execute + rescue Gem::MissingSpecError + prompt.error('Sequent gem not found. Please check your Gemfile.') + rescue Sequent8Migration::Stop => e + prompt.error(e.message) + end + end + end + end +end diff --git a/lib/sequent/cli/sequent_8_migration.rb b/lib/sequent/cli/sequent_8_migration.rb new file mode 100644 index 00000000..6e07e46b --- /dev/null +++ b/lib/sequent/cli/sequent_8_migration.rb @@ -0,0 +1,180 @@ +# frozen_string_literal: true + +module Sequent + module Cli + class Sequent8Migration + class Stop < StandardError; end + + def initialize(prompt) + @prompt = prompt + end + + # @raise Gem::MissingSpecError + def execute + print_introduction + abort_if_no('Do you wish to start the migration?') + copy_schema_files + abort_if_no('Do you which to continue?') + stop_application + migrate_data + prompt.ask('Press if the migration is done and you checked the results?') + migrated = commit_or_rollback + + if migrated + prompt.say <<~EOS + + Step 5. Deploy your Sequent 8 based application and start it. + + Congratulations! You are now running your application on Sequent 8! + EOS + else + prompt.say <<~EOS + + We are sorry the migration did not succeed. If you think this is a bug in Sequent don't hesitate to reach + out and submit an issue on Github: https://github.com/zilverline/sequent. + + Don't forget to start your application again! + EOS + end + end + + private + + attr_reader :prompt + + def print_introduction + prompt.say <<~EOS + This script will guide you through upgrading your Sequent application to Sequent 8. + + The Sequent 8 database has been further optimized for disk usage and + performance. In addition it supports partitioning the tables for aggregates, + commands, and events, making it easier to manage the database (VACUUM, + CLUSTER) since these can work on the smaller partition tables. + + It is highly recommended to test this upgrade on a copy of your production database first. + + This script consists of the following steps: + + Step 1: Copy the Sequent 8 database schema and + migration files to your project's `db/` directory. When this step is completed you + can customize these files to your liking and commit the changes. + + One decision you need to make is whether you want to define partitions. This is + mainly useful when your database tables are larger than 10 gigabytes or so. By + default Sequent 8 uses a single "default" partition. + + The `db/sequent_schema_partitions.sql` file contains the database partitions for + the `aggregates`, `commands`, and `events` tables, you can customize your + partitions here. + + Step 2: Shutdown your application. + + Step 3: Run the migration script. The script starts a transaction but DOES NOT + commit the results. + + Step 4: Check the results and COMMIT or ROLLBACK the result. If you COMMIT, + you must perform a VACUUM ANALYZE to ensure PostgreSQL can efficiently query + the new tables + + Step 5: Now you can deploy your Sequent 8 based application and start it again. + + EOS + end + + def copy_schema_files + prompt.say <<~EOS + + Step 1. First a copy of the Sequent 8 database schema and migration scripts are + added to your db/ directory. + EOS + prompt.warn <<~EOS + + WARNING: this may overwrite your existing scripts, please use your version control system to commit or abort any of the changes! + EOS + + abort_if_no('Do you which to continue?') + + FileUtils.copy_entry("#{sequent_gem_dir}/db", 'db') + + prompt.warn <<~EOS + + WARNING: The schema files have been copied, please verify and adjust the contents before committing and continuing. + EOS + end + + def stop_application + prompt.say <<~EOS + + Step 2. Please shut down your existing application. + EOS + + abort_if_no(<<~EOS) + Only proceed once your application is stopped. Is your application stopped and do you want to continue? + EOS + end + + def migrate_data + prompt.say <<~EOS + + Step 3. Open a `psql` connection to the database you wish to migrate. + EOS + prompt.warn <<~EOS + + It is highly recommended to test this on a copy of your production database first! + EOS + + prompt.say <<~EOS + + Depending on the size of your database the migration can take a long time. Open the `psql` connection from a screen session if needed. + If you run this from a screen session from another server you will need to copy all needed sql files to that server. + + ``` + psql -U myapp_user myapp_db + ``` + EOS + + prompt.ask('Press to read the next instructions once you connected to the database...') + + prompt.say <<~EOS + + Run the database migration. This doesn't commit anything yet so you can check the results first. + + ``` + psql> \\i db/sequent_8_migration.sql + ``` + EOS + end + + def commit_or_rollback + answer = prompt.yes? 'Did the migration succeed?' + if answer + prompt.say <<~EOS + + Step 4. After checking everything went OK, COMMIT and optimize the database: + + ``` + psql> COMMIT; VACUUM VERBOSE ANALYZE; + ``` + EOS + else + prompt.say <<~EOS + + Step 4. Rollback the migration: + + ``` + psql> ROLLBACK; + ``` + EOS + end + answer + end + + def sequent_gem_dir = Gem::Specification.find_by_name('sequent').gem_dir + + def abort_if_no(message, abort_message: 'Stopped at your request. You can restart this migration at any time.') + answer = prompt.yes?(message) + fail Stop, abort_message unless answer + end + end + end +end diff --git a/sequent.gemspec b/sequent.gemspec index d8ca9279..1ab5b920 100644 --- a/sequent.gemspec +++ b/sequent.gemspec @@ -31,6 +31,7 @@ Gem::Specification.new do |s| s.add_dependency 'activerecord', active_star_version s.add_dependency 'bcrypt', '~> 3.1' s.add_dependency 'csv', '~> 3.3' + s.add_dependency 'gli', '~> 2.22' s.add_dependency 'i18n' s.add_dependency 'logger', '~> 1.6' s.add_dependency 'oj', '~> 3.3' @@ -39,6 +40,7 @@ Gem::Specification.new do |s| s.add_dependency 'pg', '~> 1.2' s.add_dependency 'postgresql_cursor', '~> 0.6' s.add_dependency 'thread_safe', '~> 0.3.6' + s.add_dependency 'tty-prompt', '~> 0.23.1' s.add_dependency 'tzinfo', '>= 1.1' s.add_development_dependency 'prop_check', '~> 1.0' s.add_development_dependency 'pry', '~> 0.13' @@ -46,7 +48,7 @@ Gem::Specification.new do |s| s.add_development_dependency 'rspec', rspec_version s.add_development_dependency 'rspec-collection_matchers', '~> 1.2' s.add_development_dependency 'rspec-mocks', rspec_version - s.add_development_dependency 'rubocop', '~> 1.56', '>= 1.56.3' + s.add_development_dependency 'rubocop', '~> 1.68.0' s.add_development_dependency 'simplecov', '~> 0.21' s.add_development_dependency 'timecop', '~> 0.9' end From 79996bc2e9713e43ef00837daf46172c38d4665c Mon Sep 17 00:00:00 2001 From: Lars Vonk Date: Thu, 14 Nov 2024 12:08:08 +0100 Subject: [PATCH 125/128] Enable all new rubocops And fix the warnings. --- .rubocop.yml | 195 +----------------- .../config/environments/production.rb | 2 +- .../rails-app/spec/rails_helper.rb | 2 +- lib/sequent/cli/app.rb | 2 +- lib/sequent/core/helpers/array_with_type.rb | 2 +- .../core/helpers/association_validator.rb | 4 +- lib/sequent/core/helpers/attribute_support.rb | 15 +- lib/sequent/core/helpers/equal_support.rb | 6 +- .../helpers/message_matchers/has_attrs.rb | 2 + lib/sequent/core/helpers/message_router.rb | 4 +- lib/sequent/core/helpers/param_support.rb | 4 +- lib/sequent/core/helpers/string_support.rb | 2 +- .../persistors/active_record_persistor.rb | 2 +- .../replay_optimized_postgres_persistor.rb | 7 +- lib/sequent/core/projector.rb | 2 +- lib/sequent/generator.rb | 2 +- lib/sequent/migrations/grouper.rb | 4 +- lib/sequent/migrations/view_schema.rb | 12 +- lib/sequent/test/time_comparison.rb | 2 +- sequent.gemspec | 1 + spec/database.rb | 6 +- spec/lib/sequent/configuration_spec.rb | 4 +- .../helpers/association_validator_spec.rb | 4 +- spec/lib/sequent/core/projector_spec.rb | 2 +- spec/spec_helper.rb | 2 +- 25 files changed, 49 insertions(+), 241 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index 65e12ae3..4a60737e 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -14,6 +14,7 @@ AllCops: SuggestExtensions: rubocop-rake: false rubocop-rspec: false + NewCops: enable Exclude: # default - '**/node_modules/**/*' @@ -308,219 +309,29 @@ Style/SoleNestedConditional: Style/EvalWithLocation: Enabled: false - -Layout/SpaceBeforeBrackets: # (new in 1.7) - Enabled: false -Lint/AmbiguousAssignment: # (new in 1.7) - Enabled: false -Lint/DeprecatedConstants: # (new in 1.8) - Enabled: false -Lint/DuplicateBranch: # (new in 1.3) - Enabled: false -Lint/DuplicateRegexpCharacterClassElement: # (new in 1.1) - Enabled: false Lint/EmptyBlock: # (new in 1.1) Enabled: false Lint/EmptyClass: # (new in 1.3) Enabled: false -Lint/LambdaWithoutLiteralBlock: # (new in 1.8) - Enabled: false -Lint/NoReturnInBeginEndBlocks: # (new in 1.2) - Enabled: false -Lint/NumberedParameterAssignment: # (new in 1.9) - Enabled: false -Lint/OrAssignmentToConstant: # (new in 1.9) - Enabled: false -Lint/RedundantDirGlobSort: # (new in 1.8) - Enabled: false Lint/SymbolConversion: # (new in 1.9) Enabled: false -Lint/ToEnumArguments: # (new in 1.1) - Enabled: false -Lint/TripleQuotes: # (new in 1.9) - Enabled: false -Lint/UnexpectedBlockArity: # (new in 1.5) - Enabled: false -Lint/UnmodifiedReduceAccumulator: # (new in 1.1) - Enabled: false Style/ArgumentsForwarding: # (new in 1.1) Enabled: false -Style/CollectionCompact: # (new in 1.2) - Enabled: false Style/DocumentDynamicEvalDefinition: # (new in 1.1) Enabled: false -Style/EndlessMethod: # (new in 1.8) - Enabled: false -Style/HashExcept: # (new in 1.7) - Enabled: false -Style/IfWithBooleanLiteralBranches: # (new in 1.9) - Enabled: false Style/NegatedIfElseCondition: # (new in 1.2) Enabled: false -Style/NilLambda: # (new in 1.3) - Enabled: false -Style/RedundantArgument: # (new in 1.4) - Enabled: false -Style/SwapValues: # (new in 1.1) - Enabled: false -Style/StringConcatenation: - Enabled: false -Style/HashEachMethods: - Enabled: false -Lint/RedundantSafeNavigation: - Enabled: false -Style/HashTransformValues: - Enabled: false -Lint/FloatComparison: - Enabled: false Style/ClassVars: Enabled: false -Layout/LineEndStringConcatenationIndentation: # new in 1.18 - Enabled: false -Lint/AmbiguousRange: # new in 1.19 - Enabled: false -Lint/EmptyInPattern: # new in 1.16 - Enabled: false -Naming/InclusiveLanguage: # new in 1.18 - Enabled: false -Style/HashConversion: # new in 1.10 - Enabled: false -Style/InPatternThen: # new in 1.16 - Enabled: false -Style/MultilineInPatternThen: # new in 1.16 - Enabled: false -Style/QuotedSymbols: # new in 1.16 - Enabled: false -Style/RedundantSelfAssignmentBranch: # new in 1.19 - Enabled: false -Style/StringChars: # new in 1.12 - Enabled: false -# new since rubny 3.2 -Gemspec/DeprecatedAttributeAssignment: # new in 1.30 - Enabled: false Gemspec/DevelopmentDependencies: # new in 1.44 Enabled: false -Gemspec/RequireMFA: # new in 1.23 - Enabled: false -Layout/LineContinuationLeadingSpace: # new in 1.31 - Enabled: false -Layout/LineContinuationSpacing: # new in 1.31 - Enabled: false -Lint/AmbiguousOperatorPrecedence: # new in 1.21 - Enabled: false -Lint/ConstantOverwrittenInRescue: # new in 1.31 - Enabled: false -Lint/DuplicateMagicComment: # new in 1.37 - Enabled: false -Lint/DuplicateMatchPattern: # new in 1.50 - Enabled: false -Lint/IncompatibleIoSelectWithFiberScheduler: # new in 1.21 - Enabled: false -Lint/MixedCaseRange: # new in 1.53 - Enabled: false -Lint/NonAtomicFileOperation: # new in 1.31 - Enabled: false -Lint/RedundantRegexpQuantifiers: # new in 1.53 - Enabled: false -Lint/RefinementImportMethods: # new in 1.27 - Enabled: false -Lint/RequireRangeParentheses: # new in 1.32 - Enabled: false -Lint/RequireRelativeSelfPath: # new in 1.22 - Enabled: false -Lint/UselessRescue: # new in 1.43 - Enabled: false -Lint/UselessRuby2Keywords: # new in 1.23 - Enabled: false -Metrics/CollectionLiteralLength: # new in 1.47 - Enabled: false Naming/BlockForwarding: # new in 1.24 Enabled: false Security/CompoundHash: # new in 1.28 Enabled: false -Security/IoMethods: # new in 1.22 - Enabled: false -Style/ArrayIntersect: # new in 1.40 - Enabled: false -Style/ComparableClamp: # new in 1.44 - Enabled: false -Style/ConcatArrayLiterals: # new in 1.41 - Enabled: false -Style/DataInheritance: # new in 1.49 - Enabled: false -Style/DirEmpty: # new in 1.48 - Enabled: false -Style/EmptyHeredoc: # new in 1.32 - Enabled: false -Style/EnvHome: # new in 1.29 - Enabled: false -Style/ExactRegexpMatch: # new in 1.51 - Enabled: false Style/FetchEnvVar: # new in 1.28 Enabled: false -Style/FileEmpty: # new in 1.48 - Enabled: false -Style/FileRead: # new in 1.24 - Enabled: false -Style/FileWrite: # new in 1.24 - Enabled: false -Style/MagicCommentFormat: # new in 1.35 - Enabled: false -Style/MapCompactWithConditionalBlock: # new in 1.30 - Enabled: false -Style/MapToHash: # new in 1.24 - Enabled: false -Style/MapToSet: # new in 1.42 - Enabled: false -Style/MinMaxComparison: # new in 1.42 - Enabled: false -Style/NestedFileDirname: # new in 1.26 - Enabled: false -Style/NumberedParameters: # new in 1.22 - Enabled: false -Style/NumberedParametersLimit: # new in 1.22 - Enabled: false Style/ObjectThen: # new in 1.28 Enabled: false -Style/OpenStructUse: # new in 1.23 - Enabled: false -Style/OperatorMethodCall: # new in 1.37 - Enabled: false -Style/RedundantArrayConstructor: # new in 1.52 - Enabled: false -Style/RedundantConstantBase: # new in 1.40 - Enabled: false -Style/RedundantCurrentDirectoryInPath: # new in 1.53 - Enabled: false -Style/RedundantDoubleSplatHashBraces: # new in 1.41 - Enabled: false -Style/RedundantEach: # new in 1.38 - Enabled: false -Style/RedundantFilterChain: # new in 1.52 - Enabled: false -Style/RedundantHeredocDelimiterQuotes: # new in 1.45 - Enabled: false -Style/RedundantInitialize: # new in 1.27 - Enabled: false -Style/RedundantLineContinuation: # new in 1.49 - Enabled: false -Style/RedundantRegexpArgument: # new in 1.53 - Enabled: false -Style/RedundantRegexpConstructor: # new in 1.52 - Enabled: false -Style/RedundantStringEscape: # new in 1.37 - Enabled: false -Style/ReturnNilInPredicateMethodDefinition: # new in 1.53 - Enabled: false -Style/SelectByRegexp: # new in 1.22 - Enabled: false -Style/YAMLFileRead: # new in 1.53 - Enabled: false -Style/HashSyntax: - Enabled: false -Lint/RedundantRequireStatement: - Enabled: false -Lint/AmbiguousRegexpLiteral: - Enabled: false -Style/SafeNavigation: - Enabled: false +Style/SafeNavigationChainLength: + Enabled: false \ No newline at end of file diff --git a/integration-specs/rails-app/config/environments/production.rb b/integration-specs/rails-app/config/environments/production.rb index 14032033..a9501c09 100644 --- a/integration-specs/rails-app/config/environments/production.rb +++ b/integration-specs/rails-app/config/environments/production.rb @@ -78,7 +78,7 @@ config.active_support.report_deprecations = false # Use default logging formatter so that PID and timestamp are not suppressed. - config.log_formatter = ::Logger::Formatter.new + config.log_formatter = Logger::Formatter.new # Use a different logger for distributed setups. # require "syslog/logger" diff --git a/integration-specs/rails-app/spec/rails_helper.rb b/integration-specs/rails-app/spec/rails_helper.rb index a5c3a0d3..485afb09 100644 --- a/integration-specs/rails-app/spec/rails_helper.rb +++ b/integration-specs/rails-app/spec/rails_helper.rb @@ -33,7 +33,7 @@ end RSpec.configure do |config| # Remove this line if you're not using ActiveRecord or ActiveRecord fixtures - config.fixture_path = "#{::Rails.root}/spec/fixtures" + config.fixture_path = "#{Rails.root}/spec/fixtures" # If you're not using ActiveRecord, or you'd prefer not to run each of your # examples within a transaction, remove the following line or assign false diff --git a/lib/sequent/cli/app.rb b/lib/sequent/cli/app.rb index 205435f3..854f752e 100644 --- a/lib/sequent/cli/app.rb +++ b/lib/sequent/cli/app.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true require_relative '../generator' -require_relative './sequent8_migration' +require_relative 'sequent8_migration' module Sequent module Cli class App diff --git a/lib/sequent/core/helpers/array_with_type.rb b/lib/sequent/core/helpers/array_with_type.rb index 0c6393ac..6d20501c 100644 --- a/lib/sequent/core/helpers/array_with_type.rb +++ b/lib/sequent/core/helpers/array_with_type.rb @@ -13,7 +13,7 @@ def initialize(item_type) end def deserialize_from_json(value) - value.nil? ? nil : value.map { |item| item_type.deserialize_from_json(item) } + value&.map { |item| item_type.deserialize_from_json(item) } end def to_s diff --git a/lib/sequent/core/helpers/association_validator.rb b/lib/sequent/core/helpers/association_validator.rb index 3e79f060..8b16f015 100644 --- a/lib/sequent/core/helpers/association_validator.rb +++ b/lib/sequent/core/helpers/association_validator.rb @@ -35,7 +35,7 @@ def validate(record) value = record.instance_variable_get("@#{association}") if value && incorrect_type?(value, record, association) record.errors.add(association, "is not of type #{describe_type(record.class.types[association])}") - elsif value&.is_a?(Array) + elsif value.is_a?(Array) item_type = record.class.types.fetch(association).item_type record.errors.add(association, 'is invalid') unless validate_all(value, item_type).all? elsif value&.invalid? @@ -47,7 +47,7 @@ def validate(record) private def incorrect_type?(value, record, association) - return unless record.class.respond_to?(:types) + return false unless record.class.respond_to?(:types) type = record.class.types[association] if type.respond_to?(:candidate?) diff --git a/lib/sequent/core/helpers/attribute_support.rb b/lib/sequent/core/helpers/attribute_support.rb index 83545a9a..1173491c 100644 --- a/lib/sequent/core/helpers/attribute_support.rb +++ b/lib/sequent/core/helpers/attribute_support.rb @@ -63,12 +63,11 @@ def attrs(args) Sequent::Core::Helpers::DefaultValidators.for(type).add_validations_for(self, attribute) end - if type.instance_of?(Sequent::Core::Helpers::ArrayWithType) - associations << attribute - elsif included_modules.include?(ActiveModel::Validations) && - type.included_modules.include?(Sequent::Core::Helpers::AttributeSupport) - associations << attribute - end + next unless type.instance_of?(Sequent::Core::Helpers::ArrayWithType) || + (included_modules.include?(ActiveModel::Validations) && + type.included_modules.include?(Sequent::Core::Helpers::AttributeSupport)) + + associations << attribute end if included_modules.include?(ActiveModel::Validations) && associations.present? validates_with Sequent::Core::Helpers::AssociationValidator, associations: associations @@ -168,7 +167,7 @@ def self.included(host_class) def attributes hash = HashWithIndifferentAccess.new - self.class.types.each do |name, _| + self.class.types.each_key do |name| value = instance_variable_get("@#{name}") hash[name] = if value.respond_to?(:attributes) value.attributes @@ -181,7 +180,7 @@ def attributes def as_json(opts = {}) hash = HashWithIndifferentAccess.new - self.class.types.each do |name, _| + self.class.types.each_key do |name| value = instance_variable_get("@#{name}") hash[name] = if value.respond_to?(:as_json) value.as_json(opts) diff --git a/lib/sequent/core/helpers/equal_support.rb b/lib/sequent/core/helpers/equal_support.rb index d0a327ca..5ed7bd48 100644 --- a/lib/sequent/core/helpers/equal_support.rb +++ b/lib/sequent/core/helpers/equal_support.rb @@ -13,7 +13,7 @@ def ==(other) return false if other.nil? return false if self.class != other.class - self.class.types.each do |name, _| + self.class.types.each_key do |name| self_value = send(name) other_value = other.send(name) if self_value.class == DateTime && other_value.class == DateTime @@ -28,8 +28,8 @@ def ==(other) def hash hash = 17 - self.class.types.each do |name, _| - hash = hash * 31 + send(name).hash + self.class.types.each_key do |name| + hash = (hash * 31) + send(name).hash end hash end diff --git a/lib/sequent/core/helpers/message_matchers/has_attrs.rb b/lib/sequent/core/helpers/message_matchers/has_attrs.rb index b17043c0..0b21c8f7 100644 --- a/lib/sequent/core/helpers/message_matchers/has_attrs.rb +++ b/lib/sequent/core/helpers/message_matchers/has_attrs.rb @@ -20,10 +20,12 @@ def matches_message?(message) end def to_s + # rubocop:disable Layout/LineEndStringConcatenationIndentation 'has_attrs(' \ "#{MessageMatchers::ArgumentSerializer.serialize_value(message_matcher)}, " \ "#{AttrMatchers::ArgumentSerializer.serialize_value(expected_attrs)}" \ ')' + # rubocop:enable Layout/LineEndStringConcatenationIndentation end private diff --git a/lib/sequent/core/helpers/message_router.rb b/lib/sequent/core/helpers/message_router.rb index c45e247c..f898c728 100644 --- a/lib/sequent/core/helpers/message_router.rb +++ b/lib/sequent/core/helpers/message_router.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true -require_relative './attr_matchers/attr_matchers' -require_relative './message_matchers/message_matchers' +require_relative 'attr_matchers/attr_matchers' +require_relative 'message_matchers/message_matchers' module Sequent module Core diff --git a/lib/sequent/core/helpers/param_support.rb b/lib/sequent/core/helpers/param_support.rb index 4f794d34..d2c38589 100644 --- a/lib/sequent/core/helpers/param_support.rb +++ b/lib/sequent/core/helpers/param_support.rb @@ -79,9 +79,7 @@ def as_params def value_to_string(val) if val.is_a?(Sequent::Core::ValueObject) val.as_params - elsif val.is_a? DateTime - val.iso8601 - elsif val.is_a? Date + elsif val.is_a?(DateTime) || val.is_a?(Date) val.iso8601 elsif val.is_a? Time val.iso8601(Sequent.configuration.time_precision) diff --git a/lib/sequent/core/helpers/string_support.rb b/lib/sequent/core/helpers/string_support.rb index 9094f2c0..b19ed2b1 100644 --- a/lib/sequent/core/helpers/string_support.rb +++ b/lib/sequent/core/helpers/string_support.rb @@ -15,7 +15,7 @@ def to_s value = instance_variable_get(name.to_s) s += "#{name}=[#{value}], " end - '{' + s.chomp(', ') + '}' + "{#{s.chomp(', ')}}" end end end diff --git a/lib/sequent/core/persistors/active_record_persistor.rb b/lib/sequent/core/persistors/active_record_persistor.rb index b281110b..9581c8f7 100644 --- a/lib/sequent/core/persistors/active_record_persistor.rb +++ b/lib/sequent/core/persistors/active_record_persistor.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true require 'active_record' -require_relative './persistor' +require_relative 'persistor' module Sequent module Core diff --git a/lib/sequent/core/persistors/replay_optimized_postgres_persistor.rb b/lib/sequent/core/persistors/replay_optimized_postgres_persistor.rb index 0ccea2df..4bbbc384 100644 --- a/lib/sequent/core/persistors/replay_optimized_postgres_persistor.rb +++ b/lib/sequent/core/persistors/replay_optimized_postgres_persistor.rb @@ -1,9 +1,8 @@ # frozen_string_literal: true -require 'set' require 'active_record' require 'csv' -require_relative './persistor' +require_relative 'persistor' module Sequent module Core @@ -177,7 +176,7 @@ def initialize(insert_with_csv_size = 50, indices = {}, default_indexed_columns end indices.each do |record_class, indexed_columns| - columns = indexed_columns.flatten(1).map(&:to_sym).to_set + default_indexed_columns + columns = indexed_columns.flatten(1).to_set(&:to_sym) + default_indexed_columns @record_index[record_class] = Index.new(columns & record_class.column_names.map(&:to_sym)) end @@ -358,7 +357,7 @@ def commit def clear @record_store.clear - @record_index.values.each(&:clear) + @record_index.each_value(&:clear) end private diff --git a/lib/sequent/core/projector.rb b/lib/sequent/core/projector.rb index a69558a9..9f93949e 100644 --- a/lib/sequent/core/projector.rb +++ b/lib/sequent/core/projector.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true require_relative 'helpers/message_handler' -require_relative './persistors/active_record_persistor' +require_relative 'persistors/active_record_persistor' module Sequent module Core diff --git a/lib/sequent/generator.rb b/lib/sequent/generator.rb index affa0183..0b066d1f 100644 --- a/lib/sequent/generator.rb +++ b/lib/sequent/generator.rb @@ -1,3 +1,3 @@ # frozen_string_literal: true -require_relative './generator/generator' +require_relative 'generator/generator' diff --git a/lib/sequent/migrations/grouper.rb b/lib/sequent/migrations/grouper.rb index 6ba14a8b..b8548635 100644 --- a/lib/sequent/migrations/grouper.rb +++ b/lib/sequent/migrations/grouper.rb @@ -56,7 +56,7 @@ def self.group_partitions(partitions, target_group_size) current_size = 0 else taken = target_group_size - current_size - upper_bound = partition.lower_bound + UUID_COUNT * taken / partition.original_size + upper_bound = partition.lower_bound + (UUID_COUNT * taken / partition.original_size) result << (current_start..GroupEndpoint.new(partition.key, number_to_uuid(upper_bound - 1))) @@ -79,7 +79,7 @@ def self.number_to_uuid(number) end def self.uuid_to_number(uuid) - Integer(uuid.gsub(/-/, ''), 16) + Integer(uuid.gsub('-', ''), 16) end UUID_COUNT = 2**128 diff --git a/lib/sequent/migrations/view_schema.rb b/lib/sequent/migrations/view_schema.rb index 4b32c610..9540a782 100644 --- a/lib/sequent/migrations/view_schema.rb +++ b/lib/sequent/migrations/view_schema.rb @@ -8,7 +8,7 @@ require_relative '../sequent' require_relative '../util/timer' require_relative '../util/printer' -require_relative './projectors' +require_relative 'projectors' require_relative 'planner' require_relative 'executor' require_relative 'sql' @@ -205,17 +205,15 @@ def migrate_online Versions.end_online!(Sequent.new_version) end Sequent.logger.info("Done migrate_online for version #{Sequent.new_version}") - rescue ConcurrentMigration - # Do not rollback the migration when this is a concurrent migration as the other one is running - raise - rescue InvalidMigrationDefinition - # Do not rollback the migration when since there is nothing to rollback + rescue ConcurrentMigration, InvalidMigrationDefinition + # ConcurrentMigration: Do not rollback the migration when this is a concurrent migration + # as the other one is running + # InvalidMigrationDefinition: Do not rollback the migration when since there is nothing to rollback raise rescue Exception => e # rubocop:disable Lint/RescueException rollback_migration raise e end - ## # Last part of a view schema migration # diff --git a/lib/sequent/test/time_comparison.rb b/lib/sequent/test/time_comparison.rb index 76689757..8c41e368 100644 --- a/lib/sequent/test/time_comparison.rb +++ b/lib/sequent/test/time_comparison.rb @@ -16,7 +16,7 @@ module Compare # omit nsec in datetime comparisons def <=>(other) - if other&.is_a?(DateTimePatches::Normalize) + if other.is_a?(DateTimePatches::Normalize) precision = Sequent.configuration.time_precision return normalize.iso8601(precision) <=> other.normalize.iso8601(precision) end diff --git a/sequent.gemspec b/sequent.gemspec index 1ab5b920..4bcfd5ce 100644 --- a/sequent.gemspec +++ b/sequent.gemspec @@ -51,4 +51,5 @@ Gem::Specification.new do |s| s.add_development_dependency 'rubocop', '~> 1.68.0' s.add_development_dependency 'simplecov', '~> 0.21' s.add_development_dependency 'timecop', '~> 0.9' + s.metadata['rubygems_mfa_required'] = 'true' end diff --git a/spec/database.rb b/spec/database.rb index 4a63a22b..a489bda4 100644 --- a/spec/database.rb +++ b/spec/database.rb @@ -29,8 +29,8 @@ def test_config(database_name: 'sequent_spec_db') username: 'sequent', password: 'sequent', database: database_name, - schema_search_path: "#{Sequent.configuration.view_schema_name},"\ - "#{Sequent.configuration.event_store_schema_name},public", + schema_search_path: "#{Sequent.configuration.view_schema_name}," \ + "#{Sequent.configuration.event_store_schema_name},public", advisory_locks: false, }, }, @@ -43,7 +43,7 @@ def test_config(database_name: 'sequent_spec_db') username: 'sequent', password: 'sequent', database: database_name, - schema_search_path: "#{Sequent.configuration.view_schema_name},"\ + schema_search_path: "#{Sequent.configuration.view_schema_name}," \ "#{Sequent.configuration.event_store_schema_name},public", advisory_locks: false, }, diff --git a/spec/lib/sequent/configuration_spec.rb b/spec/lib/sequent/configuration_spec.rb index 4d4e40c2..3261675f 100644 --- a/spec/lib/sequent/configuration_spec.rb +++ b/spec/lib/sequent/configuration_spec.rb @@ -68,7 +68,7 @@ def repository config.enable_autoregistration = true config.command_handlers = [command_handler_class.new] end - end.to raise_error /is registered 2 times. A CommandHandler can only be registered once/ + end.to raise_error(/is registered 2 times. A CommandHandler can only be registered once/) end end @@ -82,7 +82,7 @@ def repository config.enable_autoregistration = true config.event_handlers = [event_handler_class.new] end - end.to raise_error /is registered 2 times. An EventHandler can only be registered once/ + end.to raise_error(/is registered 2 times. An EventHandler can only be registered once/) end end end diff --git a/spec/lib/sequent/core/helpers/association_validator_spec.rb b/spec/lib/sequent/core/helpers/association_validator_spec.rb index 19c76c9d..8b8a023f 100644 --- a/spec/lib/sequent/core/helpers/association_validator_spec.rb +++ b/spec/lib/sequent/core/helpers/association_validator_spec.rb @@ -7,12 +7,12 @@ let(:subject) { Sequent::Core::Helpers::AssociationValidator.new(options) } it 'fails when providing no associations' do - expect { subject }.to raise_error /Must provide ':associations' to validate/ + expect { subject }.to raise_error(/Must provide ':associations' to validate/) end it 'fails when provind an empty list of associations' do options[:associations] = [] - expect { subject }.to raise_error /Must provide ':associations' to validate/ + expect { subject }.to raise_error(/Must provide ':associations' to validate/) end context 'validating an array with simple types' do diff --git a/spec/lib/sequent/core/projector_spec.rb b/spec/lib/sequent/core/projector_spec.rb index e7397f04..f07a67e8 100644 --- a/spec/lib/sequent/core/projector_spec.rb +++ b/spec/lib/sequent/core/projector_spec.rb @@ -9,6 +9,6 @@ class TestProjector1 < Sequent::Core::Projector end expect do Sequent.configuration.event_handlers << TestProjector1.new - end.to raise_error /A Projector must manage at least one table/ + end.to raise_error(/A Projector must manage at least one table/) end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 1a2db7a0..28c0e80f 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -9,7 +9,7 @@ require 'timecop' require_relative '../lib/sequent' require_relative '../lib/sequent/generator' -require_relative './lib/sequent/fixtures/fixtures' +require_relative 'lib/sequent/fixtures/fixtures' require './lib/sequent/test/database_helpers' require 'simplecov' SimpleCov.start if ENV['COVERAGE'] From 4ccb60f289c1391754dad8f8383fde250e79c261 Mon Sep 17 00:00:00 2001 From: Lars Vonk Date: Fri, 15 Nov 2024 14:04:09 +0100 Subject: [PATCH 126/128] Simplify unless to if --- lib/sequent/core/helpers/attribute_support.rb | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/sequent/core/helpers/attribute_support.rb b/lib/sequent/core/helpers/attribute_support.rb index 1173491c..034c3b65 100644 --- a/lib/sequent/core/helpers/attribute_support.rb +++ b/lib/sequent/core/helpers/attribute_support.rb @@ -63,12 +63,13 @@ def attrs(args) Sequent::Core::Helpers::DefaultValidators.for(type).add_validations_for(self, attribute) end - next unless type.instance_of?(Sequent::Core::Helpers::ArrayWithType) || - (included_modules.include?(ActiveModel::Validations) && - type.included_modules.include?(Sequent::Core::Helpers::AttributeSupport)) + is_array = type.instance_of?(Sequent::Core::Helpers::ArrayWithType) + needs_validation = !is_array && included_modules.include?(ActiveModel::Validations) && + type.included_modules.include?(Sequent::Core::Helpers::AttributeSupport) - associations << attribute + associations << attribute if is_array || needs_validation end + if included_modules.include?(ActiveModel::Validations) && associations.present? validates_with Sequent::Core::Helpers::AssociationValidator, associations: associations end From 8377e28ec537b9ea880974ab852b69298e07792f Mon Sep 17 00:00:00 2001 From: Erik Rozendaal Date: Mon, 9 Dec 2024 15:35:27 +0100 Subject: [PATCH 127/128] Prepare for release 8.0.0 --- CHANGELOG.md | 2 +- gemfiles/ar_7_1.gemfile.lock | 4 ++-- gemfiles/ar_7_2.gemfile.lock | 4 ++-- lib/version.rb | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c52781ac..4fd56373 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# Changelog 8.0.x (changes since 7.0.1) +# Changelog 8.0.0 (changes since 7.2.0) - Sequent now requires at least Ruby 3.2, ActiveRecord 7.1, and PostgreSQL 14. diff --git a/gemfiles/ar_7_1.gemfile.lock b/gemfiles/ar_7_1.gemfile.lock index ea9766a7..52382d84 100644 --- a/gemfiles/ar_7_1.gemfile.lock +++ b/gemfiles/ar_7_1.gemfile.lock @@ -1,7 +1,7 @@ PATH remote: .. specs: - sequent (8.0.0.pre.dev.1) + sequent (8.0.0) activemodel (>= 7.1.3) activerecord (>= 7.1.3) bcrypt (~> 3.1) @@ -71,7 +71,7 @@ GEM postgresql_cursor (0.6.9) activerecord (>= 6.0) prop_check (1.0.0) - pry (0.14.2) + pry (0.15.0) coderay (~> 1.1) method_source (~> 1.0) racc (1.8.1) diff --git a/gemfiles/ar_7_2.gemfile.lock b/gemfiles/ar_7_2.gemfile.lock index c24c784f..5566ea01 100644 --- a/gemfiles/ar_7_2.gemfile.lock +++ b/gemfiles/ar_7_2.gemfile.lock @@ -1,7 +1,7 @@ PATH remote: .. specs: - sequent (8.0.0.pre.dev.1) + sequent (8.0.0) activemodel (>= 7.1.3) activerecord (>= 7.1.3) bcrypt (~> 3.1) @@ -71,7 +71,7 @@ GEM postgresql_cursor (0.6.9) activerecord (>= 6.0) prop_check (1.0.0) - pry (0.14.2) + pry (0.15.0) coderay (~> 1.1) method_source (~> 1.0) racc (1.8.1) diff --git a/lib/version.rb b/lib/version.rb index ee002b3d..5b9782d9 100644 --- a/lib/version.rb +++ b/lib/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module Sequent - VERSION = '8.0.0-dev.1' + VERSION = '8.0.0' end From e2ac897bc35f8e5a131494c9006f3cec7b207cc2 Mon Sep 17 00:00:00 2001 From: Lars Vonk Date: Tue, 10 Dec 2024 10:21:59 +0100 Subject: [PATCH 128/128] Small tweaks to the upgrade guide. --- docs/docs/upgrade-guide.md | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/docs/docs/upgrade-guide.md b/docs/docs/upgrade-guide.md index 381d01b1..f3bedc73 100644 --- a/docs/docs/upgrade-guide.md +++ b/docs/docs/upgrade-guide.md @@ -4,23 +4,27 @@ title: Upgrade Guide ## Upgrade to Sequent 8.x from older versions +**IMPORTANT**: Ensure you are on the latest released 7.x version first. + Sequent 8 remodels the PostgreSQL event store to allow partitioning of -the aggregates, commands, and events tables. Furthermore it contains +the aggregates, commands, and events tables. Furthermore, it contains various storage optimizations. To migrate your older Sequent database -you can use the `bundle exec sequent migrate` command. Make sure to run this after -updating Sequent in your Gemfile, running `bundle update --conservative -sequent`, and from the root directory of your project. +we advise you to **run the migration first on a copy of your production +database!** This will give you a good indication on how long the +migration will take so you can schedule downtime appropriately. +Next to that it will ensure all data in your event store is compatible +with Sequent 8. Normally this won't be a problem +unless you somehow have corrupted data in your event store. + +To run the upgrade: +1. Update Sequent to version 8 in your Gemfile +2. Run the `bundle exec sequent migrate` command from your project directory. -This command will help you perform the database upgrade by providing -you with a default schema and database upgrade script that you can +This command will help you perform the database upgrade step by step +by providing you with a default schema and database upgrade script that you can customize to match your desired partitioning setup, although the default configuration will work for many cases as well. -**IMPORTANT**: Ensure you test your migration on a copy of your database first! This will give you -a good indication on how long the migration will take so you can schedule downtime appropriately. -Next to that it will ensure all data in your event store is compatible with Sequent 8. Normally this won't be a problem -unless you somehow have corrupted data in your event store. - **IMPORTANT**: If the migration succeeds and you COMMIT the results you must vacuum (e.g. using VACUUM VERBOSE ANALYZE) the new tables to ensure good performance!