Create a migration command to migrate the data from V2 to V3

Package.swift

@@ -98,6 +98,7 @@ let package = Package(
 				.process("test_files/MovimientosCuenta.xls"),
 				.process("test_files/MovimientosTarjetaCredito.xls"),
 				.process("test_files/n26_es.csv"),
+				.process("test_files/old.sqlite3"),
 			]
 		),
 

README.md

@@ -6,32 +6,25 @@ This is the base repository, to clone and start it using docker.
 ## File formats suported
 Currently this application supports the following bank file formats:
 
-- Caixabanc Account and card Excel
 - Caixa Enginyers Account and Credit Card
 - Commerce bank
 - N26 CSV file
-- MultiBank QIF File (Thanks to [Albert Sola](https://github.com/albertsola/))
-
-(*) To create a new one, can you take a look at the changes from @AlbertSola to add the qif format [commit](https://github.com/Dracks/mr-scrooge-server/commit/630d6bafe99ba6cbbe29a92959a6954726cdcb88)
-
+- MultiBank QIF File (TBD)
 
 ## Demo
-You can run the docker image generated for the release inside her:
-https://hub.docker.com/r/dracks/mrscrooge/
 
-To run, execute the following command:
+## How to Migrate from Mr Scrooge V2
+The migration from Django to Vapor brought significant database improvements, including group data ownership capabilities that enable multiple users to operate on a single instance. This upgrade requires a complete data migration from the previous database version.
+
+1. Set up a new database for the upgraded version
+2. Update your environment or .env file with the new database connection details  
+3. Generate a new user via the "demo_user" command
+4. Execute the data migration using v2_migrate along with the group ID for the user you wish, the command demo_user automatically provides you a group Id:
 ```
-docker run --rm -p 3333:80 -e DEMODATA="true" dracks/mrscrooge:latest
+./server v2_migrate sqlite://db.sqlite 1234-1234-1234-1234
 ```
 
-It will open your port 3333 with user demo/demo
-
-You can set the following environment variables:
-- DEMODATA: Default false. this will create a random data with tags and graphs
-- DEMOUSER: default "demo". The username to create on the application
-- DEMOPWD: default "demo". The password for the user
-- ALLOWED_HOSTS: default "localhost". You can put a list of hosts separed with a comma character
-- DECIMAL_COUNT: default 2, the number of digits to show in the graphs
+** IMPORTANT **: The web-client must have been launched and logged into at least once in V2 before graphs can be imported
 
 ## Screenshots MVP
 ### Home
@@ -49,22 +42,36 @@ You can set the following environment variables:
 3. Run it
 
 ### Run the server
-1. Make sure you have python3 available in your machine
-2. Install dependencies (Recomendable use of virtual env [tutorial sample](https://www.pythonforbeginners.com/basics/how-to-use-python-virtualenv)) with:
-```pip install -r requirements.txt```
-3. Run the tests
-```./manage.py test```
-4. Run the server
-```./manage.py runserver```
+1. Run the server, with swift:
+```
+swift run
+```
+2. Migrate the data:
+```
+# ServerDebug is a softlink, to reuse the build from the swift run
+./serverDebug migrate
+```
+3. Create an admin user (by default will be demo:demo):
+```
+./serverDebug create_user --admin
+```
 
+### Run the server tests
+Simply run the tests using swift:
+```
+swift test
+```
 ### Run the view
-1. Make sure you have node 8.0 or superior
-2. Install dependencies:
-```yarn```
-or
-```npm install ```
-3. Run the tests ```yarn test``` or ```npm test``` (It will run with a watcher, this means that don't stop after run the tests waiting for changes)
-4. Run the project ```yarn start``` or ```npm start```
+1. Install dependencies
+```
+pnpm install
+```
+
+2. Build in watch mode
+```
+pnpm run start:dev
+```
+
 
 ## Thanks
 The current application icon is from VisualPharm

src/swift-server-tests/app-test.swift

@@ -123,12 +123,10 @@ final class MrScroogeServerTest: AbstractBaseTestsClass {
 	}
 
 	func testCreateUserCommand() async throws {
-		guard let app = app else {
-			throw TestError()
-		}
+		let app = try getApp()
 
 		let command = CreateUserCommand()
-		let arguments = ["demo_user"]
+		let arguments = ["create_user"]
 
 		let console = TestConsole()
 		let input = CommandInput(arguments: arguments)

src/swift-server-tests/migrate-v2-command.tests.swift

@@ -0,0 +1,206 @@
+import Fluent
+import Vapor
+import XCTVapor
+
+@testable import MrScroogeServer
+
+final class MigrateV2Test: AbstractBaseTestsClass {
+
+	func testV2MigrateCommand() async throws {
+		let app = try getApp()
+
+		let dbPath = try XCTUnwrap(
+			Bundle.module.url(forResource: "old", withExtension: "sqlite3"))
+
+		let migrateTestGroup = UserGroup(name: "Migration")
+		try await migrateTestGroup.save(on: app.db)
+		let groupId = try migrateTestGroup.requireID()
+
+		let command = V2MigrateCommand()
+		let arguments = [
+			"v2_migrate", "sqlite://\(dbPath)", groupId.uuidString,
+		]
+
+		let console = TestConsole()
+		let input = CommandInput(arguments: arguments)
+		var context = CommandContext(
+			console: console,
+			input: input
+		)
+
+		context.application = app
+
+		try await console.run(command, with: context)
+
+		/*let output = console
+			.testOutputQueue
+			.map { $0.trimmingCharacters(in: .whitespacesAndNewlines) }
+
+		XCTAssertTrue(output.count > 0)
+		XCTAssertContains(output.first, "labels create from tags")*/
+
+		let labels = try await Label.query(on: app.db).filter(\.$groupOwner.$id == groupId)
+			.all()
+		XCTAssertEqual(labels.count, 3, "Labels number is not valid")
+
+		func getLabelIdByName(_ name: String) throws -> UUID? {
+			return try labels.first { $0.name == name }?.requireID()
+		}
+		/*let labelsDict: [UUID: String] = try labels.reduce(into: [:]) {
+			partialResult, label in
+			let labelId = try label.requireID()
+			partialResult[labelId] = label.name
+		}*/
+
+		let rules = try await Rule.query(on: app.db).filter(\.$groupOwner.$id == groupId)
+			.all()
+
+		XCTAssertEqual(rules.count, 3, "Rules number is not valid")
+
+		let ruleWithParent = rules.filter { $0.name == "With parent" }.first
+		XCTAssertNotNil(ruleWithParent)
+		if let ruleWithParent {
+			try await ruleWithParent.$labels.load(on: app.db)
+			try await ruleWithParent.$conditions.load(on: app.db)
+			XCTAssertNotNil(ruleWithParent.$parent.id, "Parent rule")
+			XCTAssertEqual(ruleWithParent.labels.map({ $0.name }), ["With parent"])
+			XCTAssertEqual(
+				ruleWithParent.conditions.count, 6,
+				"It has the correct number of conditions")
+		}
+
+		let transactionsCount = try await BankTransaction.query(on: app.db).filter(
+			\.$groupOwner.$id == groupId
+		).count()
+		XCTAssertEqual(transactionsCount, 296)
+
+		let november9Transaction = try await BankTransaction.query(on: app.db)
+			.filter(\.$_date == "2019-11-09")
+			.with(\.$labels)
+			.first()
+		XCTAssertNotNil(november9Transaction)
+		XCTAssertEqual(november9Transaction?.$groupOwner.id, groupId)
+		XCTAssertEqual(november9Transaction?.description, "The description")
+		XCTAssertEqual(november9Transaction?.value, -2.88)
+		XCTAssertEqual(november9Transaction?.movementName, "OPERACIÓ VIKINI")
+		XCTAssertEqual(november9Transaction?.details, "TARGETA *5019")
+		XCTAssertEqual(november9Transaction?.kind, "caixa-enginyers/account")
+		XCTAssertEqual(november9Transaction?.dateValue?.toString(), "2019-11-09")
+
+		XCTAssertEqual(november9Transaction?.labels.count, 1)
+		XCTAssertEqual(
+			november9Transaction?.labels.map { $0.name }, ["Expenses"])
+
+		let august4Transaction = try await BankTransaction.query(on: app.db)
+			.filter(\.$_date == "2019-08-04")
+			.with(\.$labels)
+			.first()
+		XCTAssertNotNil(august4Transaction)
+		XCTAssertEqual(august4Transaction?.$groupOwner.id, groupId)
+		XCTAssertEqual(august4Transaction?.description, nil)
+		XCTAssertEqual(august4Transaction?.value, 1500.0)
+		XCTAssertEqual(august4Transaction?.movementName, "Dr Who")
+		XCTAssertEqual(august4Transaction?.details, "Transfer")
+		XCTAssertEqual(august4Transaction?.kind, "caixa-enginyers/account")
+		XCTAssertEqual(august4Transaction?.dateValue?.toString(), "2019-08-04")
+
+		XCTAssertEqual(august4Transaction?.labels.count, 1)
+		XCTAssertEqual(
+			august4Transaction?.labels.map { $0.name }, ["Revenues"])
+
+		// TODO: Add the validations
+		let graphsPage = try await app.graphService.getGraphs(
+			groupsId: [groupId], graphsIds: nil)
+		XCTAssertEqual(graphsPage.list.count, 3)
+
+		let incomeVsOutcome = graphsPage.list.first { $0.name == "Income vs outcome" }
+		XCTAssertNotNil(incomeVsOutcome)
+		if let incomeVsOutcome {
+			XCTAssertEqual(
+				incomeVsOutcome,
+				.init(
+					id: incomeVsOutcome.id, groupOwnerId: groupId.uuidString,
+					name: "Income vs outcome", kind: .bar,
+					dateRange: .oneYear,
+					group: .init(group: .sign),
+					horizontalGroup: .init(group: .month, accumulate: false)
+				),
+				"Income vs Outcome correctly transformed")
+		}
+
+		let compareByDay = graphsPage.list.first { $0.name == "Compare by day" }
+		XCTAssertNotNil(compareByDay)
+		if let compareByDay {
+			try XCTAssertEqual(
+				compareByDay,
+				.init(
+					id: compareByDay.id, groupOwnerId: groupId.uuidString,
+					name: "Compare by day", kind: .line,
+					labelFilterId: getLabelIdByName("Expenses")?.uuidString,
+					dateRange: .halfYear,
+					group: .init(group: .month),
+					horizontalGroup: .init(group: .day, accumulate: true)
+				),
+				"Compare by day graph correctly transformed")
+		}
+
+		let testPie = graphsPage.list.first { $0.name == "Test Pie" }
+		XCTAssertNotNil(testPie)
+
+		if let testPie {
+			try XCTAssertEqual(
+				testPie,
+				.init(
+					id: testPie.id, groupOwnerId: groupId.uuidString,
+					name: "Test Pie", kind: .pie,
+					dateRange: .all,
+					group: .init(
+						group: .labels,
+						hideOthers: true,
+						labels: ["With parent", "Expenses"].map {
+							try getLabelIdByName($0)?.uuidString ?? ""
+						}
+					)
+				), "Test pie graph correctly transformed")
+		}
+	}
+
+	func testInvalidUrl() async throws {
+		let app = try getApp()
+
+		let dbPath = try XCTUnwrap(
+			Bundle.module.url(forResource: "old", withExtension: "sqlite3"))
+
+		let migrateTestGroup = UserGroup(name: "Migration")
+		try await migrateTestGroup.save(on: app.db)
+		let groupId = try migrateTestGroup.requireID()
+
+		let command = V2MigrateCommand()
+		let arguments = [
+			"v2_migrate", "sqlte://\(dbPath)", groupId.uuidString,
+		]
+
+		let console = TestConsole()
+		let input = CommandInput(arguments: arguments)
+		var context = CommandContext(
+			console: console,
+			input: input
+		)
+
+		context.application = app
+
+		try await console.run(command, with: context)
+
+		// XCTAssertNil(console.testOutputQueue)
+
+		let output = console
+			.testOutputQueue
+			.map { $0.trimmingCharacters(in: .whitespacesAndNewlines) }
+
+		XCTAssertTrue(output.count > 0)
+		XCTAssertContains(
+			output.first,
+			"Invalid database URL format. Must start with \'sqlite://\' or \'postgres://\'"
+		)
+	}
+}

src/swift-server-tests/profile.tests.swift

@@ -137,11 +137,11 @@ final class ProfileTests: AbstractBaseTestsClass {
 	}
 
 	func testCreateUserAsAdmin() async throws {
-	   let app = try getApp()
-				
-				let headers = try await app.getHeaders(
-					forUser: .init(
-						username: testAdmin.username, password: "test-admin-password"))
+		let app = try getApp()
+
+		let headers = try await app.getHeaders(
+			forUser: .init(
+				username: testAdmin.username, password: "test-admin-password"))
 
 		let newUser = Components.Schemas.CreateUserParams(
 			username: "Some new user",

src/swift-server/bank-transaction/bank-transaction.models.swift

@@ -43,6 +43,7 @@ final class BankTransaction: Model, Content, @unchecked Sendable {
 	@Siblings(through: LabelTransaction.self, from: \.$transaction, to: \.$label)
 	var labels: [Label]
 
+	// TODO: change the name to comment
 	@OptionalField(key: "description")
 	var description: String?