✨(xi) add a xi command to index all courses and their content

CHANGELOG.md

@@ -11,6 +11,7 @@ and this project adheres to
 ### Added
 
 - Add dedicated views depending on LTI roles
+- Add a warren xi index all CLI command
 
 ### Changed
 

src/api/core/warren/cli.py

@@ -314,9 +314,53 @@ def xi_index_course_content(
     moodle_ws_token: str,
     ignore_errors: bool,
 ):
-    """Index LMS course content."""
+    """Index LMS content of a course."""
     asyncio.run(
         _xi_index_course_content(
             course_id, xi_url, moodle_url, moodle_ws_token, ignore_errors
         )
     )
+
+
+async def _xi_index_all(
+    xi_url: str,
+    moodle_url: str,
+    moodle_ws_token: str,
+    ignore_errors: bool,
+):
+    """Index LMS courses and their content.
+
+    Nota bene: as we are calling multiple asynchronous functions, we need
+    to wrap calls in a single async function called in a synchronous Click
+    command using the asyncio.run method. Calling asyncio.run multiple times
+    can close the execution loop unexpectedly.
+    """
+    lms = Moodle(url=moodle_url, token=moodle_ws_token)
+    xi = ExperienceIndex(url=xi_url)
+
+    indexer_courses = Courses(lms=lms, xi=xi, ignore_errors=ignore_errors)
+    await indexer_courses.execute()
+
+    experiences = await xi.experience.read(aggregation_level=AggregationLevel.THREE)
+    for experience in experiences:
+        course = await xi.experience.get(object_id=experience.id)
+        if course is None:
+            raise click.BadParameter(
+                f"Unknown course {experience.id}. Course indexation has failed!"
+            )
+        indexer_content = CourseContent(
+            course=course, lms=lms, xi=xi, ignore_errors=ignore_errors
+        )
+        await indexer_content.execute()
+
+
+@xi_index.command("all")
+@click.option("--xi-url", "-x", default="")
+@click.option("--moodle-url", "-u", default="")
+@click.option("--moodle-ws-token", "-t", default="")
+@click.option("--ignore-errors/--no-ignore-errors", "-I/-F", default=False)
+def xi_index_all(
+    xi_url: str, moodle_url: str, moodle_ws_token: str, ignore_errors: bool
+):
+    """Index all LMS courses and their content."""
+    asyncio.run(_xi_index_all(xi_url, moodle_url, moodle_ws_token, ignore_errors))

src/api/core/warren/tests/test_cli.py

@@ -463,6 +463,66 @@ def test_xi_index_course_content_command_with_unknown_course(monkeypatch):
     assert "Unknown course fake-course-id. It should be indexed first!" in result.output
 
 
+def test_xi_index_all(monkeypatch):
+    """Test warren xi index all command."""
+    runner = CliRunner()
+
+    moodle_client_mock = MagicMock(return_value=None)
+    courses_indexer_execute_mock = AsyncMock()
+
+    monkeypatch.setattr(Moodle, "__init__", moodle_client_mock)
+    monkeypatch.setattr(Courses, "execute", courses_indexer_execute_mock)
+
+    xi_experience_read_mock = AsyncMock(
+        return_value=[
+            ExperienceRead(
+                **ExperienceFactory.build_dict(
+                    exclude=set(), id="ce0927fa-5f72-4623-9d29-37ef45c39609"
+                )
+            ),
+            ExperienceRead(
+                **ExperienceFactory.build_dict(
+                    exclude=set(), id="a0fb8abc-96d8-4b32-8551-f36a064a6a3f"
+                )
+            ),
+        ]
+    )
+    xi_experience_get_mock = AsyncMock(
+        return_value=ExperienceRead(
+            **ExperienceFactory.build_dict(
+                exclude=set(), id="ce0927fa-5f72-4623-9d29-37ef45c39609"
+            )
+        )
+    )
+    content_indexer_execute_mock = AsyncMock()
+
+    monkeypatch.setattr(CRUDExperience, "read", xi_experience_read_mock)
+    monkeypatch.setattr(CRUDExperience, "get", xi_experience_get_mock)
+    monkeypatch.setattr(CourseContent, "execute", content_indexer_execute_mock)
+
+    result = runner.invoke(
+        cli,
+        [
+            "xi",
+            "index",
+            "all",
+            "--xi-url",
+            "http://xi.foo.com",
+            "--moodle-url",
+            "http://moodle.foo.com",
+            "--moodle-ws-token",
+            "faketoken",
+        ],
+    )
+
+    assert result.exit_code == 0
+    moodle_client_mock.assert_called_with(
+        url="http://moodle.foo.com", token="faketoken"
+    )
+    courses_indexer_execute_mock.assert_called()
+    content_indexer_execute_mock.assert_called()
+
+
 def test_xi_list_courses_command_when_no_course_exists(monkeypatch):
     """Test warren xi list courses command with no indexed course."""
     runner = CliRunner()

src/api/core/warren/xi/indexers/moodle/etl.py

@@ -49,7 +49,14 @@ async def _extract(self) -> List[Course]:
 
     def _transform(self, raw: List[Course]) -> Iterator[ExperienceCreate]:
         """Transform courses into experiences."""
-        return (course.to_experience(base_url=self._lms.url) for course in raw)
+        for course in raw:
+            try:
+                yield course.to_experience(base_url=self._lms.url)
+            except ValidationError as err:
+                if not self._ignore_errors:
+                    raise err
+                logger.exception("Skipping invalid course %s", course.id)
+                pass
 
     async def _load(self, data: Iterator[ExperienceCreate]) -> None:
         """Load experiences into the Experience Index (XI)."""
@@ -121,6 +128,7 @@ def _transform(self, raw: List[Section]) -> Iterator[ExperienceCreate]:
                 if not self._ignore_errors:
                     raise err
                 logger.exception("Skipping invalid module %s", module.id)
+                pass
 
     async def _load(self, data: Iterator[ExperienceCreate]) -> None:
         """Load experiences into the Experience Index (XI) and create relations."""