Squashed commit of the following:

commit 91288c1
Author: Bradley Miller <[email protected]>
Date:   Wed Aug 30 16:03:21 2023 -0500

    Show correct course on datashop dump

commit 02e8102
Author: Bradley Miller <[email protected]>
Date:   Wed Aug 30 15:37:18 2023 -0500

    Fix: make sure questions are in a runestone-sphinx container

commit 1b702f0
Author: Bradley Miller <[email protected]>
Date:   Thu Aug 24 17:09:30 2023 -0500

    New: use author server to download logs

commit e1fff3b
Author: Bradley Miller <[email protected]>
Date:   Thu Aug 24 17:09:14 2023 -0500

    New: use author server to download logs

commit 51a90a4
Author: Bradley Miller <[email protected]>
Date:   Thu Aug 24 14:59:47 2023 -0500

    New: show different message to instructors

commit 591cf24
Author: Bradley Miller <[email protected]>
Date:   Thu Aug 24 14:59:33 2023 -0500

    Fix: speed up gradebook filtering of instructors

commit 74addc0
Author: Bradley Miller <[email protected]>
Date:   Tue Aug 22 16:48:34 2023 -0500

    Allow instructor to dump one class in DS format

commit a4429f2
Author: Bradley Miller <[email protected]>
Date:   Mon Aug 21 17:21:31 2023 -0500

    customize datashop dump for instructors only

Author: bnmnetp

author.compose.yml

@@ -13,6 +13,7 @@ services:
         command: uvicorn rsptx.author_server_api.main:app --host 0.0.0.0 --port 8114
         volumes:
             - ${BOOK_PATH}:/books
+            - $HOME/downloads:/usr/src/app/downloads
         environment:
             - SERVER_CONFIG=${SERVER_CONFIG}
             - CELERY_BROKER_URL=redis://redis:6379/0
@@ -35,6 +36,7 @@ services:
         volumes:
             - ${BOOK_PATH}:/books
             - ${SSH_AUTH_SOCK:-/tmp}:/ssh-agent # forward host ssh agent
+            - $HOME/downloads:/usr/src/app/downloads
         environment:
             - SERVER_CONFIG=${SERVER_CONFIG}
             - CELERY_BROKER_URL=redis://redis:6379/0

bases/rsptx/author_server_api/main.py

@@ -33,7 +33,7 @@
 
 # Local App
 # ---------
-from rsptx.forms import LibraryForm, DatashopForm
+from rsptx.forms import LibraryForm, DatashopForm, DatashopInstForm
 from rsptx.author_server_api.worker import (
     build_runestone_book,
     clone_runestone_book,
@@ -50,6 +50,7 @@
     fetch_instructor_courses,
     fetch_books_by_author,
     fetch_course,
+    fetch_course_by_id,
     fetch_library_book,
     update_library_book,
     create_course,
@@ -150,7 +151,7 @@ async def home(request: Request, user=Depends(auth_manager)):
 async def logfiles(request: Request, user=Depends(auth_manager)):
     if await is_instructor(request):
 
-        lf_path = pathlib.Path("logfiles", user.username)
+        lf_path = pathlib.Path("downloads", "logfiles", user.username)
         logger.debug(f"WORKING DIR = {lf_path}")
         if lf_path.exists():
             ready_files = {
@@ -180,13 +181,13 @@ async def logfiles(request: Request, user=Depends(auth_manager)):
 
 @app.get("/author/getfile/{fname}")
 async def getfile(request: Request, fname: str, user=Depends(auth_manager)):
-    file_path = pathlib.Path("logfiles", user.username, fname)
+    file_path = pathlib.Path("downloads", "logfiles", user.username, fname)
     return FileResponse(file_path)
 
 
 @app.get("/author/getdatashop/{fname}")
 async def _getdshop(request: Request, fname: str, user=Depends(auth_manager)):
-    file_path = pathlib.Path("datashop", user.username, fname)
+    file_path = pathlib.Path("downloads", "datashop", user.username, fname)
     return FileResponse(file_path)
 
 
@@ -237,7 +238,9 @@ async def dump_assignments(request: Request, course: str, user=Depends(auth_mana
     """,
         eng,
     )
-    all_aq_pairs.to_csv(f"{course}_assignments.csv", index=False)
+    all_aq_pairs.to_csv(
+        f"downloads/logfiles/{user.username}/{course}_assignments.csv", index=False
+    )
 
     return JSONResponse({"detail": "success"})
 
@@ -336,17 +339,22 @@ async def editlib(request: Request, book: str, user=Depends(auth_manager)):
 @app.post("/author/anonymize_data/{book}")
 async def anondata(request: Request, book: str, user=Depends(auth_manager)):
     # Get the book and populate the form with current data
-    if not await verify_author(user):
-        return RedirectResponse(url="/notauthorized")
+    is_author = await verify_author(user)
+    is_inst = await is_instructor(request)
+
+    if not (is_author or is_inst):
+        return RedirectResponse(url="/author/notauthorized")
 
     # Create a list of courses taught by this user to validate courses they
     # can dump directly.
     course = await fetch_course(user.course_name)
     courses = await fetch_instructor_courses(user.id)
-    class_list = [c.id for c in courses]
-    class_list = [str(x) for x in class_list]
+    class_list = []
+    for c in courses:
+        the_course = await fetch_course_by_id(c.course)
+        class_list.append(the_course.course_name)
 
-    lf_path = pathlib.Path("datashop", user.username)
+    lf_path = pathlib.Path("downloads", "datashop", user.username)
     logger.debug(f"WORKING DIR = {lf_path}")
     if lf_path.exists():
         ready_files = [x for x in lf_path.iterdir()]
@@ -356,13 +364,22 @@ async def anondata(request: Request, book: str, user=Depends(auth_manager)):
     # this will either create the form with data from the submitted form or
     # from the kwargs passed if there is not form data.  So we can prepopulate
     #
-    form = await DatashopForm.from_formdata(
-        request, basecourse=book, clist=",".join(class_list)
-    )
+    if is_author:
+        form = await DatashopForm.from_formdata(
+            request, basecourse=book, clist=",".join(class_list)
+        )
+
+    elif is_inst:
+        form = await DatashopInstForm.from_formdata(
+            request,
+            basecourse=book,
+            clist=",".join(class_list),
+            specific_course=course.course_name,
+        )
+        form.specific_course.choices = class_list
     if request.method == "POST" and await form.validate():
         print(f"Got {form.authors.data}")
         print(f"FORM data = {form.data}")
-
         # return RedirectResponse(url="/", status_code=status.HTTP_303_SEE_OTHER)
     return templates.TemplateResponse(
         "author/anonymize_data.html",
@@ -373,6 +390,8 @@ async def anondata(request: Request, book: str, user=Depends(auth_manager)):
             ready_files=ready_files,
             kind="datashop",
             course=course,
+            is_author=is_author,
+            is_instructor=is_inst,
         ),
     )
 
@@ -471,7 +490,7 @@ async def dump_code(payload=Body(...), user=Depends(auth_manager)):
 @app.get("/author/dlsAvailable/{kind}", status_code=201)
 async def check_downloads(request: Request, kind: str, user=Depends(auth_manager)):
     # kind will be either logfiles or datashop
-    lf_path = pathlib.Path("logfiles", user.username)
+    lf_path = pathlib.Path("downloads", "logfiles", user.username)
     logger.debug(f"WORKING DIR = {lf_path}")
     if lf_path.exists():
         ready_files = [x.name for x in lf_path.iterdir()]

bases/rsptx/author_server_api/worker.py

@@ -277,7 +277,7 @@ def useinfo_to_csv(self, classname, username):
         params=dict(cname=classname),
         con=eng,
     )
-    p = pathlib.Path("logfiles", username)
+    p = pathlib.Path("downloads","logfiles", username)
     p.mkdir(parents=True, exist_ok=True)
     p = p / f"{classname}_useinfo.csv.zip"
     self.update_state(state="WRITING", meta={"current": "creating csv.zip file"})
@@ -299,7 +299,7 @@ def code_to_csv(self, classname, username):
         params=dict(cname=classname),
         con=eng,
     )
-    p = pathlib.Path("logfiles", username)
+    p = pathlib.Path("downloads", "logfiles", username)
     p.mkdir(parents=True, exist_ok=True)
     p = p / f"{classname}_code.csv.zip"
     self.update_state(state="WRITING", meta={"current": "creating csv.zip file"})
@@ -335,7 +335,7 @@ def anonymize_data_dump(self, **kwargs):
     )
     a.create_datashop_data()
     self.update_state(state="WORKING", meta={"current": "Writing datashop file"})
-    p = pathlib.Path("datashop", username)
+    p = pathlib.Path("downloads", "datashop", username)
     p.mkdir(parents=True, exist_ok=True)
     a.write_datashop(path=p)
     self.update_state(state="SUCCESS", meta={"current": "Ready for download"})

bases/rsptx/web2py_server/applications/runestone/controllers/dashboard.py

@@ -451,6 +451,14 @@ def grades():
         session.flash = "Your course does not exist"
         redirect(URL("dashboard", "index"))
 
+    iset = set()
+    instructors = db(db.course_instructor.course == course.id).select(
+        db.course_instructor.ALL
+    )
+    if instructors:
+        for i in instructors:
+            iset.add(i.instructor)
+
     assignments = db(db.assignments.course == course.id).select(
         db.assignments.ALL, orderby=(db.assignments.duedate, db.assignments.id)
     )
@@ -487,7 +495,7 @@ def grades():
     rows = []
     for row in trows:
         # remove instructor rows from trows
-        if not verifyInstructorStatus(auth.user.course_id, row[3]):
+        if row[3] not in iset:
             rows.append(row)
 
     studentinfo = {}
@@ -498,7 +506,7 @@ def grades():
     total_possible_points = 0
     students = []
     for s in allstudents:
-        if verifyInstructorStatus(auth.user.course_id, s.id):
+        if s.id in iset:
             # filter out instructors from the gradebook
             continue
         students.append(s)

bases/rsptx/web2py_server/applications/runestone/static/motd.html

@@ -26,6 +26,6 @@ <h4>Please Support Runestone</h4>
     already.
 </p>
 
-<div style="width: 50%; margin-left: auto; margin-right: auto;">
-<img src="/runestone/static/_images/SPP_Logo_Horizontal_Final2020.png" />
+<div style="width: 25%; margin-left: auto; margin-right: auto; margin-bottom: 10px;">
+<img src="/runestone/static/_images/SPP_Logo_Horizontal_Final2020.png" width="150" />
 </div>

bases/rsptx/web2py_server/applications/runestone/views/admin/admin.html

@@ -24,8 +24,8 @@ <h4 style="text-align: center" class="list-group-item-heading">Manage Students</
     <a id="AddInstructorTab" data-toggle="tab" href="#AddInstructor" class="list-group-item">
         <h4 style="text-align: center" class="list-group-item-heading">Add TA</h4>
     </a>
-    <a id="courselog" href="/{{=request.application}}/admin/courselog" class="list-group-item">
-        <h4 style="text-align: center" class="list-group-item-heading">Download Log</h4>
+    <a id="courselog" href="/author/anonymize_data/{{=course.base_course}}" class="list-group-item">
+        <h4 style="text-align: center" class="list-group-item-heading">Download Course Data</h4>
     </a>
     <a id="activeLink" href="/{{=request.application}}/dashboard/active" class="list-group-item">
         <h4 style="text-align: center" class="list-group-item-heading">Students Online</h4>

components/rsptx/data_extract/anonymizeCourseData.py

@@ -2,10 +2,12 @@
 # coding: utf-8
 
 import os
+import datetime
 import random
 import re
 import pandas as pd
 import pathlib
+import pdb
 from sqlalchemy import create_engine
 from tqdm import tqdm
 
@@ -96,19 +98,24 @@ def __init__(
         dburl,
         with_assess=False,
         start_date="2019-01-01",
-        end_date="2022-05-16",
+        end_date=datetime.date.today(),
         sample_size=10,
         include_basecourse=False,
         specific_course="",
+        preserve_user_ids=False,
     ):
         self.eng = create_engine(dburl)
         self.BASECOURSE = basecourse
         self.WITH_ASSESS = with_assess
         self.START_DATE = start_date
         self.END_DATE = end_date
-        self.SAMPLE_SIZE = int(sample_size)
+        if sample_size:
+            self.SAMPLE_SIZE = int(sample_size)
+        else:
+            self.SAMPLE_SIZE = 10
         print(f"include basecourse = {include_basecourse}")
         self.include_basecourse = include_basecourse
+        self.preserve_username = preserve_user_ids
         if specific_course:
             self.COURSE_LIST = [specific_course]
         else:
@@ -257,8 +264,11 @@ def anonymize_user(self, id):
             if id in self.inst_set:
                 self.user_map[id] = "REMOVEME"
             else:
-                self.user_map[id] = self.user_num
-                self.user_num += 1
+                if self.preserve_username:
+                    self.user_map[id] = id
+                else:
+                    self.user_map[id] = self.user_num
+                    self.user_num += 1
         return self.user_map[id]
 
     def anonymize_course(self, id):
@@ -435,7 +445,12 @@ def sessionize_data(self):
         # The below is very expensive for long operations.  Maybe we could rewrite it with a rolling function
 
         self.useinfo["tdiff"] = self.useinfo.timestamp.diff()
-        self.useinfo["sdiff"] = self.useinfo.sid.diff()
+        if self.preserve_username:
+            self.useinfo["sdiff"] = (
+                self.useinfo.sid.ne(self.useinfo.sid.shift()).bfill().astype(int)
+            )
+        else:
+            self.useinfo["sdiff"] = self.useinfo.sid.diff()
         self.sess_count = 0
         self.useinfo["session"] = self.useinfo.progress_apply(self.sessionize, axis=1)
         self.student_problem_ct = {}
@@ -502,7 +517,6 @@ def create_datashop_data(self):
                 "anon_institution",
             ]
         ]
-
         useinfo_w_answers.columns = [
             "Time",
             "Anon Student Id",
@@ -648,10 +662,11 @@ def write_datashop(self, path="./"):
 
 if __name__ == "__main__":
     a = Anonymizer(
-        "py4e-int",
+        "thinkcspy",
         os.environ["DBURL"],
         sample_size=3,
-        cl=["Win22-SI206", "Win21-SI206"],
+        specific_course="bl_thinkcspy_summer23",
+        preserve_user_ids=True,
     )
     print("Choosing Courses")
     a.choose_courses()

components/rsptx/db/crud.py

@@ -397,6 +397,25 @@ async def fetch_course(course_name: str) -> CoursesValidator:
         return CoursesValidator.from_orm(course)
 
 
+async def fetch_course_by_id(course_id: int) -> CoursesValidator:
+    """
+    Fetches a course by its id.
+
+    :param course_name: The id of the course to be fetched.
+    :type course_name: int
+    :return: A CoursesValidator instance representing the fetched course.
+    :rtype: CoursesValidator
+    """
+    query = select(Courses).where(Courses.id == course_id)
+    async with async_session() as session:
+        res = await session.execute(query)
+        # When selecting ORM entries it is useful to use the ``scalars`` method
+        # This modifies the result so that you are getting the ORM object
+        # instead of a Row object. `See <https://docs.sqlalchemy.org/en/14/orm/queryguide.html#selecting-orm-entities-and-attributes>`_
+        course = res.scalars().one_or_none()
+        return CoursesValidator.from_orm(course)
+
+
 async def fetch_base_course(base_course: str) -> CoursesValidator:
     """
     Fetches a base course by its name.

components/rsptx/forms/__init__.py

@@ -1,4 +1,4 @@
 from rsptx.forms import core
-from rsptx.forms.author import LibraryForm, DatashopForm
+from rsptx.forms.author import LibraryForm, DatashopForm, DatashopInstForm
 
-__all__ = ["core", "LibraryForm", "DatashopForm"]
+__all__ = ["core", "LibraryForm", "DatashopForm", "DatashopInstForm"]

components/rsptx/forms/author.py

@@ -7,6 +7,7 @@
     DateTimeField,
     DateField,
     HiddenField,
+    SelectField,
     IntegerRangeField,
 )
 
@@ -47,3 +48,10 @@ class DatashopForm(StarletteForm):
     include_basecourse = BooleanField("Include data from the open course")
     specific_course = StringField("Create data shop file for this course")
     clist = HiddenField()
+
+
+class DatashopInstForm(StarletteForm):
+    preserve_user_ids = BooleanField("Preserve User IDs")
+    specific_course = SelectField("Class Name")
+    basecourse = HiddenField()
+    clist = HiddenField()