Merge pull request #1175 from p2pu/2024-async-export

Asynchronous workflow for long running exports

direct-requirements.txt

@@ -7,6 +7,7 @@ crispy-bootstrap5
 dj-database-url
 Django==4.2.11
 django-bleach
+django-celery-results
 django-cors-headers
 django-crispy-forms
 django-filter

frontend/staff-dashboard.jsx

@@ -0,0 +1,109 @@
+import React, {useState} from 'react'
+import ReactDOM from 'react-dom'
+import { createRoot } from 'react-dom/client';
+import ErrorBoundary from './components/error-boundary'
+
+import axios from 'axios'
+
+axios.defaults.xsrfCookieName = 'csrftoken'
+axios.defaults.xsrfHeaderName = 'X-CSRFToken'
+
+
+const ExportLinks = props => {
+
+  const [requestPending, setRequestPending] = useState(false) // indicate that request to create export is pending
+  const [pollingUrl, setPollingUrl] = useState('')
+  const [exportUrl, setExportUrl] = useState('')
+  const [errorMessage, setErrorMessage] = useState('')
+
+  const startPolling = url => {
+    setPollingUrl(url)
+
+    const pollRequest = async backoff => {
+      try {
+        const pollResp = await axios({url, method: 'GET', responseType: 'json'});
+        if (pollResp.status === 200) {
+          const result = pollResp.data
+          if (result.status == 'PENDING'){
+            console.log(`keep on polling ${pollingUrl}, ${pollResp}`)
+            setTimeout(pollRequest, backoff, Math.clamp(backoff*2, 2000, 16000))
+          } else if (result.status == 'SUCCESS') {
+            console.log(`done polling ${pollingUrl}, ${pollResp}`)
+            setPollingUrl('')
+            setExportUrl(result.result.presigned_url)
+            setRequestPending(false)
+            return
+          } else if (result.status == 'FAILURE') {
+            console.log(`done polling ${pollingUrl}, ${pollResp}`)
+            setPollingUrl('')
+            setRequestPending(false)
+            setErrorMessage('Something went wrong with the export.')
+            return
+          }
+
+        }
+      } catch (e) {
+        setPollingUrl('')
+        setErrorMessage('Something went wrong while waiting for the export to finish.')
+        setRequestPending(false)
+        console.log(e)
+      }
+    }
+
+    pollRequest(2000)
+
+  }
+
+  const onClick = e => {
+    e.preventDefault()
+    const url = e.target.href
+    // Post request and start polling
+    setExportUrl('')
+    setErrorMessage('')
+    setRequestPending(true)
+    axios({url, method: 'GET', responseType: 'json'}).then(res => {
+      if (res.status === 200){
+        console.log(res.data.task_id)
+        console.log('Start polling')
+        startPolling(`/en/export/status/${res.data.task_id}/`)
+      } else {
+        console.log('Export request returned unexpected status code', res)
+        setErrorMessage('Something went wrong creating the export.')
+        setRequestPending(false)
+      }
+    }).catch(err => {
+      console.log('Export request failed', err)
+      setErrorMessage('Something went wrong creating the export.')
+      setRequestPending(false)
+    })
+  }
+
+  return (
+    <>
+    <ul className="row list-unstyled">
+      { 
+        props.exportLinks.map( ({url, text, asynchronous}, index) => 
+          <li key={index} className="col d-flex mb-3">
+            <a className={"btn btn-primary w-100 " + (requestPending?"disabled":"") } href={url} onClick={asynchronous?onClick:undefined} ><i className="fas fa-download" aria-hidden="true"></i><br/>{text}</a>
+          </li>
+        )
+      }
+    </ul>
+      { pollingUrl && <div><span className="spinner-border spinner-border-sm" role="status"></span>&nbsp;Busy creating export, don't refesh page</div>}
+      { exportUrl && <a href={exportUrl}>Download your export</a> }
+      { errorMessage && <div className="alert alert-danger">{errorMessage}</div> }
+    </>
+  )
+}
+
+const reactDataEl = document.getElementById('react-data')
+const reactData = JSON.parse(reactDataEl.textContent)
+
+const element = document.getElementById('download-links')
+const root = createRoot(element)
+root.render(
+  <ErrorBoundary scope="staff-dashboard">
+    <ExportLinks {...reactData} />
+  </ErrorBoundary>
+)
+

learnwithpeople/settings.py

@@ -50,6 +50,7 @@
     'webpack_loader',
     'tinymce',
     'django_bleach',
+    'django_celery_results',
     # own
     'studygroups',
     'backup',
@@ -257,6 +258,7 @@
 
 ####### Celery config #######
 CELERY_BROKER_URL = env('BROKER_URL', 'amqp://guest:guest@localhost//')
+CELERY_RESULT_BACKEND = 'django-db'
 
 from celery.schedules import crontab
 

requirements.txt

@@ -1,86 +1,87 @@
-aiohttp==3.9.3
-aiohttp-retry==2.8.3
-aiosignal==1.3.1
-amqp==5.2.0
-asgiref==3.8.1
-async-timeout==4.0.3
-attrs==23.2.0
-billiard==4.2.0
-bleach==5.0.1
-boto3==1.34.81
-botocore==1.34.81
-cachetools==5.3.3
-cairocffi==1.6.1
-CairoSVG==2.7.1
-celery==5.3.6
-certifi==2024.2.2
-cffi==1.16.0
-charset-normalizer==3.3.2
-click==8.1.7
-click-didyoumean==0.3.1
-click-plugins==1.1.1
-click-repl==0.3.0
-crispy-bootstrap5==2024.2
-cssselect==1.2.0
-cssselect2==0.7.0
-cssutils==2.10.2
-defusedxml==0.7.1
-dj-database-url==2.1.0
-Django==4.2.11
-django-bleach==3.1.0
-django-cors-headers==4.3.1
-django-crispy-forms==2.1
-django-filter==24.2
-django-phonenumber-field==7.3.0
-django-tinymce==4.0.0
-django-webpack-loader==3.1.0
-djangorestframework==3.15.1
-exceptiongroup==1.2.0
-freezegun==1.4.0
-frozenlist==1.4.1
-geonamescache==2.0.0
-gunicorn==21.2.0
-h11==0.14.0
-icalendar==5.0.12
-idna==3.6
-importlib_metadata==7.1.0
-jmespath==1.0.1
-kombu==5.3.6
-lxml==5.2.1
-Markdown==3.6
-multidict==6.0.5
-outcome==1.3.0.post0
-packaging==24.0
-phonenumberslite==8.13.34
-pillow==10.3.0
-premailer==3.10.0
-prompt-toolkit==3.0.43
-psycopg2-binary==2.9.9
-pycparser==2.22
-pygal==3.0.4
-PyJWT==2.8.0
-PySocks==1.7.1
-python-dateutil==2.9.0.post0
-pytz==2024.1
-requests==2.31.0
-s3-backup-rotate==0.3.3
-s3transfer==0.10.1
-selenium==4.19.0
-six==1.16.0
-sniffio==1.3.1
-sortedcontainers==2.4.0
-sqlparse==0.4.4
-tinycss2==1.1.1
-trio==0.25.0
-trio-websocket==0.11.1
-twilio==9.0.4
-typing_extensions==4.11.0
-tzdata==2024.1
-unicodecsv==0.14.1
-urllib3==1.26.18
-vine==5.1.0
-wcwidth==0.2.13
-webencodings==0.5.1
-wsproto==1.2.0
-yarl==1.9.4
-zipp==3.18.1
+aiohttp==3.9.3
+aiohttp-retry==2.8.3
+aiosignal==1.3.1
+amqp==5.2.0
+asgiref==3.8.1
+async-timeout==4.0.3
+attrs==23.2.0
+billiard==4.2.0
+bleach==5.0.1
+boto3==1.34.81
+botocore==1.34.81
+cachetools==5.3.3
+cairocffi==1.6.1
+CairoSVG==2.7.1
+celery==5.3.6
+certifi==2024.2.2
+cffi==1.16.0
+charset-normalizer==3.3.2
+click==8.1.7
+click-didyoumean==0.3.1
+click-plugins==1.1.1
+click-repl==0.3.0
+crispy-bootstrap5==2024.2
+cssselect==1.2.0
+cssselect2==0.7.0
+cssutils==2.10.2
+defusedxml==0.7.1
+dj-database-url==2.1.0
+Django==4.2.11
+django-bleach==3.1.0
+django-celery-results==2.5.1
+django-cors-headers==4.3.1
+django-crispy-forms==2.1
+django-filter==24.2
+django-phonenumber-field==7.3.0
+django-tinymce==4.0.0
+django-webpack-loader==3.1.0
+djangorestframework==3.15.1
+exceptiongroup==1.2.0
+freezegun==1.4.0
+frozenlist==1.4.1
+geonamescache==2.0.0
+gunicorn==21.2.0
+h11==0.14.0
+icalendar==5.0.12
+idna==3.6
+importlib_metadata==7.1.0
+jmespath==1.0.1
+kombu==5.3.6
+lxml==5.2.1
+Markdown==3.6
+multidict==6.0.5
+outcome==1.3.0.post0
+packaging==24.0
+phonenumberslite==8.13.34
+pillow==10.3.0
+premailer==3.10.0
+prompt-toolkit==3.0.43
+psycopg2-binary==2.9.9
+pycparser==2.22
+pygal==3.0.4
+PyJWT==2.8.0
+PySocks==1.7.1
+python-dateutil==2.9.0.post0
+pytz==2024.1
+requests==2.31.0
+s3-backup-rotate==0.3.3
+s3transfer==0.10.1
+selenium==4.19.0
+six==1.16.0
+sniffio==1.3.1
+sortedcontainers==2.4.0
+sqlparse==0.4.4
+tinycss2==1.1.1
+trio==0.25.0
+trio-websocket==0.11.1
+twilio==9.0.4
+typing_extensions==4.11.0
+tzdata==2024.1
+unicodecsv==0.14.1
+urllib3==1.26.18
+vine==5.1.0
+wcwidth==0.2.13
+webencodings==0.5.1
+wsproto==1.2.0
+yarl==1.9.4
+zipp==3.18.1