TMMA-506: Bug fix parsing OVID abstracts with a HTML tag at the start…

… of a line (#295) * Replace source of `django-clamd` egg from Github to pypi * Ignore more files * Update cypress * Add better checks around parsing start of a citation to resolve bug TMMA-506 found when parsing OVID formatted abstracts that include HTML tags at the start of a line of the body of an abstract * Update CHANGELOG
MRCIEU · Jun 20, 2024 · bb113b7 · bb113b7
1 parent 46c68eb
commit bb113b7
Show file tree

Hide file tree

Showing 13 changed files with 147,382 additions and 15 deletions.
diff --git a/.gitignore b/.gitignore
@@ -82,4 +82,7 @@ cypress/screenshots
 cypress/videos
 cypress.env.json
 
-google-chrome-stable_current_amd64.deb
+google-chrome-stable_current_amd64.deb
+
+.env
+secret.txt
diff --git a/CHANGELOG b/CHANGELOG
@@ -1,3 +1,7 @@
+9.3.0   Make use of pypi index for django-clamd
+        Update Cypress
+        TMMA-506: Bug fix parsing OVID abstracts that include HTML tags at the start of a line. 
+
 9.2.0   Apply Python dependencies security related updates
 
 9.1.0   Apply Python dependency updates

diff --git a/Jenkinsfile.dc2.demo b/Jenkinsfile.dc2.demo
@@ -16,7 +16,7 @@ pipeline {
         stage('Demo: Cypress tests') {
             agent {
                 docker {
-                    image 'cypress/included:13.10.0'
+                    image 'cypress/included:13.12.0'
                     args '--add-host app-dc2-tmpo-d0.epi.bris.ac.uk:172.26.10.141 --entrypoint=""'
                 }
             }

diff --git a/Jenkinsfile.dc2.prod b/Jenkinsfile.dc2.prod
@@ -17,7 +17,7 @@ pipeline {
         stage('Production: Cypress tests') {
             agent {
                 docker {
-                    image 'cypress/included:13.10.0'
+                    image 'cypress/included:13.12.0'
                     args '--add-host www.temmpo.org.uk:172.26.10.142 --entrypoint=""'
                 }
             }

diff --git a/Jenkinsfile.dc2.test b/Jenkinsfile.dc2.test
@@ -16,7 +16,7 @@ pipeline {
         stage('Test: Cypress tests') {
             agent {
                 docker {
-                    image 'cypress/included:13.10.0'
+                    image 'cypress/included:13.12.0'
                     args '--add-host app-dc2-tmpo-t0.epi.bris.ac.uk:172.26.10.143 --entrypoint=""'
                 }
             }

diff --git a/README.md b/README.md
@@ -212,7 +212,7 @@ NB: Some tests require these environment variables `CREDENTIALS_USR` and `CREDEN
 
 Using docker and electron browser
 
-    docker run --rm -it -v $PWD:/e2e -w /e2e cypress/included:13.10.0
+    docker run --rm -it -v $PWD:/e2e -w /e2e cypress/included:13.12.0
 
 ## Warnings
 

diff --git a/browser/matching.py b/browser/matching.py
@@ -200,7 +200,7 @@ def _ovid_medline_read_citations(abstract_file_path):
         line = line.strip(b"\r\n")
         if len(line) == 0:
             pass
-        elif line[0:1] == b"<":
+        elif line[0:1] == b"<" and line[-1:] == b">" and (line.strip(b"<").strip(b">")).decode("utf-8").isdecimal():
             # Starting a new citation, yield if one has already been set up
             if citation:
                 yield citation

diff --git a/requirements/dev.txt b/requirements/dev.txt
@@ -279,8 +279,9 @@ django==4.2.13 \
 django-autocomplete-light==3.11.0 \
     --hash=sha256:212576a17e3308ef7ca77e280b86684167916d2091d4b73640f38845d9516328
     # via -r requirements/test.txt
-django-clamd @ https://github.com/vstoykov/django-clamd/releases/download/1.0.0/django_clamd-1.0.0-py3-none-any.whl \
-    --hash=sha256:9a5c2a4215da20772fd37e617d75b57e253779dccd49d149b911cfd9be1e1748
+django-clamd==1.0.0 \
+    --hash=sha256:106429a71db571b570c88493115891a5472623263d25ebe7f92bead6c2f04e81 \
+    --hash=sha256:ee2d46ed535355aa0840f93dd8468bf9e2236e2a54ea2affa956e5c9326e328a
     # via -r requirements/test.txt
 django-debug-toolbar==4.4.2 \
     --hash=sha256:5d7afb2ea5f8730241e5b0735396e16cd1fd8c6b53a2f3e1e30bbab9abb23728 \

diff --git a/requirements/requirements.in b/requirements/requirements.in
@@ -1,8 +1,7 @@
 # This is an implicit value, here for clarity
 --index-url https://pypi.python.org/simple/
 Bottleneck>=1.3.6
-# django-clamd==1.0.0 currently not available on pypi
-https://github.com/vstoykov/django-clamd/releases/download/1.0.0/django_clamd-1.0.0-py3-none-any.whl
+django-clamd>=1.0.0
 django-mptt>=0.14.0
 django-redis-cache>=3.0.1
 django-registration-redux>=2.12

diff --git a/requirements/requirements.txt b/requirements/requirements.txt
@@ -93,8 +93,9 @@ django==4.2.13 \
 django-autocomplete-light==3.11.0 \
     --hash=sha256:212576a17e3308ef7ca77e280b86684167916d2091d4b73640f38845d9516328
     # via -r requirements/requirements.in
-django-clamd @ https://github.com/vstoykov/django-clamd/releases/download/1.0.0/django_clamd-1.0.0-py3-none-any.whl \
-    --hash=sha256:9a5c2a4215da20772fd37e617d75b57e253779dccd49d149b911cfd9be1e1748
+django-clamd==1.0.0 \
+    --hash=sha256:106429a71db571b570c88493115891a5472623263d25ebe7f92bead6c2f04e81 \
+    --hash=sha256:ee2d46ed535355aa0840f93dd8468bf9e2236e2a54ea2affa956e5c9326e328a
     # via -r requirements/requirements.in
 django-hcaptcha-field==1.4.0 \
     --hash=sha256:36001030f53af709db3be01bf1d25023706b701bb7130f67f2aa58900b20df42 \

diff --git a/requirements/test.txt b/requirements/test.txt
@@ -262,8 +262,9 @@ django==4.2.13 \
 django-autocomplete-light==3.11.0 \
     --hash=sha256:212576a17e3308ef7ca77e280b86684167916d2091d4b73640f38845d9516328
     # via -r requirements/requirements.txt
-django-clamd @ https://github.com/vstoykov/django-clamd/releases/download/1.0.0/django_clamd-1.0.0-py3-none-any.whl \
-    --hash=sha256:9a5c2a4215da20772fd37e617d75b57e253779dccd49d149b911cfd9be1e1748
+django-clamd==1.0.0 \
+    --hash=sha256:106429a71db571b570c88493115891a5472623263d25ebe7f92bead6c2f04e81 \
+    --hash=sha256:ee2d46ed535355aa0840f93dd8468bf9e2236e2a54ea2affa956e5c9326e328a
     # via -r requirements/requirements.txt
 django-hcaptcha-field==1.4.0 \
     --hash=sha256:36001030f53af709db3be01bf1d25023706b701bb7130f67f2aa58900b20df42 \