From ad183bdafb02c35f5bab12b7eb2bc6bfecd9062a Mon Sep 17 00:00:00 2001 From: EEA Jenkins Date: Tue, 12 Dec 2017 11:20:12 +0000 Subject: [PATCH 01/16] Updated version for eea.cache to 8.2.dev0 --- docs/HISTORY.txt | 5 ++++- eea/cache/version.txt | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/HISTORY.txt b/docs/HISTORY.txt index cb01b08..1bb5214 100644 --- a/docs/HISTORY.txt +++ b/docs/HISTORY.txt @@ -1,8 +1,11 @@ Changelog ========= -8.1.dev0 - (unreleased) +8.2.dev0 - (unreleased) ----------------------- + +8.1 - (2017-12-12) +------------------ * Change: Replace eeacms/zptlint with eeacms/plone-test:4 zptlint [avoinea refs #90415] diff --git a/eea/cache/version.txt b/eea/cache/version.txt index c7469fb..529bd32 100644 --- a/eea/cache/version.txt +++ b/eea/cache/version.txt @@ -1 +1 @@ -8.1.dev0 \ No newline at end of file +8.2.dev0 \ No newline at end of file From 799ed848b03b62a1fab512fbf33df796fc29e790 Mon Sep 17 00:00:00 2001 From: Alin Voinea Date: Mon, 18 Dec 2017 19:40:43 +0200 Subject: [PATCH 02/16] [JENKINS] - Cleanup --- Jenkinsfile | 85 ++++++++++++++--------------------------------------- 1 file changed, 22 insertions(+), 63 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 07bddfc..2933620 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -6,105 +6,64 @@ pipeline { } stages { - stage('Tests') { + + stage('Code') { steps { parallel( - "WWW": { + "ZPT Lint": { node(label: 'docker-1.13') { - script { - try { - sh '''docker run -i --name="$BUILD_TAG-www" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/www-devel /debug.sh bin/test -v -vv -s $GIT_NAME''' - } finally { - sh '''docker rm -v $BUILD_TAG-www''' - } - } + sh '''docker run -i --rm --name="$BUILD_TAG-zptlint" -e GIT_BRANCH="$BRANCH_NAME" -e ADDONS="$GIT_NAME" -e DEVELOP="src/$GIT_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/plone-test:4 zptlint''' } }, - "KGS": { + "JS Lint": { node(label: 'docker-1.13') { - script { - try { - sh '''docker run -i --name="$BUILD_TAG-kgs" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/kgs-devel /debug.sh bin/test --test-path /plone/instance/src/$GIT_NAME -v -vv -s $GIT_NAME''' - } finally { - sh '''docker rm -v $BUILD_TAG-kgs''' - } - } + sh '''docker run -i --rm --name="$BUILD_TAG-jslint" -e GIT_SRC="https://github.com/eea/$GIT_NAME.git" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/jslint4java''' } }, - "Plone4": { + "PyFlakes": { node(label: 'docker-1.13') { - script { - try { - sh '''docker run -i --name="$BUILD_TAG-plone4" -e GIT_BRANCH="$BRANCH_NAME" -e ADDONS="$GIT_NAME" -e DEVELOP="src/$GIT_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/plone-test:4 -v -vv -s $GIT_NAME''' - } finally { - sh '''docker rm -v $BUILD_TAG-plone4''' - } - } + sh '''docker run -i --rm --name="$BUILD_TAG-pyflakes" -e GIT_SRC="https://github.com/eea/$GIT_NAME.git" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/pyflakes''' + } + }, + + "i18n": { + node(label: 'docker-1.13') { + sh '''docker run -i --rm --name=$BUILD_TAG-i18n -e GIT_SRC="https://github.com/eea/$GIT_NAME.git" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/i18ndude''' } } ) } } - stage('Code Analysis') { + stage('Tests') { steps { parallel( - "ZPT Lint": { - node(label: 'docker-1.13') { - script { - try { - sh '''docker run -i --name="$BUILD_TAG-zptlint" -e GIT_BRANCH="$BRANCH_NAME" -e ADDONS="$GIT_NAME" -e DEVELOP="src/$GIT_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/plone-test:4 zptlint''' - } finally { - sh '''docker rm -v $BUILD_TAG-zptlint''' - } - } - } - }, - - "JS Lint": { + "WWW": { node(label: 'docker-1.13') { - script { - try { - sh '''docker run -i --name="$BUILD_TAG-jslint" -e GIT_SRC="https://github.com/eea/$GIT_NAME.git" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/jslint4java''' - } finally { - sh '''docker rm -v $BUILD_TAG-jslint''' - } - } + sh '''docker run -i --rm --name="$BUILD_TAG-www" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/www-devel /debug.sh bin/test -v -vv -s $GIT_NAME''' } }, - "PyFlakes": { + "KGS": { node(label: 'docker-1.13') { - script { - try { - sh '''docker run -i --name="$BUILD_TAG-pyflakes" -e GIT_SRC="https://github.com/eea/$GIT_NAME.git" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/pyflakes''' - } finally { - sh '''docker rm -v $BUILD_TAG-pyflakes''' - } - } + sh '''docker run -i --rm --name="$BUILD_TAG-kgs" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/kgs-devel /debug.sh bin/test --test-path /plone/instance/src/$GIT_NAME -v -vv -s $GIT_NAME''' } }, - "i18n": { + "Plone4": { node(label: 'docker-1.13') { - script { - try { - sh '''docker run -i --name=$BUILD_TAG-i18n -e GIT_SRC="https://github.com/eea/$GIT_NAME.git" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/i18ndude''' - } finally { - sh '''docker rm -v $BUILD_TAG-i18n''' - } - } + sh '''docker run -i --rm --name="$BUILD_TAG-plone4" -e GIT_BRANCH="$BRANCH_NAME" -e ADDONS="$GIT_NAME" -e DEVELOP="src/$GIT_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/plone-test:4 -v -vv -s $GIT_NAME''' } } ) } } - stage('Code Syntax') { + stage('Cosmetics') { steps { parallel( From d98a3a260ba765008d3a2b9ade2afd1ac207a494 Mon Sep 17 00:00:00 2001 From: Alin Voinea Date: Mon, 18 Dec 2017 19:42:32 +0200 Subject: [PATCH 03/16] Refs #89563 - Automatize release --- Jenkinsfile | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 2933620..8e56b87 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -127,6 +127,41 @@ pipeline { } } + stage('Pull Request') { + when { + not { + environment name: 'CHANGE_ID', value: '' + } + environment name: 'CHANGE_TARGET', value: 'master' + } + steps { + node(label: 'docker-1.13') { + script { + if ( env.CHANGE_BRANCH != "develop" && !( env.CHANGE_BRANCH.startsWith("hotfix")) ) { + error "Pipeline aborted due to PR not made from develop or hotfix branch" + } + sh '''docker run -i --rm --name="$BUILD_TAG-gitflow-pr" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" -e GIT_ORG="$GIT_ORG" -e GIT_VERSIONFILE="$GIT_VERSIONFILE" -e GIT_HISTORYFILE="$GIT_HISTORYFILE" -e GIT_NAME="$GIT_NAME" eeacms/gitflow''' + } + } + } + } + + stage('Release') { + when { + allOf { + environment name: 'CHANGE_ID', value: '' + branch 'master' + } + } + steps { + node(label: 'docker-1.13') { + withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: 'eea-jenkins', usernameVariable: 'EGGREPO_USERNAME', passwordVariable: 'EGGREPO_PASSWORD'],string(credentialsId: 'eea-jenkins-token', variable: 'GITHUB_TOKEN')]) { + sh '''docker run -i --rm --name="$BUILD_TAG-gitflow-master" -e GIT_BRANCH="$BRANCH_NAME" -e EGGREPO_USERNAME="$EGGREPO_USERNAME" -e EGGREPO_PASSWORD="$EGGREPO_PASSWORD" -e GIT_NAME="$GIT_NAME" -e GIT_VERSIONFILE="$GIT_VERSIONFILE" -e GIT_ORG="$GIT_ORG" -e GIT_TOKEN="$GITHUB_TOKEN" eeacms/gitflow''' + } + } + } + } + } post { From 00828b6bf85a600a9113d99e2182ab64da3c762a Mon Sep 17 00:00:00 2001 From: Alin Voinea Date: Mon, 18 Dec 2017 22:02:49 +0200 Subject: [PATCH 04/16] [JENKINS] - More cleanup --- Jenkinsfile | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 8e56b87..775fb17 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -71,11 +71,9 @@ pipeline { node(label: 'docker-1.13') { script { try { - sh '''docker run -i --name="$BUILD_TAG-jshint" -e GIT_SRC="https://github.com/eea/$GIT_NAME.git" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/jshint''' + sh '''docker run -i --rm --name="$BUILD_TAG-jshint" -e GIT_SRC="https://github.com/eea/$GIT_NAME.git" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/jshint''' } catch (err) { echo "Unstable: ${err}" - } finally { - sh '''docker rm -v $BUILD_TAG-jshint''' } } } @@ -85,11 +83,9 @@ pipeline { node(label: 'docker-1.13') { script { try { - sh '''docker run -i --name="$BUILD_TAG-csslint" -e GIT_SRC="https://github.com/eea/$GIT_NAME.git" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/csslint''' + sh '''docker run -i --rm --name="$BUILD_TAG-csslint" -e GIT_SRC="https://github.com/eea/$GIT_NAME.git" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/csslint''' } catch (err) { echo "Unstable: ${err}" - } finally { - sh '''docker rm -v $BUILD_TAG-csslint''' } } } @@ -99,11 +95,9 @@ pipeline { node(label: 'docker-1.13') { script { try { - sh '''docker run -i --name="$BUILD_TAG-pep8" -e GIT_SRC="https://github.com/eea/$GIT_NAME.git" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/pep8''' + sh '''docker run -i --rm --name="$BUILD_TAG-pep8" -e GIT_SRC="https://github.com/eea/$GIT_NAME.git" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/pep8''' } catch (err) { echo "Unstable: ${err}" - } finally { - sh '''docker rm -v $BUILD_TAG-pep8''' } } } @@ -113,11 +107,9 @@ pipeline { node(label: 'docker-1.13') { script { try { - sh '''docker run -i --name="$BUILD_TAG-pylint" -e GIT_SRC="https://github.com/eea/$GIT_NAME.git" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/pylint''' + sh '''docker run -i --rm --name="$BUILD_TAG-pylint" -e GIT_SRC="https://github.com/eea/$GIT_NAME.git" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/pylint''' } catch (err) { echo "Unstable: ${err}" - } finally { - sh '''docker rm -v $BUILD_TAG-pylint''' } } } From 9b40135e448a22c6b8b229de68015adfaede3315 Mon Sep 17 00:00:00 2001 From: valentinab25 Date: Thu, 18 Jan 2018 12:35:34 +0100 Subject: [PATCH 05/16] Added egg release to pypi - Refs #91780 --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 775fb17..acd01bb 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -132,7 +132,7 @@ pipeline { if ( env.CHANGE_BRANCH != "develop" && !( env.CHANGE_BRANCH.startsWith("hotfix")) ) { error "Pipeline aborted due to PR not made from develop or hotfix branch" } - sh '''docker run -i --rm --name="$BUILD_TAG-gitflow-pr" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" -e GIT_ORG="$GIT_ORG" -e GIT_VERSIONFILE="$GIT_VERSIONFILE" -e GIT_HISTORYFILE="$GIT_HISTORYFILE" -e GIT_NAME="$GIT_NAME" eeacms/gitflow''' + sh '''docker run -i --rm --name="$BUILD_TAG-gitflow-pr" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" -e GIT_ORG="$GIT_ORG" -e GIT_NAME="$GIT_NAME" eeacms/gitflow''' } } } @@ -147,8 +147,8 @@ pipeline { } steps { node(label: 'docker-1.13') { - withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: 'eea-jenkins', usernameVariable: 'EGGREPO_USERNAME', passwordVariable: 'EGGREPO_PASSWORD'],string(credentialsId: 'eea-jenkins-token', variable: 'GITHUB_TOKEN')]) { - sh '''docker run -i --rm --name="$BUILD_TAG-gitflow-master" -e GIT_BRANCH="$BRANCH_NAME" -e EGGREPO_USERNAME="$EGGREPO_USERNAME" -e EGGREPO_PASSWORD="$EGGREPO_PASSWORD" -e GIT_NAME="$GIT_NAME" -e GIT_VERSIONFILE="$GIT_VERSIONFILE" -e GIT_ORG="$GIT_ORG" -e GIT_TOKEN="$GITHUB_TOKEN" eeacms/gitflow''' + withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: 'eea-jenkins', usernameVariable: 'EGGREPO_USERNAME', passwordVariable: 'EGGREPO_PASSWORD'],string(credentialsId: 'eea-jenkins-token', variable: 'GITHUB_TOKEN'),[$class: 'UsernamePasswordMultiBinding', credentialsId: 'pypi-jenkins', usernameVariable: 'PYPI_USERNAME', passwordVariable: 'PYPI_PASSWORD']]) { + sh '''docker run -i --rm --name="$BUILD_TAG-gitflow-master" -e GIT_BRANCH="$BRANCH_NAME" -e EGGREPO_USERNAME="$EGGREPO_USERNAME" -e EGGREPO_PASSWORD="$EGGREPO_PASSWORD" -e GIT_NAME="$GIT_NAME" -e PYPI_USERNAME="$PYPI_USERNAME" -e PYPI_PASSWORD="$PYPI_PASSWORD" -e GIT_ORG="$GIT_ORG" -e GIT_TOKEN="$GITHUB_TOKEN" eeacms/gitflow''' } } } From 7364c90dd19402b651cf3f07364a04927b65a02c Mon Sep 17 00:00:00 2001 From: Alin Voinea Date: Fri, 23 Mar 2018 16:50:57 +0200 Subject: [PATCH 06/16] [JENKINS] - Jenkinsfile update where Docker jobs run --- Jenkinsfile | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index acd01bb..6534121 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -12,25 +12,25 @@ pipeline { parallel( "ZPT Lint": { - node(label: 'docker-1.13') { + node(label: 'docker') { sh '''docker run -i --rm --name="$BUILD_TAG-zptlint" -e GIT_BRANCH="$BRANCH_NAME" -e ADDONS="$GIT_NAME" -e DEVELOP="src/$GIT_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/plone-test:4 zptlint''' } }, "JS Lint": { - node(label: 'docker-1.13') { + node(label: 'docker') { sh '''docker run -i --rm --name="$BUILD_TAG-jslint" -e GIT_SRC="https://github.com/eea/$GIT_NAME.git" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/jslint4java''' } }, "PyFlakes": { - node(label: 'docker-1.13') { + node(label: 'docker') { sh '''docker run -i --rm --name="$BUILD_TAG-pyflakes" -e GIT_SRC="https://github.com/eea/$GIT_NAME.git" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/pyflakes''' } }, "i18n": { - node(label: 'docker-1.13') { + node(label: 'docker') { sh '''docker run -i --rm --name=$BUILD_TAG-i18n -e GIT_SRC="https://github.com/eea/$GIT_NAME.git" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/i18ndude''' } } @@ -43,19 +43,19 @@ pipeline { parallel( "WWW": { - node(label: 'docker-1.13') { + node(label: 'docker') { sh '''docker run -i --rm --name="$BUILD_TAG-www" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/www-devel /debug.sh bin/test -v -vv -s $GIT_NAME''' } }, "KGS": { - node(label: 'docker-1.13') { + node(label: 'docker') { sh '''docker run -i --rm --name="$BUILD_TAG-kgs" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/kgs-devel /debug.sh bin/test --test-path /plone/instance/src/$GIT_NAME -v -vv -s $GIT_NAME''' } }, "Plone4": { - node(label: 'docker-1.13') { + node(label: 'docker') { sh '''docker run -i --rm --name="$BUILD_TAG-plone4" -e GIT_BRANCH="$BRANCH_NAME" -e ADDONS="$GIT_NAME" -e DEVELOP="src/$GIT_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/plone-test:4 -v -vv -s $GIT_NAME''' } } @@ -68,7 +68,7 @@ pipeline { parallel( "JS Hint": { - node(label: 'docker-1.13') { + node(label: 'docker') { script { try { sh '''docker run -i --rm --name="$BUILD_TAG-jshint" -e GIT_SRC="https://github.com/eea/$GIT_NAME.git" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/jshint''' @@ -80,7 +80,7 @@ pipeline { }, "CSS Lint": { - node(label: 'docker-1.13') { + node(label: 'docker') { script { try { sh '''docker run -i --rm --name="$BUILD_TAG-csslint" -e GIT_SRC="https://github.com/eea/$GIT_NAME.git" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/csslint''' @@ -92,7 +92,7 @@ pipeline { }, "PEP8": { - node(label: 'docker-1.13') { + node(label: 'docker') { script { try { sh '''docker run -i --rm --name="$BUILD_TAG-pep8" -e GIT_SRC="https://github.com/eea/$GIT_NAME.git" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/pep8''' @@ -104,7 +104,7 @@ pipeline { }, "PyLint": { - node(label: 'docker-1.13') { + node(label: 'docker') { script { try { sh '''docker run -i --rm --name="$BUILD_TAG-pylint" -e GIT_SRC="https://github.com/eea/$GIT_NAME.git" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/pylint''' @@ -127,7 +127,7 @@ pipeline { environment name: 'CHANGE_TARGET', value: 'master' } steps { - node(label: 'docker-1.13') { + node(label: 'docker') { script { if ( env.CHANGE_BRANCH != "develop" && !( env.CHANGE_BRANCH.startsWith("hotfix")) ) { error "Pipeline aborted due to PR not made from develop or hotfix branch" @@ -146,7 +146,7 @@ pipeline { } } steps { - node(label: 'docker-1.13') { + node(label: 'docker') { withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: 'eea-jenkins', usernameVariable: 'EGGREPO_USERNAME', passwordVariable: 'EGGREPO_PASSWORD'],string(credentialsId: 'eea-jenkins-token', variable: 'GITHUB_TOKEN'),[$class: 'UsernamePasswordMultiBinding', credentialsId: 'pypi-jenkins', usernameVariable: 'PYPI_USERNAME', passwordVariable: 'PYPI_PASSWORD']]) { sh '''docker run -i --rm --name="$BUILD_TAG-gitflow-master" -e GIT_BRANCH="$BRANCH_NAME" -e EGGREPO_USERNAME="$EGGREPO_USERNAME" -e EGGREPO_PASSWORD="$EGGREPO_PASSWORD" -e GIT_NAME="$GIT_NAME" -e PYPI_USERNAME="$PYPI_USERNAME" -e PYPI_PASSWORD="$PYPI_PASSWORD" -e GIT_ORG="$GIT_ORG" -e GIT_TOKEN="$GITHUB_TOKEN" eeacms/gitflow''' } From 5a0d22736b3b1e7466cbf477a890b0f02a5675d6 Mon Sep 17 00:00:00 2001 From: valentinab25 Date: Tue, 3 Apr 2018 10:51:07 +0200 Subject: [PATCH 07/16] Jenkinsfile new vars for gitflow - Refs #89957 --- Jenkinsfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6534121..9410f4f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -132,7 +132,9 @@ pipeline { if ( env.CHANGE_BRANCH != "develop" && !( env.CHANGE_BRANCH.startsWith("hotfix")) ) { error "Pipeline aborted due to PR not made from develop or hotfix branch" } - sh '''docker run -i --rm --name="$BUILD_TAG-gitflow-pr" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" -e GIT_ORG="$GIT_ORG" -e GIT_NAME="$GIT_NAME" eeacms/gitflow''' + withCredentials([string(credentialsId: 'eea-jenkins-token', variable: 'GITHUB_TOKEN')]) { + sh '''docker run -i --rm --name="$BUILD_TAG-gitflow-pr" -e GIT_CHANGE_BRANCH="$CHANGE_BRANCH" -e GIT_CHANGE_AUTHOR="$CHANGE_AUTHOR" -e GIT_CHANGE_TITLE="$CHANGE_TITLE" -e GIT_TOKEN="$GITHUB_TOKEN" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" -e GIT_ORG="$GIT_ORG" -e GIT_NAME="$GIT_NAME" eeacms/gitflow''' + } } } } From fb779c6ef6fa172ab0766d527c540fd38228d45a Mon Sep 17 00:00:00 2001 From: Alin Voinea Date: Fri, 13 Apr 2018 18:23:40 +0300 Subject: [PATCH 08/16] [JENKINS] - Update Jenkins build status badge --- README.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index a2277b0..6d1f928 100644 --- a/README.rst +++ b/README.rst @@ -1,8 +1,12 @@ ========= EEA Cache ========= -.. image:: http://ci.eionet.europa.eu/job/eea/job/eea.cache/job/master/badge/icon - :target: http://ci.eionet.europa.eu/job/eea/job/eea.cache/job/master/display/redirect +.. image:: https://ci.eionet.europa.eu/buildStatus/icon?job=eea/eea.cache/develop + :target: https://ci.eionet.europa.eu/job/eea/job/eea.cache/job/develop/display/redirect + :alt: Develop +.. image:: https://ci.eionet.europa.eu/buildStatus/icon?job=eea/eea.cache/master + :target: https://ci.eionet.europa.eu/job/eea/job/eea.cache/job/master/display/redirect + :alt: Master Introduction ============ From d1e7e9f35f57cbab820c7f9ae7f49d6a539046e8 Mon Sep 17 00:00:00 2001 From: Alexandru Ghica Date: Thu, 10 May 2018 14:44:16 +0000 Subject: [PATCH 09/16] replaced invalidate_cache public method with a new method, called cache.invalidate.public, that use eea.cache refs #95020 --- docs/HISTORY.txt | 3 +++ eea/cache/browser/app/view.py | 5 +++++ eea/cache/browser/configure.zcml | 7 +++++++ 3 files changed, 15 insertions(+) diff --git a/docs/HISTORY.txt b/docs/HISTORY.txt index 1bb5214..037c65e 100644 --- a/docs/HISTORY.txt +++ b/docs/HISTORY.txt @@ -3,6 +3,9 @@ Changelog 8.2.dev0 - (unreleased) ----------------------- +* Feature: replaced invalidate_cache public method with a new method, + called cache.invalidate.public, that use eea.cache + [alecghica refs #95020] 8.1 - (2017-12-12) ------------------ diff --git a/eea/cache/browser/app/view.py b/eea/cache/browser/app/view.py index 80f257a..9174807 100644 --- a/eea/cache/browser/app/view.py +++ b/eea/cache/browser/app/view.py @@ -123,3 +123,8 @@ def __call__(self, **kwargs): invalidate_varnish() return _(u"Varnish and Memcache invalidated.") + + +class InvalidateCachePublic(BrowserView): + """ View to invalidate Varnish and Memcache + """ diff --git a/eea/cache/browser/configure.zcml b/eea/cache/browser/configure.zcml index 9f0adbb..a977369 100644 --- a/eea/cache/browser/configure.zcml +++ b/eea/cache/browser/configure.zcml @@ -27,6 +27,13 @@ allowed_attributes="relatedItems backRefs" permission="eea.cache.invalidate" /> + + Date: Tue, 15 May 2018 14:55:14 +0000 Subject: [PATCH 10/16] replaced invalidate_cache method with a new method, called cache.invalidate.footer, that use eea.cache. refs #95020 --- docs/HISTORY.txt | 4 +-- eea/cache/browser/app/view.py | 53 ++++++++++++++++++++++++++++++-- eea/cache/browser/configure.zcml | 6 ++-- 3 files changed, 56 insertions(+), 7 deletions(-) diff --git a/docs/HISTORY.txt b/docs/HISTORY.txt index 037c65e..0f3195b 100644 --- a/docs/HISTORY.txt +++ b/docs/HISTORY.txt @@ -3,8 +3,8 @@ Changelog 8.2.dev0 - (unreleased) ----------------------- -* Feature: replaced invalidate_cache public method with a new method, - called cache.invalidate.public, that use eea.cache +* Feature: replaced invalidate_cache method with a new method, + called cache.invalidate.footer, that use eea.cache [alecghica refs #95020] 8.1 - (2017-12-12) diff --git a/eea/cache/browser/app/view.py b/eea/cache/browser/app/view.py index 9174807..e1b6f7e 100644 --- a/eea/cache/browser/app/view.py +++ b/eea/cache/browser/app/view.py @@ -125,6 +125,55 @@ def __call__(self, **kwargs): return _(u"Varnish and Memcache invalidated.") -class InvalidateCachePublic(BrowserView): - """ View to invalidate Varnish and Memcache +class InvalidateCacheFooter(InvalidateCache): + """ Public view to invalidate Varnish and Memcache """ + + def __call__(self, **kwargs): + from Products.statusmessages.interfaces import IStatusMessage + from zope.component import getMultiAdapter + from Acquisition import aq_parent + from Products.CMFCore.utils import getToolByName + + msg_invalidated = _(u"Cache invalidated.") + msg_not_invalidated = _(u"Cache could not be invalidated.") + + # Get propper context for a default view + self.parent = self.context + state = getMultiAdapter((self.context, self.request), name='plone_context_state') + if state.is_default_page(): + self.parent = aq_parent(self.context) + + # Don't allow invalidation with direct link + referer = self.request.get('HTTP_REFERER', '') + if referer.endswith('/'): + referer = referer[:-1] + if (self.context.absolute_url() != referer and + self.parent.absolute_url() != referer): + return msg_not_invalidated + +# # Authenticated editors can invalidate cache from everywhere +# mtool = getToolByName(self.context, 'portal_membership') +# if mtool.checkPermission('Modify portal content', self.parent): +# return True + +# # Check eea internal ips +# addr_list = set(['127.0.0.1']) +# ptool = getToolByName(self.context, 'portal_properties') +# ips_list = getattr(ptool, 'eea_internal_ips', None) +# if ips_list: +# addr_list.update(ips_list.getProperty('allowed_ips', [])) + +# addr = self.request.get('HTTP_X_FORWARDED_FOR', '') +# addr = addr or self.request.get('REMOTE_ADDR', '') + +# for ip in addr_list: +# if addr.startswith(ip): +# return True +# return False + + super(InvalidateCacheFooter, self).__call__(**kwargs) + + IStatusMessage(self.request).addStatusMessage(msg_invalidated, type='info') + self.request.response.redirect(self.parent.absolute_url()) + return msg_invalidated \ No newline at end of file diff --git a/eea/cache/browser/configure.zcml b/eea/cache/browser/configure.zcml index a977369..25f55a4 100644 --- a/eea/cache/browser/configure.zcml +++ b/eea/cache/browser/configure.zcml @@ -28,11 +28,11 @@ permission="eea.cache.invalidate" /> + class=".app.view.InvalidateCacheFooter" + permission="eea.cache.invalidate" /> From 28888b8b0ef6b78167f5414d834da9d326e1d78e Mon Sep 17 00:00:00 2001 From: Alexandru Ghica Date: Tue, 22 May 2018 13:29:13 +0000 Subject: [PATCH 11/16] Refresh this page method redirect you back to the original context. cache.settings if called in a context of a default view it will also invalidate its parent. refs #95020 --- docs/HISTORY.txt | 8 +- eea/cache/browser/app/edit.py | 55 +++++++- eea/cache/browser/app/view.py | 224 ++++++++++++++++++++----------- eea/cache/browser/configure.zcml | 7 - 4 files changed, 206 insertions(+), 88 deletions(-) diff --git a/docs/HISTORY.txt b/docs/HISTORY.txt index 0f3195b..1b0b78e 100644 --- a/docs/HISTORY.txt +++ b/docs/HISTORY.txt @@ -3,8 +3,12 @@ Changelog 8.2.dev0 - (unreleased) ----------------------- -* Feature: replaced invalidate_cache method with a new method, - called cache.invalidate.footer, that use eea.cache +* Feature: replaced invalidate_cache method with a method that use eea.cache + [alecghica refs #95020] +* Feature: "Refresh this page" method redirect you back to the original context + [alecghica refs #95020] +* Feature: cache.settings if called in a context of a default view it will + also invalidate its parent [alecghica refs #95020] 8.1 - (2017-12-12) diff --git a/eea/cache/browser/app/edit.py b/eea/cache/browser/app/edit.py index c94db6d..5495197 100644 --- a/eea/cache/browser/app/edit.py +++ b/eea/cache/browser/app/edit.py @@ -6,6 +6,8 @@ from z3c.form import form, button, interfaces, util from plone.supermodel import model from plone.autoform.form import AutoExtensibleForm +from Products.CMFCore.utils import getToolByName +from Products.statusmessages.interfaces import IStatusMessage from eea.cache.interfaces import ICacheAware from eea.cache.browser.interfaces import VARNISH from eea.cache.config import EEAMessageFactory as _ @@ -43,6 +45,12 @@ class ISettings(model.Schema): default=False ) + redirectURL = schema.URI( + title=_(u"Redirect URL"), + description=_(u"Redirect URL when request parameter is present."), + required=False + ) + class SettingsBehavior(object): """ Cache invalidation behaviour @@ -166,12 +174,45 @@ def backRefs(self, value): if value.get('memcache') and self.invalidate_memcache: self.invalidate_memcache.backRefs() + @property + def redirectURL(self): + """ Redirect URL + """ + return '' + + @redirectURL.setter + def redirectURL(self, value): + """ Redirect URL + """ + return + class SettingsForm(AutoExtensibleForm, form.EditForm): """ Cache settings """ schema = ISettings + def updateWidgets(self): + """ Update widgets + """ + super(SettingsForm, self).updateWidgets() + self.widgets['redirectURL'].mode = interfaces.HIDDEN_MODE + + def update(self): + """ Update form + """ + super(SettingsForm, self).update() + redirect_url = self.request.get('redirect', '') + + if redirect_url: + for name, widget in self.widgets.items(): + if name != 'redirectURL': + # Select everything by default + widget.items[0]['checked'] = True + else: + # Add the redirect URL + widget.value = redirect_url + def applyChanges(self, content, data): """ Apply changes """ @@ -207,15 +248,25 @@ def invalidate(self, action): """ Invalidate cache """ self.status = u"" + msg_invalidated = _(u"Cache invalidated") + portal_url = getToolByName(self.context, 'portal_url')() + redirectURL = self.widgets['redirectURL'].value data, errors = self.extractData() + if not redirectURL.startswith(portal_url): + redirectURL = '' + if errors: self.status = self.formErrorsMessage return content = self.getContent() changes = self.applyChanges(content, data) - if changes: - self.status = _(u"Cache invalidated") + if redirectURL and changes: + IStatusMessage(self.request).addStatusMessage(msg_invalidated, + type='info') + self.request.response.redirect(redirectURL) + elif changes: + self.status = msg_invalidated else: self.status = _(u"Nothing selected to invalidate") diff --git a/eea/cache/browser/app/view.py b/eea/cache/browser/app/view.py index e1b6f7e..b2e69e0 100644 --- a/eea/cache/browser/app/view.py +++ b/eea/cache/browser/app/view.py @@ -9,14 +9,41 @@ from eea.cache.browser.interfaces import VARNISH from eea.cache.config import EEAMessageFactory as _ from Products.Five.browser import BrowserView +from zope.component import getMultiAdapter + logger = logging.getLogger('eea.cache') -class InvalidateMemCache(BrowserView): +class BaseInvalidate(BrowserView): + """ Base class for the invalidate + """ + + def __init__(self, context, request): + super(BaseInvalidate, self).__init__(context, request) + self._parent = None + self._is_default_page = None + + @property + def parent(self): + if self._parent is None: + self._parent = self.context.getParentNode() + return self._parent + + @property + def is_default_page(self): + if self._is_default_page is None: + state = getMultiAdapter((self.context, self.request), + name='plone_context_state') + self._is_default_page = state.is_default_page() + return self._is_default_page + + +class InvalidateMemCache(BaseInvalidate): """ View to invalidate memcache """ - def relatedItems(self, **kwargs): + + def related_items(self, context, **kwargs): """ Invalidate related Items """ getRelatedItems = getattr(self.context, 'getRelatedItems', lambda: []) @@ -27,10 +54,21 @@ def relatedItems(self, **kwargs): event.notify(InvalidateMemCacheEvent(raw=True, dependencies=[uid])) return _(u"Memcache invalidated for relatedItems.") - def backRefs(self, **kwargs): + def relatedItems(self, **kwargs): + """ Invalidate related Items + """ + msg = self.related_items(self.context) + + # Invalidate cache for a default view parent + if self.is_default_page: + msg = self.related_items(self.parent) + + return msg + + def back_refs(self, context, **kwargs): """ Invalidate back references """ - getBRefs = getattr(self.context, 'getBRefs', lambda: []) + getBRefs = getattr(context, 'getBRefs', lambda: []) for item in getBRefs(): uid = queryAdapter(item, IUUID) if not uid: @@ -38,22 +76,42 @@ def backRefs(self, **kwargs): event.notify(InvalidateMemCacheEvent(raw=True, dependencies=[uid])) return _(u"Memcache invalidated for back references.") - def __call__(self, **kwargs): - uid = queryAdapter(self.context, IUUID) + def backRefs(self, **kwargs): + """ Invalidate back references + """ + msg = self.back_refs(self.context) + + # Invalidate cache for a default view parent + if self.is_default_page: + msg = self.back_refs(self.parent) + + return msg + + def invalidate_cache(self, context, **kwargs): + uid = queryAdapter(context, IUUID) if not uid: return _(u"Can't invalidate memcache. Missing uid adapter.") event.notify(InvalidateMemCacheEvent(raw=True, dependencies=[uid])) return _(u"Memcache invalidated.") + def __call__(self, **kwargs): + msg = self.invalidate_cache(self.context) + + # Invalidate cache for a default view parent + if self.is_default_page and not kwargs.get("parent", None): + msg = self.invalidate_cache(self.parent) + + return msg + -class InvalidateVarnish(BrowserView): +class InvalidateVarnish(BaseInvalidate): """ View to invalidate Varnish """ - def relatedItems(self, **kwargs): + def related_items(self, context, **kwargs): """ Invalidate related Items """ - getRelatedItems = getattr(self.context, 'getRelatedItems', lambda: []) + getRelatedItems = getattr(context, 'getRelatedItems', lambda: []) for item in getRelatedItems(): invalidate_cache = queryMultiAdapter( (item, self.request), name='varnish.invalidate', @@ -61,10 +119,21 @@ def relatedItems(self, **kwargs): invalidate_cache() return _(u"Varnish invalidated for relatedItems.") - def backRefs(self, **kwargs): + def relatedItems(self, **kwargs): + """ Invalidate related Items + """ + msg = self.related_items(self.context) + + # Invalidate cache for a default view parent + if self.is_default_page: + msg = self.related_items(self.parent) + + return msg + + def back_refs(self, context, **kwargs): """ Invalidate back references """ - getBRefs = getattr(self.context, 'getBRefs', lambda: []) + getBRefs = getattr(context, 'getBRefs', lambda: []) for item in getBRefs(): invalidate_cache = queryMultiAdapter( (item, self.request), name='varnish.invalidate', @@ -72,24 +141,46 @@ def backRefs(self, **kwargs): invalidate_cache() return _(u"Varnish invalidated for back references.") - def __call__(self, **kwargs): + def backRefs(self, **kwargs): + """ Invalidate back references + """ + msg = self.back_refs(self.context) + + # Invalidate cache for a default view parent + if self.is_default_page: + msg = self.back_refs(self.parent) + + return msg + + def invalidate_cache(self, context, **kwargs): + """ Invalidate Varnish + """ if not VARNISH: return _(u"Varnish invalidated.") try: - if VARNISH.purge.isPurged(self.context): - event.notify(InvalidateVarnishEvent(self.context)) + if VARNISH.purge.isPurged(context): + event.notify(InvalidateVarnishEvent(context)) except Exception, err: logger.exception(err) return _(u"Varnish invalidated.") + def __call__(self, **kwargs): + msg = self.invalidate_cache(self.context) + + # Invalidate cache for a default view parent + if self.is_default_page and not kwargs.get("parent", None): + msg = self.invalidate_cache(self.parent) + + return msg -class InvalidateCache(BrowserView): + +class InvalidateCache(BaseInvalidate): """ View to invalidate Varnish and Memcache """ - - def relatedItems(self, **kwargs): + + def related_items(self, context, **kwargs): """ Invalidate related Items """ getRelatedItems = getattr(self.context, 'getRelatedItems', lambda: []) @@ -97,83 +188,62 @@ def relatedItems(self, **kwargs): invalidate_cache = queryMultiAdapter( (item, self.request), name='cache.invalidate', default=lambda: None) - invalidate_cache() + invalidate_cache(parent="ignore") return _(u"Cache invalidated for relatedItems.") - def backRefs(self, **kwargs): + def relatedItems(self, **kwargs): + """ Invalidate related Items + """ + msg = self.related_items(self.context) + + # Invalidate cache for a default view parent + if self.is_default_page: + msg = self.related_items(self.parent) + + return msg + + def back_refs(self, context, **kwargs): """ Invalidate back references """ - getBRefs = getattr(self.context, 'getBRefs', lambda: []) + getBRefs = getattr(context, 'getBRefs', lambda: []) for item in getBRefs(): invalidate_cache = queryMultiAdapter( (item, self.request), name='cache.invalidate', default=lambda: None) - invalidate_cache() + invalidate_cache(parent="ignore") return _(u"Cache invalidated for back references.") - def __call__(self, **kwargs): + def backRefs(self, **kwargs): + """ Invalidate back references + """ + msg = self.back_refs(self.context) + + # Invalidate cache for a default view parent + if self.is_default_page: + msg = self.back_refs(self.parent) + + return msg + + def invalidate_cache(self, context, **kwargs): + """ Invalidate Varnish and Memcache + """ # Memcache - invalidate_memcache = queryMultiAdapter((self.context, self.request), + invalidate_memcache = queryMultiAdapter((context, self.request), name='memcache.invalidate') - invalidate_memcache() + invalidate_memcache(parent="ignore") # Varnish - invalidate_varnish = queryMultiAdapter((self.context, self.request), + invalidate_varnish = queryMultiAdapter((context, self.request), name='varnish.invalidate') - invalidate_varnish() + invalidate_varnish(parent="ignore") return _(u"Varnish and Memcache invalidated.") + def __call__(self, **kwargs): + msg = self.invalidate_cache(self.context) -class InvalidateCacheFooter(InvalidateCache): - """ Public view to invalidate Varnish and Memcache - """ + # Invalidate cache for a default view parent + if self.is_default_page and not kwargs.get("parent", None): + msg = self.invalidate_cache(self.parent) - def __call__(self, **kwargs): - from Products.statusmessages.interfaces import IStatusMessage - from zope.component import getMultiAdapter - from Acquisition import aq_parent - from Products.CMFCore.utils import getToolByName - - msg_invalidated = _(u"Cache invalidated.") - msg_not_invalidated = _(u"Cache could not be invalidated.") - - # Get propper context for a default view - self.parent = self.context - state = getMultiAdapter((self.context, self.request), name='plone_context_state') - if state.is_default_page(): - self.parent = aq_parent(self.context) - - # Don't allow invalidation with direct link - referer = self.request.get('HTTP_REFERER', '') - if referer.endswith('/'): - referer = referer[:-1] - if (self.context.absolute_url() != referer and - self.parent.absolute_url() != referer): - return msg_not_invalidated - -# # Authenticated editors can invalidate cache from everywhere -# mtool = getToolByName(self.context, 'portal_membership') -# if mtool.checkPermission('Modify portal content', self.parent): -# return True - -# # Check eea internal ips -# addr_list = set(['127.0.0.1']) -# ptool = getToolByName(self.context, 'portal_properties') -# ips_list = getattr(ptool, 'eea_internal_ips', None) -# if ips_list: -# addr_list.update(ips_list.getProperty('allowed_ips', [])) - -# addr = self.request.get('HTTP_X_FORWARDED_FOR', '') -# addr = addr or self.request.get('REMOTE_ADDR', '') - -# for ip in addr_list: -# if addr.startswith(ip): -# return True -# return False - - super(InvalidateCacheFooter, self).__call__(**kwargs) - - IStatusMessage(self.request).addStatusMessage(msg_invalidated, type='info') - self.request.response.redirect(self.parent.absolute_url()) - return msg_invalidated \ No newline at end of file + return msg diff --git a/eea/cache/browser/configure.zcml b/eea/cache/browser/configure.zcml index 25f55a4..9f0adbb 100644 --- a/eea/cache/browser/configure.zcml +++ b/eea/cache/browser/configure.zcml @@ -27,13 +27,6 @@ allowed_attributes="relatedItems backRefs" permission="eea.cache.invalidate" /> - - Date: Thu, 24 May 2018 10:42:35 +0000 Subject: [PATCH 12/16] fixed case under all invalidate methods when a broken relations is present and None is returned instead of an object, refs #95020 --- docs/HISTORY.txt | 3 ++ eea/cache/browser/app/view.py | 67 ++++++++++++++++++++++------------- 2 files changed, 45 insertions(+), 25 deletions(-) diff --git a/docs/HISTORY.txt b/docs/HISTORY.txt index 1b0b78e..7afd431 100644 --- a/docs/HISTORY.txt +++ b/docs/HISTORY.txt @@ -10,6 +10,9 @@ Changelog * Feature: cache.settings if called in a context of a default view it will also invalidate its parent [alecghica refs #95020] +* Bug fix: fixed case under all invalidate methods when a broken relations + is present and None is found instead of an object + [alecghica refs #95020] 8.1 - (2017-12-12) ------------------ diff --git a/eea/cache/browser/app/view.py b/eea/cache/browser/app/view.py index b2e69e0..55f8789 100644 --- a/eea/cache/browser/app/view.py +++ b/eea/cache/browser/app/view.py @@ -14,7 +14,6 @@ logger = logging.getLogger('eea.cache') - class BaseInvalidate(BrowserView): """ Base class for the invalidate """ @@ -48,10 +47,13 @@ def related_items(self, context, **kwargs): """ getRelatedItems = getattr(self.context, 'getRelatedItems', lambda: []) for item in getRelatedItems(): - uid = queryAdapter(item, IUUID) - if not uid: - continue - event.notify(InvalidateMemCacheEvent(raw=True, dependencies=[uid])) + try: + uid = queryAdapter(item, IUUID) + if not uid: + continue + event.notify(InvalidateMemCacheEvent(raw=True, dependencies=[uid])) + except TypeError, err: + logger.exception(err) return _(u"Memcache invalidated for relatedItems.") def relatedItems(self, **kwargs): @@ -70,10 +72,13 @@ def back_refs(self, context, **kwargs): """ getBRefs = getattr(context, 'getBRefs', lambda: []) for item in getBRefs(): - uid = queryAdapter(item, IUUID) - if not uid: - continue - event.notify(InvalidateMemCacheEvent(raw=True, dependencies=[uid])) + try: + uid = queryAdapter(item, IUUID) + if not uid: + continue + event.notify(InvalidateMemCacheEvent(raw=True, dependencies=[uid])) + except TypeError, err: + logger.exception(err) return _(u"Memcache invalidated for back references.") def backRefs(self, **kwargs): @@ -113,10 +118,13 @@ def related_items(self, context, **kwargs): """ getRelatedItems = getattr(context, 'getRelatedItems', lambda: []) for item in getRelatedItems(): - invalidate_cache = queryMultiAdapter( - (item, self.request), name='varnish.invalidate', - default=lambda: None) - invalidate_cache() + try: + invalidate_cache = queryMultiAdapter( + (item, self.request), name='varnish.invalidate', + default=lambda: None) + invalidate_cache() + except TypeError, err: + logger.exception(err) return _(u"Varnish invalidated for relatedItems.") def relatedItems(self, **kwargs): @@ -135,10 +143,13 @@ def back_refs(self, context, **kwargs): """ getBRefs = getattr(context, 'getBRefs', lambda: []) for item in getBRefs(): - invalidate_cache = queryMultiAdapter( - (item, self.request), name='varnish.invalidate', - default=lambda: None) - invalidate_cache() + try: + invalidate_cache = queryMultiAdapter( + (item, self.request), name='varnish.invalidate', + default=lambda: None) + invalidate_cache() + except TypeError, err: + logger.exception(err) return _(u"Varnish invalidated for back references.") def backRefs(self, **kwargs): @@ -185,10 +196,13 @@ def related_items(self, context, **kwargs): """ getRelatedItems = getattr(self.context, 'getRelatedItems', lambda: []) for item in getRelatedItems(): - invalidate_cache = queryMultiAdapter( - (item, self.request), name='cache.invalidate', - default=lambda: None) - invalidate_cache(parent="ignore") + try: + invalidate_cache = queryMultiAdapter( + (item, self.request), name='cache.invalidate', + default=lambda: None) + invalidate_cache(parent="ignore") + except TypeError, err: + logger.exception(err) return _(u"Cache invalidated for relatedItems.") def relatedItems(self, **kwargs): @@ -207,10 +221,13 @@ def back_refs(self, context, **kwargs): """ getBRefs = getattr(context, 'getBRefs', lambda: []) for item in getBRefs(): - invalidate_cache = queryMultiAdapter( - (item, self.request), name='cache.invalidate', - default=lambda: None) - invalidate_cache(parent="ignore") + try: + invalidate_cache = queryMultiAdapter( + (item, self.request), name='cache.invalidate', + default=lambda: None) + invalidate_cache(parent="ignore") + except TypeError, err: + logger.exception(err) return _(u"Cache invalidated for back references.") def backRefs(self, **kwargs): From 10de18e5675d55a69d740b1c6c6c496d73aecb2d Mon Sep 17 00:00:00 2001 From: EEA Jenkins Date: Thu, 24 May 2018 19:00:08 +0300 Subject: [PATCH 13/16] Updated version to 8.2 --- eea/cache/version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eea/cache/version.txt b/eea/cache/version.txt index 529bd32..2983cad 100644 --- a/eea/cache/version.txt +++ b/eea/cache/version.txt @@ -1 +1 @@ -8.2.dev0 \ No newline at end of file +8.2 From ab6c80df8d0d090dfc34eecc9c6afc103a5aaebb Mon Sep 17 00:00:00 2001 From: EEA Jenkins Date: Thu, 24 May 2018 19:04:33 +0300 Subject: [PATCH 14/16] Updated changelog - removed develop information --- docs/HISTORY.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/HISTORY.txt b/docs/HISTORY.txt index 7afd431..dbeb06a 100644 --- a/docs/HISTORY.txt +++ b/docs/HISTORY.txt @@ -1,7 +1,7 @@ Changelog ========= -8.2.dev0 - (unreleased) +8.2 - (2018-05-24) ----------------------- * Feature: replaced invalidate_cache method with a method that use eea.cache [alecghica refs #95020] From 53f7e17c71d777f0f9a5277750711a0ca5140a1f Mon Sep 17 00:00:00 2001 From: Alin Voinea Date: Fri, 25 May 2018 13:28:22 +0300 Subject: [PATCH 15/16] Refs #95020 - Cleanup duplicate code --- eea/cache/browser/app/view.py | 100 +++++++++++----------------------- 1 file changed, 33 insertions(+), 67 deletions(-) diff --git a/eea/cache/browser/app/view.py b/eea/cache/browser/app/view.py index 55f8789..29c406c 100644 --- a/eea/cache/browser/app/view.py +++ b/eea/cache/browser/app/view.py @@ -38,6 +38,38 @@ def is_default_page(self): return self._is_default_page + def relatedItems(self, **kwargs): + """ Invalidate related Items + """ + msg = self.related_items(self.context) + + # Invalidate cache for a default view parent + if self.is_default_page: + msg = self.related_items(self.parent) + + return msg + + def backRefs(self, **kwargs): + """ Invalidate back references + """ + msg = self.back_refs(self.context) + + # Invalidate cache for a default view parent + if self.is_default_page: + msg = self.back_refs(self.parent) + + return msg + + def related_items(self, context, **kwargs): + """ Related items invalidation. To be implemented + """ + raise NotImplementedError + + def back_refs(self, context, **kwargs): + """ Invalidate back references. To be implemented + """ + raise NotImplementedError + class InvalidateMemCache(BaseInvalidate): """ View to invalidate memcache """ @@ -56,17 +88,6 @@ def related_items(self, context, **kwargs): logger.exception(err) return _(u"Memcache invalidated for relatedItems.") - def relatedItems(self, **kwargs): - """ Invalidate related Items - """ - msg = self.related_items(self.context) - - # Invalidate cache for a default view parent - if self.is_default_page: - msg = self.related_items(self.parent) - - return msg - def back_refs(self, context, **kwargs): """ Invalidate back references """ @@ -81,17 +102,6 @@ def back_refs(self, context, **kwargs): logger.exception(err) return _(u"Memcache invalidated for back references.") - def backRefs(self, **kwargs): - """ Invalidate back references - """ - msg = self.back_refs(self.context) - - # Invalidate cache for a default view parent - if self.is_default_page: - msg = self.back_refs(self.parent) - - return msg - def invalidate_cache(self, context, **kwargs): uid = queryAdapter(context, IUUID) if not uid: @@ -127,17 +137,6 @@ def related_items(self, context, **kwargs): logger.exception(err) return _(u"Varnish invalidated for relatedItems.") - def relatedItems(self, **kwargs): - """ Invalidate related Items - """ - msg = self.related_items(self.context) - - # Invalidate cache for a default view parent - if self.is_default_page: - msg = self.related_items(self.parent) - - return msg - def back_refs(self, context, **kwargs): """ Invalidate back references """ @@ -152,17 +151,6 @@ def back_refs(self, context, **kwargs): logger.exception(err) return _(u"Varnish invalidated for back references.") - def backRefs(self, **kwargs): - """ Invalidate back references - """ - msg = self.back_refs(self.context) - - # Invalidate cache for a default view parent - if self.is_default_page: - msg = self.back_refs(self.parent) - - return msg - def invalidate_cache(self, context, **kwargs): """ Invalidate Varnish """ @@ -190,7 +178,7 @@ def __call__(self, **kwargs): class InvalidateCache(BaseInvalidate): """ View to invalidate Varnish and Memcache """ - + def related_items(self, context, **kwargs): """ Invalidate related Items """ @@ -205,17 +193,6 @@ def related_items(self, context, **kwargs): logger.exception(err) return _(u"Cache invalidated for relatedItems.") - def relatedItems(self, **kwargs): - """ Invalidate related Items - """ - msg = self.related_items(self.context) - - # Invalidate cache for a default view parent - if self.is_default_page: - msg = self.related_items(self.parent) - - return msg - def back_refs(self, context, **kwargs): """ Invalidate back references """ @@ -230,17 +207,6 @@ def back_refs(self, context, **kwargs): logger.exception(err) return _(u"Cache invalidated for back references.") - def backRefs(self, **kwargs): - """ Invalidate back references - """ - msg = self.back_refs(self.context) - - # Invalidate cache for a default view parent - if self.is_default_page: - msg = self.back_refs(self.parent) - - return msg - def invalidate_cache(self, context, **kwargs): """ Invalidate Varnish and Memcache """ From 4459e1d06e52e7068ea07625e43bd93827c259fd Mon Sep 17 00:00:00 2001 From: Alin Voinea Date: Fri, 25 May 2018 16:44:58 +0300 Subject: [PATCH 16/16] Refs #95020 - Speed-up cache invalidation for backward-references --- docs/HISTORY.txt | 2 ++ eea/cache/browser/app/view.py | 12 ++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/HISTORY.txt b/docs/HISTORY.txt index dbeb06a..f4eb458 100644 --- a/docs/HISTORY.txt +++ b/docs/HISTORY.txt @@ -3,6 +3,8 @@ Changelog 8.2 - (2018-05-24) ----------------------- +* Bug fix: Speed-up cache invalidation for backward-references + [avoinea refs #95020] * Feature: replaced invalidate_cache method with a method that use eea.cache [alecghica refs #95020] * Feature: "Refresh this page" method redirect you back to the original context diff --git a/eea/cache/browser/app/view.py b/eea/cache/browser/app/view.py index 29c406c..73f6e5c 100644 --- a/eea/cache/browser/app/view.py +++ b/eea/cache/browser/app/view.py @@ -91,8 +91,8 @@ def related_items(self, context, **kwargs): def back_refs(self, context, **kwargs): """ Invalidate back references """ - getBRefs = getattr(context, 'getBRefs', lambda: []) - for item in getBRefs(): + getBRefs = getattr(context, 'getBRefs', lambda r: []) + for item in getBRefs('relatesTo'): try: uid = queryAdapter(item, IUUID) if not uid: @@ -140,8 +140,8 @@ def related_items(self, context, **kwargs): def back_refs(self, context, **kwargs): """ Invalidate back references """ - getBRefs = getattr(context, 'getBRefs', lambda: []) - for item in getBRefs(): + getBRefs = getattr(context, 'getBRefs', lambda r: []) + for item in getBRefs('relatesTo'): try: invalidate_cache = queryMultiAdapter( (item, self.request), name='varnish.invalidate', @@ -196,8 +196,8 @@ def related_items(self, context, **kwargs): def back_refs(self, context, **kwargs): """ Invalidate back references """ - getBRefs = getattr(context, 'getBRefs', lambda: []) - for item in getBRefs(): + getBRefs = getattr(context, 'getBRefs', lambda r: []) + for item in getBRefs('relatesTo'): try: invalidate_cache = queryMultiAdapter( (item, self.request), name='cache.invalidate',