diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index e39a7fc479..0000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2.1 - -setup: true - -orbs: - path-filtering: circleci/path-filtering@0.1.1 - -workflows: - regular: - jobs: - # the path-filtering/filter job determines which pipeline - # parameters to update, i.e. which builds to run. - - path-filtering/filter: - name: check-updated-files - base-revision: main - config-path: .circleci/continue_config.yml - # - mapping: | - package/.* backend_change true - .circleci/.* ci_change true - ^((?!package/).)*$ frontend_change true - filters: - tags: - only: /^v[0-9]+\.[0-9]+\.[0-9]+$/ diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml deleted file mode 100755 index 308f3ce6f9..0000000000 --- a/.circleci/continue_config.yml +++ /dev/null @@ -1,415 +0,0 @@ -version: 2.1 - -# Backend CI jobs are the heavy ones, so we only run them if the backend -# or CI config has changed. -parameters: - backend_change: - type: boolean - default: false - ci_change: - type: boolean - default: false - frontend_change: - type: boolean - default: false - -orbs: - win: circleci/windows@4.1.1 - node: circleci/node@5.2.0 - -# No windows executor is listed here since windows builds use win/default and modify -# the Python version through the conda environment. -executors: - docker: - parameters: - python_version: - type: string - docker: - - image: cimg/python:<>-node - working_directory: ~/repo - -commands: - setup_python_env: - steps: - - run: - name: Install Python dependencies - command: | - pip install git+https://github.com/kedro-org/kedro@main - pip install -r package/test_requirements.txt -r demo-project/src/docker_requirements.txt -U - - run: - name: Echo package versions - command: | - python -V - pip freeze - - install_node_dependencies: - steps: - - node/install: - node-version: '18.20.0' - # using install-packages command from node orb - - node/install-packages: - override-ci-command: npm install - - nvm_use_npm_install: - steps: - - run: - name: Use NVM to run Node v18 - command: | - nvm install v18.20.0 - node -v - nvm alias default v18.20.0 - nvm use v18.20.0 - - run: - name: Install Node dependencies - command: | - node -v - npm install - - npm_build: - steps: - - run: - name: Build React application - command: | - node -v - make build - - win_setup_python_env: - parameters: - python_version: - type: string - steps: - - run: - name: Install and initialize Miniconda - command: | - choco install miniconda3 -y - refreshenv - C:\tools\miniconda3\Scripts\conda init powershell - - run: - name: Create 'kedro-viz' conda environment - command: conda create --name kedro-viz python=<> -y - - run: - name: Install Kedro-Viz dependencies - command: | - conda activate kedro-viz - pip install git+https://github.com/kedro-org/kedro@main - pip install -r package/test_requirements.txt -U - - run: - name: Echo package versions - command: | - conda activate kedro-viz - python -V - pip freeze - - run: - name: Install 'make' command - command: choco install make - - setup: - steps: - - checkout - - setup_python_env - - install_node_dependencies - - npm_build - - win_setup: - parameters: - python_version: - type: string - executor: - steps: - - checkout - - win_setup_python_env: - python_version: <> - - nvm_use_npm_install - - npm_build - - setup_cypress_requirements: - description: Install cypress requirements and dependencies - steps: - - run: - name: Install cypress system dependencies - command: | - sudo sed -i 's/archive.ubuntu.com/us-east-1.ec2.archive.ubuntu.com/g' /etc/apt/sources.list - sudo apt-get update - sudo apt-get install libgtk2.0-0 libgtk-3-0 libgbm-dev libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 libxtst6 xauth xvfb - -jobs: - e2e_tests: - parameters: - python_version: - type: string - executor: - name: docker - python_version: <> - steps: - - setup - - run: - name: Run all end to end tests - command: make e2e-tests - - win_e2e_tests: - parameters: - python_version: - type: string - executor: win/default - steps: - - win_setup: - python_version: <> - - run: - name: Run all end to end tests on Windows - command: conda activate kedro-viz; make e2e-tests - - unit_tests: - parameters: - python_version: - type: string - executor: - name: docker - python_version: <> - steps: - - setup - - run: - name: Run Python tests - command: make pytest - - win_unit_tests: - parameters: - python_version: - type: string - executor: win/default - steps: - - win_setup: - python_version: <> - - run: - name: Run Python tests on Windows - command: conda activate kedro-viz; make pytest - - lint: - parameters: - python_version: - type: string - executor: - name: docker - python_version: <> - steps: - - checkout - - setup_python_env - - run: - name: Run secret scan - command: make secret-scan - - run: - name: Run security scan - command: make security-scan - - run: - name: Verify GraphQL schema is up to date - command: make schema-check - - run: - name: Run Python formatters and linters - command: make format-check lint-check - - javascript_lint_and_tests: - executor: - name: docker - python_version: '3.9' - steps: - - checkout - - setup_python_env - - install_node_dependencies - - setup_cypress_requirements - - run: - name: Test lib transpilation - command: npm run lib - - run: - name: Test JS library imports - command: | - npm run lib-test:setup - cd tools/test-lib/react-app - npm run test:ci - - run: - name: Run Eslint - command: npm run lint - - run: - name: Run JavaScript tests - command: npm run test:ci - - run: - name: Run Javascript end to end tests - command: npm run cy:ci - - release_to_npm: - executor: - name: docker - python_version: '3.9' - steps: - - checkout - - install_node_dependencies - - npm_build - - run: - name: Authenticate with registry - command: echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" > ~/repo/.npmrc - - run: - name: Publish to npm - command: npm publish - - release_to_pypi: - executor: - name: docker - python_version: '3.9' - steps: - - setup - - run: - name: Make Python package - command: make package - - run: - name: Install twine - command: python -m pip install -U twine - - run: - name: Publish to PyPI - command: python -m twine upload package/dist/* - - deploy_demo: - executor: - name: docker - python_version: '3.9' - steps: - - checkout - - setup_remote_docker - - run: - name: Setup environment - command: | - cd demo-project - echo "AWS_ECR_URL=public.ecr.aws/g0x0s3o2/kedro-viz-live-demo" >> $BASH_ENV - echo "KEDRO_VIZ_VERSION=$(cat .version)" >> $BASH_ENV - echo "cd demo-project" >> $BASH_ENV - - run: - name: Install AWS CLI - command: pip3 install awscli - - run: - name: Build demo container image - command: | - echo "kedro_viz==$KEDRO_VIZ_VERSION" >> src/docker_requirements.txt - docker build -t $AWS_ECR_URL:$KEDRO_VIZ_VERSION . - aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws - docker push $AWS_ECR_URL:$KEDRO_VIZ_VERSION - - run: - name: Create a new lightsail deployment - command: | - # install lightsail cli - # run https://docs.aws.amazon.com/cli/latest/reference/lightsail/create-container-service-deployment.html#create-container-service-deployment - aws lightsail create-container-service-deployment --region eu-west-2 --cli-input-json file://./lightsail.json - - all_circleci_checks_succeeded: - docker: - - image: circleci/python # any light-weight image - steps: - - run: - name: Success! - command: echo "All checks passed" - -release_filter: &release_filter - filters: - tags: - only: /^v[0-9]+\.[0-9]+\.[0-9]+$/ - branches: - ignore: /.*/ - -workflows: - version: 2.1 - - deploy_demo: - jobs: - - deploy_demo: - context: - - kedro-ecr-publish - filters: - branches: - only: demo - - build_frontend: - when: - or: - - <> - - <> - jobs: - - javascript_lint_and_tests - - all_circleci_checks_succeeded: - requires: - - javascript_lint_and_tests - - build_backend: - when: - or: - - <> - - <> - jobs: - - e2e_tests: - matrix: - parameters: - python_version: ['3.9', '3.10', '3.11'] - - win_e2e_tests: - matrix: - parameters: - python_version: ['3.9', '3.10', '3.11'] - filters: - branches: - only: - - main - - demo - - unit_tests: - matrix: - parameters: - python_version: ['3.9', '3.10', '3.11'] - - win_unit_tests: - matrix: - parameters: - python_version: ['3.9', '3.10', '3.11'] - filters: - branches: - only: - - main - - demo - - lint: - matrix: - parameters: - python_version: ['3.9', '3.10', '3.11'] - - all_circleci_checks_succeeded: - requires: - - e2e_tests - - unit_tests - - lint - - release: - jobs: - - release_to_npm: - <<: *release_filter - - release_to_pypi: - <<: *release_filter - - daily: - triggers: - - schedule: - cron: '0 1 * * *' - filters: - branches: - only: - - main - jobs: - - e2e_tests: - matrix: - parameters: - python_version: ['3.9', '3.10', '3.11'] - - win_e2e_tests: - matrix: - parameters: - python_version: ['3.9', '3.10', '3.11'] - - unit_tests: - matrix: - parameters: - python_version: ['3.9', '3.10', '3.11'] - - win_unit_tests: - matrix: - parameters: - python_version: ['3.9', '3.10', '3.11'] - - lint: - matrix: - parameters: - python_version: ['3.9', '3.10', '3.11'] - - javascript_lint_and_tests diff --git a/.github/actions/install_kedro_and_python_dependencies/action.yml b/.github/actions/install_kedro_and_python_dependencies/action.yml new file mode 100644 index 0000000000..3874ccdc20 --- /dev/null +++ b/.github/actions/install_kedro_and_python_dependencies/action.yml @@ -0,0 +1,15 @@ +name: Install Kedro and other Python Dependencies +description: Installs Kedro from the main branch and other Python dependencies, then prints the Python version and installed packages. +runs: + using: composite + steps: + - name: Install Python dependencies + run: |- + pip install git+https://github.com/kedro-org/kedro@main + pip install -r package/test_requirements.txt -r demo-project/src/docker_requirements.txt -U + shell: bash + - name: Echo package versions + run: |- + python -V + pip freeze + shell: bash \ No newline at end of file diff --git a/.github/actions/install_node_dependencies/action.yml b/.github/actions/install_node_dependencies/action.yml new file mode 100644 index 0000000000..b36a41e83a --- /dev/null +++ b/.github/actions/install_node_dependencies/action.yml @@ -0,0 +1,37 @@ +name: Setup Node.js and Install Dependencies +description: Sets up a specific Node.js version, caches Node modules, and installs Node dependencies. + +inputs: + node-version: + description: 'Node.js version' + required: false + default: '18.20.0' + + package-path: + description: 'Path to package.json file' + required: false + default: '.' + +runs: + using: composite + steps: + - name: Setup Node.js + uses: actions/setup-node@v4.0.0 + with: + node-version: ${{ inputs.node-version }} + + - name: Get NPM Cache Directory + id: npm-cache-dir + run: echo "dir=$(npm config get cache)" >> $GITHUB_OUTPUT + shell: bash + + - name: Cache Node.js packages + uses: actions/cache@v2 + with: + path: "${{ steps.npm-cache-dir.outputs.dir }}" + key: "${{ runner.os }}-node-${{ hashFiles(format('{0}/package-lock.json', inputs.package-path)) }}" + restore-keys: "${{ runner.os }}-node-" + + - name: Install Node Dependencies + run: npm install + shell: bash diff --git a/.github/actions/setup_tests/action.yml b/.github/actions/setup_tests/action.yml new file mode 100644 index 0000000000..4de29e785a --- /dev/null +++ b/.github/actions/setup_tests/action.yml @@ -0,0 +1,46 @@ +name: Setup Tests +description: Sets up the testing environment by setting up Python and Node.js, caching Python packages, installing Kedro and other Python dependencies, and building the React application. + +inputs: + os: + description: 'Operating system' + required: false + default: 'ubuntu-latest' + python-version: + description: 'Python version' + required: false + default: '3.9' + +runs: + using: "composite" + steps: + - name: Set up Python ${{inputs.python-version}} + uses: actions/setup-python@v5 + with: + python-version: ${{inputs.python-version}} + + - name: Cache python packages for Linux + if: inputs.os == 'ubuntu-latest' + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{inputs.os}}-python-${{inputs.python-version}} + + - name: Cache python packages for Windows + if: inputs.os == 'windows-latest' + uses: actions/cache@v4 + with: + path: ~\AppData\Local\pip\Cache + key: ${{inputs.os}}-python-${{inputs.python-version}} + + - name: Install Kedro and other Python Dependencies + uses: "./.github/actions/install_kedro_and_python_dependencies" + + - name: Setup Node.js and Install Dependencies + uses: "./.github/actions/install_node_dependencies" + + - name: Build React application + run: |- + node -v + make build + shell: bash \ No newline at end of file diff --git a/.github/styles/Kedro-viz/Spellings.yml b/.github/styles/Kedro-viz/Spellings.yml new file mode 100644 index 0000000000..6a1c9c9b64 --- /dev/null +++ b/.github/styles/Kedro-viz/Spellings.yml @@ -0,0 +1,5 @@ +extends: spelling +message: "Did you really mean '%s'?" +level: warning +ignore: + - Kedro-viz/ignore.txt \ No newline at end of file diff --git a/.github/styles/Kedro-viz/abbreviations.yml b/.github/styles/Kedro-viz/abbreviations.yml new file mode 100644 index 0000000000..986b14c67b --- /dev/null +++ b/.github/styles/Kedro-viz/abbreviations.yml @@ -0,0 +1,11 @@ +extends: substitution +message: "Use '%s' instead of abbreviations like '%s'." +ignorecase: true +level: warning +nonword: true +action: + name: replace +swap: + '\b(?:eg|e\.g\.|eg\.)[\s,]': for example + '\b(?:ie|i\.e\.|ie\.)[\s,]': that is + '\b(?:etc)[\s\n,.]': and more \ No newline at end of file diff --git a/.github/styles/Kedro-viz/gender.yml b/.github/styles/Kedro-viz/gender.yml new file mode 100644 index 0000000000..2e00326f0f --- /dev/null +++ b/.github/styles/Kedro-viz/gender.yml @@ -0,0 +1,13 @@ +extends: existence +message: "Use a gender-neutral pronoun instead of '%s'." +level: error +ignorecase: true +tokens: + - he/she + - s/he + - \(s\)he + - \bhe\b + - \bhim\b + - \bhis\b + - \bshe\b + - \bher\b \ No newline at end of file diff --git a/.github/styles/Kedro-viz/headings.yml b/.github/styles/Kedro-viz/headings.yml new file mode 100644 index 0000000000..e250d1df25 --- /dev/null +++ b/.github/styles/Kedro-viz/headings.yml @@ -0,0 +1,56 @@ +extends: capitalization +message: "'%s' should use sentence-style capitalization." +level: warning +scope: heading +match: $sentence +exceptions: + - Kedro + - Kedro-Viz + - Airflow + - Amazon + - Amazon Web Services + - AWS Step Functions + - AWS Systems Manager + - Azure + - Azure App Service + - Azure App Service Plan + - Azure Blob Storage + - Azure Event Hub + - CI/CD + - DataCatalog + - Data Catalog + - Docker + - Docker Compose + - Docker Swarm + - Dockerfile + - GitHub + - GitHub Actions + - Google + - Google Analytics + - Google Cloud + - Google Cloud Functions + - GraphQL + - Hook + - Hooks + - IDs + - Jenkins + - JFrog + - JFrog Artifactory + - Jira + - Kafka + - Kubernetes + - Kubernetes Engine + - Kubernetes Pod + - Kubernetes Service + - Lambda + - Linux + - MySQL + - Python + - QuantumBlack + - QuantumBlack Labs + - Red Hat + - Redis + - Slack + - Ubuntu + - Unix + - URLs \ No newline at end of file diff --git a/.github/styles/Kedro-viz/ignore.txt b/.github/styles/Kedro-viz/ignore.txt new file mode 100644 index 0000000000..21d52adda3 --- /dev/null +++ b/.github/styles/Kedro-viz/ignore.txt @@ -0,0 +1,33 @@ +Kedro +kedro +Kedro's +kedro's +Kubeflow +Databricks +Conda +conda +Cookiecutter +config +fsspec +Kaggle +namespace +namespaces +namespaced +regressors +repo +Repo +dbx +MLflow +csv +yaml +matplotlib +Matplotlib +IPython +APIs +networkx +Plotly +Pylint +SQLAlchemy +Astro +Xebia +pytest \ No newline at end of file diff --git a/.github/styles/Kedro-viz/inclusive.yml b/.github/styles/Kedro-viz/inclusive.yml new file mode 100644 index 0000000000..aa79a3f2c1 --- /dev/null +++ b/.github/styles/Kedro-viz/inclusive.yml @@ -0,0 +1,12 @@ +extends: substitution +message: "Use '%s' instead of '%s'." +ignorecase: true +level: error +action: + name: replace +swap: + # bad: good + 'black ?list': 'disallow list|exclude list' + 'master': primary + 'slave': secondary + 'white ?list': 'allow list|include list' \ No newline at end of file diff --git a/.github/styles/Kedro-viz/links.yml b/.github/styles/Kedro-viz/links.yml new file mode 100644 index 0000000000..aa79a3f2c1 --- /dev/null +++ b/.github/styles/Kedro-viz/links.yml @@ -0,0 +1,12 @@ +extends: substitution +message: "Use '%s' instead of '%s'." +ignorecase: true +level: error +action: + name: replace +swap: + # bad: good + 'black ?list': 'disallow list|exclude list' + 'master': primary + 'slave': secondary + 'white ?list': 'allow list|include list' \ No newline at end of file diff --git a/.github/styles/Kedro-viz/oxfordcomma.yml b/.github/styles/Kedro-viz/oxfordcomma.yml new file mode 100644 index 0000000000..5b9c7dbff8 --- /dev/null +++ b/.github/styles/Kedro-viz/oxfordcomma.yml @@ -0,0 +1,6 @@ +extends: existence +message: "Use the Oxford comma in '%s'." +scope: sentence +level: suggestion +tokens: + - '(?:[^,]+,){1,}\s\w+\s(?:and|or)' \ No newline at end of file diff --git a/.github/styles/Kedro-viz/pronouns.yml b/.github/styles/Kedro-viz/pronouns.yml new file mode 100644 index 0000000000..d6ad378172 --- /dev/null +++ b/.github/styles/Kedro-viz/pronouns.yml @@ -0,0 +1,11 @@ +extends: existence +message: "Avoid first-person singular pronouns such as '%s'." +level: warning +nonword: true +tokens: + - (?<=^|\s)I(?=\s) + - (?<=^|\s)I,(?=\s) + - \bI'm\b + - (?<=\s)[Mm]e\b + - (?<=\s)[Mm]y\b + - (?<=\s)[Mm]ine\b \ No newline at end of file diff --git a/.github/styles/Kedro-viz/sentencelength.yml b/.github/styles/Kedro-viz/sentencelength.yml new file mode 100644 index 0000000000..4a8042ea13 --- /dev/null +++ b/.github/styles/Kedro-viz/sentencelength.yml @@ -0,0 +1,11 @@ +extends: occurrence +message: "Try to keep your sentence length to 30 words or fewer." +level: suggestion +# Here, we're counting the number of words +# in a sentence. +# +# If there are more than 30, we'll flag it. +scope: sentence +ignorecase: false +max: 30 +token: (\w+) \ No newline at end of file diff --git a/.github/styles/Kedro-viz/toowordy.yml b/.github/styles/Kedro-viz/toowordy.yml new file mode 100644 index 0000000000..320bfaf3fe --- /dev/null +++ b/.github/styles/Kedro-viz/toowordy.yml @@ -0,0 +1,218 @@ +# Write Good's "Too wordy" rule https://github.com/testthedocs/vale-styles/blob/master/write-good/TooWordy.yml +extends: existence +message: "'%s' is too wordy" +ignorecase: true +level: warning +tokens: + - a number of + - abundance + - accede to + - accelerate + - accentuate + - accompany + - accomplish + - accorded + - accrue + - acquiesce + - acquire + - adjacent to + - adjustment + - admissible + - advantageous + - adversely impact + - advise + - aforementioned + - aggregate + - aircraft + - all of + - all things considered + - alleviate + - allocate + - along the lines of + - already existing + - alternatively + - amazing + - ameliorate + - anticipate + - apparent + - appreciable + - as a matter of fact + - as a means of + - as far as I'm concerned + - as of yet + - as to + - as yet + - ascertain + - assistance + - at the present time + - at this time + - attain + - attributable to + - because of the fact that + - belated + - benefit from + - bestow + - by means of + - by virtue of the fact that + - by virtue of + - cease + - close proximity + - commence + - comply with + - concerning + - consequently + - consolidate + - constitutes + - demonstrate + - depart + - designate + - discontinue + - due to the fact that + - each and every + - economical + - eliminate + - elucidate + - employ + - endeavor + - enumerate + - equitable + - equivalent + - evaluate + - evidenced + - exclusively + - expedite + - expend + - expiration + - facilitate + - factual evidence + - feasible + - finalise + - first and foremost + - for all intents and purposes + - for the most part + - for the purpose of + - forfeit + - formulate + - have a tendency to + - honest truth + - however + - if and when + - impacted + - implement + - in a manner of speaking + - in a timely manner + - in a very real sense + - in accordance with + - in addition + - in all likelihood + - in an effort to + - in between + - in excess of + - in lieu of + - in light of the fact that + - in many cases + - in my opinion + - in order to + - in regard to + - in some instances + - in terms of + - in the case of + - in the event that + - in the final analysis + - in the nature of + - in the near future + - in the process of + - inception + - incumbent upon + - indicate + - indication + - initiate + - irregardless + - is applicable to + - is authorised to + - is responsible for + - it is essential + - it seems that + - it was + - magnitude + - maximum + - methodology + - minimise + - modify + - monitor + - multiple + - necessitate + - nevertheless + - not certain + - not many + - not often + - not unless + - not unlike + - notwithstanding + - null and void + - numerous + - objective + - obligate + - obtain + - on the contrary + - on the other hand + - one particular + - optimum + - overall + - owing to the fact that + - participate + - particulars + - pass away + - pertaining to + - point in time + - portion + - possess + - preclude + - previously + - prior to + - prioritise + - procure + - proficiency + - provided that + - purchase + - put simply + - readily apparent + - refer back + - regarding + - relocate + - remainder + - remuneration + - requirement + - reside + - residence + - retain + - satisfy + - shall + - should you wish + - similar to + - solicit + - span across + - strategise + - subsequent + - substantial + - successfully complete + - sufficient + - terminate + - the month of + - the point I am trying to make + - therefore + - time period + - took advantage of + - transmit + - transpire + - type of + - until such time as + - utilisation + - utilise + - validate + - various different + - what I mean to say is + - whether or not + - with respect to + - with the exception of + - witnessed \ No newline at end of file diff --git a/.github/styles/Kedro-viz/ukspelling.yml b/.github/styles/Kedro-viz/ukspelling.yml new file mode 100644 index 0000000000..10c8439ca8 --- /dev/null +++ b/.github/styles/Kedro-viz/ukspelling.yml @@ -0,0 +1,28 @@ +extends: existence +message: "In general, use UK English spelling instead of '%s'." +link: 'https://github.com/kedro-org/kedro/wiki/Kedro-documentation-style-guide' +ignorecase: true +level: warning +tokens: + - '(?:\w+)zation' + - '(?:\w+)izing' + - '(?:\w+)izer' + - '(?:\w+)ized' + - '(?:\w+)ize' + - '(?:\w+)log' + - '(?:\w+)lor' + - '(?:\w+)lyze' +exceptions: + - backlog + - blog + - capsize + - catalog + - Catalog + - DataCatalog + - dialog + - log + - maize + - prize + - seize + - size + - tailor \ No newline at end of file diff --git a/.github/styles/Kedro-viz/weaselwords.yml b/.github/styles/Kedro-viz/weaselwords.yml new file mode 100644 index 0000000000..bb8001fbc1 --- /dev/null +++ b/.github/styles/Kedro-viz/weaselwords.yml @@ -0,0 +1,207 @@ +# Write Good's Weasel Words rule https://github.com/testthedocs/vale-styles/blob/master/write-good/Weasel.yml +extends: existence +message: "'%s' is a weasel word!" +ignorecase: true +level: warning +tokens: + - absolutely + - accidentally + - additionally + - allegedly + - alternatively + - angrily + - anxiously + - approximately + - awkwardly + - badly + - barely + - beautifully + - blindly + - boldly + - bravely + - brightly + - briskly + - bristly + - bubbly + - busily + - calmly + - carefully + - carelessly + - cautiously + - cheerfully + - clearly + - closely + - coldly + - completely + - consequently + - correctly + - courageously + - crinkly + - cruelly + - crumbly + - cuddly + - currently + - daily + - daringly + - deadly + - definitely + - deliberately + - doubtfully + - dumbly + - eagerly + - easily + - elegantly + - enormously + - enthusiastically + - equally + - especially + - eventually + - exactly + - exceedingly + - exclusively + - extremely + - fairly + - faithfully + - fatally + - fiercely + - finally + - fondly + - few + - foolishly + - fortunately + - frankly + - frantically + - generously + - gently + - giggly + - gladly + - gracefully + - greedily + - happily + - hardly + - hastily + - healthily + - heartily + - helpfully + - honestly + - hourly + - hungrily + - hurriedly + - immediately + - impatiently + - inadequately + - ingeniously + - innocently + - inquisitively + - interestingly + - irritably + - jiggly + - joyously + - justly + - kindly + - largely + - lately + - lazily + - likely + - literally + - lonely + - loosely + - loudly + - loudly + - luckily + - madly + - many + - mentally + - mildly + - monthly + - mortally + - mostly + - mysteriously + - neatly + - nervously + - nightly + - noisily + - normally + - obediently + - occasionally + - only + - openly + - painfully + - particularly + - patiently + - perfectly + - politely + - poorly + - powerfully + - presumably + - previously + - promptly + - punctually + - quarterly + - quickly + - quietly + - rapidly + - rarely + - really + - recently + - recklessly + - regularly + - remarkably + - relatively + - reluctantly + - repeatedly + - rightfully + - roughly + - rudely + - sadly + - safely + - selfishly + - sensibly + - seriously + - sharply + - shortly + - shyly + - significantly + - silently + - simply + - sleepily + - slowly + - smartly + - smelly + - smoothly + - softly + - solemnly + - sparkly + - speedily + - stealthily + - sternly + - stupidly + - substantially + - successfully + - suddenly + - surprisingly + - suspiciously + - swiftly + - tenderly + - tensely + - thoughtfully + - tightly + - timely + - truthfully + - unexpectedly + - unfortunately + - usually + - very + - victoriously + - violently + - vivaciously + - warmly + - waverly + - weakly + - wearily + - weekly + - wildly + - wisely + - worldly + - wrinkly + - yearly \ No newline at end of file diff --git a/.github/styles/Kedro-viz/words.yml b/.github/styles/Kedro-viz/words.yml new file mode 100644 index 0000000000..0a9a1d3433 --- /dev/null +++ b/.github/styles/Kedro-viz/words.yml @@ -0,0 +1,53 @@ +extends: substitution +message: "Use '%s' instead of '%s'." +ignorecase: false +level: warning +action: + name: replace +swap: + # bad: good + 'acknowledgement': 'acknowledgment' + 'auto-complete': 'autocomplete' + 'a number of': 'few|several|many' + 'and/or': 'and|or|either or' + 'back end': 'backend' + 'bear in mind': 'keep in mind' + 'culprit': 'cause' + 'data set': 'dataset' + 'drill down|drilling down|drill into|drilling into': 'examine|investigate|analyze' + 'figure out': 'determine' + 'fine tune|fine-tune': 'customize|optimize|refine' + 'for the most part': 'generally|usually' + 'front end': 'frontend' + 'highly|very': '' + 'hit': 'click|select' + 'in order to': 'to' + 'keep in mind': 'consider' + 'left up to': 'determined by' + 'leverage': 'use' + 'multi-alert': 'multi alert' + 'Note that': '**Note**:' + 'obviously|obvious': '' + 'on the fly': 'real-time|real time' + 'once': 'after' + 'play a hand': 'influence' + 'please|just': '' + 'easily|easy': '' + 'quickly|quick': '' + 'screen board': 'screenboard' + 'simply|simple': '' + 'stand for': 'represents|means' + 'reenable': 're-enable' + 'run time': 'runtime' + 'refer to|visit': 'see|read|follow' + 'time board': 'timeboard' + 'time series': 'timeseries' + 'toplist': 'top list' + 'tradeoff': 'trade-off' + 'turnkey': 'ready to use' + 'under the hood': '' + 'utilize': 'use' + 'via': 'with|through' + 'visit': 'see|read' + 'webserver': 'web server' + 'web site': 'website' \ No newline at end of file diff --git a/.github/workflows/all-checks.yml b/.github/workflows/all-checks.yml new file mode 100644 index 0000000000..4c981d6c8b --- /dev/null +++ b/.github/workflows/all-checks.yml @@ -0,0 +1,44 @@ +name: Run all checks on Kedro-Viz +# Runs end-to-end tests, unit tests, linting and JavaScript +# linting & tests on Kedro-Viz for different +# operating systems and Python versions. + +on: + workflow_call: + workflow_dispatch: + schedule: + # Run every day at 1:00 AM(UTC time) + - cron: 0 1 * * * +jobs: + e2e_tests: + strategy: + matrix: + os: [ windows-latest, ubuntu-latest ] + python-version: [ "3.9", "3.10", "3.11" ] + uses: ./.github/workflows/e2e-tests.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + + unit_tests: + strategy: + matrix: + os: [ windows-latest, ubuntu-latest ] + python-version: [ "3.9", "3.10", "3.11" ] + uses: ./.github/workflows/unit-tests.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + + lint: + strategy: + matrix: + os: [ ubuntu-latest ] + python-version: [ "3.9", "3.10", "3.11" ] + uses: ./.github/workflows/lint.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + + javascript_lint_and_tests: + uses: ./.github/workflows/javascript-lint-and-tests.yml diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml new file mode 100644 index 0000000000..70a8691854 --- /dev/null +++ b/.github/workflows/build-backend.yml @@ -0,0 +1,44 @@ +name: Build backend +# Runs end-to-end tests, unit tests, and linting on the backend code +# for different operating systems and Python versions. + +on: + push: + paths: + - 'package/**' + - '.github/**' + pull_request: + paths: + - 'package/**' + - '.github/**' + workflow_dispatch: +jobs: + e2e_tests: + strategy: + matrix: + os: [ windows-latest, ubuntu-latest ] + python-version: [ "3.9", "3.10", "3.11" ] + uses: ./.github/workflows/e2e-tests.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + + unit_tests: + strategy: + matrix: + os: [ windows-latest, ubuntu-latest ] + python-version: [ "3.9", "3.10", "3.11" ] + uses: ./.github/workflows/unit-tests.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + + lint: + strategy: + matrix: + os: [ ubuntu-latest ] + python-version: [ "3.9", "3.10", "3.11" ] + uses: ./.github/workflows/lint.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/check-release.yml b/.github/workflows/check-release.yml index 8b73b1cc8d..9b4c94df69 100644 --- a/.github/workflows/check-release.yml +++ b/.github/workflows/check-release.yml @@ -4,17 +4,21 @@ on: push: branches: - main + workflow_dispatch: + +env: + PYTHON_VERSION: '3.9' jobs: check-version: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: ${{ env.PYTHON_VERSION }} - name: Install dependencies run: | @@ -36,17 +40,25 @@ jobs: package_name: ${{ steps.version_check.outputs.package_name }} package_version: ${{ steps.version_check.outputs.package_version }} - prepare-release: - needs: [check-version] + test-kedro-viz: + needs: check-version if: ${{ needs.check-version.outputs.new_release == 'true' }} + uses: ./.github/workflows/all-checks.yml + + prepare-release: + needs: [check-version, test-kedro-viz] + if: | + !contains(needs.*.result, 'failure') && + !contains(needs.*.result, 'cancelled') && + needs.check-version.outputs.new_release == 'true' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: ${{ env.PYTHON_VERSION }} - name: Extract release notes from ${{needs.check-version.outputs.package_name}}/RELEASE.md id: extract diff --git a/.github/workflows/deploy-demo.yml b/.github/workflows/deploy-demo.yml new file mode 100644 index 0000000000..80c188adf0 --- /dev/null +++ b/.github/workflows/deploy-demo.yml @@ -0,0 +1,41 @@ +name: Deploy demo +# Builds a Docker image of the demo project, pushes it to +# AWS ECR, and deploys it to AWS Lightsail. + +on: + workflow_dispatch: + push: + branches: + - demo +jobs: + deploy_demo: + runs-on: ubuntu-latest + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup environment + run: |- + cd demo-project + echo "AWS_ECR_URL=public.ecr.aws/g0x0s3o2/kedro-viz-live-demo" >> $GITHUB_ENV + echo "KEDRO_VIZ_VERSION=$(cat .version)" >> $GITHUB_ENV + + - name: Install AWS CLI + run: pip3 install awscli + + - name: Build demo container image + run: |- + cd demo-project + echo "kedro_viz==$KEDRO_VIZ_VERSION" >> src/docker_requirements.txt + docker build -t $AWS_ECR_URL:$KEDRO_VIZ_VERSION . + aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws + docker push $AWS_ECR_URL:$KEDRO_VIZ_VERSION + + - name: Create a new lightsail deployment + run: |- + cd demo-project + # run https://docs.aws.amazon.com/cli/latest/reference/lightsail/create-container-service-deployment.html#create-container-service-deployment + aws lightsail create-container-service-deployment --region eu-west-2 --cli-input-json file://./lightsail.json diff --git a/.github/workflows/docs-language-linter.yml b/.github/workflows/docs-language-linter.yml new file mode 100644 index 0000000000..20980c3f49 --- /dev/null +++ b/.github/workflows/docs-language-linter.yml @@ -0,0 +1,17 @@ +name: Language Linter for Kedro-viz Docs +on: + workflow_dispatch: + pull_request: + paths: + - "docs/**" + - '**.md' + +jobs: + vale: + name: runner / vale + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: errata-ai/vale-action@reviewdog + with: + reporter: github-pr-check \ No newline at end of file diff --git a/.github/workflows/docs-only-checks.yml b/.github/workflows/docs-only-checks.yml new file mode 100644 index 0000000000..bd09e387e8 --- /dev/null +++ b/.github/workflows/docs-only-checks.yml @@ -0,0 +1,26 @@ +name: Run linter on Kedro-viz Docs + +on: + push: + branches: + - main + paths: + - "docs/**" + - '**.md' + pull_request: + branches: + - main + paths: + - "docs/**" + - '**.md' + +jobs: + lint: + strategy: + matrix: + os: [ ubuntu-latest ] + python-version: ["3.9", "3.10", "3.11"] + uses: ./.github/workflows/lint.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} \ No newline at end of file diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml new file mode 100644 index 0000000000..5eee3d98cd --- /dev/null +++ b/.github/workflows/e2e-tests.yml @@ -0,0 +1,39 @@ +name: Run e2e tests on Kedro-Viz +# Runs end-to-end tests on Kedro-Viz for different +# operating systems and Python versions. + +on: + workflow_call: + inputs: + os: + type: string + python-version: + type: string +jobs: + e2e_tests: + runs-on: ${{ inputs.os }} + + # below condition checks if the operating system is Ubuntu, or + # if the operating system is Windows and the branch is main/demo + if: > + inputs.os == 'ubuntu-latest' || + ( + ( + github.ref == 'refs/heads/main' || + github.ref == 'refs/heads/demo' + ) && + inputs.os == 'windows-latest' + ) + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Tests + uses: "./.github/actions/setup_tests" + with: + os: ${{ inputs.os }} + python-version: ${{ inputs.python-version }} + + - name: Run all end to end tests + run: make e2e-tests \ No newline at end of file diff --git a/.github/workflows/javascript-lint-and-tests.yml b/.github/workflows/javascript-lint-and-tests.yml new file mode 100644 index 0000000000..ca880da94e --- /dev/null +++ b/.github/workflows/javascript-lint-and-tests.yml @@ -0,0 +1,64 @@ +name: Run javascript linters and tests on Kedro-Viz +# Runs JavaScript linting, unit tests, and end-to-end tests on +# Kedro-Viz for ubuntu-latest operating systems and Python 3.9. + +on: + push: + paths-ignore: + - 'package/**' + pull_request: + paths-ignore: + - 'package/**' + workflow_dispatch: + workflow_call: + +env: + PYTHON_VERSION: '3.9' + +jobs: + javascript_lint_and_tests: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Cache python packages for Linux + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ubuntu-latest-python-3.9 + + - name: Install Kedro and other Python Dependencies + uses: "./.github/actions/install_kedro_and_python_dependencies" + + - name: Setup Node.js and Install Dependencies + uses: "./.github/actions/install_node_dependencies" + + - name: Setup Cypress requirements + run: |- + sudo sed -i 's/archive.ubuntu.com/us-east-1.ec2.archive.ubuntu.com/g' /etc/apt/sources.list + sudo apt-get update + sudo apt-get install libgtk2.0-0 libgtk-3-0 libgbm-dev libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 libxtst6 xauth xvfb + + - name: Test lib transpilation + run: npm run lib + + - name: Test JS library imports + run: |- + npm run lib-test:setup + cd tools/test-lib/react-app + npm run test:ci + + - name: Run Eslint + run: npm run lint + + - name: Run JavaScript tests + run: npm run test:ci + + - name: Run Javascript end to end tests + run: npm run cy:ci diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000000..02489d36dd --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,45 @@ +name: Run linters on Kedro-Viz +# Runs secret scan, security scan, GraphQL schema check, +# and Python formatters and linters on Kedro-Viz for +# different operating systems and Python versions. + +on: + workflow_call: + inputs: + os: + type: string + python-version: + type: string +jobs: + lint: + runs-on: ${{ inputs.os }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python ${{inputs.python-version}} + uses: actions/setup-python@v5 + with: + python-version: ${{inputs.python-version}} + + - name: Cache python packages for Linux + if: inputs.os == 'ubuntu-latest' + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{inputs.os}}-python-${{inputs.python-version}} + + - name: Install Kedro and other Python Dependencies + uses: "./.github/actions/install_kedro_and_python_dependencies" + + - name: Run secret scan + run: make secret-scan + + - name: Run security scan + run: make security-scan + + - name: Verify GraphQL schema is up to date + run: make schema-check + + - name: Run Python formatters and linters + run: make format-check lint-check diff --git a/.github/workflows/merge-gatekeeper.yml b/.github/workflows/merge-gatekeeper.yml new file mode 100644 index 0000000000..2d1c5a22a5 --- /dev/null +++ b/.github/workflows/merge-gatekeeper.yml @@ -0,0 +1,30 @@ +name: Merge Gatekeeper + +on: + pull_request: + branches: + - main + +env: + TIMEOUT: 3600 + INTERVAL: 30 + +jobs: + merge-gatekeeper: + runs-on: ubuntu-latest + # Restrict permissions of the GITHUB_TOKEN. + # Docs: https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs + permissions: + checks: read + statuses: read + steps: + - name: Run Merge Gatekeeper + # NOTE: v1 is updated to reflect the latest v1.x.y. Please use any tag/branch that suits your needs: + # https://github.com/upsidr/merge-gatekeeper/tags + # https://github.com/upsidr/merge-gatekeeper/branches + uses: upsidr/merge-gatekeeper@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + timeout: ${{ env.TIMEOUT }} + interval: ${{ env.INTERVAL }} + ignored: 'ci/circleci: win_unit_tests-3.9,ci/circleci: win_unit_tests-3.10,ci/circleci: win_unit_tests-3.8,ci/circleci: unit_tests-3.10,ci/circleci: unit_tests-3.8,ci/circleci: unit_tests-3.9,ci/circleci: win_e2e_tests-3.10,ci/circleci: win_e2e_tests-3.9,ci/circleci: win_e2e_tests-3.8,ci/circleci: e2e_tests-3.8,ci/circleci: e2e_tests-3.9,ci/circleci: e2e_tests-3.10,ci/circleci: lint-3.8,ci/circleci: lint-3.9,ci/circleci: lint-3.10,ci/circleci: javascript_lint_and_tests,ci/circleci: check-updated-files,ci/circleci: all_circleci_checks_succeeded,CircleCI Pipeline' \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000000..abd88b88bb --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,59 @@ +name: Release to NPM and PyPI +# Builds and releases the application to NPM and PyPI +# when a new release is published or the workflow is manually triggered. + +on: + release: + types: [published] + workflow_dispatch: + +env: + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + NODE_VERSION: 18.20.0 + REGISTRY_URL: 'https://registry.npmjs.org' + +jobs: + release_to_npm: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + registry-url: ${{ env.REGISTRY_URL }} + + - name: Install Node dependencies + run: npm install + + - name: Build React application + run: |- + node -v + make build + + - name: Publish to npm + run: npm publish + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + + release_to_pypi: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Tests + uses: "./.github/actions/setup_tests" + + - name: Install wheel + run: pip install wheel + + - name: Make Python package + run: make package + + - name: Publish distribution to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: package/dist + password: ${{ secrets.KEDRO_VIZ_PYPI_TOKEN }} diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml new file mode 100644 index 0000000000..fe970683c3 --- /dev/null +++ b/.github/workflows/unit-tests.yml @@ -0,0 +1,39 @@ +name: Run unit-tests on Kedro-Viz +# Runs unit tests on Kedro-Viz across different +# OS and Python versions after environment setup. + +on: + workflow_call: + inputs: + os: + type: string + python-version: + type: string +jobs: + unit_tests: + runs-on: ${{ inputs.os }} + + # below condition checks if the operating system is Ubuntu, or + # if the operating system is Windows and the branch is main/demo + if: > + inputs.os == 'ubuntu-latest' || + ( + ( + github.ref == 'refs/heads/main' || + github.ref == 'refs/heads/demo' + ) && + inputs.os == 'windows-latest' + ) + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Tests + uses: "./.github/actions/setup_tests" + with: + os: ${{ inputs.os }} + python-version: ${{ inputs.python-version }} + + - name: Run Python tests + run: make pytest \ No newline at end of file diff --git a/.gitpod.yml b/.gitpod.yml index c9ffc8cfbc..c7986d0e48 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,5 +1,5 @@ -image: gitpod/workspace-full:2023-01-16-03-31-28 - +image: gitpod/workspace-full:2023-09-29-11-03-42 # image with Python 3.11 and Node 18 + tasks: - name: frontend init: | @@ -14,7 +14,7 @@ tasks: - name: backend before: echo PIP_USER=no >> ~/.bashrc && export PIP_USER=no init: | - pip install "pip==23.0.1" --user + pip install "pip==23.3.1" --user pip install -r package/test_requirements.txt -r demo-project/src/docker_requirements.txt --user # Install latest kedro and all its dependencies. pip install https://github.com/kedro-org/kedro/archive/main.zip --user diff --git a/.readthedocs.yml b/.readthedocs.yml index 57e08a91fc..4faaef3ab6 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -8,7 +8,13 @@ build: os: ubuntu-22.04 tools: python: "3.9" - nodejs: "19" + jobs: + post_checkout: + - git fetch --unshallow || true + pre_build: + - pip freeze + - python -m sphinx -WETan -j auto -D language=en -b linkcheck -d _build/doctrees docs/source _build/linkcheck + sphinx: builder: html diff --git a/.vale.ini b/.vale.ini new file mode 100644 index 0000000000..5f4e3c3a7c --- /dev/null +++ b/.vale.ini @@ -0,0 +1,7 @@ +StylesPath = .github/styles + +MinAlertLevel = suggestion + +[*.md] +BasedOnStyles = Vale, Kedro-viz +Vale.Spelling = NO \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 08673a0f62..89c575d110 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -318,6 +318,23 @@ Python dependencies in Kedro-Viz are usually updated automatically through tools If the lower-bound e2e test fails, it indicates that some dependencies may not work correctly with Kedro-Viz. To resolve this, update the problematic dependency in both `requirements.txt` and `lower-requirements.txt`. +### Testing UI using Gitpod +**Please ensure you have installed Gitpod on your browser extensions - [Gitpod browser extension](https://www.gitpod.io/docs/configure/user-settings/browser-extension)** + +**1. Once you open a pull request, click on the `Open` icon on the right** +Screenshot 2024-04-17 at 15 54 36 + + +**2. It will open the gitpod workspace automatically** +Screenshot 2024-04-17 at 15 57 41 + +**3. Please wait until the frontend and backend builds are successful. You can check it by clicking on `frontend` and `backend` tabs in the terminal. (Estimated build time : 5 minutes)** +Screenshot 2024-04-17 at 15 59 46 +Screenshot 2024-04-17 at 16 04 38 + +**4. Unblock pop-up ads and refresh your browser. Kedro-Viz should be running on your gitpod workspace** +Screenshot 2024-04-17 at 16 06 05 + # Release guidelines - Practice frequent, staggered releases. diff --git a/RELEASE.md b/RELEASE.md index 48cba0cad2..842b09c916 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -5,6 +5,32 @@ Please follow the established format: - Use present tense (e.g. 'Add new feature') - Include the ID number for the related PR (or PRs) in parentheses --> +# Next release + +# Release 9.1.0 + +## Major features and improvements +- Introduce the toggle to expand and collapse all pipelines button in the utility bar. (#1858) +- Allow Kedro-Viz commands to run from any sub directory within Kedro project. (#1871) + +## Bug fixes and other changes +- Fix broken URL when active pipeline name changes on initial load. (#1914) +- Fix bug related to tag filtering and sharing with stateful URL. (#1878) +- Update settings panel design (#1875) +- Migrate from CircleCi to GitHub Actions. (#1876) +- Include expandAllPipelines in initial state. (#1896) +- Refactor backend integration with Kedro by replacing bootstrap_project with configure_project. (#1796) +- Enhance kedro-viz doc integration. (#1874) +- Enhance Kedro-Viz documentation by using Kedro-sphinx-theme. (#1898) +- Remove default props from functional components. (#1906) +- Fix for schema change in strawberry-graphql JSON scalar. (#1903) +- Fix messaging level when package compatibility is not satisfied. (#1904) +- Upgrade GitPod to include Node 18 and Python 3.11. (#1862) +- Add utility functions related to Transcoding to Kedro viz. (#1928) +- Include JSON dataset in the demo-project. (#1930) +- Update intersphinx_mapping of kedro-datasets. (#1911) +- Add GitPod to the local hosts list. (#1923) + # Release 9.0.0 ## Major features and improvements diff --git a/cypress/tests/ui/toolbar/global-toolbar.cy.js b/cypress/tests/ui/toolbar/global-toolbar.cy.js index 95f07b127d..a0bcc80e91 100644 --- a/cypress/tests/ui/toolbar/global-toolbar.cy.js +++ b/cypress/tests/ui/toolbar/global-toolbar.cy.js @@ -146,38 +146,24 @@ describe('Global Toolbar', () => { }); }); - it('verifies that users can expand all modular pipelines on first load. #TC-7', () => { + it('verifies that users can expand all modular pipelines directly from the toolbar. #TC-7', () => { const modularPipelineChildNodeText = 'Create Derived Features'; - // Alias - cy.get('[data-test="pipeline-toggle-input-expandAllPipelines"]').as( - 'isExpandAllPipelinesCheckBox' - ); + // Alias for better readability + cy.get('[data-test="expand-all-pipelines-toggle"]').as('expandAllPipelinesToggle'); // Assert before action - cy.get('@isExpandAllPipelinesCheckBox').should('not.be.checked'); - cy.get('.pipeline-node__text').should( - 'not.contain', - modularPipelineChildNodeText - ); - cy.get('[role="treeitem"]') - .should('have.attr', 'aria-expanded') - .should('eq', 'false'); + cy.get('@expandAllPipelinesToggle').should('not.be.checked'); + cy.get('.pipeline-node__text').should('not.contain', modularPipelineChildNodeText); + cy.get('[role="treeitem"]').should('have.attr', 'aria-expanded', 'false'); - // Action - cy.get('@isExpandAllPipelinesCheckBox').check({ force: true }); - cy.get('[data-test="Apply changes and close in Settings Modal"]').click({ - force: true, - }); + // Action - toggling the expand all pipelines directly from the toolbar + cy.get('@expandAllPipelinesToggle').click(); // Assert after action - cy.get('[role="treeitem"]', { timeout: 5000 }) - .should('have.attr', 'aria-expanded') - .should('eq', 'true'); - cy.get('.pipeline-node__text').should( - 'contain', - modularPipelineChildNodeText - ); + cy.get('[role="treeitem"]') + .should('have.attr', 'aria-expanded', 'true'); + cy.get('.pipeline-node__text').should('contain', modularPipelineChildNodeText); }); }); }); diff --git a/demo-project/.version b/demo-project/.version index f7ee06693c..47da986f86 100755 --- a/demo-project/.version +++ b/demo-project/.version @@ -1 +1 @@ -9.0.0 +9.1.0 diff --git a/demo-project/conf/base/catalog_08_reporting.yml b/demo-project/conf/base/catalog_08_reporting.yml index 2cd3d21411..148c0e1246 100644 --- a/demo-project/conf/base/catalog_08_reporting.yml +++ b/demo-project/conf/base/catalog_08_reporting.yml @@ -38,3 +38,10 @@ reporting.confusion_matrix: type: matplotlib.MatplotlibWriter filepath: ${_base_location}/08_reporting/confusion_matrix.png versioned: true + +reporting.top_shuttle_data: + type: json.JSONDataset + filepath: ${_base_location}/08_reporting/top_shuttle_data.json + metadata: + kedro-viz: + layer: reporting diff --git a/demo-project/data/08_reporting/top_shuttle_data.json b/demo-project/data/08_reporting/top_shuttle_data.json new file mode 100644 index 0000000000..4e6afa6e00 --- /dev/null +++ b/demo-project/data/08_reporting/top_shuttle_data.json @@ -0,0 +1,147 @@ +[ + { + "shuttle_id": 63561, + "shuttle_location": "Niue", + "shuttle_type": "Type V5", + "engine_type": "Quantum", + "engine_vendor": "ThetaBase Services", + "engines": 1.0, + "passenger_capacity": 2, + "cancellation_policy": "strict", + "crew": 1.0, + "d_check_complete": false, + "moon_clearance_complete": false, + "price": 1325.0, + "company_id": 35029, + "review_scores_rating": 97, + "review_scores_comfort": 10, + "review_scores_amenities": 9, + "review_scores_trip": 10, + "review_scores_crew": 10, + "review_scores_location": 9, + "review_scores_price": 10, + "number_of_reviews": 133, + "reviews_per_month": 1.65, + "review_id": 1, + "company_rating": 1.0, + "company_location": "Niue", + "total_fleet_count": 4.0, + "iata_approved": false + }, + { + "shuttle_id": 53260, + "shuttle_location": "Niue", + "shuttle_type": "Type V5", + "engine_type": "Quantum", + "engine_vendor": "Banks, Wood and Phillips", + "engines": 1.0, + "passenger_capacity": 2, + "cancellation_policy": "strict", + "crew": 1.0, + "d_check_complete": false, + "moon_clearance_complete": false, + "price": 1325.0, + "company_id": 35029, + "review_scores_rating": 98, + "review_scores_comfort": 10, + "review_scores_amenities": 9, + "review_scores_trip": 10, + "review_scores_crew": 10, + "review_scores_location": 9, + "review_scores_price": 10, + "number_of_reviews": 37, + "reviews_per_month": 0.48, + "review_id": 1354, + "company_rating": 1.0, + "company_location": "Niue", + "total_fleet_count": 4.0, + "iata_approved": false + }, + { + "shuttle_id": 51019, + "shuttle_location": "Niue", + "shuttle_type": "Type V5", + "engine_type": "Quantum", + "engine_vendor": "ThetaBase Services", + "engines": 1.0, + "passenger_capacity": 2, + "cancellation_policy": "flexible", + "crew": 1.0, + "d_check_complete": false, + "moon_clearance_complete": false, + "price": 1260.0, + "company_id": 35029, + "review_scores_rating": 92, + "review_scores_comfort": 10, + "review_scores_amenities": 9, + "review_scores_trip": 10, + "review_scores_crew": 10, + "review_scores_location": 9, + "review_scores_price": 9, + "number_of_reviews": 10, + "reviews_per_month": 0.15, + "review_id": 1985, + "company_rating": 1.0, + "company_location": "Niue", + "total_fleet_count": 4.0, + "iata_approved": false + }, + { + "shuttle_id": 53898, + "shuttle_location": "Niue", + "shuttle_type": "Type V5", + "engine_type": "Plasma", + "engine_vendor": "ThetaBase Services", + "engines": 3.0, + "passenger_capacity": 5, + "cancellation_policy": "strict", + "crew": 3.0, + "d_check_complete": false, + "moon_clearance_complete": false, + "price": 2196.0, + "company_id": 35029, + "review_scores_rating": 98, + "review_scores_comfort": 10, + "review_scores_amenities": 9, + "review_scores_trip": 10, + "review_scores_crew": 10, + "review_scores_location": 9, + "review_scores_price": 10, + "number_of_reviews": 11, + "reviews_per_month": 0.21, + "review_id": 4879, + "company_rating": 1.0, + "company_location": "Niue", + "total_fleet_count": 4.0, + "iata_approved": false + }, + { + "shuttle_id": 36260, + "shuttle_location": "Anguilla", + "shuttle_type": "Type V5", + "engine_type": "Quantum", + "engine_vendor": "ThetaBase Services", + "engines": 1.0, + "passenger_capacity": 2, + "cancellation_policy": "strict", + "crew": 1.0, + "d_check_complete": true, + "moon_clearance_complete": false, + "price": 1780.0, + "company_id": 30292, + "review_scores_rating": 90, + "review_scores_comfort": 8, + "review_scores_amenities": 9, + "review_scores_trip": 10, + "review_scores_crew": 9, + "review_scores_location": 9, + "review_scores_price": 9, + "number_of_reviews": 3, + "reviews_per_month": 0.09, + "review_id": 2, + "company_rating": 0.67, + "company_location": "Anguilla", + "total_fleet_count": 6.0, + "iata_approved": false + } +] \ No newline at end of file diff --git a/demo-project/lightsail.json b/demo-project/lightsail.json index 2eed52918e..d96bfade72 100644 --- a/demo-project/lightsail.json +++ b/demo-project/lightsail.json @@ -2,7 +2,7 @@ "serviceName": "kedro-viz-live-demo", "containers": { "kedro-viz-live-demo": { - "image": "public.ecr.aws/g0x0s3o2/kedro-viz-live-demo:9.0.0", + "image": "public.ecr.aws/g0x0s3o2/kedro-viz-live-demo:9.1.0", "ports": { "4141": "HTTP" } diff --git a/demo-project/src/demo_project/pipelines/data_ingestion/pipeline.py b/demo-project/src/demo_project/pipelines/data_ingestion/pipeline.py index 4ed0f9f676..1acbdf9531 100755 --- a/demo-project/src/demo_project/pipelines/data_ingestion/pipeline.py +++ b/demo-project/src/demo_project/pipelines/data_ingestion/pipeline.py @@ -42,7 +42,6 @@ def create_pipeline(**kwargs) -> Pipeline: inputs=["reviews", "params:typing.reviews.columns_as_floats"], outputs="int_typed_reviews", name='apply_types_to_reviews' - ), node( func=aggregate_company_data, diff --git a/demo-project/src/demo_project/pipelines/reporting/nodes.py b/demo-project/src/demo_project/pipelines/reporting/nodes.py index b758540302..cd4796ceb1 100644 --- a/demo-project/src/demo_project/pipelines/reporting/nodes.py +++ b/demo-project/src/demo_project/pipelines/reporting/nodes.py @@ -9,7 +9,7 @@ import plotly.express as px import seaborn as sn from plotly import graph_objects as go - +from typing import Dict from .image_utils import DrawTable @@ -119,3 +119,21 @@ def create_matplotlib_chart(companies: pd.DataFrame) -> plt: ) sn.heatmap(confusion_matrix, annot=True) return plt + + +def get_top_shuttles_data(model_input_table: pd.DataFrame) -> Dict: + """This function retrieves the head from the input table + and converts them into a JSON dataset. + + Args: + model_input_table (pd.DataFrame): The data to retrieve the top N rows from + top_n (int, optional): The number of top rows to retrieve. Defaults to 5. + + Returns: + str: A JSON string representing the top N rows of the dataset. + """ + + # Get the top N rows of the model input table + top_shuttle_df = model_input_table.head(5) + top_shuttle_json = top_shuttle_df.to_dict(orient="records") + return top_shuttle_json diff --git a/demo-project/src/demo_project/pipelines/reporting/pipeline.py b/demo-project/src/demo_project/pipelines/reporting/pipeline.py index 18a4a5d3b9..4b6eb4e6de 100644 --- a/demo-project/src/demo_project/pipelines/reporting/pipeline.py +++ b/demo-project/src/demo_project/pipelines/reporting/pipeline.py @@ -11,6 +11,7 @@ make_cancel_policy_bar_chart, make_price_analysis_image, make_price_histogram, + get_top_shuttles_data, ) @@ -43,6 +44,11 @@ def create_pipeline(**kwargs) -> Pipeline: inputs="prm_shuttle_company_reviews", outputs="confusion_matrix", ), + node( + func=get_top_shuttles_data, + inputs="prm_shuttle_company_reviews", + outputs="top_shuttle_data", + ), ], inputs=["prm_shuttle_company_reviews", "feature_importance_output"], namespace="reporting", diff --git a/demo-project/src/demo_project/requirements.in b/demo-project/src/demo_project/requirements.in index af486a91a6..dff89fa8d9 100644 --- a/demo-project/src/demo_project/requirements.in +++ b/demo-project/src/demo_project/requirements.in @@ -16,4 +16,4 @@ wheel>=0.35, <0.37 pillow~=9.0 matplotlib==3.5.0 pre-commit~=1.17 -seaborn~=0.11.2 +seaborn>=0.13.0 diff --git a/demo-project/src/docker_requirements.txt b/demo-project/src/docker_requirements.txt index 407cdfb038..5f220c35cf 100644 --- a/demo-project/src/docker_requirements.txt +++ b/demo-project/src/docker_requirements.txt @@ -2,4 +2,4 @@ kedro>=0.18.0 kedro-datasets[pandas.CSVDataset,pandas.ExcelDataset, pandas.ParquetDataset, plotly.PlotlyDataset, matplotlib.MatplotlibWriter]>=2.1.0 scikit-learn~=1.0 pillow~=9.0 -seaborn~=0.11.2 +seaborn>=0.13.0 diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index d0c3cbf102..0000000000 --- a/docs/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line, and also -# from the environment for the first two. -SPHINXOPTS ?= -SPHINXBUILD ?= sphinx-build -SOURCEDIR = source -BUILDDIR = build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/build-docs.sh b/docs/build-docs.sh new file mode 100644 index 0000000000..89cecc80cb --- /dev/null +++ b/docs/build-docs.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -e + +# Exit script if you try to use an uninitialized variable. +set -o nounset + +action=$1 + +if [ "$action" == "linkcheck" ]; then + sphinx-build -WETan -j auto -D language=en -b linkcheck -d docs/build/doctrees docs/source docs/build/linkcheck +elif [ "$action" == "docs" ]; then + sphinx-build -WETa --keep-going -j auto -D language=en -b html -d docs/build/doctrees docs/source docs/build/html +fi diff --git a/docs/make.bat b/docs/make.bat deleted file mode 100644 index 747ffb7b30..0000000000 --- a/docs/make.bat +++ /dev/null @@ -1,35 +0,0 @@ -@ECHO OFF - -pushd %~dp0 - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set SOURCEDIR=source -set BUILDDIR=build - -%SPHINXBUILD% >NUL 2>NUL -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.https://www.sphinx-doc.org/ - exit /b 1 -) - -if "%1" == "" goto help - -%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% -goto end - -:help -%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% - -:end -popd diff --git a/docs/source/_templates/layout.html b/docs/source/_templates/layout.html deleted file mode 100644 index 5e2cd48b45..0000000000 --- a/docs/source/_templates/layout.html +++ /dev/null @@ -1,76 +0,0 @@ -{% extends "!layout.html" %} {%- block extrabody %} - -
- - -
- Kedro - Kedro-Viz - Kedro-Datasets -
- - {%- include "searchbox.html" %} -
- -{% endblock %} {%- block extrahead %} - - - -{% endblock %} diff --git a/docs/source/conf.py b/docs/source/conf.py index be246d7c16..7a21f6e72e 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -54,7 +54,7 @@ intersphinx_mapping = { "kedro": ("https://docs.kedro.org/en/stable/", None), - "kedro-datasets": ("https://docs.kedro.org/projects/kedro-datasets/en/kedro-datasets-1.7.1/", None), + "kedro-datasets": ("https://docs.kedro.org/projects/kedro-datasets/en/kedro-datasets-3.0.0/", None), } # -- Options for HTML output ------------------------------------------------- @@ -62,7 +62,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = "sphinx_rtd_theme" +html_theme = "kedro-sphinx-theme" # Theme options are theme-specific and customise the look and feel of a theme # further. For a list of options available for each theme, see the diff --git a/package-lock.json b/package-lock.json index dfc8620df7..c3cf40ca18 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "@quantumblack/kedro-viz", - "version": "9.0.0", + "version": "9.1.0", "lockfileVersion": 3, "requires": true, "packages": { diff --git a/package.json b/package.json index fd72c4d33d..beb265955f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@quantumblack/kedro-viz", - "version": "9.0.0", + "version": "9.1.0", "description": "Kedro-Viz is an interactive development tool for building data science pipelines with Kedro.", "main": "lib/components/app/index.js", "files": [ diff --git a/package/kedro_viz/__init__.py b/package/kedro_viz/__init__.py index b419657da6..822cc0d6f3 100644 --- a/package/kedro_viz/__init__.py +++ b/package/kedro_viz/__init__.py @@ -2,7 +2,7 @@ import sys import warnings -__version__ = "9.0.0" +__version__ = "9.1.0" class KedroVizPythonVersionWarning(UserWarning): diff --git a/package/kedro_viz/api/rest/responses.py b/package/kedro_viz/api/rest/responses.py index 13aaa05a6d..7850d091ad 100644 --- a/package/kedro_viz/api/rest/responses.py +++ b/package/kedro_viz/api/rest/responses.py @@ -372,16 +372,17 @@ def get_selected_pipeline_response(registered_pipeline_id: str): def get_package_compatibilities_response( - package_requirements: Dict[str, str], + package_requirements: Dict[str, Dict[str, str]], ) -> List[PackageCompatibilityAPIResponse]: """API response for `/api/package_compatibility`.""" package_requirements_response = [] - for package_name, compatible_version in package_requirements.items(): + for package_name, package_info in package_requirements.items(): + compatible_version = package_info["min_compatible_version"] try: package_version = get_package_version(package_name) - except PackageNotFoundError as exc: - logger.exception("Failed to get package version. Error: %s", str(exc)) + except PackageNotFoundError: + logger.warning(package_info["warning_message"]) package_version = "0.0.0" is_compatible = packaging.version.parse( diff --git a/package/kedro_viz/constants.py b/package/kedro_viz/constants.py index 9afb7eddd4..620fa951e3 100644 --- a/package/kedro_viz/constants.py +++ b/package/kedro_viz/constants.py @@ -14,4 +14,14 @@ SHAREABLEVIZ_SUPPORTED_PLATFORMS = ["aws", "azure", "gcp"] -PACKAGE_REQUIREMENTS = {"fsspec": "2023.9.0", "kedro-datasets": "2.1.0"} +PACKAGE_REQUIREMENTS = { + "fsspec": { + "min_compatible_version": "2023.9.0", + "warning_message": "Publish and share Kedro-Viz requires fsspec >= 2023.9.0", + }, + "kedro-datasets": { + "min_compatible_version": "2.1.0", + "warning_message": "Experiment Tracking is exclusively supported " + "for users with kedro-datasets >= 2.1.0", + }, +} diff --git a/package/kedro_viz/data_access/managers.py b/package/kedro_viz/data_access/managers.py index 4e4e772e5a..98fe36e78d 100644 --- a/package/kedro_viz/data_access/managers.py +++ b/package/kedro_viz/data_access/managers.py @@ -9,7 +9,6 @@ from kedro.io import DataCatalog from kedro.pipeline import Pipeline as KedroPipeline from kedro.pipeline.node import Node as KedroNode -from kedro.pipeline.pipeline import _strip_transcoding from sqlalchemy.orm import sessionmaker from kedro_viz.constants import DEFAULT_REGISTERED_PIPELINE_ID, ROOT_MODULAR_PIPELINE_ID @@ -26,6 +25,7 @@ TranscodedDataNode, ) from kedro_viz.services import layers_services, modular_pipelines_services +from kedro_viz.utils import _strip_transcoding from .repositories import ( CatalogRepository, diff --git a/package/kedro_viz/data_access/repositories/catalog.py b/package/kedro_viz/data_access/repositories/catalog.py index 427a13cf0d..663e2d2230 100644 --- a/package/kedro_viz/data_access/repositories/catalog.py +++ b/package/kedro_viz/data_access/repositories/catalog.py @@ -6,17 +6,10 @@ from typing import TYPE_CHECKING, Dict, Optional from kedro.io import DataCatalog - -try: - # kedro 0.19.4 onwards - from kedro.pipeline._transcoding import TRANSCODING_SEPARATOR, _strip_transcoding -except ImportError: # pragma: no cover - # older versions - from kedro.pipeline.pipeline import TRANSCODING_SEPARATOR, _strip_transcoding - from packaging.version import parse from kedro_viz.constants import KEDRO_VERSION +from kedro_viz.utils import TRANSCODING_SEPARATOR, _strip_transcoding try: # kedro 0.18.11 onwards diff --git a/package/kedro_viz/data_access/repositories/modular_pipelines.py b/package/kedro_viz/data_access/repositories/modular_pipelines.py index b887d020b8..b32b62ecb8 100644 --- a/package/kedro_viz/data_access/repositories/modular_pipelines.py +++ b/package/kedro_viz/data_access/repositories/modular_pipelines.py @@ -1,5 +1,6 @@ """`kedro_viz.data_access.repositories.modular_pipelines` defines repository to centralise access to modular pipelines data.""" + from typing import Dict, Optional, Union from kedro_viz.constants import ROOT_MODULAR_PIPELINE_ID @@ -215,6 +216,8 @@ def extract_from_node(self, node: GraphNode) -> Optional[str]: # so does the modular pipeline. modular_pipeline.pipelines.update(node.pipelines) + modular_pipeline.tags.update(node.tags) + # Since we extract the modular pipeline from the node's namespace, # the node is by definition a child of the modular pipeline. self.add_child( diff --git a/package/kedro_viz/integrations/kedro/data_loader.py b/package/kedro_viz/integrations/kedro/data_loader.py index f3506d74fb..4662951781 100644 --- a/package/kedro_viz/integrations/kedro/data_loader.py +++ b/package/kedro_viz/integrations/kedro/data_loader.py @@ -3,7 +3,7 @@ load data from projects created in a range of Kedro versions. """ -# pylint: disable=import-outside-toplevel, protected-access +# pylint: disable=protected-access import json import logging @@ -11,8 +11,10 @@ from typing import Any, Dict, Optional, Tuple from kedro import __version__ +from kedro.framework.project import configure_project, pipelines from kedro.framework.session import KedroSession from kedro.framework.session.store import BaseSessionStore +from kedro.framework.startup import bootstrap_project from kedro.io import DataCatalog from kedro.pipeline import Pipeline @@ -69,14 +71,16 @@ def load_data( project_path: Path, env: Optional[str] = None, include_hooks: bool = False, + package_name: Optional[str] = None, extra_params: Optional[Dict[str, Any]] = None, ) -> Tuple[DataCatalog, Dict[str, Pipeline], BaseSessionStore, Dict]: """Load data from a Kedro project. Args: - project_path: the path whether the Kedro project is located. + project_path: the path where the Kedro project is located. env: the Kedro environment to load the data. If not provided. it will use Kedro default, which is local. include_hooks: A flag to include all registered hooks in your Kedro Project. + package_name: The name of the current package extra_params: Optional dictionary containing extra project parameters for underlying KedroContext. If specified, will update (and therefore take precedence over) the parameters retrieved from the project @@ -85,10 +89,11 @@ def load_data( A tuple containing the data catalog and the pipeline dictionary and the session store. """ - from kedro.framework.project import pipelines - from kedro.framework.startup import bootstrap_project - - bootstrap_project(project_path) + if package_name: + configure_project(package_name) + else: + # bootstrap project when viz is run in dev mode + bootstrap_project(project_path) with KedroSession.create( project_path=project_path, diff --git a/package/kedro_viz/integrations/kedro/hooks.py b/package/kedro_viz/integrations/kedro/hooks.py index d68c5eea70..f062c04ea8 100644 --- a/package/kedro_viz/integrations/kedro/hooks.py +++ b/package/kedro_viz/integrations/kedro/hooks.py @@ -12,12 +12,7 @@ from kedro.io import DataCatalog from kedro.io.core import get_filepath_str -try: - # kedro 0.19.4 onwards - from kedro.pipeline._transcoding import TRANSCODING_SEPARATOR, _strip_transcoding -except ImportError: # pragma: no cover - # older versions - from kedro.pipeline.pipeline import TRANSCODING_SEPARATOR, _strip_transcoding +from kedro_viz.utils import TRANSCODING_SEPARATOR, _strip_transcoding logger = logging.getLogger(__name__) diff --git a/package/kedro_viz/launchers/cli.py b/package/kedro_viz/launchers/cli.py index a97d01e60c..5c2bcac75f 100644 --- a/package/kedro_viz/launchers/cli.py +++ b/package/kedro_viz/launchers/cli.py @@ -9,6 +9,7 @@ from click_default_group import DefaultGroup from kedro.framework.cli.project import PARAMS_ARG_HELP from kedro.framework.cli.utils import KedroCliError, _split_params +from kedro.framework.project import PACKAGE_NAME from packaging.version import parse from watchgod import RegExpWatcher, run_process @@ -22,7 +23,9 @@ from kedro_viz.integrations.deployment.deployer_factory import DeployerFactory from kedro_viz.integrations.pypi import get_latest_version, is_running_outdated_version from kedro_viz.launchers.utils import ( + _PYPROJECT, _check_viz_up, + _find_kedro_project, _start_browser, _wait_for, viz_deploy_progress_timer, @@ -128,6 +131,17 @@ def run( """Launch local Kedro Viz instance""" from kedro_viz.server import run_server + kedro_project_path = _find_kedro_project(Path.cwd()) + + if kedro_project_path is None: + display_cli_message( + "ERROR: Failed to start Kedro-Viz : " + "Could not find the project configuration " + f"file '{_PYPROJECT}' at '{Path.cwd()}'. ", + "red", + ) + return + installed_version = parse(__version__) latest_version = get_latest_version() if is_running_outdated_version(installed_version, latest_version): @@ -151,15 +165,15 @@ def run( "save_file": save_file, "pipeline_name": pipeline, "env": env, + "project_path": kedro_project_path, "autoreload": autoreload, "include_hooks": include_hooks, + "package_name": PACKAGE_NAME, "extra_params": params, } if autoreload: - project_path = Path.cwd() - run_server_kwargs["project_path"] = project_path run_process_kwargs = { - "path": project_path, + "path": kedro_project_path, "target": run_server, "kwargs": run_server_kwargs, "watcher_cls": RegExpWatcher, @@ -268,6 +282,7 @@ def create_shareableviz_process( endpoint, bucket_name, include_hooks, + PACKAGE_NAME, process_completed, exception_queue, ), @@ -338,11 +353,19 @@ def create_shareableviz_process( def load_and_deploy_viz( - platform, endpoint, bucket_name, include_hooks, process_completed, exception_queue + platform, + endpoint, + bucket_name, + include_hooks, + package_name, + process_completed, + exception_queue, ): """Loads Kedro Project data, creates a deployer and deploys to a platform""" try: - load_and_populate_data(Path.cwd(), include_hooks=include_hooks) + load_and_populate_data( + Path.cwd(), include_hooks=include_hooks, package_name=package_name + ) # Start the deployment deployer = DeployerFactory.create_deployer(platform, endpoint, bucket_name) diff --git a/package/kedro_viz/launchers/jupyter.py b/package/kedro_viz/launchers/jupyter.py index d44493f424..f51f6ce7eb 100644 --- a/package/kedro_viz/launchers/jupyter.py +++ b/package/kedro_viz/launchers/jupyter.py @@ -1,6 +1,7 @@ """`kedro_viz.launchers.jupyter` provides line_magic to launch the viz server from a jupyter notebook. """ + # pragma: no cover import logging import multiprocessing @@ -12,6 +13,7 @@ import IPython from IPython.display import HTML, display +from kedro.framework.project import PACKAGE_NAME from watchgod import RegExpWatcher, run_process from kedro_viz.launchers.utils import _check_viz_up, _wait_for @@ -140,6 +142,7 @@ def run_viz( # pylint: disable=too-many-locals "env": env, "autoreload": autoreload, "include_hooks": include_hooks, + "package_name": PACKAGE_NAME, "extra_params": params, "project_path": project_path, } diff --git a/package/kedro_viz/launchers/utils.py b/package/kedro_viz/launchers/utils.py index 1ade2b426c..bf79602349 100755 --- a/package/kedro_viz/launchers/utils.py +++ b/package/kedro_viz/launchers/utils.py @@ -3,12 +3,14 @@ import logging import webbrowser +from pathlib import Path from time import sleep, time -from typing import Any, Callable +from typing import Any, Callable, Union import requests logger = logging.getLogger(__name__) +_PYPROJECT = "pyproject.toml" class WaitForException(Exception): @@ -105,3 +107,23 @@ def viz_deploy_progress_timer(process_completed, timeout): ) sleep(1) elapsed_time += 1 + + +def _is_project(project_path: Union[str, Path]) -> bool: + metadata_file = Path(project_path).expanduser().resolve() / _PYPROJECT + if not metadata_file.is_file(): + return False + + try: + return "[tool.kedro]" in metadata_file.read_text(encoding="utf-8") + # pylint: disable=broad-exception-caught + except Exception: + return False + + +def _find_kedro_project(current_dir: Path) -> Any: + paths_to_check = [current_dir] + list(current_dir.parents) + for project_dir in paths_to_check: + if _is_project(project_dir): + return project_dir + return None diff --git a/package/kedro_viz/models/flowchart.py b/package/kedro_viz/models/flowchart.py index e93418323c..d82dcc9bec 100644 --- a/package/kedro_viz/models/flowchart.py +++ b/package/kedro_viz/models/flowchart.py @@ -11,14 +11,6 @@ from typing import Any, Dict, List, Optional, Set, Union, cast from kedro.pipeline.node import Node as KedroNode - -try: - # kedro 0.19.4 onwards - from kedro.pipeline._transcoding import TRANSCODING_SEPARATOR, _strip_transcoding -except ImportError: # pragma: no cover - # older versions - from kedro.pipeline.pipeline import TRANSCODING_SEPARATOR, _strip_transcoding - from pydantic import ( BaseModel, ConfigDict, @@ -29,6 +21,7 @@ ) from kedro_viz.models.utils import get_dataset_type +from kedro_viz.utils import TRANSCODING_SEPARATOR, _strip_transcoding try: # kedro 0.18.11 onwards diff --git a/package/kedro_viz/server.py b/package/kedro_viz/server.py index 2c5b205576..eb31f7a9c9 100644 --- a/package/kedro_viz/server.py +++ b/package/kedro_viz/server.py @@ -53,14 +53,19 @@ def load_and_populate_data( path: Path, env: Optional[str] = None, include_hooks: bool = False, - extra_params: Optional[Dict[str, Any]] = None, + package_name: Optional[str] = None, pipeline_name: Optional[str] = None, + extra_params: Optional[Dict[str, Any]] = None, ): """Loads underlying Kedro project data and populates Kedro Viz Repositories""" # Loads data from underlying Kedro Project catalog, pipelines, session_store, stats_dict = kedro_data_loader.load_data( - path, env, include_hooks, extra_params + path, + env, + include_hooks, + package_name, + extra_params, ) pipelines = ( @@ -83,6 +88,7 @@ def run_server( project_path: Optional[str] = None, autoreload: bool = False, include_hooks: bool = False, + package_name: Optional[str] = None, extra_params: Optional[Dict[str, Any]] = None, ): # pylint: disable=redefined-outer-name """Run a uvicorn server with a FastAPI app that either launches API response data from a file @@ -101,15 +107,24 @@ def run_server( project_path: the optional path of the Kedro project that contains the pipelines to visualise. If not supplied, the current working directory will be used. include_hooks: A flag to include all registered hooks in your Kedro Project. + package_name: The name of the current package extra_params: Optional dictionary containing extra project parameters for underlying KedroContext. If specified, will update (and therefore take precedence over) the parameters retrieved from the project configuration. """ + path = Path(project_path) if project_path else Path.cwd() if load_file is None: - load_and_populate_data(path, env, include_hooks, extra_params, pipeline_name) + load_and_populate_data( + path, + env, + include_hooks, + package_name, + pipeline_name, + extra_params, + ) if save_file: save_api_responses_to_fs(save_file, fsspec.filesystem("file")) @@ -127,8 +142,6 @@ def run_server( if __name__ == "__main__": # pragma: no cover import argparse - from kedro.framework.startup import bootstrap_project - parser = argparse.ArgumentParser(description="Launch a development viz server") parser.add_argument("project_path", help="Path to a Kedro project") parser.add_argument( @@ -140,7 +153,6 @@ def run_server( args = parser.parse_args() project_path = (Path.cwd() / args.project_path).absolute() - bootstrap_project(project_path) run_process_kwargs = { "path": project_path, diff --git a/package/kedro_viz/services/modular_pipelines.py b/package/kedro_viz/services/modular_pipelines.py index e15f381000..bc568e99ff 100644 --- a/package/kedro_viz/services/modular_pipelines.py +++ b/package/kedro_viz/services/modular_pipelines.py @@ -1,6 +1,7 @@ """`kedro_viz.services.modular_pipelines` defines modular pipelines-related business logic. The service layer consist of pure functions operating on domain models. """ + from typing import Dict from kedro_viz.constants import ROOT_MODULAR_PIPELINE_ID @@ -78,6 +79,7 @@ def expand_tree( ) expanded_tree[parent_id].pipelines.update(modular_pipeline_node.pipelines) + expanded_tree[parent_id].tags.update(modular_pipeline_node.tags) expanded_tree[parent_id].children.add( ModularPipelineChild( id=f"{parent_id}.{chunks[i]}", diff --git a/package/kedro_viz/utils.py b/package/kedro_viz/utils.py new file mode 100644 index 0000000000..2d919a0c82 --- /dev/null +++ b/package/kedro_viz/utils.py @@ -0,0 +1,39 @@ +"""Transcoding related utility functions.""" +from typing import Tuple + +TRANSCODING_SEPARATOR = "@" + + +def _transcode_split(element: str) -> Tuple[str, str]: + """Split the name by the transcoding separator. + If the transcoding part is missing, empty string will be put in. + + Returns: + Node input/output name before the transcoding separator, if present. + Raises: + ValueError: Raised if more than one transcoding separator + is present in the name. + """ + split_name = element.split(TRANSCODING_SEPARATOR) + + if len(split_name) > 2: # noqa: PLR2004 + raise ValueError( # pragma: no cover + f"Expected maximum 1 transcoding separator, found {len(split_name) - 1} " + f"instead: '{element}'." + ) + if len(split_name) == 1: + split_name.append("") + + return tuple(split_name) # type: ignore + + +def _strip_transcoding(element: str) -> str: + """Strip out the transcoding separator and anything that follows. + + Returns: + Node input/output name before the transcoding separator, if present. + Raises: + ValueError: Raised if more than one transcoding separator + is present in the name. + """ + return _transcode_split(element)[0] diff --git a/package/setup.py b/package/setup.py index d2257d6f6d..e04cda700f 100755 --- a/package/setup.py +++ b/package/setup.py @@ -52,11 +52,7 @@ }, extras_require={ "docs": [ - "sphinx>=5.3,<7.3", - "sphinx_copybutton==0.5.2", - "sphinx-notfound-page", - "sphinx_rtd_theme==1.3.0", - "myst-parser>=1.0,<2.1", + "kedro-sphinx-theme==2024.4.0", ], "aws": ["s3fs>=2021.4"], "azure": ["adlfs>=2021.4"], diff --git a/package/tests/test_api/test_rest/test_responses.py b/package/tests/test_api/test_rest/test_responses.py index 5e5d32a4c2..79c05092e4 100644 --- a/package/tests/test_api/test_rest/test_responses.py +++ b/package/tests/test_api/test_rest/test_responses.py @@ -1,4 +1,5 @@ # pylint: disable=too-many-lines +import logging import operator from pathlib import Path from typing import Any, Dict, Iterable, List @@ -7,7 +8,6 @@ import pytest from fastapi.testclient import TestClient -from importlib_metadata import PackageNotFoundError from kedro_viz.api import apps from kedro_viz.api.rest.responses import ( @@ -184,7 +184,7 @@ def assert_example_data(response_data): { "id": "uk.data_processing", "name": "uk.data_processing", - "tags": [], + "tags": ["split"], "pipelines": ["__default__"], "type": "modularPipeline", "modular_pipelines": None, @@ -195,7 +195,7 @@ def assert_example_data(response_data): { "id": "uk.data_science", "name": "uk.data_science", - "tags": [], + "tags": ["train"], "pipelines": ["__default__"], "type": "modularPipeline", "modular_pipelines": None, @@ -206,7 +206,7 @@ def assert_example_data(response_data): { "id": "uk", "name": "uk", - "tags": [], + "tags": ["split", "train"], "pipelines": ["__default__"], "type": "modularPipeline", "modular_pipelines": None, @@ -731,7 +731,7 @@ def test_get_pipeline(self, client): { "id": "uk", "name": "uk", - "tags": [], + "tags": ["train"], "pipelines": ["data_science"], "type": "modularPipeline", "modular_pipelines": None, @@ -742,7 +742,7 @@ def test_get_pipeline(self, client): { "id": "uk.data_science", "name": "uk.data_science", - "tags": [], + "tags": ["train"], "pipelines": ["data_science"], "type": "modularPipeline", "modular_pipelines": None, @@ -829,10 +829,30 @@ class TestPackageCompatibilities: @pytest.mark.parametrize( "package_name, package_version, package_requirements, expected_compatibility_response", [ - ("fsspec", "2023.9.1", {"fsspec": "2023.0.0"}, True), - ("fsspec", "2023.9.1", {"fsspec": "2024.0.0"}, False), - ("kedro-datasets", "2.1.0", {"kedro-datasets": "2.1.0"}, True), - ("kedro-datasets", "1.8.0", {"kedro-datasets": "2.1.0"}, False), + ( + "fsspec", + "2023.9.1", + {"fsspec": {"min_compatible_version": "2023.0.0"}}, + True, + ), + ( + "fsspec", + "2023.9.1", + {"fsspec": {"min_compatible_version": "2024.0.0"}}, + False, + ), + ( + "kedro-datasets", + "2.1.0", + {"kedro-datasets": {"min_compatible_version": "2.1.0"}}, + True, + ), + ( + "kedro-datasets", + "1.8.0", + {"kedro-datasets": {"min_compatible_version": "2.1.0"}}, + False, + ), ], ) def test_get_package_compatibilities_response( @@ -854,16 +874,27 @@ def test_get_package_compatibilities_response( assert package_response.package_version == package_version assert package_response.is_compatible is expected_compatibility_response - def test_get_package_compatibilities_exception_response( - self, - mocker, - ): - mocker.patch( - "kedro_viz.api.rest.responses.get_package_compatibilities_response", - side_effect=PackageNotFoundError("random-package"), - ) - package_name = "random-package" - response = get_package_compatibilities_response({package_name: "1.0.0"}) + def test_get_package_compatibilities_exception_response(self, caplog): + mock_package_requirement = { + "random-package": { + "min_compatible_version": "1.0.0", + "warning_message": "random-package is not available", + } + } + + with caplog.at_level(logging.WARNING): + response = get_package_compatibilities_response(mock_package_requirement) + + assert len(caplog.records) == 1 + + record = caplog.records[0] + + assert record.levelname == "WARNING" + assert ( + mock_package_requirement["random-package"]["warning_message"] + in record.message + ) + expected_response = PackageCompatibilityAPIResponse( package_name="random-package", package_version="0.0.0", is_compatible=False ) diff --git a/package/tests/test_launchers/test_cli.py b/package/tests/test_launchers/test_cli.py index 83baeb917f..4ad1b6dd86 100755 --- a/package/tests/test_launchers/test_cli.py +++ b/package/tests/test_launchers/test_cli.py @@ -9,6 +9,7 @@ from kedro_viz import __version__ from kedro_viz.constants import SHAREABLEVIZ_SUPPORTED_PLATFORMS, VIZ_DEPLOY_TIME_LIMIT from kedro_viz.launchers import cli +from kedro_viz.launchers.utils import _PYPROJECT from kedro_viz.server import run_server @@ -85,8 +86,10 @@ def mock_project_path(mocker): "save_file": None, "pipeline_name": None, "env": None, + "project_path": "testPath", "autoreload": False, "include_hooks": False, + "package_name": None, "extra_params": {}, }, ), @@ -99,8 +102,10 @@ def mock_project_path(mocker): "save_file": None, "pipeline_name": None, "env": None, + "project_path": "testPath", "autoreload": False, "include_hooks": False, + "package_name": None, "extra_params": {}, }, ), @@ -118,8 +123,10 @@ def mock_project_path(mocker): "save_file": None, "pipeline_name": None, "env": None, + "project_path": "testPath", "autoreload": False, "include_hooks": False, + "package_name": None, "extra_params": {}, }, ), @@ -148,8 +155,10 @@ def mock_project_path(mocker): "save_file": "save_dir", "pipeline_name": "data_science", "env": "local", + "project_path": "testPath", "autoreload": False, "include_hooks": False, + "package_name": None, "extra_params": {"extra_param": "param"}, }, ), @@ -162,8 +171,10 @@ def mock_project_path(mocker): "save_file": None, "pipeline_name": None, "env": None, + "project_path": "testPath", "autoreload": False, "include_hooks": True, + "package_name": None, "extra_params": {}, }, ), @@ -180,6 +191,11 @@ def test_kedro_viz_command_run_server( runner = CliRunner() # Reduce the timeout argument from 600 to 1 to make test run faster. mocker.patch("kedro_viz.launchers.cli._wait_for.__defaults__", (True, 1, True, 1)) + # Mock finding kedro project + mocker.patch( + "kedro_viz.launchers.cli._find_kedro_project", + return_value=run_server_args["project_path"], + ) with runner.isolated_filesystem(): runner.invoke(cli.viz_cli, command_options) @@ -190,8 +206,30 @@ def test_kedro_viz_command_run_server( assert run_server_args["port"] in cli._VIZ_PROCESSES +def test_kedro_viz_command_should_log_project_not_found( + mocker, mock_project_path, mock_click_echo +): + # Reduce the timeout argument from 600 to 1 to make test run faster. + mocker.patch("kedro_viz.launchers.cli._wait_for.__defaults__", (True, 1, True, 1)) + # Mock finding kedro project + mocker.patch("kedro_viz.launchers.cli._find_kedro_project", return_value=None) + runner = CliRunner() + with runner.isolated_filesystem(): + runner.invoke(cli.viz_cli, ["viz", "run"]) + + mock_click_echo_calls = [ + call( + "\x1b[31mERROR: Failed to start Kedro-Viz : " + "Could not find the project configuration " + f"file '{_PYPROJECT}' at '{mock_project_path}'. \x1b[0m" + ) + ] + + mock_click_echo.assert_has_calls(mock_click_echo_calls) + + def test_kedro_viz_command_should_log_outdated_version( - mocker, mock_http_response, mock_click_echo + mocker, mock_http_response, mock_click_echo, mock_project_path ): installed_version = parse(__version__) mock_version = f"{installed_version.major + 1}.0.0" @@ -204,6 +242,10 @@ def test_kedro_viz_command_should_log_outdated_version( # Reduce the timeout argument from 600 to 1 to make test run faster. mocker.patch("kedro_viz.launchers.cli._wait_for.__defaults__", (True, 1, True, 1)) + # Mock finding kedro project + mocker.patch( + "kedro_viz.launchers.cli._find_kedro_project", return_value=mock_project_path + ) runner = CliRunner() with runner.isolated_filesystem(): runner.invoke(cli.viz_cli, ["viz", "run"]) @@ -223,7 +265,7 @@ def test_kedro_viz_command_should_log_outdated_version( def test_kedro_viz_command_should_not_log_latest_version( - mocker, mock_http_response, mock_click_echo + mocker, mock_http_response, mock_click_echo, mock_project_path ): requests_get = mocker.patch("requests.get") requests_get.return_value = mock_http_response( @@ -233,6 +275,10 @@ def test_kedro_viz_command_should_not_log_latest_version( mocker.patch("kedro_viz.server.run_server") # Reduce the timeout argument from 600 to 1 to make test run faster. mocker.patch("kedro_viz.launchers.cli._wait_for.__defaults__", (True, 1, True, 1)) + # Mock finding kedro project + mocker.patch( + "kedro_viz.launchers.cli._find_kedro_project", return_value=mock_project_path + ) runner = CliRunner() with runner.isolated_filesystem(): runner.invoke(cli.viz_cli, ["viz", "run"]) @@ -243,7 +289,7 @@ def test_kedro_viz_command_should_not_log_latest_version( def test_kedro_viz_command_should_not_log_if_pypi_is_down( - mocker, mock_http_response, mock_click_echo + mocker, mock_http_response, mock_click_echo, mock_project_path ): requests_get = mocker.patch("requests.get") requests_get.side_effect = requests.exceptions.RequestException("PyPI is down") @@ -251,6 +297,10 @@ def test_kedro_viz_command_should_not_log_if_pypi_is_down( mocker.patch("kedro_viz.server.run_server") # Reduce the timeout argument from 600 to 1 to make test run faster. mocker.patch("kedro_viz.launchers.cli._wait_for.__defaults__", (True, 1, True, 1)) + # Mock finding kedro project + mocker.patch( + "kedro_viz.launchers.cli._find_kedro_project", return_value=mock_project_path + ) runner = CliRunner() with runner.isolated_filesystem(): runner.invoke(cli.viz_cli, ["viz", "run"]) @@ -267,6 +317,10 @@ def test_kedro_viz_command_with_autoreload( # Reduce the timeout argument from 600 to 1 to make test run faster. mocker.patch("kedro_viz.launchers.cli._wait_for.__defaults__", (True, 1, True, 1)) + # Mock finding kedro project + mocker.patch( + "kedro_viz.launchers.cli._find_kedro_project", return_value=mock_project_path + ) runner = CliRunner() with runner.isolated_filesystem(): runner.invoke(cli.viz_cli, ["viz", "run", "--autoreload"]) @@ -284,6 +338,7 @@ def test_kedro_viz_command_with_autoreload( "autoreload": True, "project_path": mock_project_path, "include_hooks": False, + "package_name": None, "extra_params": {}, }, "watcher_cls": RegExpWatcher, @@ -579,6 +634,7 @@ def test_create_shareableviz_process( endpoint, bucket_name, include_hooks, + None, mock_process_completed.return_value, mock_exception_queue.return_value, ), @@ -614,22 +670,24 @@ def test_create_shareableviz_process( @pytest.mark.parametrize( - "platform, endpoint, bucket_name, include_hooks", + "platform, endpoint, bucket_name, include_hooks, package_name", [ ( "azure", "https://example-bucket.web.core.windows.net", "example-bucket", False, + "demo_project", ), ( "aws", "http://example-bucket.s3-website.us-east-2.amazonaws.com/", "example-bucket", True, + "demo_project", ), - ("gcp", "http://34.120.87.227/", "example-bucket", False), - ("local", None, None, True), + ("gcp", "http://34.120.87.227/", "example-bucket", False, "demo_project"), + ("local", None, None, True, "demo_project"), ], ) def test_load_and_deploy_viz_success( @@ -637,6 +695,7 @@ def test_load_and_deploy_viz_success( endpoint, bucket_name, include_hooks, + package_name, mock_DeployerFactory, mock_load_and_populate_data, mock_process_completed, @@ -651,12 +710,13 @@ def test_load_and_deploy_viz_success( endpoint, bucket_name, include_hooks, + package_name, mock_process_completed, mock_exception_queue, ) mock_load_and_populate_data.assert_called_once_with( - mock_project_path, include_hooks=include_hooks + mock_project_path, include_hooks=include_hooks, package_name=package_name ) mock_DeployerFactory.create_deployer.assert_called_once_with( platform, endpoint, bucket_name diff --git a/package/tests/test_launchers/test_jupyter.py b/package/tests/test_launchers/test_jupyter.py index af41f9b884..dd489778ca 100644 --- a/package/tests/test_launchers/test_jupyter.py +++ b/package/tests/test_launchers/test_jupyter.py @@ -35,6 +35,7 @@ def test_run_viz(self, mocker, patched_check_viz_up): "env": None, "autoreload": False, "include_hooks": False, + "package_name": None, "extra_params": "", }, ) @@ -59,6 +60,7 @@ def test_run_viz(self, mocker, patched_check_viz_up): "env": None, "autoreload": False, "include_hooks": True, + "package_name": None, "extra_params": "", }, ) @@ -105,6 +107,7 @@ def test_run_viz_on_databricks(self, mocker, patched_check_viz_up, monkeypatch): "env": None, "autoreload": False, "include_hooks": False, + "package_name": None, "extra_params": "", }, ) diff --git a/package/tests/test_launchers/test_utils.py b/package/tests/test_launchers/test_utils.py index e9dead2f79..04425cfd09 100644 --- a/package/tests/test_launchers/test_utils.py +++ b/package/tests/test_launchers/test_utils.py @@ -1,3 +1,4 @@ +from pathlib import Path from unittest import mock from unittest.mock import Mock, call, patch @@ -7,6 +8,8 @@ from kedro_viz.constants import VIZ_DEPLOY_TIME_LIMIT from kedro_viz.launchers.utils import ( _check_viz_up, + _find_kedro_project, + _is_project, _start_browser, viz_deploy_progress_timer, ) @@ -69,3 +72,50 @@ def test_viz_deploy_progress_timer(capsys): for second in range(1, VIZ_DEPLOY_TIME_LIMIT + 1): expected_output = f"...Creating your build/deploy Kedro-Viz ({second}s)" assert expected_output in captured.out + + +class TestIsProject: + project_path = Path.cwd() + + def test_no_metadata_file(self, mocker): + mocker.patch.object(Path, "is_file", return_value=False) + + assert not _is_project(self.project_path) + + def test_toml_invalid_format(self, tmp_path): + """Test for loading context from an invalid path.""" + toml_path = tmp_path / "pyproject.toml" + toml_path.write_text("!!") # Invalid TOML + + assert not _is_project(tmp_path) + + def test_non_kedro_project(self, mocker): + mocker.patch.object(Path, "is_file", return_value=True) + mocker.patch.object(Path, "read_text", return_value="[tool]") + + assert not _is_project(self.project_path) + + def test_valid_toml_file(self, mocker): + mocker.patch.object(Path, "is_file", return_value=True) + pyproject_toml_payload = "[tool.kedro]" # \nproject_name = 'proj'" + mocker.patch.object(Path, "read_text", return_value=pyproject_toml_payload) + + assert _is_project(self.project_path) + + def test_toml_bad_encoding(self, mocker): + mocker.patch.object(Path, "is_file", return_value=True) + mocker.patch.object(Path, "read_text", side_effect=UnicodeDecodeError) + + assert not _is_project(self.project_path) + + +@pytest.mark.parametrize( + "project_dir, is_project_found, expected", + [ + ("/path/to/valid/project", True, Path("/path/to/valid/project")), + ("/path/to/nonexistent/project", False, None), + ], +) +def test_find_kedro_project(project_dir, is_project_found, expected, mocker): + mocker.patch("kedro_viz.launchers.utils._is_project", return_value=is_project_found) + assert _find_kedro_project(Path(project_dir)) == expected diff --git a/src/actions/actions.test.js b/src/actions/actions.test.js index 4778ed741b..fde1e1b162 100644 --- a/src/actions/actions.test.js +++ b/src/actions/actions.test.js @@ -16,6 +16,7 @@ import { TOGGLE_CODE, TOGGLE_MODULAR_PIPELINE_FOCUS_MODE, TOGGLE_HOVERED_FOCUS_MODE, + TOGGLE_EXPAND_ALL_PIPELINES, changeFlag, resetData, toggleIgnoreLargeWarning, @@ -32,6 +33,7 @@ import { updateChartSize, toggleFocusMode, toggleHoveredFocusMode, + toggleExpandAllPipelines, } from '../actions'; import { TOGGLE_NODE_CLICKED, @@ -75,7 +77,18 @@ describe('actions', () => { expect(toggleLayers(visible)).toEqual(expectedAction); }); - it('should create an action to toggle whether to show layers', () => { + it('should create an action to toggle whether to expand all modular pipelines or collapse', () => { + const shouldExpandAllPipelines = false; + const expectedAction = { + type: TOGGLE_EXPAND_ALL_PIPELINES, + shouldExpandAllPipelines, + }; + expect(toggleExpandAllPipelines(shouldExpandAllPipelines)).toEqual( + expectedAction + ); + }); + + it('should create an action to toggle whether to show minimap', () => { const visible = false; const expectedAction = { type: TOGGLE_MINIMAP, diff --git a/src/actions/index.js b/src/actions/index.js index 973e16d531..47e83197af 100644 --- a/src/actions/index.js +++ b/src/actions/index.js @@ -24,6 +24,19 @@ export function toggleLayers(visible) { }; } +export const TOGGLE_EXPAND_ALL_PIPELINES = 'TOGGLE_EXPAND_ALL_PIPELINES'; + +/** + * Toggle whether to expand all modular pipelines or collapse + * @param {Boolean} shouldExpandAllPipelines + */ +export function toggleExpandAllPipelines(shouldExpandAllPipelines) { + return { + type: TOGGLE_EXPAND_ALL_PIPELINES, + shouldExpandAllPipelines, + }; +} + export const TOGGLE_EXPORT_MODAL = 'TOGGLE_EXPORT_MODAL'; /** diff --git a/src/actions/pipelines.js b/src/actions/pipelines.js index 06e1bc30e6..8882b425d8 100644 --- a/src/actions/pipelines.js +++ b/src/actions/pipelines.js @@ -99,7 +99,7 @@ export function loadInitialPipelineData() { // obtain the status of expandAllPipelines to decide whether it needs to overwrite the // list of visible nodes const expandAllPipelines = - state.display.expandAllPipelines || state.flags.expandAllPipelines; + state.display.expandAllPipelines || state.expandAllPipelines; let newState = await loadJsonData(url).then((data) => preparePipelineState(data, true, expandAllPipelines) ); @@ -122,7 +122,7 @@ export function loadInitialPipelineData() { */ export function loadPipelineData(pipelineID) { return async function (dispatch, getState) { - const { dataSource, pipeline, display, flags } = getState(); + const { dataSource, pipeline, display, expandAllPipelines } = getState(); if (pipelineID && pipelineID === pipeline.active) { return; @@ -136,10 +136,10 @@ export function loadPipelineData(pipelineID) { active: pipelineID, }); - const expandAllPipelines = - display.expandAllPipelines || flags.expandAllPipelines; + const shouldExpandAllPipelines = + display.expandAllPipelines || expandAllPipelines; const newState = await loadJsonData(url).then((data) => - preparePipelineState(data, false, expandAllPipelines) + preparePipelineState(data, false, shouldExpandAllPipelines) ); // Set active pipeline here rather than dispatching two separate actions, diff --git a/src/apollo/schema.graphql b/src/apollo/schema.graphql index e48274ebb0..2fd62c4204 100644 --- a/src/apollo/schema.graphql +++ b/src/apollo/schema.graphql @@ -1,7 +1,7 @@ """ -The `JSON` scalar type represents JSON values as specified by [ECMA-404](http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-404.pdf). +The `JSON` scalar type represents JSON values as specified by [ECMA-404](https://ecma-international.org/wp-content/uploads/ECMA-404_2nd_edition_december_2017.pdf). """ -scalar JSON @specifiedBy(url: "http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-404.pdf") +scalar JSON @specifiedBy(url: "https://ecma-international.org/wp-content/uploads/ECMA-404_2nd_edition_december_2017.pdf") """Metric data""" type MetricPlotDataset { diff --git a/src/components/app/app.js b/src/components/app/app.js index 6948da6558..ba70cfb8af 100644 --- a/src/components/app/app.js +++ b/src/components/app/app.js @@ -97,6 +97,7 @@ App.propTypes = { labelBtn: PropTypes.bool, layerBtn: PropTypes.bool, exportBtn: PropTypes.bool, + pipelineBtn: PropTypes.bool, sidebar: PropTypes.bool, }), /** diff --git a/src/components/experiment-tracking/run-details-modal/run-details-modal.scss b/src/components/experiment-tracking/run-details-modal/run-details-modal.scss index ed6b438fbc..c43b8b4dae 100644 --- a/src/components/experiment-tracking/run-details-modal/run-details-modal.scss +++ b/src/components/experiment-tracking/run-details-modal/run-details-modal.scss @@ -18,3 +18,15 @@ text-align: right; width: 100%; } + +.version-reminder-and-run-details-button-wrapper { + align-items: baseline; + display: flex; + justify-content: space-between; + width: 100%; + margin-top: 50px; + + .button:first-of-type { + margin-right: 20px; + } +} diff --git a/src/components/flowchart-primary-toolbar/flowchart-primary-toolbar.js b/src/components/flowchart-primary-toolbar/flowchart-primary-toolbar.js index 2d7932fc57..d40e6ecfeb 100644 --- a/src/components/flowchart-primary-toolbar/flowchart-primary-toolbar.js +++ b/src/components/flowchart-primary-toolbar/flowchart-primary-toolbar.js @@ -5,13 +5,18 @@ import { toggleLayers, toggleSidebar, toggleTextLabels, + toggleExpandAllPipelines, } from '../../actions'; +import { loadInitialPipelineData } from '../../actions/pipelines'; import IconButton from '../ui/icon-button'; import LabelIcon from '../icons/label'; import ExportIcon from '../icons/export'; import LayersIcon from '../icons/layers'; import PrimaryToolbar from '../primary-toolbar'; import { getVisibleLayerIDs } from '../../selectors/disabled'; +import ExpandPipelinesIcon from '../icons/expand-pipelines'; +import CollapsePipelinesIcon from '../icons/collapse-pipelines'; +import { useGeneratePathname } from '../../utils/hooks/use-generate-pathname'; /** * Main controls for filtering the chart data @@ -28,47 +33,76 @@ export const FlowchartPrimaryToolbar = ({ textLabels, visible, visibleLayers, -}) => ( - <> - - onToggleTextLabels(!textLabels)} - visible={visible.labelBtn} - /> - onToggleLayers(!visibleLayers)} - visible={visible.layerBtn} - /> - onToggleExportModal(true)} - visible={visible.exportBtn} - /> - - -); + expandedPipelines, + onToggleExpandAllPipelines, +}) => { + const { toSetQueryParam } = useGeneratePathname(); + + const handleToggleExpandAllPipelines = () => { + const isExpanded = !expandedPipelines; + onToggleExpandAllPipelines(isExpanded); + toSetQueryParam('expandAllPipelines', isExpanded.toString()); + }; + + return ( + <> + + onToggleTextLabels(!textLabels)} + visible={visible.labelBtn} + /> + onToggleLayers(!visibleLayers)} + visible={visible.layerBtn} + /> + + onToggleExportModal(true)} + visible={visible.exportBtn} + /> + + + ); +}; export const mapStateToProps = (state) => ({ disableLayerBtn: !state.layer.ids.length, @@ -76,6 +110,7 @@ export const mapStateToProps = (state) => ({ textLabels: state.textLabels, visible: state.visible, visibleLayers: Boolean(getVisibleLayerIDs(state).length), + expandedPipelines: state.expandAllPipelines, }); export const mapDispatchToProps = (dispatch) => ({ @@ -91,6 +126,10 @@ export const mapDispatchToProps = (dispatch) => ({ onToggleTextLabels: (value) => { dispatch(toggleTextLabels(Boolean(value))); }, + onToggleExpandAllPipelines: (isExpanded) => { + dispatch(toggleExpandAllPipelines(isExpanded)); + dispatch(loadInitialPipelineData()); + }, }); export default connect( diff --git a/src/components/flowchart-primary-toolbar/flowchart-primary-toolbar.test.js b/src/components/flowchart-primary-toolbar/flowchart-primary-toolbar.test.js index f9c332ca2e..a6f328c8a5 100644 --- a/src/components/flowchart-primary-toolbar/flowchart-primary-toolbar.test.js +++ b/src/components/flowchart-primary-toolbar/flowchart-primary-toolbar.test.js @@ -6,10 +6,16 @@ import ConnectedFlowchartPrimaryToolbar, { } from './flowchart-primary-toolbar'; import { mockState, setup } from '../../utils/state.mock'; +jest.mock('../../utils/hooks/use-generate-pathname', () => ({ + useGeneratePathname: () => ({ + toSetQueryParam: jest.fn(), + }), +})); + describe('PrimaryToolbar', () => { it('renders without crashing', () => { const wrapper = setup.mount(); - expect(wrapper.find('.pipeline-icon-toolbar__button').length).toBe(4); + expect(wrapper.find('.pipeline-icon-toolbar__button').length).toBe(5); }); it('hides all buttons (except menu button) when visible prop is false for each of them', () => { @@ -17,6 +23,7 @@ describe('PrimaryToolbar', () => { labelBtn: false, layerBtn: false, exportBtn: false, + pipelineBtn: false, }; const wrapper = setup.mount(, { visible, @@ -31,7 +38,7 @@ describe('PrimaryToolbar', () => { const wrapper = setup.mount(, { visible, }); - expect(wrapper.find('.pipeline-icon-toolbar__button').length).toBe(3); + expect(wrapper.find('.pipeline-icon-toolbar__button').length).toBe(4); }); const functionCalls = [ @@ -39,6 +46,7 @@ describe('PrimaryToolbar', () => { ['.pipeline-menu-button--labels', 'onToggleTextLabels'], ['.pipeline-menu-button--export', 'onToggleExportModal'], ['.pipeline-menu-button--layers', 'onToggleLayers'], + ['.pipeline-menu-button--pipeline', 'onToggleExpandAllPipelines'], ]; test.each(functionCalls)( @@ -62,6 +70,7 @@ describe('PrimaryToolbar', () => { const expectedResult = { disableLayerBtn: expect.any(Boolean), textLabels: expect.any(Boolean), + expandedPipelines: expect.any(Boolean), displaySidebar: true, visible: expect.objectContaining({ exportBtn: expect.any(Boolean), @@ -70,6 +79,7 @@ describe('PrimaryToolbar', () => { settingsModal: expect.any(Boolean), labelBtn: expect.any(Boolean), layerBtn: expect.any(Boolean), + pipelineBtn: expect.any(Boolean), sidebar: expect.any(Boolean), }), visibleLayers: expect.any(Boolean), @@ -113,5 +123,14 @@ describe('PrimaryToolbar', () => { type: 'TOGGLE_TEXT_LABELS', }); }); + + it('onToggleExpandAllPipelines', () => { + const dispatch = jest.fn(); + mapDispatchToProps(dispatch).onToggleExpandAllPipelines(true); + expect(dispatch.mock.calls[0][0]).toEqual({ + type: 'TOGGLE_EXPAND_ALL_PIPELINES', + shouldExpandAllPipelines: true, + }); + }); }); }); diff --git a/src/components/flowchart-wrapper/flowchart-wrapper.js b/src/components/flowchart-wrapper/flowchart-wrapper.js index b305bc73ea..bbc04812f1 100644 --- a/src/components/flowchart-wrapper/flowchart-wrapper.js +++ b/src/components/flowchart-wrapper/flowchart-wrapper.js @@ -92,7 +92,7 @@ export const FlowChartWrapper = ({ if (localStorageParams) { const paramActions = { pipeline: (value) => { - if (!searchParams.has(params.pipeline) && activePipeline) { + if (activePipeline) { toSetQueryParam(params.pipeline, value.active || activePipeline); } }, @@ -110,9 +110,9 @@ export const FlowChartWrapper = ({ disabledKeys && toSetQueryParam(params.types, mappedDisabledNodes); } }, - flags: (value) => { + expandAllPipelines: (value) => { if (!searchParams.has(params.expandAll)) { - toSetQueryParam(params.expandAll, value.expandAllPipelines); + toSetQueryParam(params.expandAll, value); } }, }; diff --git a/src/components/global-toolbar/global-toolbar.scss b/src/components/global-toolbar/global-toolbar.scss index 5a0cb30971..b117e90442 100644 --- a/src/components/global-toolbar/global-toolbar.scss +++ b/src/components/global-toolbar/global-toolbar.scss @@ -46,6 +46,10 @@ } } +.pipeline-menu-button--pipeline svg { + opacity: 0.7; +} + .pipeline-global-routes-toolbar a.active .pipeline-menu-button--link { background-color: var(--color-global-toolbar-active-btn); border-right: 1px solid var(--color-border-line); diff --git a/src/components/global-toolbar/global-toolbar.test.js b/src/components/global-toolbar/global-toolbar.test.js index f591299f08..74e9012844 100644 --- a/src/components/global-toolbar/global-toolbar.test.js +++ b/src/components/global-toolbar/global-toolbar.test.js @@ -56,6 +56,7 @@ describe('GlobalToolbar', () => { miniMapBtn: true, modularPipelineFocusMode: null, metadataModal: false, + pipelineBtn: true, settingsModal: false, shareableUrlModal: false, sidebar: true, diff --git a/src/components/icons/collapse-pipelines.js b/src/components/icons/collapse-pipelines.js new file mode 100644 index 0000000000..e23687c878 --- /dev/null +++ b/src/components/icons/collapse-pipelines.js @@ -0,0 +1,9 @@ +import React from 'react'; + +const CollapsePipelinesIcon = ({ className }) => ( + + + +); + +export default CollapsePipelinesIcon; diff --git a/src/components/icons/expand-pipelines.js b/src/components/icons/expand-pipelines.js new file mode 100644 index 0000000000..d280fa2638 --- /dev/null +++ b/src/components/icons/expand-pipelines.js @@ -0,0 +1,9 @@ +import React from 'react'; + +const ExpandPipelinesIcon = ({ className }) => ( + + + +); + +export default ExpandPipelinesIcon; diff --git a/src/components/settings-modal/settings-modal.js b/src/components/settings-modal/settings-modal.js index c3f081f8e3..aeb0fb5656 100644 --- a/src/components/settings-modal/settings-modal.js +++ b/src/components/settings-modal/settings-modal.js @@ -8,14 +8,9 @@ import { } from '../../actions'; import { getFlagsState } from '../../utils/flags'; import SettingsModalRow from './settings-modal-row'; -import { - settings as settingsConfig, - localStorageName, - params, -} from '../../config'; +import { settings as settingsConfig, localStorageName } from '../../config'; import { saveLocalStorage } from '../../store/helpers'; import { localStorageKeyFeatureHintsStep } from '../../components/feature-hints/feature-hints'; -import { useGeneratePathname } from '../../utils/hooks/use-generate-pathname'; import Button from '../ui/button'; import Modal from '../ui/modal'; @@ -47,8 +42,6 @@ const SettingsModal = ({ useState(showFeatureHints); const [toggleFlags, setToggleFlags] = useState(flags); - const { toSetQueryParam } = useGeneratePathname(); - useEffect(() => { setShowFeatureHintsValue(showFeatureHints); }, [showFeatureHints]); @@ -66,9 +59,6 @@ const SettingsModal = ({ const updatedFlags = Object.entries(toggleFlags); updatedFlags.map((each) => { const [name, value] = each; - if (name === params.expandAll) { - toSetQueryParam(params.expandAll, value); - } return onToggleFlag(name, value); }); @@ -95,7 +85,6 @@ const SettingsModal = ({ onToggleIsPrettyName, showSettingsModal, toggleFlags, - toSetQueryParam, ]); const resetStateCloseModal = () => { @@ -115,14 +104,6 @@ const SettingsModal = ({ >
-
General
-
-
Name
-
State
-
- Description -
-
-
-
-
Experiments
{flagData.map(({ name, value, description }) => ( ))} +
+
{isRunningLocally() ? ( isOutdated ? (
@@ -190,33 +170,34 @@ const SettingsModal = ({
) ) : null} -
-
- - +
+ + +
diff --git a/src/components/settings-modal/settings-modal.scss b/src/components/settings-modal/settings-modal.scss index ea67b59031..b1a5bd0ca2 100644 --- a/src/components/settings-modal/settings-modal.scss +++ b/src/components/settings-modal/settings-modal.scss @@ -2,13 +2,17 @@ @use '../../styles/mixins' as mixins; .kui-theme--light { - --color-modal-header-text: #{colors.$black-700}; + --color-modal-header-text: #{colors.$black-0}; --color-upgrade-reminder-text: #{colors.$blue-300}; + --color-modal-table-text: #{colors.$black-600}; + --color-modal-table-description: #{colors.$black-900-70}; } .kui-theme--dark { - --color-modal-header-text: #{colors.$black-0}; + --color-modal-header-text: #{colors.$black-300}; --color-upgrade-reminder-text: #{colors.$yellow-300}; + --color-modal-table-text: #{colors.$white-900-85}; + --color-modal-table-description: #{colors.$white-900-70}; } .pipeline-settings-modal { @@ -45,21 +49,12 @@ .pipeline-settings-modal__column { margin: 1.145em 0; -} - -.pipeline-settings-modal__column, -.pipeline-settings-modal__header { align-items: baseline; display: flex; flex-wrap: wrap; font-size: 1.4em; } -.pipeline-settings-modal__header { - margin-bottom: 10px; - color: var(--color-modal-header-text); -} - .pipeline-settings-modal__subtitle { margin-bottom: 24px; font-size: 1.8em; @@ -67,10 +62,16 @@ .pipeline-settings-modal__name { @include mixins.flexColumn(3); + + color: var(--color-modal-table-text); + font-weight: 500; } .pipeline-settings-modal__description { @include mixins.flexColumn(6); + + color: var(--color-modal-table-description); + font-weight: 500; } .pipeline-settings-modal__state { diff --git a/src/components/ui/button/button.js b/src/components/ui/button/button.js index 3bc486ce35..84f009a7ce 100644 --- a/src/components/ui/button/button.js +++ b/src/components/ui/button/button.js @@ -6,7 +6,14 @@ import './button.scss'; /** * Generic Kedro Button */ -const Button = ({ children, dataTest, disabled, onClick, size, mode }) => ( +const Button = ({ + children, + dataTest = 'TestDefaultDataValue', + disabled = false, + onClick, + size = 'regular', + mode = 'primary', +}) => (