diff --git a/.gitignore b/.gitignore index 8eac626..cf7b8aa 100644 --- a/.gitignore +++ b/.gitignore @@ -19,4 +19,5 @@ venv .*.sw? -.env \ No newline at end of file +.env +.DS_Store diff --git a/app/data_sources/trello.py b/app/data_sources/trello.py new file mode 100644 index 0000000..ca7cf0c --- /dev/null +++ b/app/data_sources/trello.py @@ -0,0 +1,161 @@ +import datetime +import logging +from typing import Dict, List +import requests +from dataclasses import dataclass +import json + +from data_source_api.base_data_source import BaseDataSource, ConfigField, HTMLInputType +from data_source_api.basic_document import DocumentType, BasicDocument +from data_source_api.exception import InvalidDataSourceConfig +from index_queue import IndexQueue +from parsers.html import html_to_text +from data_source_api.utils import parse_with_workers + +logger = logging.getLogger(__name__) + +@dataclass +class TrelloConfig(): + organization_name: str + api_key: str + api_token: str + + def __post_init__(self): + self.organization_name = self.organization_name.lower().replace(' ','') + +class TrelloDataSource(BaseDataSource): + @staticmethod + def get_config_fields() -> List[ConfigField]: + return [ + ConfigField(label="Trello Organization Name", name="organization_name", type="text"), + ConfigField(label="API Key", name="api_key", type=HTMLInputType.PASSWORD), + ConfigField(label="API Token", name="api_token", type=HTMLInputType.PASSWORD), + ] + + @staticmethod + def validate_config(config: Dict) -> None: + try: + trello_config = TrelloConfig(**config) + url = f"https://api.trello.com/1/organizations/{trello_config.organization_name}/boards" + + headers = { + "Accept": "application/json" + } + + query = { + 'key': trello_config.api_key, + 'token': trello_config.api_token + } + response = requests.request("GET", url, headers=headers, params=query) + if response.status_code != 200: + raise Exception(f"None 200 status code returned. {response.status_code}") + except Exception as e: + raise InvalidDataSourceConfig from e + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._trello_config = TrelloConfig(**self._config) + self._headers = { + "Accept": "application/json" + } + + def _fetch_board_name(self, card_id) -> str: + url = f"https://api.trello.com/1/cards/{card_id}/board" + query = { + 'key': self._trello_config.api_key, + 'token': self._trello_config.api_token + } + + response = requests.request( + "GET", + url, + params=query + ) + return json.loads(response.text)['name'] + + def _fetch_card_comments(self, card_id) -> List[Dict]: + url = f"https://api.trello.com/1/cards/{card_id}/actions?filter=commentCard" + query = { + 'key': self._trello_config.api_key, + 'token': self._trello_config.api_token + } + + response = requests.request( + "GET", + url, + params=query + ) + return json.loads(response.text) + + def _parse_documents_worker(self, raw_docs: List[Dict]): + logging.info(f'Worker parsing {len(raw_docs)} documents') + parsed_docs = [] + total_fed = 0 + + for raw_page in raw_docs: + comments = self._fetch_card_comments(raw_page['id']) + if len(comments): + for comment in comments: + last_modified = datetime.datetime.strptime(comment['date'], "%Y-%m-%dT%H:%M:%S.%fZ") + if last_modified < self._last_index_time: + continue + + html_content = comment['data']['text'] + plain_text = html_to_text(html_content) + + parsed_docs.append(BasicDocument(title=raw_page['name'], + content=plain_text, + author=comment['memberCreator']['fullName'], + author_image_url=comment['memberCreator']['avatarUrl'], + timestamp=last_modified, + id=comment['id'], + data_source_id=self._data_source_id, + url=raw_page['shortUrl'], + type=DocumentType.COMMENT, + location=self._fetch_board_name(raw_page['id']))) + if len(parsed_docs) >= 50: + total_fed += len(parsed_docs) + IndexQueue.get_instance().put(docs=parsed_docs) + parsed_docs = [] + + IndexQueue.get_instance().put(docs=parsed_docs) + total_fed += len(parsed_docs) + if total_fed > 0: + logging.info(f'Worker fed {total_fed} documents') + + + def _list_boards(self) -> List[Dict]: + url = f"https://api.trello.com/1/organizations/{self._trello_config.organization_name}/boards" + + headers = { + "Accept": "application/json" + } + + query = { + 'key': self._trello_config.api_key, + 'token': self._trello_config.api_token + } + return json.loads(requests.request("GET", url, headers=headers, params=query).text) + + def _feed_new_documents(self) -> None: + logger.info('Feeding new Trello Cards') + boards = self._list_boards() + raw_docs = [] + for i in range(0, len(boards), 1): + url = f"https://api.trello.com/1/boards/{boards[i]['id']}/cards" + query = { + 'key': self._trello_config.api_key, + 'token': self._trello_config.api_token + } + response = requests.request( + "GET", + url, + params=query + ) + card_results = json.loads(response.text) + for card in card_results: + raw_docs.append(card) + parse_with_workers(self._parse_documents_worker, raw_docs) + + + \ No newline at end of file diff --git a/app/static/data_source_icons/trello.png b/app/static/data_source_icons/trello.png new file mode 100644 index 0000000..d0df984 Binary files /dev/null and b/app/static/data_source_icons/trello.png differ diff --git a/docs/data-sources/trello/APIKey.png b/docs/data-sources/trello/APIKey.png new file mode 100644 index 0000000..ae2a7db Binary files /dev/null and b/docs/data-sources/trello/APIKey.png differ diff --git a/docs/data-sources/trello/NewAPIKey.png b/docs/data-sources/trello/NewAPIKey.png new file mode 100644 index 0000000..1be5faf Binary files /dev/null and b/docs/data-sources/trello/NewAPIKey.png differ diff --git a/docs/data-sources/trello/NewPage.png b/docs/data-sources/trello/NewPage.png new file mode 100644 index 0000000..38ae122 Binary files /dev/null and b/docs/data-sources/trello/NewPage.png differ diff --git a/docs/data-sources/trello/PowerUpForm.png b/docs/data-sources/trello/PowerUpForm.png new file mode 100644 index 0000000..b36cf8c Binary files /dev/null and b/docs/data-sources/trello/PowerUpForm.png differ diff --git a/docs/data-sources/trello/trello.md b/docs/data-sources/trello/trello.md new file mode 100644 index 0000000..570a60a --- /dev/null +++ b/docs/data-sources/trello/trello.md @@ -0,0 +1,15 @@ +# Setting up Trello data source + +Please note that all cards on all boards you have access to will be indexed. +1. Navigate to the [Trello Docs](https://developer.atlassian.com/cloud/trello/guides/rest-api/api-introduction/) and navigate to the Power Ups page. +2. Once on the Power-Ups page click the new button. +![New Page](./NewPage.png) +3. Fill out the form on the page, and save your Power-Up. +![Power-Up form](PowerUpForm.png) +4. On the next page click the "Generate a new API key" option. +![New API Key](NewAPIKey.png) +5. Copy the API Key and paste it into Gerev. +6. Click on the "Token" link, and authorize the app on the next page. +![API Key](APIKey.png) +7. Copy the token that appears on the next page and paste it into the API Token field. +8. Specify your Organization name and save the data source. \ No newline at end of file diff --git a/ui/src/components/data-source-panel.tsx b/ui/src/components/data-source-panel.tsx index 7e2c1d3..4299aeb 100644 --- a/ui/src/components/data-source-panel.tsx +++ b/ui/src/components/data-source-panel.tsx @@ -297,6 +297,17 @@ export default class DataSourcePanel extends React.ComponentNote that the url must begin with either http:// or https://

)} + + {this.state.selectedDataSource.value === 'trello' && ( + + 1. {'Navigate to https://developer.atlassian.com/cloud/trello/guides/rest-api/api-introduction/ and follow the steps here'} + 2. {'Once you are on your Power Ups page Click "New"'} + 3. {'Fill out the form and click create'} + 4. {'Click Generate a New API Key on the page you were redirected to'} + 5. {'Copy the API Key and paste it here'} + 6. {'Click "Token" in the paragraph next to the API Key, Authorize your app, and paste the API Token provided here'} + + )}