diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 6a6c589a..bea38a36 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -75,9 +75,7 @@ jobs: EQ_BASIC_USER_NAME: ${{ secrets.EQ_BASIC_USER_NAME }} EQ_BASIC_USER_PWD: ${{ secrets.EQ_BASIC_USER_PWD }} EQ_SECRET: ${{ secrets.EQ_SECRET_DEV }} - JSON_PAGE_SIZE: 1000 MAX_QUERY_SIZE: 1000000 - MAX_VALUES_QUERY_SIZE: 100 SERVER_BASE_PATH: /expertquery SERVER_URL: https://owapps-dev.app.cloud.gov/expertquery STREAM_BATCH_SIZE: 2000 @@ -146,9 +144,7 @@ jobs: cf set-env $APP_NAME "EQ_BASIC_USER_NAME" "$EQ_BASIC_USER_NAME" > /dev/null cf set-env $APP_NAME "EQ_BASIC_USER_PWD" "$EQ_BASIC_USER_PWD" > /dev/null cf set-env $APP_NAME "EQ_SECRET" "$EQ_SECRET" > /dev/null - cf set-env $APP_NAME "JSON_PAGE_SIZE" "$JSON_PAGE_SIZE" > /dev/null cf set-env $APP_NAME "MAX_QUERY_SIZE" "$MAX_QUERY_SIZE" > /dev/null - cf set-env $APP_NAME "MAX_VALUES_QUERY_SIZE" "$MAX_VALUES_QUERY_SIZE" > /dev/null cf set-env $APP_NAME "PUBLIC_URL" "$SERVER_URL" > /dev/null cf set-env $APP_NAME "SERVER_BASE_PATH" "$SERVER_BASE_PATH" > /dev/null cf set-env $APP_NAME "SERVER_URL" "$SERVER_URL" > /dev/null diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index a06d76d9..fbdfced0 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -76,9 +76,7 @@ jobs: EQ_BASIC_USER_NAME: ${{ secrets.EQ_BASIC_USER_NAME }} EQ_BASIC_USER_PWD: ${{ secrets.EQ_BASIC_USER_PWD }} EQ_SECRET: ${{ secrets.EQ_SECRET_STAGING }} - JSON_PAGE_SIZE: 1000 MAX_QUERY_SIZE: 1000000 - MAX_VALUES_QUERY_SIZE: 100 SERVER_BASE_PATH: /expertquery SERVER_URL: https://owapps-stage.app.cloud.gov/expertquery STREAM_BATCH_SIZE: 2000 @@ -147,9 +145,7 @@ jobs: cf set-env $APP_NAME "EQ_BASIC_USER_NAME" "$EQ_BASIC_USER_NAME" > /dev/null cf set-env $APP_NAME "EQ_BASIC_USER_PWD" "$EQ_BASIC_USER_PWD" > /dev/null cf set-env $APP_NAME "EQ_SECRET" "$EQ_SECRET" > /dev/null - cf set-env $APP_NAME "JSON_PAGE_SIZE" "$JSON_PAGE_SIZE" > /dev/null cf set-env $APP_NAME "MAX_QUERY_SIZE" "$MAX_QUERY_SIZE" > /dev/null - cf set-env $APP_NAME "MAX_VALUES_QUERY_SIZE" "$MAX_VALUES_QUERY_SIZE" > /dev/null cf set-env $APP_NAME "PUBLIC_URL" "$SERVER_URL" > /dev/null cf set-env $APP_NAME "SERVER_BASE_PATH" "$SERVER_BASE_PATH" > /dev/null cf set-env $APP_NAME "SERVER_URL" "$SERVER_URL" > /dev/null diff --git a/app/client/public/css/styles.css b/app/client/public/css/styles.css index 55647c34..fd3b9e38 100644 --- a/app/client/public/css/styles.css +++ b/app/client/public/css/styles.css @@ -92,6 +92,14 @@ cursor: pointer; } +.layout-fixed { + table-layout: fixed; +} + +.maxh-90vh { + max-height: 90vh; +} + .sr-only { border: 0; clip: rect(0, 0, 0, 0); @@ -115,6 +123,10 @@ text-shadow: 0 0 3px rgba(0, 0, 0, 0.5); } +.whitespace-wrap tbody td { + white-space: normal; +} + .width-fit { width: fit-content; } diff --git a/app/client/public/scss/_uswds-theme.scss b/app/client/public/scss/_uswds-theme.scss index d80cbc3a..2da28055 100644 --- a/app/client/public/scss/_uswds-theme.scss +++ b/app/client/public/scss/_uswds-theme.scss @@ -16,6 +16,10 @@ in the form $setting: value, 'output': true, 'responsive': true, ), + $max-height-settings: ( + 'output': true, + 'responsive': true, + ), $theme-font-weight-semibold: 600, $theme-show-notifications: false, $theme-utility-breakpoints: ( diff --git a/app/client/src/components/app.tsx b/app/client/src/components/app.tsx index a9d6a815..9ad11917 100644 --- a/app/client/src/components/app.tsx +++ b/app/client/src/components/app.tsx @@ -16,7 +16,7 @@ import { cloudSpace, serverBasePath, serverUrl } from 'config'; // utils import { getData } from 'utils'; // types -import type { Content, JsonContent } from 'contexts/content'; +import type { Content } from 'contexts/content'; declare global { interface Window { @@ -27,19 +27,6 @@ declare global { } } -// Map profile columns arrays to Sets -function parseProfileConfig(jsonConfig: JsonContent['profileConfig']) { - return Object.entries(jsonConfig).reduce((current, [key, config]) => { - return { - ...current, - [key]: { - ...config, - columns: new Set(config.columns), - }, - }; - }, {}) as Content['profileConfig']; -} - /** Custom hook to fetch static content */ function useFetchedContent() { const contentDispatch = useContentDispatch(); @@ -48,17 +35,14 @@ function useFetchedContent() { const controller = new AbortController(); contentDispatch({ type: 'FETCH_CONTENT_REQUEST' }); - getData({ + getData({ url: `${serverUrl}/api/lookupFiles`, signal: controller.signal, }) .then((res) => { contentDispatch({ type: 'FETCH_CONTENT_SUCCESS', - payload: { - ...res, - profileConfig: parseProfileConfig(res.profileConfig), - }, + payload: res, }); }) .catch((err: Error) => { diff --git a/app/client/src/components/downloadModal.tsx b/app/client/src/components/downloadModal.tsx index ad909fec..5bd95889 100644 --- a/app/client/src/components/downloadModal.tsx +++ b/app/client/src/components/downloadModal.tsx @@ -12,13 +12,13 @@ import { clientUrl } from 'config'; // styles import '@reach/dialog/styles.css'; // types -import type { FetchState, Status, Value } from 'types'; +import type { FetchState, QueryData, Status } from 'types'; /* ## Components */ -export function DownloadModal({ +export function DownloadModal({ apiKey, dataId, downloadStatus, @@ -231,7 +231,7 @@ export function DownloadModal({ ## Types */ -type DownloadModalProps = { +type DownloadModalProps = { apiKey: string; dataId?: string; downloadStatus: Status; @@ -241,13 +241,3 @@ type DownloadModalProps = { queryUrl: string | null; setDownloadStatus: (status: Status) => void; }; - -type PostData = { - columns: string[]; - filters: { - [field: string]: Value | Value[]; - }; - options: { - [field: string]: Value; - }; -}; diff --git a/app/client/src/components/inPageNav.tsx b/app/client/src/components/inPageNav.tsx index a62a9a12..d03451ae 100644 --- a/app/client/src/components/inPageNav.tsx +++ b/app/client/src/components/inPageNav.tsx @@ -113,7 +113,9 @@ function useInPageNavDispatch() { ## Components */ -export function InPageNavLayout({ children }: Readonly<{ children: ReactNode }>) { +export function InPageNavLayout({ + children, +}: Readonly<{ children: ReactNode }>) { return ( {children} diff --git a/app/client/src/components/page.tsx b/app/client/src/components/page.tsx index 917130b6..fe4d6739 100644 --- a/app/client/src/components/page.tsx +++ b/app/client/src/components/page.tsx @@ -94,7 +94,12 @@ type HeaderLinkProps = { href: string; }; -function HeaderLink({ className, children, icon, href }: Readonly) { +function HeaderLink({ + className, + children, + icon, + href, +}: Readonly) { const Icon = icon; return ( diff --git a/app/client/src/components/previewModal.tsx b/app/client/src/components/previewModal.tsx new file mode 100644 index 00000000..5674b5bc --- /dev/null +++ b/app/client/src/components/previewModal.tsx @@ -0,0 +1,234 @@ +import { Dialog } from '@reach/dialog'; +import Close from 'images/close.svg?react'; +import { uniqueId } from 'lodash'; +import { useEffect, useMemo, useState } from 'react'; +// components +import { Alert } from 'components/alert'; +import { Loading } from 'components/loading'; +import { Table } from 'components/table'; +// utils +import { isAbort, postData, useAbort } from 'utils'; +// styles +import '@reach/dialog/styles.css'; +// types +import type { ColumnConfig, FetchState, QueryData, Value } from 'types'; + +const RANK_KEY = 'rankPercent'; + +export function PreviewModal({ + apiKey, + columns, + limit, + onClose, + ranked = true, + queryData, + queryUrl, +}: Readonly>) { + const { abort, getSignal } = useAbort(); + + const tableColumns = useMemo(() => { + const columnsToShow = columns.filter((column) => + column.hasOwnProperty('preview'), + ) as Array>; + const columnsOrdered = columnsToShow.toSorted( + (a, b) => (a.preview.order ?? Infinity) - (b.preview.order ?? Infinity), + ); + const columnDefs = columnsOrdered.map((column) => ({ + id: column.key, + name: column.preview.label || column.key, + sortable: column.preview.sortable ?? false, + width: column.preview.width ?? 100, + })); + if (ranked) { + return [ + { + id: RANK_KEY, + name: 'Rank (%)', + sortable: true, + width: 100, + }, + ...columnDefs, + ]; + } + + return columnDefs; + }, [columns]); + + const closeModal = () => { + abort(); + onClose(); + }; + + const [id] = useState(uniqueId('modal-')); + + // Data to be displayed in the preview table. + const [preview, setPreview] = useState | string>>({ + data: null, + status: 'idle', + }); + + useEffect(() => { + setPreview({ data: null, status: 'pending' }); + postData({ + url: queryUrl, + apiKey, + data: { + ...queryData, + options: { + ...queryData.options, + format: 'json', + pageSize: limit, + }, + }, + signal: getSignal(), + }) + .then((res) => { + if (!Array.isArray(res.data)) { + setPreview({ data: res.message, status: 'success' }); + return; + } + + const columnMap = columns.reduce>( + (acc, column) => { + acc[column.key] = column; + return acc; + }, + {}, + ); + const data = res.data.map((row: DataRow) => + Object.entries(row) + .filter( + ([key, _value]) => + key === RANK_KEY || columnMap[key].hasOwnProperty('preview'), + ) + .toSorted(([a], [b]) => { + if (ranked) { + if (a === RANK_KEY) return -1; + if (b === RANK_KEY) return 1; + } + return ( + (columnMap[a].preview?.order ?? Infinity) - + (columnMap[b].preview?.order ?? Infinity) + ); + }) + .map(([key, value]) => { + const column = columnMap[key]; + if (column?.preview?.transform?.type === 'link') { + const textColumn = column.preview.transform.args[0]; + return { + sortValue: row[textColumn], + value: ( + + {row[textColumn]} + + ), + }; + } else { + return value; + } + }), + ); + setPreview({ data, status: 'success' }); + }) + .catch((err) => { + if (isAbort(err)) return; + console.error(err); + setPreview({ data: null, status: 'failure' }); + }); + }, [apiKey, queryData, queryUrl]); + + return ( + +
+
+

+ Results Preview{' '} +

+ {preview.status === 'pending' && ( +
+ +

Searching, please wait...

+
+ )} + {preview.status === 'failure' && ( + + The specified query could not be executed at this time. + + )} + {preview.status === 'success' && ( + <> + {typeof preview.data === 'string' ? ( + {preview.data} + ) : ( + <> + {preview.data.length === 0 ? ( + No results found + ) : ( + <> + {preview.data.length >= limit && ( + + Limited to {limit} rows + + )} + + + )} + + )} + + )} + + + + + ); +} + +/* +## Types +*/ + +type DataRow = Record; + +type PreviewModalProps = { + apiKey: string; + columns: ColumnConfig[]; + limit: number; + onClose: () => void; + ranked?: boolean; + queryData: D; + queryUrl: string; +}; + +export default PreviewModal; diff --git a/app/client/src/components/table.tsx b/app/client/src/components/table.tsx new file mode 100644 index 00000000..978b32e9 --- /dev/null +++ b/app/client/src/components/table.tsx @@ -0,0 +1,235 @@ +/** Adapted from https://github.com/MetroStar/comet/blob/main/packages/comet-uswds/src/components/table/table.tsx */ +import table from '@uswds/uswds/js/usa-table'; +import classNames from 'classnames'; +import { useState } from 'react'; +// types +import type { ReactNode } from 'react'; + +function isCellSpec(value: any): value is TableCell { + return ( + typeof value === 'object' && value !== null && value.hasOwnProperty('value') + ); +} + +export const Table = ({ + id, + caption, + columns, + data, + sortable = false, + initialSortIndex = 0, + initialSortDir = 'ascending', + scrollable = false, + borderless = false, + stacked = false, + stickyHeader = false, + striped = false, + className, + tabIndex = -1, +}: TableProps): React.ReactElement => { + // Swap sort direction. + const getSortDirection = (prevSortDir: 'ascending' | 'descending') => { + if (prevSortDir === 'descending') { + return 'ascending'; + } else { + return 'descending'; + } + }; + + const [sortDir, setSortDir] = useState<'ascending' | 'descending'>( + getSortDirection(initialSortDir), // FIXME: This is a bug (possible race condition with `epa.js`), it should be `initialSortDir` + ); + const [sortIndex, setSortIndex] = useState(initialSortIndex); + + // If a header of a sortable column is clicked, sort the column or change the sort direction. + const handleHeaderClick = (index: number) => { + const column = columns[index]; + if (column?.sortable) { + if (sortIndex === index) { + setSortDir((prevSortDir) => getSortDirection(prevSortDir)); + } else { + setSortIndex(index); + } + } + }; + + return ( +
{ + if (node && sortable) { + table.on(node); + } + }} + > +
+ + + + {columns + .map((obj) => ({ + ...obj, + sortable: obj.sortable !== undefined ? obj.sortable : true, + })) + .map((column: TableColumn, index: number) => ( + + ))} + + + + {data.map((row, i: number) => { + const rowData: TableCell[] = []; + row.forEach((cell: string | number | TableCell) => { + if (sortable) { + rowData.push({ + value: isCellSpec(cell) ? cell.value : cell, + sortValue: isCellSpec(cell) + ? (cell.sortValue ?? cell.value ?? '').toString() + : cell, + }); + } else { + rowData.push({ + value: isCellSpec(cell) ? cell.value : cell, + }); + } + }); + + return ( + + {rowData.map((col, j) => ( + + ))} + + ); + })} + +
handleHeaderClick(index)} + style={{ width: column.width ? `${column.width}px` : 'auto' }} + > + {column.name} +
+ {col.value} +
+ {sortable && ( +
+ )} +
+ ); +}; + +/* +## Types +*/ + +type TableProps = { + /** + * The unique identifier for this component + */ + id: string; + /** + * The table header details for the table + */ + columns: TableColumn[]; + /** + * The data to display in the table rows + */ + data: T[]; + /** + * An optional caption to display above the table + */ + caption?: string; + /** + * A boolean indicating if the table is sortable or not + */ + sortable?: boolean; + /** + * The column index to set as the default sort + */ + initialSortIndex?: number; + /** + * The default sort direction if sortIndex is provided + */ + initialSortDir?: 'ascending' | 'descending'; + /** + * A function to call when the table is sorted + */ + onSort?: () => void; + /** + * A boolean indicating if the table is scrollable or not + */ + scrollable?: boolean; + /** + * A boolean indicating if the table is borderless or not + */ + borderless?: boolean; + /** + * A boolean indicating if the table should use a stacked layout or not + */ + stacked?: boolean; + /** + * A boolean indicating if the table has a sticky header or not + */ + stickyHeader?: boolean; + /** + * A boolean indicating if the table is striped or not + */ + striped?: boolean; + /** + * Additional class names for the table + */ + className?: string; + /** + * Used primarily to make table focusable + */ + tabIndex?: number; +}; + +type TableColumn = { + id: string; + name: string; + sortable?: boolean; + width?: number; +}; + +type TableCell = { + value: ReactNode; + sortValue?: string | number; +}; + +export default Table; diff --git a/app/client/src/contexts/content.tsx b/app/client/src/contexts/content.tsx index 51b7a9de..a1aaf54c 100644 --- a/app/client/src/contexts/content.tsx +++ b/app/client/src/contexts/content.tsx @@ -1,6 +1,7 @@ import { createContext, useContext, useReducer } from 'react'; import type { Dispatch, ReactNode } from 'react'; import type { + ColumnConfig, DomainOptions, MultiOptionField, SingleOptionField, @@ -66,13 +67,14 @@ export type Content = { }; parameters: { debounceMilliseconds: number; + searchPreviewPageSize: number; selectOptionsPageSize: number; }; profileConfig: { [key: string]: { key: string; description: string; - columns: Set; + columns: Array; label: string; resource: string; }; @@ -88,17 +90,6 @@ export type Content = { }; }; -export type JsonContent = Omit & { - profileConfig: { - [key: string]: { - description: string; - columns: string[]; - label: string; - resource: string; - }; - }; -}; - type State = { content: | { status: 'idle'; data: Record } diff --git a/app/client/src/images/search.svg b/app/client/src/images/search.svg new file mode 100644 index 00000000..cd9fd53c --- /dev/null +++ b/app/client/src/images/search.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/app/client/src/routes/home.tsx b/app/client/src/routes/home.tsx index f68de772..654e336b 100644 --- a/app/client/src/routes/home.tsx +++ b/app/client/src/routes/home.tsx @@ -9,6 +9,7 @@ import { import Select from 'react-select'; import { AsyncPaginate, wrapMenuList } from 'react-select-async-paginate'; import Download from 'images/file_download.svg?react'; +import Search from 'images/search.svg?react'; // components import { AccordionItem } from 'components/accordion'; import { Alert } from 'components/alert'; @@ -20,6 +21,7 @@ import { Loading } from 'components/loading'; import { DownloadModal } from 'components/downloadModal'; import { ClearSearchModal } from 'components/clearSearchModal'; import { MenuList as CustomMenuList } from 'components/menuList'; +import { PreviewModal } from 'components/previewModal'; import { RadioButtons } from 'components/radioButtons'; import { SourceSelect } from 'components/sourceSelect'; import { StepIndicator } from 'components/stepIndicator'; @@ -41,6 +43,8 @@ import type { Option, SingleOptionField, SingleValueField, + SingleValueRangeField, + SingleValueTextField, StaticOptions, Status, Value, @@ -50,8 +54,6 @@ import type { ## Components */ -export default Home; - function HomeContent({ content }: Readonly<{ content: Content }>) { const { domainValues, @@ -216,6 +218,7 @@ function HomeContent({ content }: Readonly<{ content: Content }>) { format, formatHandler: setFormat, glossary, + previewLimit: content.parameters.searchPreviewPageSize, profile, queryParams, resetFilters, @@ -265,6 +268,7 @@ export function QueryBuilder() { format, formatHandler, glossary, + previewLimit, profile, resetFilters, sourceFields, @@ -274,16 +278,22 @@ export function QueryBuilder() { } = useHomeContext(); const { - clearConfirmationVisible, - closeClearConfirmation, - openClearConfirmation, - } = useClearConfirmationVisibility(); + visible: clearConfirmationVisible, + close: closeClearConfirmation, + open: openClearConfirmation, + } = useModalVisibility(); const { - closeDownloadConfirmation, - downloadConfirmationVisible, - openDownloadConfirmation, - } = useDownloadConfirmationVisibility(); + close: closeDownloadConfirmation, + visible: downloadConfirmationVisible, + open: openDownloadConfirmation, + } = useModalVisibility(); + + const { + close: closeSearchPreview, + visible: searchPreviewVisible, + open: openSearchPreview, + } = useModalVisibility(); const [downloadStatus, setDownloadStatus] = useDownloadStatus( profile, @@ -296,33 +306,21 @@ export function QueryBuilder() { return ( <> - {downloadConfirmationVisible && ( - - )} - {clearConfirmationVisible && ( - { - resetFilters(); - navigate('/attains', { replace: true }); - }} - onClose={closeClearConfirmation} - /> - )}
+ {clearConfirmationVisible && ( + { + resetFilters(); + navigate('/attains', { replace: true }); + }} + onClose={closeClearConfirmation} + /> + )} + {profile.key === 'actionDocuments' && ( + <> +
+ +
+ {searchPreviewVisible && ( + { + const column = profile.columns.find((col) => col.key === key); + return column?.ranked === true; + })} + /> + )} + + )} + Download + {downloadConfirmationVisible && ( + + )} {downloadStatus === 'success' && ( = removeNulls( - fields.map((fieldConfig) => { - const sourceFieldConfig = - 'source' in fieldConfig && - (fieldConfig.source as string) in sourceFields - ? sourceFields[fieldConfig.source as string] - : null; - - const tooltip = - fieldConfig.label in glossary - ? glossary[fieldConfig.label].definition - : null; - - switch (fieldConfig.type) { - case 'multiselect': - case 'select': - if ( - !sourceFieldConfig && - fieldConfig.type === 'multiselect' && - fieldConfig.key in staticOptions && - staticOptions[fieldConfig.key].length <= 5 - ) { + const fieldsJsx: Array<[JSX.Element, string, string | undefined]> = + removeNulls( + fields.map((fieldConfig) => { + const sourceFieldConfig = + 'source' in fieldConfig && + (fieldConfig.source as string) in sourceFields + ? sourceFields[fieldConfig.source as string] + : null; + + const tooltip = + fieldConfig.label in glossary + ? glossary[fieldConfig.label].definition + : null; + + switch (fieldConfig.type) { + case 'multiselect': + case 'select': + if ( + !sourceFieldConfig && + fieldConfig.type === 'multiselect' && + fieldConfig.key in staticOptions && + staticOptions[fieldConfig.key].length <= 5 + ) { + return [ + , + fieldConfig.key, + fieldConfig.size, + ]; + } + + const sourceKey = sourceFieldConfig?.key ?? null; + const sourceValue = sourceFieldConfig + ? sourceState[sourceFieldConfig.id] + : null; + const selectProps = { + additionalOptions: 'additionalOptions' in fieldConfig ? fieldConfig.additionalOptions : [], + apiKey, + apiUrl, + contextFilters: getContextFilters( + fieldConfig, + Object.values(filterFields).concat(Object.values(sourceFields)), + profile, + { + ...queryParams.filters, + ...(sourceKey && sourceValue + ? { [sourceKey]: sourceValue.value } + : {}), + }, + ), + defaultOption: + 'default' in fieldConfig ? fieldConfig.default : null, + filterHandler: filterHandlers[fieldConfig.key], + filterKey: fieldConfig.key, + filterLabel: fieldConfig.label, + filterValue: filterState[fieldConfig.key], + isMulti: isMultiOptionField(fieldConfig), + profile, + secondaryFilterKey: + 'secondaryKey' in fieldConfig ? fieldConfig.secondaryKey : null, + sortDirection: + 'direction' in fieldConfig + ? (fieldConfig.direction as SortDirection) + : 'asc', + sourceKey, + sourceValue, + staticOptions, + } as SelectFilterProps; + return [ - + + + {tooltip && ( + + )} + +
+ {sourceFieldConfig ? ( + + ) : ( + + )} +
+
, + fieldConfig.key, + fieldConfig.size, + ]; + case 'date': + case 'year': + // Prevents range fields from rendering twice + if (fieldConfig.boundary === 'high') return null; + + const pairedField = fields.find( + (otherField) => + otherField.key !== fieldConfig.key && + otherField.type === fieldConfig.type && + otherField.domain === fieldConfig.domain, + ); + // All range inputs should have a high and a low boundary field + if (!pairedField || !isSingleValueRangeField(pairedField)) + return null; + + return [ + , - fieldConfig.key, + fieldConfig.domain, + fieldConfig.size, ]; - } - - const sourceKey = sourceFieldConfig?.key ?? null; - const sourceValue = sourceFieldConfig - ? sourceState[sourceFieldConfig.id] - : null; - const selectProps = { - additionalOptions: 'additionalOptions' in fieldConfig ? fieldConfig.additionalOptions : [], - apiKey, - apiUrl, - contextFilters: getContextFilters( - fieldConfig, - Object.values(filterFields).concat(Object.values(sourceFields)), - profile, - { - ...queryParams.filters, - ...(sourceKey && sourceValue - ? { [sourceKey]: sourceValue.value } - : {}), - }, - ), - defaultOption: - 'default' in fieldConfig ? fieldConfig.default : null, - filterHandler: filterHandlers[fieldConfig.key], - filterKey: fieldConfig.key, - filterLabel: fieldConfig.label, - filterValue: filterState[fieldConfig.key], - isMulti: isMultiOptionField(fieldConfig), - profile, - secondaryFilterKey: - 'secondaryKey' in fieldConfig ? fieldConfig.secondaryKey : null, - sortDirection: - 'direction' in fieldConfig - ? (fieldConfig.direction as SortDirection) - : 'asc', - sourceKey, - sourceValue, - staticOptions, - } as SelectFilterProps; - - return [ -
- - - {tooltip && ( - - )} - -
- {sourceFieldConfig ? ( - + + + {tooltip && ( + + )} + +
+ - ) : ( - - )} -
-
, - fieldConfig.key, - ]; - case 'date': - case 'year': - // Prevents range fields from rendering twice - if (fieldConfig.boundary === 'high') return null; - - const pairedField = fields.find( - (otherField) => - otherField.key !== fieldConfig.key && - otherField.type === fieldConfig.type && - otherField.domain === fieldConfig.domain, - ); - // All range inputs should have a high and a low boundary field - if (!pairedField || !isSingleValueField(pairedField)) return null; - - return [ - , - fieldConfig.domain, - ]; - default: - return null; - } - }), - ); +
+
, + fieldConfig.key, + fieldConfig.size, + ]; + default: + return null; + } + }), + ); return (
- {fieldsJsx.map(([field, key]) => ( -
+ {fieldsJsx.map(([field, key, size]) => ( +
{field}
))} @@ -948,41 +1045,21 @@ function SelectFilter({ ## Hooks */ -function useClearConfirmationVisibility() { - const [clearConfirmationVisible, setClearConfirmationVisible] = - useState(false); - - const closeClearConfirmation = useCallback(() => { - setClearConfirmationVisible(false); - }, []); - - const openClearConfirmation = useCallback(() => { - setClearConfirmationVisible(true); - }, []); - - return { - clearConfirmationVisible, - closeClearConfirmation, - openClearConfirmation, - }; -} - -function useDownloadConfirmationVisibility() { - const [downloadConfirmationVisible, setDownloadConfirmationVisible] = - useState(false); +function useModalVisibility() { + const [visible, setVisible] = useState(false); - const closeDownloadConfirmation = useCallback(() => { - setDownloadConfirmationVisible(false); + const close = useCallback(() => { + setVisible(false); }, []); - const openDownloadConfirmation = useCallback(() => { - setDownloadConfirmationVisible(true); + const open = useCallback(() => { + setVisible(true); }, []); return { - closeDownloadConfirmation, - downloadConfirmationVisible, - openDownloadConfirmation, + visible, + close, + open, }; } @@ -1151,7 +1228,7 @@ function useQueryParams({ return { filters: buildFilterData(filterFields, filterState, profile), options: { format }, - columns: Array.from(profile.columns), + columns: profile.columns.map((column) => column.key), }; }, [filterFields, filterState, format, profile]); @@ -1636,6 +1713,8 @@ async function getUrlInputs( newState[key] = matchDate(params[key] ?? null); } else if (isYearField(filterField)) { newState[key] = matchYear(params[key] ?? null); + } else if (isSingleValueTextField(filterField)) { + newState[key] = (params[key] ?? '').toString(); } }), ]); @@ -1680,8 +1759,9 @@ function isOption(maybeOption: Option | string): maybeOption is Option { function isProfileField(field: FilterField, profile: Profile) { const profileColumns = profile.columns; - if (profileColumns.has(field.key)) return true; - if ('domain' in field && profileColumns.has(field.domain)) return true; + if (profileColumns.some((c) => c.key === field.key)) return true; + if ('domain' in field && profileColumns.some((c) => c.key === field.domain)) + return true; return false; } @@ -1692,9 +1772,23 @@ function isSingleOptionField(field: FilterField): field is SingleOptionField { // Type narrowing function isSingleValueField(field: FilterField): field is SingleValueField { + return isSingleValueTextField(field) || isSingleValueRangeField(field); +} + +// Type narrowing +function isSingleValueRangeField( + field: FilterField, +): field is SingleValueRangeField { return field.type === 'date' || field.type === 'year'; } +// Type narrowing +function isSingleValueTextField( + field: FilterField, +): field is SingleValueTextField { + return field.type === 'text'; +} + // Type narrowing function isYearField(field: FilterField) { return field.type === 'year'; @@ -1926,6 +2020,8 @@ type FilterQueryData = { }; type HomeContext = { + apiKey: string; + apiUrl: string; filterFields: FilterFields; filterGroups: FilterGroup[]; filterGroupLabels: FilterGroupLabels; @@ -1934,10 +2030,9 @@ type HomeContext = { format: Option; formatHandler: (format: Option) => void; glossary: Content['glossary']; + previewLimit: number; profile: Profile; queryParams: QueryData; - apiKey: string; - apiUrl: string; resetFilters: () => void; sourceFields: SourceFields; sourceHandlers: SourceFieldInputHandlers; @@ -2047,3 +2142,5 @@ type SourceSelectFilterProps = SelectFilterProps & { }; type UrlQueryParam = [Value, Value]; + +export default Home; diff --git a/app/client/src/routes/nationalDownloads.tsx b/app/client/src/routes/nationalDownloads.tsx index 2658b8e1..38121f4b 100644 --- a/app/client/src/routes/nationalDownloads.tsx +++ b/app/client/src/routes/nationalDownloads.tsx @@ -69,7 +69,9 @@ type NationalDownloadsDataProps = { content: FetchState; }; -function NationalDownloadsData({ content }: Readonly) { +function NationalDownloadsData({ + content, +}: Readonly) { if (content.status !== 'success') return null; return ( @@ -144,7 +146,7 @@ function NationalDownloadsData({ content }: Readonly function ParagraphNoMargin( props: React.ClassAttributes & - React.HTMLAttributes + React.HTMLAttributes, ) { return

{props.children}

; } diff --git a/app/client/src/types/index.ts b/app/client/src/types/index.ts index 50de6a10..3e54afa5 100644 --- a/app/client/src/types/index.ts +++ b/app/client/src/types/index.ts @@ -26,12 +26,29 @@ type BaseFilterFieldConfig = { label: string; secondaryKey?: string; source?: string; - type: 'date' | 'multiselect' | 'select' | 'year'; + type: 'date' | 'multiselect' | 'select' | 'text' | 'year'; + size?: 'small' | 'medium' | 'large'; +}; + +export type ColumnConfig = { + key: string; + preview?: { + label?: string; + order?: number; + sortable?: boolean; + transform?: { + type: 'link'; + args: string[]; + }; + width?: number; + }; + ranked?: boolean; }; // Fields provided in the `domainValues` of the Content context export type ConcreteField = | 'actionAgency' + | 'actionDocumentType' | 'actionType' | 'assessmentTypes' | 'assessmentUnitStatus' @@ -80,11 +97,27 @@ export type Option = { value: Value; }; +export type QueryData = { + columns: string[]; + filters: { + [field: string]: Value | Value[]; + }; + options: { + [field: string]: Value; + }; +}; + export type SingleOptionField = BaseFilterFieldConfig & { type: 'select'; }; -export type SingleValueField = BaseFilterFieldConfig & { +export type SingleValueField = SingleValueTextField | SingleValueRangeField; + +export type SingleValueTextField = BaseFilterFieldConfig & { + type: 'text'; +}; + +export type SingleValueRangeField = BaseFilterFieldConfig & { boundary: 'low' | 'high'; domain: string; type: 'date' | 'year'; diff --git a/app/client/src/types/uswds.d.ts b/app/client/src/types/uswds.d.ts new file mode 100644 index 00000000..ad983596 --- /dev/null +++ b/app/client/src/types/uswds.d.ts @@ -0,0 +1 @@ +declare module '@uswds/uswds/js/usa-table'; diff --git a/app/server/.env.example b/app/server/.env.example index f31f26e2..468d47d6 100644 --- a/app/server/.env.example +++ b/app/server/.env.example @@ -18,5 +18,4 @@ DB_POOL_MAX=20 STREAM_BATCH_SIZE=2000 STREAM_HIGH_WATER_MARK=10000 MAX_QUERY_SIZE=1000000 -MAX_VALUES_QUERY_SIZE=100 -JSON_PAGE_SIZE=1000 +MAX_PAGE_SIZE=500 diff --git a/app/server/app/app.js b/app/server/app/app.js index 33ca2f0b..b033e108 100644 --- a/app/server/app/app.js +++ b/app/server/app/app.js @@ -96,8 +96,6 @@ const requiredEnvVars = [ 'STREAM_BATCH_SIZE', 'STREAM_HIGH_WATER_MARK', 'MAX_QUERY_SIZE', - 'MAX_VALUES_QUERY_SIZE', - 'JSON_PAGE_SIZE', ]; if (isLocal || isTest) { diff --git a/app/server/app/content/config/fields.json b/app/server/app/content/config/fields.json index f51d241f..baf8a1b8 100644 --- a/app/server/app/content/config/fields.json +++ b/app/server/app/content/config/fields.json @@ -5,6 +5,11 @@ "label": "Action Agency", "type": "multiselect" }, + "actionDocumentType": { + "key": "actionDocumentType", + "label": "Document Type", + "type": "multiselect" + }, "actionId": { "key": "actionId", "label": "Action ID (Name)", @@ -210,6 +215,19 @@ "label": "Delisted Reason", "type": "multiselect" }, + "documentName": { + "key": "documentName", + "label": "Document Name (File Name)", + "type": "multiselect", + "secondaryKey": "documentFileName", + "size": "medium" + }, + "documentQuery": { + "key": "documentQuery", + "label": "Search Text / Keyword", + "type": "text", + "size": "large" + }, "epaIrCategory": { "key": "epaIrCategory", "label": "EPA IR Category", @@ -496,6 +514,7 @@ "assessmentUnit": "Search for a specific Assessment Unit", "associatedAction": "Search by Associated Action", "catchmentAssessmentUnit": "Search by Assessment Unit or NHDPlus Catchment", + "documentText": "Search by Document", "impairmentCause": "Search by Cause of Impairment", "impairmentSource": "Search by Probable Source of Impairment", "overallStatus": "Search by Overall Status", @@ -533,11 +552,31 @@ ] }, { "key": "actionsAssessmentUnit", "fields": ["assessmentUnitId"] }, + { + "key": "timeFrame", + "fields": ["fiscalYearEstablishedLo", "fiscalYearEstablishedHi"] + } + ], + "actionDocuments": [ + { + "key": "documentText", + "fields": ["documentQuery", "documentName", "actionDocumentType"] + }, + { + "key": "areaOfInterest", + "fields": ["region", "state", "organizationType", "organizationId"] + }, + { + "key": "action", + "fields": ["actionType", "actionId"] + }, { "key": "timeFrame", "fields": [ - "fiscalYearEstablishedLo", - "fiscalYearEstablishedHi" + "completionDateLo", + "completionDateHi", + "tmdlDateLo", + "tmdlDateHi" ] } ], @@ -709,7 +748,13 @@ }, { "key": "pollutantParameter", - "fields": ["pollutantGroup", "pollutant", "addressedParameterGroup", "addressedParameter", "sourceType"] + "fields": [ + "pollutantGroup", + "pollutant", + "addressedParameterGroup", + "addressedParameter", + "sourceType" + ] }, { "key": "permitId", "fields": ["npdesIdentifier", "otherIdentifier"] }, { diff --git a/app/server/app/content/config/listOptions.json b/app/server/app/content/config/listOptions.json index ae0fdcfc..31afd685 100644 --- a/app/server/app/content/config/listOptions.json +++ b/app/server/app/content/config/listOptions.json @@ -10,6 +10,7 @@ ], "dataProfile": [ { "value": "actions", "label": "Actions" }, + { "value": "actionDocuments", "label": "Actions Document Search" }, { "value": "assessmentUnits", "label": "Assessment Units" }, { "value": "assessmentUnitsMonitoringLocations", diff --git a/app/server/app/content/config/parameters.json b/app/server/app/content/config/parameters.json index fade84ac..2d773bd4 100644 --- a/app/server/app/content/config/parameters.json +++ b/app/server/app/content/config/parameters.json @@ -1,4 +1,5 @@ { "debounceMilliseconds": 250, + "searchPreviewPageSize": 500, "selectOptionsPageSize": 20 } diff --git a/app/server/app/content/config/profiles.json b/app/server/app/content/config/profiles.json index 2b021e6b..99210ebc 100644 --- a/app/server/app/content/config/profiles.json +++ b/app/server/app/content/config/profiles.json @@ -3,93 +3,172 @@ "key": "actions", "description": "Contains detailed information on plans to restore and protect water quality including Total Maximum Daily Loads (TMDLs), 4b Plans, 5R Plans, Protection Approaches, and other plans.", "columns": [ - "objectId", - "region", - "state", - "organizationType", - "organizationId", - "organizationName", - "waterType", - "parameterGroup", - "parameter", - "actionType", - "actionId", - "actionName", - "actionAgency", - "inIndianCountry", - "includeInMeasure", - "completionDate", - "assessmentUnitId", - "assessmentUnitName", - "fiscalYearEstablished", - "locationDescription", - "waterSize", - "waterSizeUnits", - "planSummaryLink" + { "key": "objectId" }, + { "key": "region" }, + { "key": "state" }, + { "key": "organizationType" }, + { "key": "organizationId" }, + { "key": "organizationName" }, + { "key": "waterType" }, + { "key": "parameterGroup" }, + { "key": "parameter" }, + { "key": "actionType" }, + { "key": "actionId" }, + { "key": "actionName" }, + { "key": "actionAgency" }, + { "key": "inIndianCountry" }, + { "key": "includeInMeasure" }, + { "key": "completionDate" }, + { "key": "assessmentUnitId" }, + { "key": "assessmentUnitName" }, + { "key": "fiscalYearEstablished" }, + { "key": "locationDescription" }, + { "key": "waterSize" }, + { "key": "waterSizeUnits" }, + { "key": "planSummaryLink" } ], "label": "Actions", "resource": "actions" }, + "actionDocuments": { + "key": "actionDocuments", + "description": "Contains information on documents associated with actions to restore and protect water quality.", + "columns": [ + { "key": "objectId" }, + { "key": "actionDocumentType" }, + { + "key": "actionDocumentUrl", + "preview": { + "label": "Document", + "order": 1, + "sortable": true, + "transform": { + "type": "link", + "args": ["documentName"] + }, + "width": 300 + } + }, + { + "key": "actionId", + "preview": { + "label": "Action ID", + "order": 2, + "sortable": true, + "width": 110 + } + }, + { + "key": "actionName", + "preview": { + "label": "Action Name", + "order": 3, + "sortable": true, + "width": 300 + } + }, + { "key": "actionType" }, + { "key": "completionDate" }, + { "key": "documentDesc" }, + { "key": "documentFileName" }, + { "key": "documentFileTypeName" }, + { "key": "documentKey" }, + { "key": "documentName" }, + { "key": "documentQuery", "ranked": true }, + { "key": "organizationType" }, + { + "key": "organizationId", + "preview": { + "label": "Organization ID", + "order": 6, + "sortable": true, + "width": 160 + } + }, + { "key": "organizationName" }, + { + "key": "region", + "preview": { + "label": "Region", + "order": 4, + "sortable": true, + "width": 110 + } + }, + { + "key": "state", + "preview": { + "label": "State", + "order": 5, + "sortable": true, + "width": 100 + } + }, + { "key": "tmdlDate" } + ], + "label": "Actions Document Search", + "resource": "actionDocuments" + }, "assessments": { "key": "assessments", "description": "Contains detailed information on waters assessed under Section 305(b) of the Clean Water Act and waters listed as impaired under Section 303(d) of the Clean Water Act. This includes assessed uses and parameter attainments.", "columns": [ - "objectId", - "region", - "state", - "organizationType", - "organizationId", - "organizationName", - "waterType", - "reportingCycle", - "cycleLastAssessed", - "assessmentUnitId", - "assessmentUnitName", - "assessmentUnitStatus", - "overallStatus", - "epaIrCategory", - "stateIrCategory", - "useGroup", - "useName", - "useClassName", - "useSupport", - "useIrCategory", - "useStateIrCategory", - "monitoringStartDate", - "monitoringEndDate", - "assessmentDate", - "assessmentTypes", - "assessmentMethods", - "assessmentBasis", - "parameterGroup", - "parameterName", - "parameterStatus", - "parameterAttainment", - "parameterIrCategory", - "parameterStateIrCategory", - "delisted", - "delistedReason", - "pollutantIndicator", - "cycleFirstListed", - "alternateListingIdentifier", - "vision303dPriority", - "cwa303dPriorityRanking", - "cycleScheduledForTmdl", - "cycleExpectedToAttain", - "consentDecreeCycle", - "cycleId", - "seasonStartDate", - "seasonEndDate", - "associatedActionId", - "associatedActionName", - "associatedActionType", - "associatedActionStatus", - "associatedActionAgency", - "locationDescription", - "sizeSource", - "sourceScale", - "waterSize", - "waterSizeUnits" + { "key": "objectId" }, + { "key": "region" }, + { "key": "state" }, + { "key": "organizationType" }, + { "key": "organizationId" }, + { "key": "organizationName" }, + { "key": "waterType" }, + { "key": "reportingCycle" }, + { "key": "cycleLastAssessed" }, + { "key": "assessmentUnitId" }, + { "key": "assessmentUnitName" }, + { "key": "assessmentUnitStatus" }, + { "key": "overallStatus" }, + { "key": "epaIrCategory" }, + { "key": "stateIrCategory" }, + { "key": "useGroup" }, + { "key": "useName" }, + { "key": "useClassName" }, + { "key": "useSupport" }, + { "key": "useIrCategory" }, + { "key": "useStateIrCategory" }, + { "key": "monitoringStartDate" }, + { "key": "monitoringEndDate" }, + { "key": "assessmentDate" }, + { "key": "assessmentTypes" }, + { "key": "assessmentMethods" }, + { "key": "assessmentBasis" }, + { "key": "parameterGroup" }, + { "key": "parameterName" }, + { "key": "parameterStatus" }, + { "key": "parameterAttainment" }, + { "key": "parameterIrCategory" }, + { "key": "parameterStateIrCategory" }, + { "key": "delisted" }, + { "key": "delistedReason" }, + { "key": "pollutantIndicator" }, + { "key": "cycleFirstListed" }, + { "key": "alternateListingIdentifier" }, + { "key": "vision303dPriority" }, + { "key": "cwa303dPriorityRanking" }, + { "key": "cycleScheduledForTmdl" }, + { "key": "cycleExpectedToAttain" }, + { "key": "consentDecreeCycle" }, + { "key": "cycleId" }, + { "key": "seasonStartDate" }, + { "key": "seasonEndDate" }, + { "key": "associatedActionId" }, + { "key": "associatedActionName" }, + { "key": "associatedActionType" }, + { "key": "associatedActionStatus" }, + { "key": "associatedActionAgency" }, + { "key": "locationDescription" }, + { "key": "sizeSource" }, + { "key": "sourceScale" }, + { "key": "waterSize" }, + { "key": "waterSizeUnits" } ], "label": "Assessments", "resource": "assessments" @@ -98,26 +177,26 @@ "key": "assessmentUnits", "description": "Contains detailed information on assessment unit location and waterbody types. Please note, some waters may contain more than one water type.", "columns": [ - "objectId", - "region", - "state", - "organizationType", - "organizationId", - "organizationName", - "waterType", - "locationTypeCode", - "locationText", - "useClassName", - "assessmentUnitId", - "assessmentUnitName", - "assessmentUnitStatus", - "reportingCycle", - "cycleId", - "locationDescription", - "sizeSource", - "sourceScale", - "waterSize", - "waterSizeUnits" + { "key": "objectId" }, + { "key": "region" }, + { "key": "state" }, + { "key": "organizationType" }, + { "key": "organizationId" }, + { "key": "organizationName" }, + { "key": "waterType" }, + { "key": "locationTypeCode" }, + { "key": "locationText" }, + { "key": "useClassName" }, + { "key": "assessmentUnitId" }, + { "key": "assessmentUnitName" }, + { "key": "assessmentUnitStatus" }, + { "key": "reportingCycle" }, + { "key": "cycleId" }, + { "key": "locationDescription" }, + { "key": "sizeSource" }, + { "key": "sourceScale" }, + { "key": "waterSize" }, + { "key": "waterSizeUnits" } ], "label": "Assessment Units", "resource": "assessmentUnits" @@ -126,27 +205,27 @@ "key": "assessmentUnitsMonitoringLocations", "description": "Contains information on the monitoring locations used to make assessment determinations at specific assessment units.", "columns": [ - "objectId", - "region", - "state", - "organizationType", - "organizationId", - "organizationName", - "waterType", - "useClassName", - "monitoringLocationId", - "monitoringLocationOrgId", - "assessmentUnitId", - "assessmentUnitName", - "assessmentUnitStatus", - "reportingCycle", - "cycleId", - "locationDescription", - "monitoringLocationDataLink", - "sizeSource", - "sourceScale", - "waterSize", - "waterSizeUnits" + { "key": "objectId" }, + { "key": "region" }, + { "key": "state" }, + { "key": "organizationType" }, + { "key": "organizationId" }, + { "key": "organizationName" }, + { "key": "waterType" }, + { "key": "useClassName" }, + { "key": "monitoringLocationId" }, + { "key": "monitoringLocationOrgId" }, + { "key": "assessmentUnitId" }, + { "key": "assessmentUnitName" }, + { "key": "assessmentUnitStatus" }, + { "key": "reportingCycle" }, + { "key": "cycleId" }, + { "key": "locationDescription" }, + { "key": "monitoringLocationDataLink" }, + { "key": "sizeSource" }, + { "key": "sourceScale" }, + { "key": "waterSize" }, + { "key": "waterSizeUnits" } ], "label": "Assessment Units with Monitoring Locations", "resource": "assessmentUnitsMonitoringLocations" @@ -155,17 +234,17 @@ "key": "catchmentCorrespondence", "description": "Contains the association between assessment units and National Hydrography Dataset Plus (NHDPlus) high resolution catchments.", "columns": [ - "objectId", - "region", - "state", - "organizationType", - "organizationId", - "organizationName", - "assessmentUnitId", - "assessmentUnitName", - "catchmentNhdPlusId", - "reportingCycle", - "cycleId" + { "key": "objectId" }, + { "key": "region" }, + { "key": "state" }, + { "key": "organizationType" }, + { "key": "organizationId" }, + { "key": "organizationName" }, + { "key": "assessmentUnitId" }, + { "key": "assessmentUnitName" }, + { "key": "catchmentNhdPlusId" }, + { "key": "reportingCycle" }, + { "key": "cycleId" } ], "label": "Catchment Correspondence", "resource": "catchmentCorrespondence" @@ -174,27 +253,27 @@ "key": "sources", "description": "Identifies sources of impairment for assessed waters.", "columns": [ - "objectId", - "region", - "state", - "organizationType", - "organizationId", - "organizationName", - "waterType", - "assessmentUnitId", - "assessmentUnitName", - "reportingCycle", - "overallStatus", - "epaIrCategory", - "stateIrCategory", - "parameterGroup", - "causeName", - "sourceName", - "confirmed", - "cycleId", - "locationDescription", - "waterSize", - "waterSizeUnits" + { "key": "objectId" }, + { "key": "region" }, + { "key": "state" }, + { "key": "organizationType" }, + { "key": "organizationId" }, + { "key": "organizationName" }, + { "key": "waterType" }, + { "key": "assessmentUnitId" }, + { "key": "assessmentUnitName" }, + { "key": "reportingCycle" }, + { "key": "overallStatus" }, + { "key": "epaIrCategory" }, + { "key": "stateIrCategory" }, + { "key": "parameterGroup" }, + { "key": "causeName" }, + { "key": "sourceName" }, + { "key": "confirmed" }, + { "key": "cycleId" }, + { "key": "locationDescription" }, + { "key": "waterSize" }, + { "key": "waterSizeUnits" } ], "label": "Sources", "resource": "sources" @@ -203,40 +282,40 @@ "key": "tmdl", "description": "Contains detailed information on Total Maximum Daily Load (TMDL) plans.", "columns": [ - "objectId", - "region", - "state", - "organizationType", - "organizationId", - "organizationName", - "waterType", - "pollutantGroup", - "pollutant", - "addressedParameterGroup", - "addressedParameter", - "sourceType", - "npdesIdentifier", - "otherIdentifier", - "actionId", - "actionName", - "actionAgency", - "inIndianCountry", - "explicitMarginOfSafety", - "implicitMarginOfSafety", - "includeInMeasure", - "completionDate", - "tmdlDate", - "fiscalYearEstablished", - "assessmentUnitId", - "assessmentUnitName", - "loadAllocation", - "loadAllocationUnits", - "locationDescription", - "tmdlEndpoint", - "waterSize", - "waterSizeUnits", - "wasteLoadAllocation", - "planSummaryLink" + { "key": "objectId" }, + { "key": "region" }, + { "key": "state" }, + { "key": "organizationType" }, + { "key": "organizationId" }, + { "key": "organizationName" }, + { "key": "waterType" }, + { "key": "pollutantGroup" }, + { "key": "pollutant" }, + { "key": "addressedParameterGroup" }, + { "key": "addressedParameter" }, + { "key": "sourceType" }, + { "key": "npdesIdentifier" }, + { "key": "otherIdentifier" }, + { "key": "actionId" }, + { "key": "actionName" }, + { "key": "actionAgency" }, + { "key": "inIndianCountry" }, + { "key": "explicitMarginOfSafety" }, + { "key": "implicitMarginOfSafety" }, + { "key": "includeInMeasure" }, + { "key": "completionDate" }, + { "key": "tmdlDate" }, + { "key": "fiscalYearEstablished" }, + { "key": "assessmentUnitId" }, + { "key": "assessmentUnitName" }, + { "key": "loadAllocation" }, + { "key": "loadAllocationUnits" }, + { "key": "locationDescription" }, + { "key": "tmdlEndpoint" }, + { "key": "waterSize" }, + { "key": "waterSizeUnits" }, + { "key": "wasteLoadAllocation" }, + { "key": "planSummaryLink" } ], "label": "Total Maximum Daily Load", "resource": "tmdl" diff --git a/app/server/app/content/swagger/api-private.json b/app/server/app/content/swagger/api-private.json index bcb043f2..9c91a02a 100644 --- a/app/server/app/content/swagger/api-private.json +++ b/app/server/app/content/swagger/api-private.json @@ -158,6 +158,31 @@ } } }, + "/attains/actionDocuments/values/{column}": { + "post": { + "tags": ["ATTAINS - Values"], + "summary": "Query distinct column values", + "parameters": [ + { + "$ref": "#/components/parameters/columnParam" + } + ], + "requestBody": { + "$ref": "#/components/requestBodies/ValuesQueryRequestBody" + }, + "responses": { + "200": { + "$ref": "#/components/responses/ValuesQuerySuccess" + }, + "404": { + "$ref": "#/components/responses/ColumnNotFoundError" + }, + "500": { + "$ref": "#/components/responses/ServerError" + } + } + } + }, "/attains/assessments/values/{column}": { "post": { "tags": ["ATTAINS - Values"], @@ -874,7 +899,46 @@ "columns": { "type": "array", "items": { - "type": "string" + "type": "object", + "properties": { + "key": { + "type": "string" + }, + "preview": { + "type": "object", + "properties": { + "label": { + "type": "string" + }, + "order": { + "type": "integer" + }, + "sortable": { + "type": "boolean" + }, + "transform": { + "type": "object", + "properties": { + "type": { + "type": "string" + }, + "args": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "width": { + "type": "integer" + } + } + }, + "ranked": { + "type": "boolean" + } + } } }, "description": { diff --git a/app/server/app/content/swagger/api-public.json b/app/server/app/content/swagger/api-public.json index 12a73e97..b26a77c0 100644 --- a/app/server/app/content/swagger/api-public.json +++ b/app/server/app/content/swagger/api-public.json @@ -34,6 +34,232 @@ } }, "paths": { + "/attains/actionDocuments": { + "get": { + "tags": ["ATTAINS - Action Documents"], + "summary": "Query Action Documents profile data", + "description": "Query the Action Documents profile on the ATTAINS Query page.", + "parameters": [ + { + "$ref": "#/components/parameters/apiKeyHeader" + }, + { + "$ref": "#/components/parameters/outputFormatParam" + }, + { + "$ref": "#/components/parameters/pageNumberParam" + }, + { + "$ref": "#/components/parameters/pageSizeParam" + }, + { + "$ref": "#/components/parameters/columnsParam" + }, + { + "$ref": "#/components/parameters/objectIdParam" + }, + { + "$ref": "#/components/parameters/actionDocumentTypeParam" + }, + { + "$ref": "#/components/parameters/actionDocumentUrlParam" + }, + { + "$ref": "#/components/parameters/actionIdParam" + }, + { + "$ref": "#/components/parameters/actionNameParam" + }, + { + "$ref": "#/components/parameters/actionTypeParam" + }, + { + "$ref": "#/components/parameters/completionDateLoParam" + }, + { + "$ref": "#/components/parameters/completionDateHiParam" + }, + { + "$ref": "#/components/parameters/documentFileNameParam" + }, + { + "$ref": "#/components/parameters/documentFileTypeNameParam" + }, + { + "$ref": "#/components/parameters/documentKeyParam" + }, + { + "$ref": "#/components/parameters/documentNameParam" + }, + { + "$ref": "#/components/parameters/documentQueryParam" + }, + { + "$ref": "#/components/parameters/organizationIdParam" + }, + { + "$ref": "#/components/parameters/organizationNameParam" + }, + { + "$ref": "#/components/parameters/organizationTypeParam" + }, + { + "$ref": "#/components/parameters/regionParam" + }, + { + "$ref": "#/components/parameters/stateParam" + }, + { + "$ref": "#/components/parameters/tmdlDateLoParam" + }, + { + "$ref": "#/components/parameters/tmdlDateHiParam" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/ActionDocumentsQuerySuccess" + }, + "400": { + "$ref": "#/components/responses/InvalidRequestError" + }, + "500": { + "$ref": "#/components/responses/ServerError" + } + } + }, + "post": { + "tags": ["ATTAINS - Action Documents"], + "summary": "Query Action Documents profile data", + "description": "Query the Action Documents profile on the ATTAINS Query page.", + "parameters": [ + { + "$ref": "#/components/parameters/apiKeyHeader" + } + ], + "requestBody": { + "$ref": "#/components/requestBodies/ActionDocumentsQueryRequestBody" + }, + "responses": { + "200": { + "$ref": "#/components/responses/ActionDocumentsQuerySuccess" + }, + "400": { + "$ref": "#/components/responses/InvalidRequestError" + }, + "500": { + "$ref": "#/components/responses/ServerError" + } + } + } + }, + "/attains/actionDocuments/count": { + "get": { + "summary": "Row count of Action Documents profile query", + "description": "Query the Action Documents profile on the ATTAINS Query page.", + "tags": ["ATTAINS - Action Documents"], + "parameters": [ + { + "$ref": "#/components/parameters/apiKeyHeader" + }, + { + "$ref": "#/components/parameters/objectIdParam" + }, + { + "$ref": "#/components/parameters/actionDocumentTypeParam" + }, + { + "$ref": "#/components/parameters/actionDocumentUrlParam" + }, + { + "$ref": "#/components/parameters/actionIdParam" + }, + { + "$ref": "#/components/parameters/actionNameParam" + }, + { + "$ref": "#/components/parameters/actionTypeParam" + }, + { + "$ref": "#/components/parameters/completionDateLoParam" + }, + { + "$ref": "#/components/parameters/completionDateHiParam" + }, + { + "$ref": "#/components/parameters/documentFileNameParam" + }, + { + "$ref": "#/components/parameters/documentFileTypeNameParam" + }, + { + "$ref": "#/components/parameters/documentKeyParam" + }, + { + "$ref": "#/components/parameters/documentNameParam" + }, + { + "$ref": "#/components/parameters/documentQueryParam" + }, + { + "$ref": "#/components/parameters/organizationIdParam" + }, + { + "$ref": "#/components/parameters/organizationNameParam" + }, + { + "$ref": "#/components/parameters/organizationTypeParam" + }, + { + "$ref": "#/components/parameters/regionParam" + }, + { + "$ref": "#/components/parameters/stateParam" + }, + { + "$ref": "#/components/parameters/tmdlDateLoParam" + }, + { + "$ref": "#/components/parameters/tmdlDateHiParam" + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/QueryCountSuccess" + }, + "400": { + "$ref": "#/components/responses/InvalidRequestError" + }, + "500": { + "$ref": "#/components/responses/ServerError" + } + } + }, + "post": { + "summary": "Row count of Action Documents profile query", + "description": "Query the Action Documents profile on the ATTAINS Query page.", + "tags": ["ATTAINS - Action Documents"], + "parameters": [ + { + "$ref": "#/components/parameters/apiKeyHeader" + } + ], + "requestBody": { + "$ref": "#/components/requestBodies/ActionDocumentsCountRequestBody" + }, + "responses": { + "200": { + "$ref": "#/components/responses/QueryCountSuccess" + }, + "400": { + "$ref": "#/components/responses/InvalidRequestError" + }, + "500": { + "$ref": "#/components/responses/ServerError" + } + } + } + }, "/attains/actions": { "get": { "tags": ["ATTAINS - Actions"], @@ -47,7 +273,10 @@ "$ref": "#/components/parameters/outputFormatParam" }, { - "$ref": "#/components/parameters/startIdParam" + "$ref": "#/components/parameters/pageNumberParam" + }, + { + "$ref": "#/components/parameters/pageSizeParam" }, { "$ref": "#/components/parameters/columnsParam" @@ -97,6 +326,9 @@ { "$ref": "#/components/parameters/organizationNameParam" }, + { + "$ref": "#/components/parameters/organizationTypeParam" + }, { "$ref": "#/components/parameters/parameterParam" }, @@ -204,6 +436,9 @@ { "$ref": "#/components/parameters/organizationNameParam" }, + { + "$ref": "#/components/parameters/organizationTypeParam" + }, { "$ref": "#/components/parameters/parameterParam" }, @@ -270,7 +505,10 @@ "$ref": "#/components/parameters/outputFormatParam" }, { - "$ref": "#/components/parameters/startIdParam" + "$ref": "#/components/parameters/pageNumberParam" + }, + { + "$ref": "#/components/parameters/pageSizeParam" }, { "$ref": "#/components/parameters/columnsParam" @@ -383,6 +621,9 @@ { "$ref": "#/components/parameters/organizationNameParam" }, + { + "$ref": "#/components/parameters/organizationTypeParam" + }, { "$ref": "#/components/parameters/overallStatusParam" }, @@ -604,6 +845,9 @@ { "$ref": "#/components/parameters/organizationNameParam" }, + { + "$ref": "#/components/parameters/organizationTypeParam" + }, { "$ref": "#/components/parameters/overallStatusParam" }, @@ -721,7 +965,10 @@ "$ref": "#/components/parameters/outputFormatParam" }, { - "$ref": "#/components/parameters/startIdParam" + "$ref": "#/components/parameters/pageNumberParam" + }, + { + "$ref": "#/components/parameters/pageSizeParam" }, { "$ref": "#/components/parameters/columnsParam" @@ -753,6 +1000,9 @@ { "$ref": "#/components/parameters/organizationNameParam" }, + { + "$ref": "#/components/parameters/organizationTypeParam" + }, { "$ref": "#/components/parameters/regionParam" }, @@ -842,6 +1092,9 @@ { "$ref": "#/components/parameters/organizationNameParam" }, + { + "$ref": "#/components/parameters/organizationTypeParam" + }, { "$ref": "#/components/parameters/regionParam" }, @@ -908,7 +1161,10 @@ "$ref": "#/components/parameters/outputFormatParam" }, { - "$ref": "#/components/parameters/startIdParam" + "$ref": "#/components/parameters/pageNumberParam" + }, + { + "$ref": "#/components/parameters/pageSizeParam" }, { "$ref": "#/components/parameters/columnsParam" @@ -940,6 +1196,9 @@ { "$ref": "#/components/parameters/organizationNameParam" }, + { + "$ref": "#/components/parameters/organizationTypeParam" + }, { "$ref": "#/components/parameters/regionParam" }, @@ -1029,6 +1288,9 @@ { "$ref": "#/components/parameters/organizationNameParam" }, + { + "$ref": "#/components/parameters/organizationTypeParam" + }, { "$ref": "#/components/parameters/regionParam" }, @@ -1095,7 +1357,10 @@ "$ref": "#/components/parameters/outputFormatParam" }, { - "$ref": "#/components/parameters/startIdParam" + "$ref": "#/components/parameters/pageNumberParam" + }, + { + "$ref": "#/components/parameters/pageSizeParam" }, { "$ref": "#/components/parameters/columnsParam" @@ -1121,6 +1386,9 @@ { "$ref": "#/components/parameters/organizationNameParam" }, + { + "$ref": "#/components/parameters/organizationTypeParam" + }, { "$ref": "#/components/parameters/regionParam" }, @@ -1198,6 +1466,9 @@ { "$ref": "#/components/parameters/organizationNameParam" }, + { + "$ref": "#/components/parameters/organizationTypeParam" + }, { "$ref": "#/components/parameters/regionParam" }, @@ -1258,7 +1529,10 @@ "$ref": "#/components/parameters/outputFormatParam" }, { - "$ref": "#/components/parameters/startIdParam" + "$ref": "#/components/parameters/pageNumberParam" + }, + { + "$ref": "#/components/parameters/pageSizeParam" }, { "$ref": "#/components/parameters/columnsParam" @@ -1290,6 +1564,9 @@ { "$ref": "#/components/parameters/organizationNameParam" }, + { + "$ref": "#/components/parameters/organizationTypeParam" + }, { "$ref": "#/components/parameters/overallStatusParam" }, @@ -1388,6 +1665,9 @@ { "$ref": "#/components/parameters/organizationNameParam" }, + { + "$ref": "#/components/parameters/organizationTypeParam" + }, { "$ref": "#/components/parameters/overallStatusParam" }, @@ -1463,7 +1743,10 @@ "$ref": "#/components/parameters/outputFormatParam" }, { - "$ref": "#/components/parameters/startIdParam" + "$ref": "#/components/parameters/pageNumberParam" + }, + { + "$ref": "#/components/parameters/pageSizeParam" }, { "$ref": "#/components/parameters/columnsParam" @@ -1525,6 +1808,9 @@ { "$ref": "#/components/parameters/organizationNameParam" }, + { + "$ref": "#/components/parameters/organizationTypeParam" + }, { "$ref": "#/components/parameters/otherIdentifierParam" }, @@ -1656,6 +1942,9 @@ { "$ref": "#/components/parameters/organizationNameParam" }, + { + "$ref": "#/components/parameters/organizationTypeParam" + }, { "$ref": "#/components/parameters/otherIdentifierParam" }, @@ -1740,20 +2029,210 @@ } } } - }, - "500": { - "$ref": "#/components/responses/ServerError" + }, + "500": { + "$ref": "#/components/responses/ServerError" + } + } + } + } + }, + "externalDocs": { + "description": "Find out more about Expert Query", + "url": "" + }, + "components": { + "schemas": { + "ActionDocumentsFilters": { + "type": "object", + "properties": { + "objectId": { + "type": "array", + "items": { + "type": "integer" + } + }, + "actionDocumentType": { + "type": "array", + "items": { + "type": "string" + } + }, + "actionDocumentUrl": { + "type": "array", + "items": { + "type": "string" + } + }, + "actionId": { + "type": "array", + "items": { + "type": "string" + } + }, + "actionName": { + "type": "array", + "items": { + "type": "string" + } + }, + "actionType": { + "type": "array", + "items": { + "type": "string" + } + }, + "completionDateLo": { + "type": "string", + "format": "date" + }, + "completionDateHi": { + "type": "string", + "format": "date" + }, + "documentFileName": { + "type": "array", + "items": { + "type": "string" + } + }, + "documentFileTypeName": { + "type": "array", + "items": { + "type": "string" + } + }, + "documentKey": { + "type": "array", + "items": { + "type": "integer" + } + }, + "documentName": { + "type": "array", + "items": { + "type": "string" + } + }, + "documentQuery": { + "type": "string" + }, + "organizationId": { + "type": "array", + "items": { + "type": "string" + } + }, + "organizationName": { + "type": "array", + "items": { + "type": "string" + } + }, + "organizationType": { + "type": "array", + "items": { + "type": "string" + } + }, + "region": { + "type": "array", + "items": { + "type": "string" + } + }, + "state": { + "type": "array", + "items": { + "type": "string" + } + }, + "tmdlDateLo": { + "type": "string", + "format": "date" + }, + "tmdlDateHi": { + "type": "string", + "format": "date" + } + } + }, + "ActionDocumentsQueryOutput": { + "type": "object", + "properties": { + "pageNumber": { + "type": "integer" + }, + "pageSize": { + "type": "integer" + }, + "data": { + "type": "array", + "items": { + "type": "object", + "properties": { + "objectId": { + "type": "integer" + }, + "actionDocumentType": { + "type": "string" + }, + "actionDocumentUrl": { + "type": "string" + }, + "actionId": { + "type": "string" + }, + "actionName": { + "type": "string" + }, + "actionType": { + "type": "string" + }, + "completionDate": { + "type": "string", + "format": "date" + }, + "documentDesc": { + "type": "string" + }, + "documentFileName": { + "type": "string" + }, + "documentFileTypeName": { + "type": "string" + }, + "documentKey": { + "type": "integer" + }, + "documentName": { + "type": "string" + }, + "organizationId": { + "type": "string" + }, + "organizationName": { + "type": "string" + }, + "organizationType": { + "type": "string" + }, + "region": { + "type": "string" + }, + "state": { + "type": "string" + }, + "tmdlDate": { + "type": "string", + "format": "date" + } + } + } } - } - } - } - }, - "externalDocs": { - "description": "Find out more about Expert Query", - "url": "" - }, - "components": { - "schemas": { + }, + "required": ["data", "pageNumber", "pageSize"] + }, "ActionsFilters": { "type": "object", "properties": { @@ -1837,6 +2316,12 @@ "type": "string" } }, + "organizationType": { + "type": "array", + "items": { + "type": "string" + } + }, "parameter": { "type": "array", "items": { @@ -1872,7 +2357,10 @@ "ActionsQueryOutput": { "type": "object", "properties": { - "nextId": { + "pageNumber": { + "type": "integer" + }, + "pageSize": { "type": "integer" }, "data": { @@ -1955,7 +2443,7 @@ } } }, - "required": ["data"] + "required": ["data", "pageNumber", "pageSize"] }, "AssessmentsFilters": { "type": "object", @@ -2134,6 +2622,12 @@ "type": "string" } }, + "organizationType": { + "type": "array", + "items": { + "type": "string" + } + }, "overallStatus": { "type": "array", "items": { @@ -2272,7 +2766,10 @@ "AssessmentsQueryOutput": { "type": "object", "properties": { - "nextId": { + "pageNumber": { + "type": "integer" + }, + "pageSize": { "type": "integer" }, "data": { @@ -2456,7 +2953,7 @@ } } }, - "required": ["data"] + "required": ["data", "pageNumber", "pageSize"] }, "AssessmentUnitsFilters": { "type": "object", @@ -2515,6 +3012,12 @@ "type": "string" } }, + "organizationType": { + "type": "array", + "items": { + "type": "string" + } + }, "region": { "type": "array", "items": { @@ -2601,6 +3104,12 @@ "type": "string" } }, + "organizationType": { + "type": "array", + "items": { + "type": "string" + } + }, "region": { "type": "array", "items": { @@ -2633,7 +3142,10 @@ "AssessmentUnitsQueryOutput": { "type": "object", "properties": { - "nextId": { + "pageNumber": { + "type": "integer" + }, + "pageSize": { "type": "integer" }, "data": { @@ -2706,12 +3218,15 @@ } } }, - "required": ["data"] + "required": ["data", "pageNumber", "pageSize"] }, "AssessmentUnitsMonitoringLocationsQueryOutput": { "type": "object", "properties": { - "nextId": { + "pageNumber": { + "type": "integer" + }, + "pageSize": { "type": "integer" }, "data": { @@ -2787,7 +3302,7 @@ } } }, - "required": ["data"] + "required": ["data", "pageNumber", "pageSize"] }, "CatchmentCorrespondenceFilters": { "type": "object", @@ -2834,6 +3349,12 @@ "type": "string" } }, + "organizationType": { + "type": "array", + "items": { + "type": "string" + } + }, "region": { "type": "array", "items": { @@ -2854,7 +3375,10 @@ "CatchmentCorrespondenceQueryOutput": { "type": "object", "properties": { - "nextId": { + "pageNumber": { + "type": "integer" + }, + "pageSize": { "type": "integer" }, "data": { @@ -2899,7 +3423,7 @@ } } }, - "required": ["data"] + "required": ["data", "pageNumber", "pageSize"] }, "FileAttachmentCsv": { "type": "string", @@ -2917,7 +3441,10 @@ "Options": { "type": "object", "properties": { - "startId": { + "pageNumber": { + "type": "integer" + }, + "pageSize": { "type": "integer" }, "format": { @@ -2997,6 +3524,12 @@ "type": "string" } }, + "organizationType": { + "type": "array", + "items": { + "type": "string" + } + }, "overallStatus": { "type": "array", "items": { @@ -3047,7 +3580,10 @@ "SourcesQueryOutput": { "type": "object", "properties": { - "nextId": { + "pageNumber": { + "type": "integer" + }, + "pageSize": { "type": "integer" }, "data": { @@ -3123,7 +3659,7 @@ } } }, - "required": ["data"] + "required": ["data", "pageNumber", "pageSize"] }, "TmdlFilters": { "type": "object", @@ -3232,6 +3768,12 @@ "type": "string" } }, + "organizationType": { + "type": "array", + "items": { + "type": "string" + } + }, "otherIdentifier": { "type": "array", "items": { @@ -3287,7 +3829,10 @@ "TmdlQueryOutput": { "type": "object", "properties": { - "nextId": { + "pageNumber": { + "type": "integer" + }, + "pageSize": { "type": "integer" }, "data": { @@ -3406,7 +3951,7 @@ } } }, - "required": ["data"] + "required": ["data", "pageNumber", "pageSize"] } }, "parameters": { @@ -3422,6 +3967,32 @@ } } }, + "actionDocumentTypeParam": { + "name": "actionDocumentType", + "in": "query", + "explode": true, + "example": ["TMDL Report"], + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "actionDocumentUrlParam": { + "name": "actionDocumentUrl", + "in": "query", + "explode": true, + "example": [ + "https://attains.epa.gov/attains-public/api/documents/actions/21MSWQ/22502/102354" + ], + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + }, "actionIdParam": { "name": "actionId", "in": "query", @@ -3849,6 +4420,62 @@ } } }, + "documentFileNameParam": { + "name": "documentFileName", + "in": "query", + "explode": true, + "example": ["54524_Hatchie_North Indpendent ph TMDL final.pdf"], + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "documentFileTypeNameParam": { + "name": "documentFileTypeName", + "in": "query", + "explode": true, + "example": ["application/pdf"], + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "documentKeyParam": { + "name": "documentKey", + "in": "query", + "explode": true, + "example": [102600], + "schema": { + "type": "array", + "items": { + "type": "integer" + } + } + }, + "documentNameParam": { + "name": "documentName", + "in": "query", + "explode": true, + "example": ["North Indpendent ph TMDL final ERRATA"], + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "documentQueryParam": { + "name": "documentQuery", + "in": "query", + "example": ["Tuscumbia River Canal"], + "schema": { + "type": "string" + } + }, "epaIrCategoryParam": { "name": "epaIrCategory", "in": "query", @@ -4062,6 +4689,18 @@ } } }, + "organizationTypeParam": { + "name": "organizationType", + "in": "query", + "explode": true, + "example": ["State"], + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + }, "outputFormatParam": { "name": "format", "description": "The format in which data is returned. JSON data is returned inline, other types are returned as a file attachment.", @@ -4096,6 +4735,22 @@ } } }, + "pageNumberParam": { + "name": "pageNumber", + "in": "query", + "example": 2, + "schema": { + "type": "integer" + } + }, + "pageSizeParam": { + "name": "pageSize", + "in": "query", + "example": 20, + "schema": { + "type": "integer" + } + }, "parameterAttainmentParam": { "name": "parameterAttainment", "in": "query", @@ -4298,14 +4953,6 @@ } } }, - "startIdParam": { - "name": "startId", - "in": "query", - "example": 20, - "schema": { - "type": "integer" - } - }, "stateIrCategoryParam": { "name": "stateIrCategory", "in": "query", @@ -4467,6 +5114,45 @@ } }, "requestBodies": { + "ActionDocumentsCountRequestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "filters": { + "$ref": "#/components/schemas/ActionDocumentsFilters" + } + } + } + } + }, + "required": true + }, + "ActionDocumentsQueryRequestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "options": { + "$ref": "#/components/schemas/Options" + }, + "columns": { + "type": "array", + "items": { + "type": "string" + } + }, + "filters": { + "$ref": "#/components/schemas/ActionDocumentsFilters" + } + } + } + } + }, + "required": true + }, "ActionsCountRequestBody": { "content": { "application/json": { @@ -4742,6 +5428,59 @@ } }, "responses": { + "ActionDocumentsQuerySuccess": { + "description": "The query executed successfully. The response contains the query results or a message indicating that the maximum query size was exceeded.", + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/ActionDocumentsQueryOutput" + }, + { + "$ref": "#/components/schemas/QuerySizeExceededMessage" + } + ] + } + }, + "application/octet-stream": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/FileAttachmentXlsx" + }, + { + "$ref": "#/components/schemas/QuerySizeExceededMessage" + } + ] + } + }, + "text/csv": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/FileAttachmentCsv" + }, + { + "$ref": "#/components/schemas/QuerySizeExceededMessage" + } + ] + } + }, + "text/tsv": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/FileAttachmentTsv" + }, + { + "$ref": "#/components/schemas/QuerySizeExceededMessage" + } + ] + } + } + } + }, "ActionsQuerySuccess": { "description": "The query executed successfully. The response contains the query results or a message indicating that the maximum query size was exceeded.", "content": { diff --git a/app/server/app/routes/attains.js b/app/server/app/routes/attains.js index bcecf491..34d11002 100644 --- a/app/server/app/routes/attains.js +++ b/app/server/app/routes/attains.js @@ -17,8 +17,9 @@ import { getPrivateConfig, getS3Client } from '../utilities/s3.js'; import StreamingService from '../utilities/streamingService.js'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); -const jsonPageSize = parseInt(process.env.JSON_PAGE_SIZE); -const maxQuerySize = parseInt(process.env.MAX_QUERY_SIZE); +const defaultPageSize = 20; +const maxPageSize = parseInt(process.env.MAX_PAGE_SIZE || 500); +const maxQuerySize = parseInt(process.env.MAX_QUERY_SIZE || 1_000_000); const minDateTime = new Date(-8640000000000000); const maxDateTime = new Date(8640000000000000); @@ -37,18 +38,18 @@ class DuplicateParameterException extends Error { } class InvalidParameterException extends Error { - constructor(parameter) { + constructor(parameter, context) { super(); this.httpStatusCode = 400; - this.message = `The parameter '${parameter}' is not valid for the specified profile`; + this.message = `The parameter '${parameter}' is not valid for the specified ${context}`; } } class LimitExceededException extends Error { - constructor(limit) { + constructor(value, maximum = maxQuerySize) { super(); this.httpStatusCode = 400; - this.message = `The provided limit (${limit}) exceeds the maximum ${process.env.MAX_VALUES_QUERY_SIZE} allowable limit.`; + this.message = `The provided limit (${value.toLocaleString()}) exceeds the maximum ${maximum.toLocaleString()} allowable limit.`; } } @@ -63,7 +64,7 @@ class NoParametersException extends Error { /** * Searches for a materialized view, associated with the profile, that is applicable to the provided columns/filters. * @param {Object} profile definition of the profile being queried - * @param {Array} columns definitions of columns to return, where the first is the primary column + * @param {Array} columns definitions of columns to return * @param {Array} columnsForFilter names of columns that can be used to filter * @returns definition of a materialized view that is applicable to the desired columns/filters or null if none are suitable */ @@ -78,6 +79,49 @@ function findMaterializedView(profile, columns, columnsForFilter) { }); } +/** + * Searches for a view, associated with the profile, that is applicable to the provided columns. + * @param {Object} profile definition of the profile being queried + * @param {Array} columns parameter names of columns to return + * @returns definition of a view that is applicable to the desired columns, or null if none are suitable + */ +function findView(profile, columns) { + if (!profile.views) return; + + const expandedViews = profile.views?.map((view) => ({ + ...view, + columns: view.columns.map((vCol) => { + const pCol = ( + vCol.table + ? Object.values(privateConfig.tableConfig).find( + (p) => p.tableName === vCol.table, + ) + : profile + )?.columns.find((c) => c.name === vCol.name); + if (!pCol) { + throw new Error( + `The view column ${vCol.name} does not exist on the specified profile`, + ); + } + + return pCol; + }), + })); + + return expandedViews.find((view) => { + for (const col of columns) { + if ( + !view.columns.find((vCol) => + [vCol.alias, vCol.lowParam, vCol.highParam].includes(col), + ) + ) { + return; + } + } + return view; + }); +} + /** * Finds full column definitions for the provided array of column aliases * @param {Array} columnAliases array of column aliases to get full column definitions for @@ -88,7 +132,7 @@ function getColumnsFromAliases(columnAliases, profile) { const columns = []; for (const alias of columnAliases) { const column = profile.columns - .concat(profile.materializedViewColumns ?? []) + .concat(profile.referencedColumns ?? []) .find((col) => col.alias === alias); if (!column) { throw new Error(alias); @@ -100,10 +144,12 @@ function getColumnsFromAliases(columnAliases, profile) { } /** Get a subquery if "Latest" is used - * @param {Object} query KnexJS query object + * @param {Express.Request} req + * @param {Object} profile definition of the profile being queried + * @param {Object} params URL query value * @param {Object} columnName name of the "Latest" column * @param {Object} columnType data type of the "Latest" column - * @returns {Object} a different KnexJS query object + * @returns {Object} an updated KnexJS query object */ function createLatestSubquery(req, profile, params, columnName, columnType) { if (!['date', 'numeric', 'timestamptz'].includes(columnType)) return; @@ -155,13 +201,12 @@ function createLatestSubquery(req, profile, params, columnName, columnType) { /** * Creates a stream object from a query. * @param {Object} query KnexJS query object - * @param {Express.Response} req * @param {Express.Response} res * @param {string} format the format of the file attachment * @param {Object} excelDoc Excel workbook and worksheet objects - * @param {number} nextId starting objectid for the next page + * @param {Object} pageOptions page number and page size for paginated JSON */ -async function createStream(query, req, res, format, wbObject, nextId) { +async function createStream(query, res, format, wbObject, pageOptions) { pool.connect((err, client, done) => { if (err) throw err; @@ -172,7 +217,7 @@ async function createStream(query, req, res, format, wbObject, nextId) { const stream = client.query(qStream); stream.on('end', done); - StreamingService.streamResponse(res, stream, format, wbObject, nextId); + StreamingService.streamResponse(res, stream, format, wbObject, pageOptions); }); } @@ -199,37 +244,24 @@ async function streamFile(query, req, res, format, baseName) { }); const worksheet = workbook.addWorksheet('data'); - createStream(query, req, res, format, { workbook, worksheet }); + createStream(query, res, format, { workbook, worksheet }); } else { - createStream(query, req, res, format); + createStream(query, res, format); } } /** * Streams the results of a query as paginated JSON. * @param {Object} query KnexJS query object - * @param {Express.Response} req * @param {Express.Response} res - * @param {number} startId current objectid to start returning results from + * @param {number} pageNumber current page of results + * @param {number} pageSize number of results per page */ -async function streamJson(query, req, res, startId) { - if (startId) query.where('objectid', '>=', startId); - - const nextId = - ( - await queryPool( - knex - .select('objectId') - .from(query.clone().limit(maxQuerySize).as('q')) - .offset(jsonPageSize) - .limit(1), - true, - ) - )?.objectId ?? null; +async function streamJson(query, res, pageNumber, pageSize) { + if (pageNumber > 1) query.offset((pageNumber - 1) * pageSize); + query.limit(pageSize); - query.limit(jsonPageSize); - - createStream(query, req, res, 'json', null, nextId); + createStream(query, res, 'json', null, { pageNumber, pageSize }); } /** @@ -262,8 +294,8 @@ function appendRangeToWhere(query, column, lowParamValue, highParamValue) { /** * Creates an ISO date string with no timezone offset from a given date string. * @param {string} value the date string to be converted to ISO format - * @param {boolean} whether the returned time should represent midnight at the start or end of day - * @returns {string} + * @param {boolean} endOfDay whether the returned time should represent midnight at the start or end of day + * @returns {string | null} */ function dateToUtcTime(value, endOfDay = false) { if (!value) return null; @@ -292,7 +324,7 @@ function getQueryParams(req) { } // organize GET parameters to follow what we expect from POST - const optionsParams = ['f', 'format', 'startId']; + const optionsParams = ['f', 'format', 'limit', 'pageNumber', 'pageSize']; const parameters = { filters: {}, options: {}, @@ -311,6 +343,7 @@ function getQueryParams(req) { /** * Builds the select clause and where clause of the query based on the provided * profile name. + * @param {express.Request} req * @param {Object} query KnexJS query object * @param {Object} profile definition of the profile being queried * @param {Object} queryParams URL query value @@ -350,7 +383,7 @@ function parseCriteria(req, query, profile, queryParams, countOnly = false) { const selectColumns = columnsToReturn.length > 0 ? columnsToReturn : profile.columns; const selectText = selectColumns.map((col) => - col.name === col.alias ? col.name : `${col.name} AS ${col.alias}`, + col.name === col.alias ? col.name : `${col.name} AS "${col.alias}"`, ); query.select(selectText).orderBy('objectid', 'asc'); } @@ -366,7 +399,7 @@ function parseCriteria(req, query, profile, queryParams, countOnly = false) { if (lowArg || highArg) { appendRangeToWhere(query, col, lowArg, highArg); } else if (exactArg !== undefined) { - appendToWhere(query, col.name, queryParams.filters[col.alias]); + appendToWhere(query, col.name, exactArg); } }); @@ -376,6 +409,77 @@ function parseCriteria(req, query, profile, queryParams, countOnly = false) { } } +function parseDocumentSearchCriteria(req, query, profile, queryParams) { + const columnsForFilter = Object.keys(queryParams.filters); + let columnsToReturn = queryParams.columns ?? []; + const view = findView(profile, columnsForFilter.concat(columnsToReturn)); + if (view) query.from(view.name); + const target = view ?? profile; + // NOTE:XXX: This will need to change if we ever have multiple `tsvector` columns in a single table. + const documentQueryColumn = target.columns.find( + (col) => col.type === 'tsvector', + ); + const documentQuery = queryParams.filters[documentQueryColumn.alias]; + const isDocumentSearch = + documentQueryColumn && columnsForFilter.includes(documentQueryColumn.alias); + if (!isDocumentSearch && !columnsToReturn.includes('objectId')) { + columnsToReturn.push('objectId'); + } + const selectColumns = ( + columnsToReturn.length > 0 + ? target.columns.filter((col) => columnsToReturn.includes(col.alias)) + : target.columns + ).filter((col) => col.type !== 'tsvector'); + + // Build the select query, filtering down to requested columns, if the user provided that option. + const asAlias = (col) => + col.name === col.alias ? col.name : `${col.name} AS ${col.alias}`; + if (isDocumentSearch) { + query + .with('ranked', (qb) => { + qb.select( + selectColumns + .map((col) => col.name) + .concat( + knex.raw( + `ts_rank_cd(${documentQueryColumn.name}, websearch_to_tsquery(?), 1 | 32) AS rank`, + [documentQuery], + ), + ), + ) + .withSchema(req.activeSchema) + .from(target.tableName ?? target.name) + .whereRaw(`${documentQueryColumn.name} @@ websearch_to_tsquery(?)`, [ + documentQuery, + ]); + }) + .withSchema() + .from('ranked') + .select([ + knex.raw('ROUND((AVG(rank) * 100)::numeric, 1) AS "rankPercent"'), + ...selectColumns.map(asAlias), + ]) + .orderBy('rankPercent', 'desc') + .groupBy(selectColumns.map((col) => col.name)); + } else { + query.select(selectColumns.map(asAlias)).orderBy('objectid', 'asc'); + } + + // build where clause of the query + target.columns.forEach((col) => { + if (col.type === 'tsvector') return; + + const lowArg = 'lowParam' in col && queryParams.filters[col.lowParam]; + const highArg = 'highParam' in col && queryParams.filters[col.highParam]; + const exactArg = queryParams.filters[col.alias]; + if (lowArg || highArg) { + appendRangeToWhere(query, col, lowArg, highArg); + } else if (exactArg !== undefined) { + appendToWhere(query, col.name, exactArg); + } + }); +} + /** * Runs a query against the provided profile name and streams the result to the * client as csv, tsv, xlsx, json file, or inline json. @@ -391,7 +495,7 @@ async function executeQuery(profile, req, res) { const query = knex .withSchema(req.activeSchema) .from(profile.tableName) - .limit(parseInt(process.env.MAX_QUERY_SIZE)); + .limit(maxQuerySize); const queryParams = getQueryParams(req); @@ -406,7 +510,12 @@ async function executeQuery(profile, req, res) { throw new NoParametersException('Please provide at least one parameter'); } - parseCriteria(req, query, profile, queryParams); + // TODO: Merge this into one function. + if (profile.id === 'actionDocuments') { + parseDocumentSearchCriteria(req, query, profile, queryParams); + } else { + parseCriteria(req, query, profile, queryParams); + } // Check that the query doesn't exceed the MAX_QUERY_SIZE. if (await exceedsMaxSize(query)) { @@ -417,14 +526,23 @@ async function executeQuery(profile, req, res) { }); } + // Check if the query result is empty. + if (await isEmptyResult(query)) { + return res.status(200).json({ + message: `No results found for the current query. Please refine the search.`, + }); + } + const format = queryParams.options.format ?? queryParams.options.f; if (['csv', 'tsv', 'xlsx'].includes(format)) { await streamFile(query, req, res, format, profile.tableName); } else { - const startId = queryParams.options.startId - ? parseInt(queryParams.options.startId) - : null; - await streamJson(query, req, res, startId); + await streamJson( + query, + res, + parseInt(queryParams.options.pageNumber || 1), + parseInt(queryParams.options.pageSize || defaultPageSize), + ); } } catch (error) { log.error( @@ -443,19 +561,35 @@ async function executeQuery(profile, req, res) { /** * Throws an error if multiple instances of a parameter were provided * for an option or filter that accepts a single argument only - * @param {Object} queryFilters URL query value for filters + * @param {Object} queryParams URL query value for filters * @param {Object} profile definition of the profile being queried */ function validateQueryParams(queryParams, profile) { Object.entries(queryParams.options).forEach(([name, value]) => { + // Each option should only be used once. if (Array.isArray(value)) throw new DuplicateParameterException(name); + + // 'pageNumber' and 'pageSize' are only allowed to be used with 'json' format. + const format = queryParams.options.format ?? queryParams.options.f; + if ( + ['pageNumber', 'pageSize'].includes(name) && + ['csv', 'tsv', 'xlsx'].includes(format) + ) { + throw new InvalidParameterException(name, 'response format'); + } + + // 'pageSize' must be less than or equal to the maximum page size. + if (name === 'pageSize' && parseInt(value) > maxPageSize) { + throw new LimitExceededException(value, maxPageSize); + } }); + + const target = findView(profile, Object.keys(queryParams.filters)) ?? profile; Object.entries(queryParams.filters).forEach(([name, value]) => { - const column = profile.columns.find((c) => { + const column = target.columns.find((c) => { if (c.lowParam === name || c.highParam === name || c.alias === name) return c; }); - if (!column) throw new InvalidParameterException(name); if (Array.isArray(value)) { if ( column.lowParam === name || @@ -464,6 +598,7 @@ function validateQueryParams(queryParams, profile) { ) throw new DuplicateParameterException(name); } + if (!column) throw new InvalidParameterException(name, 'profile'); }); } @@ -488,6 +623,20 @@ async function exceedsMaxSize(query) { return count.count > maxQuerySize; } +/** + * Checks if the query result is empty. + * @param {Object} query KnexJS query object + * @returns {Promise} true if the query result is empty + */ +async function isEmptyResult(query) { + const count = await queryPool( + knex.from(query.clone().limit(1).as('q')).count(), + true, + ); + + return count.count === 0; +} + /** * Runs a query against the provided profile name and returns the number of records. * @param {Object} profile definition of the profile being queried @@ -514,9 +663,16 @@ async function executeQueryCountOnly(profile, req, res) { validateQueryParams(queryParams, profile); - parseCriteria(req, query, profile, queryParams, true); + // TODO: Merge this into one function. + if (profile.id === 'actionDocuments') { + parseDocumentSearchCriteria(req, query, profile, queryParams); + } else { + parseCriteria(req, query, profile, queryParams, true); + } - const count = (await queryPool(query.count(), true)).count; + const count = ( + await queryPool(knex.from(query.clone().as('q')).count(), true) + ).count; return res.status(200).json({ count, maxCount: maxQuerySize }); } catch (error) { log.error( @@ -554,7 +710,7 @@ async function executeQueryCountPerOrgCycle(profile, req, res) { log.error( formatLogMsg( metadataObj, - `Failed to get counts per organizaiton and reporting cycle from the "${profile.tableName}" table:`, + `Failed to get counts per organization and reporting cycle from the "${profile.tableName}" table:`, error, ), ); @@ -566,25 +722,19 @@ async function executeQueryCountPerOrgCycle(profile, req, res) { /** * Retrieves the domain values for a single table column. + * @param {Object} profile definition of the profile being queried * @param {express.Request} req * @param {express.Response} res */ -async function executeValuesQuery(req, res) { +async function executeValuesQuery(profile, req, res) { const metadataObj = populateMetdataObjFromRequest(req); try { - const profile = privateConfig.tableConfig[req.params.profile]; - if (!profile) { - return res - .status(404) - .json({ message: 'The requested profile does not exist' }); - } - const { additionalColumns, ...params } = getQueryParamsValues(req); if (!params.text && !params.limit) { throw new NoParametersException( - `Please provide either a text filter or a limit that does not exceed ${process.env.MAX_VALUES_QUERY_SIZE}.`, + `Please provide either a text filter or a limit that does not exceed ${maxPageSize}.`, ); } @@ -602,12 +752,20 @@ async function executeValuesQuery(req, res) { }); } - const values = await queryColumnValues( - profile, - columns, - params, - req.activeSchema, - ); + let values; + try { + values = await queryColumnValues( + res, + profile, + columns, + params, + req.activeSchema, + ); + } catch (err) { + return res.status(400).json({ + message: err.message, + }); + } return res.status(200).json(values); } catch (error) { log.error( @@ -650,12 +808,13 @@ function getQueryParamsValues(req) { /** * Craft the database query for distinct column values + * @param {express.Response} res * @param {Object} profile definition of the profile being queried * @param {Array} columns definitions of columns to return, where the first is the primary column * @param {Object} params parameters to apply to the query * @param {string} schema the currently active database schema */ -async function queryColumnValues(profile, columns, params, schema) { +async function queryColumnValues(res, profile, columns, params, schema) { const primaryColumn = columns[0]; // get columns for where clause @@ -668,19 +827,20 @@ async function queryColumnValues(profile, columns, params, schema) { // search through tableconfig.materializedViews to see if the column // we need is in here - const materializedView = findMaterializedView( - profile, - columns, - columnsForFilter, - ); + const view = + findMaterializedView(profile, columns, columnsForFilter) ?? + findView( + profile, + Object.keys(params.filters).concat(columns.map((c) => c.alias)), + ); - // ensure no mv-only columns exist if no mv was found - if (!materializedView) { + // ensure no view-only columns exist if no view was found + if (!view) { for (const col of columns) { if (!profile.columns.find((c) => c.name === col.name)) { - return res.status(404).json({ - message: `The column ${col.alias} is not available with the current query`, - }); + throw new Error( + `The column ${col.alias} is not available with the current query`, + ); } } } @@ -688,7 +848,7 @@ async function queryColumnValues(profile, columns, params, schema) { // query table directly if a suitable materialized view was not found const query = knex .withSchema(schema) - .from(materializedView ? materializedView.name : profile.tableName) + .from(view?.name ?? profile.tableName) .column( columns.reduce( (current, col) => ({ ...current, [col.alias]: col.name }), @@ -735,11 +895,11 @@ async function queryColumnValues(profile, columns, params, schema) { }); } - const maxValuesQuerySize = parseInt(process.env.MAX_VALUES_QUERY_SIZE); - if (params.limit > maxValuesQuerySize) { - throw new LimitExceededException(params.limit); + const limit = params.limit ?? maxPageSize; + if (limit > maxPageSize) { + throw new LimitExceededException(params.limit, maxPageSize); } - query.limit(params.limit ?? maxValuesQuerySize); + query.limit(limit); return await queryPool(query); } @@ -918,6 +1078,13 @@ export default function (app, basePath) { Object.entries(privateConfig.tableConfig).forEach( ([profileName, profile]) => { + if (profile.hidden) return; + + // get column domain values + router.post(`/${profileName}/values/:column`, async function (req, res) { + await executeValuesQuery(profile, req, res); + }); + // create get requests router.get(`/${profileName}`, async function (req, res) { await executeQuery(profile, req, res); @@ -934,31 +1101,31 @@ export default function (app, basePath) { await executeQueryCountOnly(profile, req, res); }); - // get column domain values - router.post('/:profile/values/:column', async function (req, res) { - await executeValuesQuery(req, res); - }); - - // get bean counts - router.get(`/${profileName}/countPerOrgCycle`, async function (req, res) { - await executeQueryCountPerOrgCycle(profile, req, res); - }); - router.post( - `/${profileName}/countPerOrgCycle`, - async function (req, res) { - await executeQueryCountPerOrgCycle(profile, req, res); - }, - ); - - router.get('/health/etlDatabase', async function (req, res) { - await checkDatabaseHealth(req, res); - }); - - router.get('/health/etlDomainValues', async function (req, res) { - await checkDomainValuesHealth(req, res); - }); + if (profile.includeCycleCount) { + // get bean counts + router.get( + `/${profileName}/countPerOrgCycle`, + async function (req, res) { + await executeQueryCountPerOrgCycle(profile, req, res); + }, + ); + router.post( + `/${profileName}/countPerOrgCycle`, + async function (req, res) { + await executeQueryCountPerOrgCycle(profile, req, res); + }, + ); + } }, ); + router.get('/health/etlDatabase', async function (req, res) { + await checkDatabaseHealth(req, res); + }); + + router.get('/health/etlDomainValues', async function (req, res) { + await checkDomainValuesHealth(req, res); + }); + app.use(`${basePath}api/attains`, router); } diff --git a/app/server/app/utilities/database.js b/app/server/app/utilities/database.js index 0ca53ca8..09a9146c 100644 --- a/app/server/app/utilities/database.js +++ b/app/server/app/utilities/database.js @@ -33,7 +33,6 @@ log.info( `STREAM_HIGH_WATER_MARK: ${parseInt(process.env.STREAM_HIGH_WATER_MARK)}`, ); log.info(`MAX_QUERY_SIZE: ${parseInt(process.env.MAX_QUERY_SIZE)}`); -log.info(`JSON_PAGE_SIZE: ${parseInt(process.env.JSON_PAGE_SIZE)}`); // Setup parsers for ensuring output matches datatype in database // (i.e., return count as 124 instead of "124") diff --git a/app/server/app/utilities/streamingService.js b/app/server/app/utilities/streamingService.js index 3ab2f584..b239c415 100644 --- a/app/server/app/utilities/streamingService.js +++ b/app/server/app/utilities/streamingService.js @@ -79,12 +79,17 @@ export default class StreamingService { /** * Transforms the streaming data to json. * @param {function} preHook function for writing initial headers - * @param {number} nextId starting objectid for the next page + * @param {Object} pageOptions page number and page size for paginated JSON * @returns Transform object */ - static getJsonTransform = (preHook, nextId) => { + static getJsonTransform = (preHook, pageOptions = {}) => { + const { pageNumber, pageSize } = pageOptions; const start = '{ "data": ['; - const end = ']' + (nextId ? `, "nextId": ${nextId}` : '') + '}'; + const end = + ']' + + (pageNumber ? `, "pageNumber": ${pageNumber}` : '') + + (pageSize ? `, "pageSize": ${pageSize}` : '') + + '}'; return new Transform({ writableObjectMode: true, transform(data, _encoding, callback) { @@ -153,14 +158,14 @@ export default class StreamingService { * @param {Transform} inStream readable stream from database query * @param {'csv'|'tsv'|'xlsx'|'json'|''} format export format file type * @param {Object} excelDoc Excel workbook and worksheet objects - * @param {number} nextId starting objectid for the next page + * @param {Object} pageOptions page number and page size for paginated JSON */ static streamResponse = ( outStream, inStream, format, - excelDoc = null, - nextId = null, + excelDoc, + pageOptions, ) => { const { preHook, errorHook, errorHandler } = StreamingService.getOptions( outStream, @@ -173,7 +178,7 @@ export default class StreamingService { outStream.end(); }); - let transform = StreamingService.getJsonTransform(preHook, nextId); + let transform = StreamingService.getJsonTransform(preHook, pageOptions); if (format === 'csv' || format === 'tsv') { transform = StreamingService.getBasicTransform(preHook, format); } diff --git a/etl/app/content-private/domainValueMappings.json b/etl/app/content-private/domainValueMappings.json index d117ef59..88ca37fb 100644 --- a/etl/app/content-private/domainValueMappings.json +++ b/etl/app/content-private/domainValueMappings.json @@ -4,6 +4,10 @@ "columns": ["actionAgency", "associatedActionAgency"], "valueField": "name" }, + "actionDocumentType": { + "domainName": "ActionDocumentType", + "columns": ["actionDocumentType"] + }, "actionType": { "domainName": "ActionType", "columns": ["actionType", "associatedActionType"], diff --git a/etl/app/content-private/tableConfig.json b/etl/app/content-private/tableConfig.json index 395ae26b..0ce4cae7 100644 --- a/etl/app/content-private/tableConfig.json +++ b/etl/app/content-private/tableConfig.json @@ -1,8 +1,11 @@ { "actions": { + "id": "actions", + "source": "attains", "tableName": "actions", "idColumn": "objectid", - "createQuery": "CREATE TABLE IF NOT EXISTS actions ( objectid INTEGER PRIMARY KEY, state VARCHAR(4000), region VARCHAR(2), organizationid VARCHAR(30) NOT NULL, organizationname VARCHAR(150) NOT NULL, organizationtype VARCHAR(30) NOT NULL, assessmentunitid VARCHAR(50), assessmentunitname VARCHAR(255), actionid VARCHAR(45) NOT NULL, actionname VARCHAR(255) NOT NULL, completiondate DATE, fiscalyearestablished VARCHAR (4), parameter VARCHAR(240), parametergroup VARCHAR(60), locationdescription VARCHAR(2000), actiontype VARCHAR(50) NOT NULL, watertype VARCHAR(40), watersize NUMERIC(18,4), watersizeunits VARCHAR(15), actionagency VARCHAR(10) NOT NULL, inindiancountry VARCHAR(1), includeinmeasure VARCHAR(1), plansummarylink VARCHAR(116) )", + "createQuery": "CREATE TABLE IF NOT EXISTS actions ( objectid INTEGER PRIMARY KEY, state VARCHAR, region VARCHAR(2), organizationid VARCHAR(30) NOT NULL, organizationname VARCHAR(150) NOT NULL, organizationtype VARCHAR(30) NOT NULL, assessmentunitid VARCHAR(50), assessmentunitname VARCHAR(255), actionid VARCHAR(45) NOT NULL, actionname VARCHAR(255) NOT NULL, completiondate DATE, fiscalyearestablished VARCHAR (4), parameter VARCHAR(240), parametergroup VARCHAR(60), locationdescription VARCHAR(2000), actiontype VARCHAR(50) NOT NULL, watertype VARCHAR(40), watersize NUMERIC(18,4), watersizeunits VARCHAR(15), actionagency VARCHAR(10) NOT NULL, inindiancountry VARCHAR(1), includeinmeasure VARCHAR(1), plansummarylink VARCHAR(116) )", + "includeCycleCount": true, "columns": [ { "name": "objectid", @@ -111,7 +114,7 @@ "alias": "waterType" } ], - "materializedViewColumns": [ + "referencedColumns": [ { "name": "statename", "alias": "stateName" @@ -195,10 +198,258 @@ } ] }, + "actionDocuments": { + "id": "actionDocuments", + "source": "attains", + "tableName": "action_documents", + "idColumn": "objectid", + "createQuery": "CREATE TABLE IF NOT EXISTS action_documents ( objectid INTEGER PRIMARY KEY, documentkey INTEGER, actionid VARCHAR(45), actiontypename VARCHAR(50), organizationid VARCHAR(30), organizationtype VARCHAR(30) NOT NULL, organizationname VARCHAR(150) NOT NULL, regionid VARCHAR(2), state VARCHAR, actionname VARCHAR(255), completiondate DATE, tmdldate DATE, actiondocumenturl TEXT )", + "columns": [ + { + "name": "objectid", + "alias": "objectId", + "skipIndex": true + }, + { + "name": "actiondocumenturl", + "alias": "actionDocumentUrl" + }, + { + "name": "actionid", + "alias": "actionId" + }, + { + "name": "actionname", + "alias": "actionName" + }, + { + "name": "actiontypename", + "alias": "actionType" + }, + { + "name": "completiondate", + "alias": "completionDate", + "lowParam": "completionDateLo", + "highParam": "completionDateHi", + "type": "date", + "indexOrder": "desc" + }, + { + "name": "documentkey", + "alias": "documentKey", + "type": "numeric" + }, + { + "name": "organizationid", + "alias": "organizationId" + }, + { + "name": "organizationname", + "alias": "organizationName" + }, + { + "name": "organizationtype", + "alias": "organizationType", + "skipIndex": true + }, + { + "name": "regionid", + "alias": "region" + }, + { + "name": "state", + "alias": "state" + }, + { + "name": "tmdldate", + "alias": "tmdlDate", + "lowParam": "tmdlDateLo", + "highParam": "tmdlDateHi", + "type": "date", + "indexOrder": "desc" + } + ], + "referencedColumns": [ + { + "name": "documentdesc", + "alias": "documentDesc" + }, + { + "name": "documentfilename", + "alias": "documentFileName" + }, + { + "name": "documentfiletypename", + "alias": "documentFileTypeName" + }, + { + "name": "documentkey", + "alias": "documentKey" + }, + { + "name": "documentname", + "alias": "documentName" + }, + { + "name": "documenttsv", + "alias": "documentQuery" + }, + { + "name": "documenttypename", + "alias": "actionDocumentType" + }, + { + "name": "statename", + "alias": "stateName" + } + ], + "materializedViews": [ + { + "name": "actiondocuments_actions", + "columns": [ + { + "name": "actionid" + }, + { + "name": "actionname" + }, + { + "name": "organizationid" + }, + { + "name": "regionid" + }, + { + "name": "state" + }, + { + "name": "statename" + } + ], + "joins": [ + { + "table": "states", + "joinKey": ["state", "statecode"] + } + ] + }, + { + "name": "actiondocuments_documents", + "columns": [ + { + "name": "documentfilename" + }, + { + "name": "documentname" + } + ], + "joins": [ + { + "table": "documents_text", + "joinKey": [ + "documents_text.documentkey", + "action_documents.documentkey" + ] + } + ] + } + ], + "views": [ + { + "name": "action_documents_view", + "columns": [ + { + "name": "objectid", + "table": "action_documents" + }, + { + "name": "actiondocumenturl" + }, + { + "name": "actionid" + }, + { + "name": "actionname" + }, + { + "name": "actiontypename" + }, + { + "name": "completiondate" + }, + { + "name": "organizationid" + }, + { + "name": "organizationname" + }, + { + "name": "organizationtype" + }, + { + "name": "regionid" + }, + { + "name": "state" + }, + { + "name": "tmdldate" + }, + { + "name": "documentdesc", + "table": "documents_text" + }, + { + "name": "documentfilename", + "table": "documents_text" + }, + { + "name": "documentfiletypename", + "table": "documents_text" + }, + { + "name": "documentkey", + "table": "documents_text" + }, + { + "name": "documentname", + "table": "documents_text" + }, + { + "name": "documenttsv", + "table": "documents_text_search" + }, + { + "name": "documenttypename", + "table": "documents_text" + } + ], + "joins": [ + { + "table": "documents_text", + "joinKey": [ + "documents_text.documentkey", + "action_documents.documentkey" + ] + }, + { + "table": "documents_text_search", + "joinKey": [ + "documents_text_search.documentid", + "documents_text.objectid" + ] + } + ] + } + ] + }, "assessments": { + "id": "assessments", + "source": "attains", "tableName": "assessments", "idColumn": "objectid", - "createQuery": "CREATE TABLE IF NOT EXISTS assessments ( objectid INTEGER PRIMARY KEY, state VARCHAR(4000), region VARCHAR(2), organizationid VARCHAR(30) NOT NULL, organizationname VARCHAR(150) NOT NULL, organizationtype VARCHAR(30) NOT NULL, reportingcycle NUMERIC(4,0) NOT NULL, cycleid NUMERIC(38,0) NOT NULL, assessmentunitid VARCHAR(50), assessmentunitname VARCHAR(255), cyclelastassessed NUMERIC(4,0) NOT NULL, overallstatus VARCHAR(4000), epaircategory VARCHAR(5), stateircategory VARCHAR(5), parametergroup VARCHAR(60), parametername VARCHAR(240), parameterstatus VARCHAR(240), usegroup VARCHAR(500), usename VARCHAR(255), useircategory VARCHAR(5), usestateircategory VARCHAR(5), usesupport VARCHAR(1), parameterattainment VARCHAR(50), parameterircategory VARCHAR(5), parameterstateircategory VARCHAR(5), cyclefirstlisted NUMERIC(4,0), associatedactionid VARCHAR(45), associatedactionname VARCHAR(255), associatedactiontype VARCHAR(50), locationdescription VARCHAR(2000), watertype VARCHAR(40), watersize NUMERIC(18,4), watersizeunits VARCHAR(15), sizesource VARCHAR(100), sourcescale VARCHAR(30), assessmentunitstatus VARCHAR(1), useclassname VARCHAR(50), assessmentdate DATE, assessmentbasis VARCHAR(30), monitoringstartdate DATE, monitoringenddate DATE, assessmentmethods VARCHAR(150), assessmenttypes VARCHAR(30), delisted VARCHAR(1), delistedreason VARCHAR(100), seasonstartdate DATE, seasonenddate DATE, pollutantindicator VARCHAR(1), cyclescheduledfortmdl NUMERIC(4,0), cycleexpectedtoattain NUMERIC(4,0), cwa303dpriorityranking VARCHAR(25), vision303dpriority VARCHAR(1), alternatelistingidentifier VARCHAR(50), consentdecreecycle NUMERIC(4,0), associatedactionstatus VARCHAR(30), associatedactionagency VARCHAR(10) )", + "createQuery": "CREATE TABLE IF NOT EXISTS assessments ( objectid INTEGER PRIMARY KEY, state VARCHAR, region VARCHAR(2), organizationid VARCHAR(30) NOT NULL, organizationname VARCHAR(150) NOT NULL, organizationtype VARCHAR(30) NOT NULL, reportingcycle NUMERIC(4,0) NOT NULL, cycleid NUMERIC(38,0) NOT NULL, assessmentunitid VARCHAR(50), assessmentunitname VARCHAR(255), cyclelastassessed NUMERIC(4,0) NOT NULL, overallstatus VARCHAR(4000), epaircategory VARCHAR(5), stateircategory VARCHAR(5), parametergroup VARCHAR(60), parametername VARCHAR(240), parameterstatus VARCHAR(240), usegroup VARCHAR(500), usename VARCHAR(255), useircategory VARCHAR(5), usestateircategory VARCHAR(5), usesupport VARCHAR(1), parameterattainment VARCHAR(50), parameterircategory VARCHAR(5), parameterstateircategory VARCHAR(5), cyclefirstlisted NUMERIC(4,0), associatedactionid VARCHAR(45), associatedactionname VARCHAR(255), associatedactiontype VARCHAR(50), locationdescription VARCHAR(2000), watertype VARCHAR(40), watersize NUMERIC(18,4), watersizeunits VARCHAR(15), sizesource VARCHAR(100), sourcescale VARCHAR(30), assessmentunitstatus VARCHAR(1), useclassname VARCHAR(50), assessmentdate DATE, assessmentbasis VARCHAR(30), monitoringstartdate DATE, monitoringenddate DATE, assessmentmethods VARCHAR(150), assessmenttypes VARCHAR(30), delisted VARCHAR(1), delistedreason VARCHAR(100), seasonstartdate DATE, seasonenddate DATE, pollutantindicator VARCHAR(1), cyclescheduledfortmdl NUMERIC(4,0), cycleexpectedtoattain NUMERIC(4,0), cwa303dpriorityranking VARCHAR(25), vision303dpriority VARCHAR(1), alternatelistingidentifier VARCHAR(50), consentdecreecycle NUMERIC(4,0), associatedactionstatus VARCHAR(30), associatedactionagency VARCHAR(10) )", + "includeCycleCount": true, "overrideWorkMemory": "1GB", "columns": [ { @@ -478,7 +729,7 @@ "alias": "waterType" } ], - "materializedViewColumns": [ + "referencedColumns": [ { "name": "statename", "alias": "stateName" @@ -673,9 +924,12 @@ ] }, "assessmentUnits": { + "id": "assessmentUnits", + "source": "attains", "tableName": "assessment_units", "idColumn": "objectid", - "createQuery": "CREATE TABLE IF NOT EXISTS assessment_units ( objectid INTEGER PRIMARY KEY, state VARCHAR(4000), region VARCHAR(2), organizationid VARCHAR(30) NOT NULL, organizationname VARCHAR(150) NOT NULL, organizationtype VARCHAR(30) NOT NULL, reportingcycle NUMERIC(4,0) NOT NULL, cycleid NUMERIC(38,0) NOT NULL, assessmentunitid VARCHAR(50) NOT NULL, assessmentunitname VARCHAR(255) NOT NULL, locationdescription VARCHAR(2000) NOT NULL, watertype VARCHAR(40) NOT NULL, watersize NUMERIC(18,4) NOT NULL, watersizeunits VARCHAR(15) NOT NULL, assessmentunitstatus VARCHAR(1) NOT NULL, useclassname VARCHAR(50), sizesource VARCHAR(100), sourcescale VARCHAR(30), locationtypecode VARCHAR(22), locationtext VARCHAR(100) )", + "createQuery": "CREATE TABLE IF NOT EXISTS assessment_units ( objectid INTEGER PRIMARY KEY, state VARCHAR, region VARCHAR(2), organizationid VARCHAR(30) NOT NULL, organizationname VARCHAR(150) NOT NULL, organizationtype VARCHAR(30) NOT NULL, reportingcycle NUMERIC(4,0) NOT NULL, cycleid NUMERIC(38,0) NOT NULL, assessmentunitid VARCHAR(50) NOT NULL, assessmentunitname VARCHAR(255) NOT NULL, locationdescription VARCHAR(2000) NOT NULL, watertype VARCHAR(40) NOT NULL, watersize NUMERIC(18,4) NOT NULL, watersizeunits VARCHAR(15) NOT NULL, assessmentunitstatus VARCHAR(1) NOT NULL, useclassname VARCHAR(50), sizesource VARCHAR(100), sourcescale VARCHAR(30), locationtypecode VARCHAR(22), locationtext VARCHAR(100) )", + "includeCycleCount": true, "columns": [ { "name": "objectid", @@ -769,7 +1023,7 @@ "alias": "waterType" } ], - "materializedViewColumns": [ + "referencedColumns": [ { "name": "statename", "alias": "stateName" @@ -840,9 +1094,12 @@ ] }, "assessmentUnitsMonitoringLocations": { + "id": "assessmentUnitsMonitoringLocations", + "source": "attains", "tableName": "assessment_units_monitoring_locations", "idColumn": "objectid", - "createQuery": "CREATE TABLE IF NOT EXISTS assessment_units_monitoring_locations ( objectid INTEGER PRIMARY KEY, state VARCHAR(4000), region VARCHAR(2), organizationid VARCHAR(30) NOT NULL, organizationname VARCHAR(150) NOT NULL, organizationtype VARCHAR(30) NOT NULL, reportingcycle NUMERIC(4,0) NOT NULL, cycleid NUMERIC(38,0) NOT NULL, assessmentunitid VARCHAR(50) NOT NULL, assessmentunitname VARCHAR(255) NOT NULL, locationdescription VARCHAR(2000) NOT NULL, watertype VARCHAR(40) NOT NULL, watersize NUMERIC(18,4) NOT NULL, watersizeunits VARCHAR(15) NOT NULL, monitoringlocationorgid VARCHAR(30), monitoringlocationid VARCHAR(35), monitoringlocationdatalink VARCHAR(255), assessmentunitstatus VARCHAR(1) NOT NULL, useclassname VARCHAR(50), sizesource VARCHAR(100), sourcescale VARCHAR(30) )", + "createQuery": "CREATE TABLE IF NOT EXISTS assessment_units_monitoring_locations ( objectid INTEGER PRIMARY KEY, state VARCHAR, region VARCHAR(2), organizationid VARCHAR(30) NOT NULL, organizationname VARCHAR(150) NOT NULL, organizationtype VARCHAR(30) NOT NULL, reportingcycle NUMERIC(4,0) NOT NULL, cycleid NUMERIC(38,0) NOT NULL, assessmentunitid VARCHAR(50) NOT NULL, assessmentunitname VARCHAR(255) NOT NULL, locationdescription VARCHAR(2000) NOT NULL, watertype VARCHAR(40) NOT NULL, watersize NUMERIC(18,4) NOT NULL, watersizeunits VARCHAR(15) NOT NULL, monitoringlocationorgid VARCHAR(30), monitoringlocationid VARCHAR(35), monitoringlocationdatalink VARCHAR(255), assessmentunitstatus VARCHAR(1) NOT NULL, useclassname VARCHAR(50), sizesource VARCHAR(100), sourcescale VARCHAR(30) )", + "includeCycleCount": true, "columns": [ { "name": "objectid", @@ -942,7 +1199,7 @@ "alias": "waterType" } ], - "materializedViewColumns": [ + "referencedColumns": [ { "name": "statename", "alias": "stateName" @@ -1013,9 +1270,12 @@ ] }, "catchmentCorrespondence": { + "id": "catchmentCorrespondence", + "source": "attains", "tableName": "catchment_correspondence", "idColumn": "objectid", - "createQuery": "CREATE TABLE IF NOT EXISTS catchment_correspondence ( objectid INTEGER PRIMARY KEY, state VARCHAR(4000), region VARCHAR(2), organizationid VARCHAR(30) NOT NULL, organizationname VARCHAR(150) NOT NULL, organizationtype VARCHAR(30) NOT NULL, reportingcycle NUMERIC(4,0) NOT NULL, cycleid NUMERIC(38,0) NOT NULL, assessmentunitid VARCHAR(50) NOT NULL, assessmentunitname VARCHAR(255) NOT NULL, catchmentnhdplusid NUMERIC(38,0) )", + "createQuery": "CREATE TABLE IF NOT EXISTS catchment_correspondence ( objectid INTEGER PRIMARY KEY, state VARCHAR, region VARCHAR(2), organizationid VARCHAR(30) NOT NULL, organizationname VARCHAR(150) NOT NULL, organizationtype VARCHAR(30) NOT NULL, reportingcycle NUMERIC(4,0) NOT NULL, cycleid NUMERIC(38,0) NOT NULL, assessmentunitid VARCHAR(50) NOT NULL, assessmentunitname VARCHAR(255) NOT NULL, catchmentnhdplusid NUMERIC(38,0) )", + "includeCycleCount": true, "overrideWorkMemory": "790MB", "columns": [ { @@ -1070,7 +1330,7 @@ "alias": "state" } ], - "materializedViewColumns": [ + "referencedColumns": [ { "name": "statename", "alias": "stateName" @@ -1138,10 +1398,98 @@ } ] }, + "documentsText": { + "id": "documentsText", + "source": "attains", + "tableName": "documents_text", + "idColumn": "objectid", + "createQuery": "CREATE TABLE IF NOT EXISTS documents_text ( objectid INTEGER PRIMARY KEY, documentkey INTEGER, documentname TEXT, documentdesc TEXT, documentfilename TEXT, documentfiletypename TEXT, documenttypename TEXT, documenttext TEXT )", + "hidden": true, + "columns": [ + { + "name": "objectid", + "alias": "objectId", + "skipIndex": true + }, + { + "name": "documentdesc", + "alias": "documentDesc" + }, + { + "name": "documentfilename", + "alias": "documentFileName" + }, + { + "name": "documentfiletypename", + "alias": "documentFileTypeName" + }, + { + "name": "documentkey", + "alias": "documentKey", + "type": "numeric" + }, + { + "name": "documentname", + "alias": "documentName" + }, + { + "name": "documenttext", + "alias": "documentText", + "skipIndex": true + }, + { + "name": "documenttypename", + "alias": "actionDocumentType" + } + ], + "referencedColumns": [], + "materializedViews": [ + { + "name": "documentstext_documentname", + "columns": [ + { + "name": "documentname" + }, + { + "name": "documentfilename" + } + ] + } + ] + }, + "documentsTextSearch": { + "id": "documentsTextSearch", + "tableName": "documents_text_search", + "idColumn": "objectid", + "createQuery": "CREATE TABLE IF NOT EXISTS documents_text ( objectid SERIAL PRIMARY KEY, documentid INTEGER, documenttsv TSVECTOR )", + "hidden": true, + "columns": [ + { + "name": "objectid", + "alias": "objectId", + "skipIndex": true + }, + { + "name": "documentid", + "alias": "documentId" + }, + { + "name": "documenttsv", + "alias": "documentQuery", + "type": "tsvector", + "acceptsMultiple": false + } + ], + "referencedColumns": [], + "materializedViews": [] + }, "sources": { + "id": "sources", + "source": "attains", "tableName": "sources", "idColumn": "objectid", - "createQuery": "CREATE TABLE IF NOT EXISTS sources ( objectid INTEGER PRIMARY KEY, state VARCHAR(4000), region VARCHAR(2), organizationid VARCHAR(30) NOT NULL, organizationname VARCHAR(150) NOT NULL, organizationtype VARCHAR(30) NOT NULL, reportingcycle NUMERIC(4,0) NOT NULL, cycleid NUMERIC(38,0) NOT NULL, assessmentunitid VARCHAR(50) NOT NULL, assessmentunitname VARCHAR(255) NOT NULL, overallstatus VARCHAR(4000), epaircategory VARCHAR(5), stateircategory VARCHAR(5), sourcename VARCHAR(240) NOT NULL, confirmed VARCHAR(1) NOT NULL, parametergroup VARCHAR(60) NOT NULL, causename VARCHAR(240) NOT NULL, locationdescription VARCHAR(2000) NOT NULL, watertype VARCHAR(40) NOT NULL, watersize NUMERIC(18,4) NOT NULL, watersizeunits VARCHAR(15) NOT NULL )", + "createQuery": "CREATE TABLE IF NOT EXISTS sources ( objectid INTEGER PRIMARY KEY, state VARCHAR, region VARCHAR(2), organizationid VARCHAR(30) NOT NULL, organizationname VARCHAR(150) NOT NULL, organizationtype VARCHAR(30) NOT NULL, reportingcycle NUMERIC(4,0) NOT NULL, cycleid NUMERIC(38,0) NOT NULL, assessmentunitid VARCHAR(50) NOT NULL, assessmentunitname VARCHAR(255) NOT NULL, overallstatus VARCHAR(4000), epaircategory VARCHAR(5), stateircategory VARCHAR(5), sourcename VARCHAR(240) NOT NULL, confirmed VARCHAR(1) NOT NULL, parametergroup VARCHAR(60) NOT NULL, causename VARCHAR(240) NOT NULL, locationdescription VARCHAR(2000) NOT NULL, watertype VARCHAR(40) NOT NULL, watersize NUMERIC(18,4) NOT NULL, watersizeunits VARCHAR(15) NOT NULL )", + "includeCycleCount": true, "columns": [ { "name": "objectid", @@ -1238,7 +1586,7 @@ "alias": "waterType" } ], - "materializedViewColumns": [ + "referencedColumns": [ { "name": "statename", "alias": "stateName" @@ -1322,9 +1670,12 @@ ] }, "tmdl": { + "id": "tmdl", + "source": "attains", "tableName": "tmdl", "idColumn": "objectid", - "createQuery": "CREATE TABLE IF NOT EXISTS tmdl ( objectid INTEGER PRIMARY KEY, state VARCHAR(4000), region VARCHAR(2), organizationid VARCHAR(30) NOT NULL, organizationname VARCHAR(150) NOT NULL, organizationtype VARCHAR(30) NOT NULL, assessmentunitid VARCHAR(50), assessmentunitname VARCHAR(255), actionid VARCHAR(45) NOT NULL, actionname VARCHAR(255) NOT NULL, completiondate DATE, tmdldate DATE, fiscalyearestablished VARCHAR(4), pollutant VARCHAR(240), pollutantgroup VARCHAR(60), sourcetype VARCHAR(40), addressedparameter VARCHAR(240), addressedparametergroup VARCHAR(60), locationdescription VARCHAR(2000), watertype VARCHAR(40), watersize NUMERIC(18,4), watersizeunits VARCHAR(15), actionagency VARCHAR(10) NOT NULL, loadallocation NUMERIC(21,3), loadallocationunits VARCHAR(40), explicitmarginofsafety VARCHAR(255), implicitmarginofsafety VARCHAR(255), tmdlendpoint1 TEXT, tmdlendpoint2 TEXT, tmdlendpoint3 TEXT, npdesidentifier VARCHAR(60), otheridentifier VARCHAR(4000), wasteloadallocation NUMERIC(24,3), inindiancountry VARCHAR(1), includeinmeasure VARCHAR(1), plansummarylink VARCHAR(116), tmdlendpoint TEXT GENERATED ALWAYS AS (coalesce(tmdlendpoint1, '') || coalesce(tmdlendpoint2, '') || coalesce(tmdlendpoint3, '')) STORED )", + "createQuery": "CREATE TABLE IF NOT EXISTS tmdl ( objectid INTEGER PRIMARY KEY, state VARCHAR, region VARCHAR(2), organizationid VARCHAR(30) NOT NULL, organizationname VARCHAR(150) NOT NULL, organizationtype VARCHAR(30) NOT NULL, assessmentunitid VARCHAR(50), assessmentunitname VARCHAR(255), actionid VARCHAR(45) NOT NULL, actionname VARCHAR(255) NOT NULL, completiondate DATE, tmdldate DATE, fiscalyearestablished VARCHAR(4), pollutant VARCHAR(240), pollutantgroup VARCHAR(60), sourcetype VARCHAR(40), addressedparameter VARCHAR(240), addressedparametergroup VARCHAR(60), locationdescription VARCHAR(2000), watertype VARCHAR(40), watersize NUMERIC(18,4), watersizeunits VARCHAR(15), actionagency VARCHAR(10) NOT NULL, loadallocation NUMERIC(21,3), loadallocationunits VARCHAR(40), explicitmarginofsafety VARCHAR(255), implicitmarginofsafety VARCHAR(255), tmdlendpoint1 TEXT, tmdlendpoint2 TEXT, tmdlendpoint3 TEXT, npdesidentifier VARCHAR(60), otheridentifier VARCHAR(4000), wasteloadallocation NUMERIC(24,3), inindiancountry VARCHAR(1), includeinmeasure VARCHAR(1), plansummarylink VARCHAR(116), tmdlendpoint TEXT GENERATED ALWAYS AS (coalesce(tmdlendpoint1, '') || coalesce(tmdlendpoint2, '') || coalesce(tmdlendpoint3, '')) STORED )", + "includeCycleCount": true, "overrideWorkMemory": "790MB", "columns": [ { @@ -1489,7 +1840,7 @@ "alias": "waterType" } ], - "materializedViewColumns": [ + "referencedColumns": [ { "name": "statename", "alias": "stateName" diff --git a/etl/app/server/database.js b/etl/app/server/database.js index c35ec78b..78d55ea5 100644 --- a/etl/app/server/database.js +++ b/etl/app/server/database.js @@ -587,6 +587,151 @@ async function loadUtilityTables(pool, s3Config, schemaName) { log.info('Utility tables finished updating'); } +async function createProfileMaterializedViews(pool, schemaName, profile) { + const { columns, includeCycleCount, materializedViews, tableName } = profile; + + const client = await getClient(pool); + try { + await client.query(`SET search_path TO ${schemaName}`); + let count = 0; + for (const mv of materializedViews) { + // optionally join columns from other tables + const joinClause = (join) => + `LEFT JOIN ${join.table} ON ${join.joinKey[0]} = ${join.joinKey[1]}`; + await client.query(` + CREATE MATERIALIZED VIEW IF NOT EXISTS ${mv.name} + AS + SELECT DISTINCT ${mv.columns.map((col) => col.name).join(', ')} + FROM ${tableName} ${mv.joins ? mv.joins.map(joinClause).join(' ') : ''} + + WITH DATA; + `); + count += 1; + log.info( + `${tableName}: Created materialized view (${count} of ${materializedViews.length}): ${mv.name}`, + ); + + const indexableColumnsMv = mv.columns.filter((col) => !col.skipIndex); + + // create indexes for the materialized view + let mvIndexCount = 0; + const mvIndexTableName = mv.name.replaceAll('_', ''); + for (const column of indexableColumnsMv) { + mvIndexCount = await createIndividualIndex( + client, + column, + mvIndexCount, + indexableColumnsMv.length, + mvIndexTableName, + mv.name, + ); + } + } + + // create countPerOrgCycle view + const groupByColumns = []; + const hasOrgId = columns.find((c) => c.name === 'organizationid'); + const hasReportingCycleId = columns.find( + (c) => c.name === 'reportingcycle', + ); + const hasCycleId = columns.find((c) => c.name === 'cycleid'); + + const orderByArray = []; + if (hasOrgId) { + groupByColumns.push('organizationid'); + orderByArray.push({ column: 'organizationid', order: 'ASC' }); + } + if (hasReportingCycleId) { + groupByColumns.push('reportingcycle'); + orderByArray.push({ column: 'reportingcycle', order: 'DESC' }); + } + if (hasCycleId) { + groupByColumns.push('cycleid'); + orderByArray.push({ column: 'cycleid', order: 'ASC' }); + } + + let mvName = `${tableName}_countperorgcycle`; + if (includeCycleCount) { + await client.query(` + CREATE MATERIALIZED VIEW IF NOT EXISTS ${mvName} + AS + SELECT ${groupByColumns.join( + ', ', + )}, count(*), count(distinct "assessmentunitid") as "assessmentUnitIdCount" + FROM ${tableName} + ${ + tableName === 'catchment_correspondence' + ? 'WHERE catchmentnhdplusid IS NOT NULL' + : '' + } + GROUP BY ${groupByColumns.join(', ')} + ORDER BY ${orderByArray + .map((col) => `${col.column} ${col.order}`) + .join(', ')} + + WITH DATA; + `); + + log.info(`${tableName}: Created countPerOrgCycle materialized view`); + } + + mvName = `${tableName}_count`; + const indexTableName = tableName.replaceAll('_', ''); + await client.query(` + CREATE MATERIALIZED VIEW IF NOT EXISTS ${mvName} + AS + SELECT count(*) + FROM ${tableName} + ${ + hasReportingCycleId + ? `WHERE ("organizationid", "reportingcycle") IN ( + SELECT "organizationid", MAX("reportingcycle") + FROM ${indexTableName}_reportingcycle + GROUP BY "organizationid" + ) + ` + : '' + } + + WITH DATA; + `); + + log.info(`${tableName}: Created count materialized view`); + } finally { + client.release(); + } +} + +async function createProfileViews(pool, schemaName, profile) { + if (!profile.views) return; + + const client = await getClient(pool); + try { + await client.query(`SET search_path TO ${schemaName}`); + let count = 0; + for (const view of profile.views) { + const joinClause = (join) => + `JOIN ${join.table} ON ${join.joinKey[0]} = ${join.joinKey[1]}`; + await client.query(` + CREATE OR REPLACE VIEW ${view.name} + AS + SELECT ${view.columns + .map((col) => (col.table ? `${col.table}.${col.name}` : col.name)) + .join(', ')} + FROM ${profile.tableName} ${ + view.joins ? view.joins.map(joinClause).join(' ') : '' + } + `); + count++; + log.info( + `${profile.tableName}: Created view (${count} of ${profile.views.length}): ${view.name}`, + ); + } + } finally { + client.release(); + } +} + export async function runLoad(pool, s3Config, s3Julian, logId) { log.info('Running ETL process!'); @@ -602,11 +747,21 @@ export async function runLoad(pool, s3Config, s3Julian, logId) { await loadUtilityTables(pool, s3Config, schemaName); // Add tables to schema and import new data - const loadTasks = Object.values(s3Config.tableConfig).map((profile) => { - return loadProfile(profile, pool, schemaName, s3Config, s3Julian); - }); + const loadTasks = Object.values(s3Config.tableConfig) + .filter((profile) => profile.source?.toLowerCase() === 'attains') + .map((profile) => { + return loadProfile(profile, pool, schemaName, s3Config, s3Julian); + }); await Promise.all(loadTasks); + // Create views and materialized views. + await Promise.all( + Object.values(s3Config.tableConfig).map(async (profile) => { + await createProfileMaterializedViews(pool, schemaName, profile); + await createProfileViews(pool, schemaName, profile); + }), + ); + const profileStats = await getProfileStats(pool, schemaName, s3Julian); // Verify the etl was successfull and the data matches what we expect. @@ -838,109 +993,6 @@ async function createIndexes(s3Config, client, overrideWorkMemory, tableName) { tableName, ); } - - // create materialized views for the table - count = 0; - for (const mv of table.materializedViews) { - // optionally join columns from other tables - const joinClause = (join) => - `LEFT JOIN ${join.table} ON ${join.joinKey[0]} = ${join.joinKey[1]}`; - await client.query(` - CREATE MATERIALIZED VIEW IF NOT EXISTS ${mv.name} - AS - SELECT DISTINCT ${mv.columns.map((col) => col.name).join(', ')} - FROM ${tableName} ${mv.joins ? mv.joins.map(joinClause).join(' ') : ''} - - WITH DATA; - `); - count += 1; - log.info( - `${tableName}: Created materialized view (${count} of ${table.materializedViews.length}): ${mv.name}`, - ); - - const indexableColumnsMv = mv.columns.filter((col) => !col.skipIndex); - - // create indexes for the materialized view - let mvIndexCount = 0; - const mvIndexTableName = mv.name.replaceAll('_', ''); - for (const column of indexableColumnsMv) { - mvIndexCount = await createIndividualIndex( - client, - column, - mvIndexCount, - indexableColumnsMv.length, - mvIndexTableName, - mv.name, - ); - } - } - - // create countPerOrgCycle view - const groupByColumns = []; - const hasOrgId = table.columns.find((c) => c.name === 'organizationid'); - const hasReportingCycleId = table.columns.find( - (c) => c.name === 'reportingcycle', - ); - const hasCycleId = table.columns.find((c) => c.name === 'cycleid'); - - const orderByArray = []; - if (hasOrgId) { - groupByColumns.push('organizationid'); - orderByArray.push({ column: 'organizationid', order: 'ASC' }); - } - if (hasReportingCycleId) { - groupByColumns.push('reportingcycle'); - orderByArray.push({ column: 'reportingcycle', order: 'DESC' }); - } - if (hasCycleId) { - groupByColumns.push('cycleid'); - orderByArray.push({ column: 'cycleid', order: 'ASC' }); - } - - let mvName = `${tableName}_countperorgcycle`; - await client.query(` - CREATE MATERIALIZED VIEW IF NOT EXISTS ${mvName} - AS - SELECT ${groupByColumns.join( - ', ', - )}, count(*), count(distinct "assessmentunitid") as "assessmentUnitIdCount" - FROM ${tableName} - ${ - tableName === 'catchment_correspondence' - ? 'WHERE catchmentnhdplusid IS NOT NULL' - : '' - } - GROUP BY ${groupByColumns.join(', ')} - ORDER BY ${orderByArray - .map((col) => `${col.column} ${col.order}`) - .join(', ')} - - WITH DATA; - `); - - log.info(`${tableName}: Created countPerOrgCycle materialized view`); - - mvName = `${tableName}_count`; - await client.query(` - CREATE MATERIALIZED VIEW IF NOT EXISTS ${mvName} - AS - SELECT count(*) - FROM ${tableName} - ${ - hasReportingCycleId - ? `WHERE ("organizationid", "reportingcycle") IN ( - SELECT "organizationid", MAX("reportingcycle") - FROM ${indexTableName}_reportingcycle - GROUP BY "organizationid" - ) - ` - : '' - } - - WITH DATA; - `); - - log.info(`${tableName}: Created count materialized view`); } // Extracts data from ordspub services @@ -991,9 +1043,118 @@ async function transform(tableName, columns, data) { return pgp.helpers.insert(rows, insertColumns, tableName); } +// TODO: Make this a bit more configurable. +// Create the documents_text_search table and triggers. +async function setupTextSearch(client) { + try { + await client.query('DROP TABLE IF EXISTS documents_text_search'); + + // Create the table. + await client.query(` + CREATE TABLE documents_text_search ( + objectid SERIAL PRIMARY KEY, + documentid INTEGER, + documenttsv TSVECTOR + ) + `); + + // Create the index on the vector column. + await client.query(` + CREATE INDEX IF NOT EXISTS documentstextsearch_documenttsv + ON documents_text_search USING gin (documenttsv) + TABLESPACE pg_default + `); + + // Create the trigger function. + await client.query(` + CREATE OR REPLACE FUNCTION process_document (new_row RECORD, chunk_size integer) + RETURNS VOID + AS $$ + DECLARE + tsv tsvector; + tsv_list tsvector[] := '{}'; + chunk text; + start_idx integer := 1; + end_idx integer; + text_length integer := octet_length(new_row.documenttext); + BEGIN + -- Process the document in chunks + WHILE start_idx <= text_length LOOP + end_idx := LEAST (start_idx + chunk_size - 1, text_length); + chunk := substr(new_row.documenttext, start_idx, end_idx - start_idx + 1); + tsv := to_tsvector('pg_catalog.english', coalesce(chunk, '')); + tsv_list := array_append(tsv_list, tsv); + start_idx := end_idx + 1; + END LOOP; + -- Add document description search vector + tsv := to_tsvector('pg_catalog.english', coalesce(new_row.documentdesc, '')); + tsv_list := array_append(tsv_list, tsv); + BEGIN + FOR i IN array_lower(tsv_list, 1)..array_upper(tsv_list, 1) + LOOP + INSERT INTO documents_text_search (documentid, documenttsv) + VALUES (new_row.objectid, tsv_list[i]); + END LOOP; + END; + RETURN; + END + $$ + LANGUAGE plpgsql; + `); + + await client.query(` + CREATE OR REPLACE FUNCTION documentstext_trigger_fn () + RETURNS TRIGGER + AS $$ + DECLARE + -- Initial chunk size of 1 MB. + chunk_size integer := 1024 * 1024; + BEGIN + LOOP + BEGIN + -- Call the external function to process the document + PERFORM + process_document (NEW, chunk_size); + -- Exit on success + RETURN NEW; + EXCEPTION + WHEN OTHERS THEN + -- Reduce the chunk size and retry + chunk_size := chunk_size / 2; + -- If chunk size is too small, raise an error + IF chunk_size < 1024 THEN + RAISE EXCEPTION 'Chunk size too small, unable to process document: %', NEW.objectid; + END IF; + END; + END LOOP; + END + $$ + LANGUAGE plpgsql; + `); + + // Create the trigger. + await client.query(` + CREATE TRIGGER documentstext_trigger + AFTER INSERT ON documents_text + FOR EACH ROW + EXECUTE FUNCTION documentstext_trigger_fn (); + `); + } catch (err) { + log.warn('Failed to create documents_text_search table'); + throw err; + } +} + // Get the ETL task for a particular profile function getProfileEtl( - { createQuery, columns, maxChunksOverride, overrideWorkMemory, tableName }, + { + createQuery, + columns, + id, + maxChunksOverride, + overrideWorkMemory, + tableName, + }, s3Config, s3Julian, ) { @@ -1002,7 +1163,7 @@ function getProfileEtl( try { await client.query(`SET search_path TO ${schemaName}`); - await client.query(`DROP TABLE IF EXISTS ${tableName}`); + await client.query(`DROP TABLE IF EXISTS ${tableName} CASCADE`); await client.query(createQuery); log.info(`Table ${tableName} created`); @@ -1011,22 +1172,39 @@ function getProfileEtl( throw err; } + // Create the search vector table and triggers for `documents_text`. + if (tableName === 'documents_text') { + await setupTextSearch(client); + } + // Extract, transform, and load the new data try { if (isLocal) { - const profileName = `profile_${tableName}`; - let res = await extract(profileName, s3Config); - let chunksProcessed = 0; - const maxChunks = maxChunksOverride ?? process.env.MAX_CHUNKS; - while ( - res.data !== null && - (!maxChunks || chunksProcessed < maxChunks) - ) { - const query = await transform(tableName, columns, res.data); - await client.query(query); - log.info(`Next record offset for table ${tableName}: ${res.next}`); - res = await extract(profileName, s3Config, res.next); - chunksProcessed += 1; + // TODO: Remove this once we have a better way to load data in local environment. + if (['actionDocuments', 'documentsText'].includes(id)) { + const fileLocation = process.env[`LOCAL_${tableName.toUpperCase()}`]; + if (!fileLocation) { + log.warn(`No local data found for ${tableName}`); + return; + } + await client.query(` + COPY ${tableName} FROM '${fileLocation}' DELIMITER ',' CSV HEADER; + `); + } else { + const profileName = `profile_${tableName}`; + let res = await extract(profileName, s3Config); + let chunksProcessed = 0; + const maxChunks = maxChunksOverride ?? process.env.MAX_CHUNKS; + while ( + res.data !== null && + (!maxChunks || chunksProcessed < maxChunks) + ) { + const query = await transform(tableName, columns, res.data); + await client.query(query); + log.info(`Next record offset for table ${tableName}: ${res.next}`); + res = await extract(profileName, s3Config, res.next); + chunksProcessed += 1; + } } } else { await client.query( @@ -1034,7 +1212,7 @@ function getProfileEtl( SELECT aws_s3.table_import_from_s3( table_name := $1, column_list := '', - options := '(format csv, header true)', + options := '(format csv, header true, encoding utf8)', s3_info := aws_commons.create_s3_uri( bucket := $2, file_path := $3,