Skip to content

Commit

Permalink
feature/FSR-159 (#43)
Browse files Browse the repository at this point in the history 
* FSR-1159 | refactor. sonar issue

* FSR-1159 | sonar issues

* FSR-1159 | sonar issue

* FSR-1159 | refactor function. sonar issues
  • Loading branch information
@ashleyjtaylor
ashleyjtaylor authored Mar 25, 2024
1 parent a1b3688 commit 01184f8
Showing 1 changed file with 80 additions and 54 deletions.
134 changes: 80 additions & 54 deletions lib/models/rloi.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ const directly = require('directly')
// telemetry parameters to import (Water level is if rloi station exists)
const parameters = ['Rainfall']

const processedValueDecimalPlace = 3

function getValuesCount (stations) {
return stations.reduce((count, station) => count + (station.SetofValues || []).length, 0)
}
Expand All @@ -21,9 +23,77 @@ function removePostfix (name) {
}
}

async function fetchStation (s3, bucket, key) {
return await s3.getObject({
Bucket: bucket,
Key: key
})
}

function setSlsTelemetryStation (item) {
// Convert ngr (national grid reference eg SK078993) to bng (British national grid eg 407800 399300)
const point = item.$.ngr && ngrToBng(item.$.ngr)

return [
item.$.stationReference,
item.$.region,
removePostfix(item.$.stationName),
item.$.ngr,
point?.easting,
point?.northing
]
}

function setSlsTelemetryValueItem (index, res, station, setOfValues) {
const value = {
telemetry_value_parent_id: res.rows[0].telemetry_value_parent_id,
value: parseFloat(setOfValues.Value[index]._),
processed_value: parseFloat(setOfValues.Value[index]._),
value_timestamp: (new Date(`${setOfValues.Value[index].$.date}T${setOfValues.Value[index].$.time}Z`)).toJSON(),
error: false
}

// Process values if they're Water Level
if (setOfValues.$.parameter === 'Water Level') {
// Subtract value if post process required
if (station.Post_Process.toLowerCase() === 'y' || station.Post_Process.toLowerCase() === 'yes') {
value.processed_value = station.Subtract ? parseFloat(util.toFixed(value.value - parseFloat(station.Subtract), processedValueDecimalPlace)) : value.value
}
if (!util.isNumeric(value.processed_value)) {
value.processed_value = null
value.error = true
}
}

return value
}

function setSlsTelemetryParent (key, station, item, setOfValues) {
return [
key,
new Date(),
parseInt(station.RLOI_ID),
item.$.stationReference,
station.Region,
new Date(`${setOfValues.$.startDate}T${setOfValues.$.startTime}Z`),
new Date(`${setOfValues.$.endDate}T${setOfValues.$.endTime}Z`),
setOfValues.$.parameter ? setOfValues.$.parameter : '',
setOfValues.$.qualifier ? setOfValues.$.qualifier : '',
setOfValues.$.units ? setOfValues.$.units : '',
(station.Post_Process.toLowerCase() === 'y' || station.Post_Process.toLowerCase() === 'yes'),
parseFloat(station.Subtract),
parseFloat(station.POR_Max_Value),
station.Station_Type,
parseFloat(station.percentile_5),
setOfValues.$.dataType ? setOfValues.$.dataType : '',
setOfValues.$.period ? setOfValues.$.period : ''
]
}

module.exports = {
async save (value, bucket, key, client, s3) {
let processed = 0
const concurrence = 3

const valuesCount = getValuesCount(value.EATimeSeriesDataExchangeFormat.Station)

Expand All @@ -32,21 +102,20 @@ module.exports = {
if (valuesCount === 0) {
return
}

for (const item of value.EATimeSeriesDataExchangeFormat.Station) {
if (item.SetofValues) {
// Update region to match station region as telemetry file region slightly differs,
// so keep consistent with station data
item.$.telemetryRegion = item.$.region
item.$.region = regions[item.$.region] ? regions[item.$.region] : item.$.region

await directly(3, item.SetofValues.map(setOfValues => async () => {
await directly(concurrence, item.SetofValues.map(setOfValues => async () => {
let station

try {
station = await s3.getObject({
Bucket: bucket,
Key: `rloi/${item.$.region}/${item.$.stationReference}/station.json`
})
station = JSON.parse(station.Body)
const result = await fetchStation(s3, bucket, `rloi/${item.$.region}/${item.$.stationReference}/station.json`)
station = JSON.parse(result.Body)
} catch (err) {
// the console log is commented out so as not to spam the cloudwatch lambda
// logging, as the s3.getObject throws an error when it can't find the object, and there
Expand All @@ -66,64 +135,21 @@ module.exports = {
Post_Process: 'n',
Station_Type: 'R'
}
// Convert ngr (national grid reference eg SK078993) to bng (British national grid eg 407800 399300)
const point = item.$.ngr && ngrToBng(item.$.ngr)

const telemetryStation = [
item.$.stationReference,
item.$.region,
removePostfix(item.$.stationName),
item.$.ngr,
point && point.easting,
point && point.northing
]

const telemetryStation = setSlsTelemetryStation(item)

await client.query('slsTelemetryStation', telemetryStation)
}

// Store parent details in sls_telemetry_value_parent
const parent = [
key,
new Date(),
parseInt(station.RLOI_ID),
item.$.stationReference,
station.Region,
new Date(`${setOfValues.$.startDate}T${setOfValues.$.startTime}Z`),
new Date(`${setOfValues.$.endDate}T${setOfValues.$.endTime}Z`),
setOfValues.$.parameter ? setOfValues.$.parameter : '',
setOfValues.$.qualifier ? setOfValues.$.qualifier : '',
setOfValues.$.units ? setOfValues.$.units : '',
(station.Post_Process.toLowerCase() === 'y' || station.Post_Process.toLowerCase() === 'yes'),
parseFloat(station.Subtract),
parseFloat(station.POR_Max_Value),
station.Station_Type,
parseFloat(station.percentile_5),
setOfValues.$.dataType ? setOfValues.$.dataType : '',
setOfValues.$.period ? setOfValues.$.period : ''
]
const parent = setSlsTelemetryParent(key, station, item, setOfValues)

const res = await client.query('slsTelemetryValueParent', parent)
const values = []
// console.log(`Loaded parent: ${station.RLOI_ID} | ${setOfValues.$.parameter} | ${setOfValues.$.qualifier}`)

for (let i = 0; i < setOfValues.Value.length; i++) {
values[i] = {
telemetry_value_parent_id: res.rows[0].telemetry_value_parent_id,
value: parseFloat(setOfValues.Value[i]._),
processed_value: parseFloat(setOfValues.Value[i]._),
value_timestamp: (new Date(`${setOfValues.Value[i].$.date}T${setOfValues.Value[i].$.time}Z`)).toJSON(),
error: false
}
// Process values if they're Water Level
if (setOfValues.$.parameter === 'Water Level') {
// Subtract value if post process required
if (station.Post_Process.toLowerCase() === 'y' || station.Post_Process.toLowerCase() === 'yes') {
values[i].processed_value = station.Subtract ? parseFloat(util.toFixed(values[i].value - parseFloat(station.Subtract), 3)) : values[i].value
}
if (!util.isNumeric(values[i].processed_value)) {
values[i].processed_value = null
values[i].error = true
}
}
values[i] = setSlsTelemetryValueItem(i, res, station, setOfValues)
}

// Note: this previously passed a single parameter as a query built using sql-node
Expand Down

0 comments on commit 01184f8

Please sign in to comment.