-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathtransform_save_data.py
155 lines (122 loc) · 5.11 KB
/
transform_save_data.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
# -*- coding: cp1252 -*-
import pickle
import os
import metrics
import numpy as np
import parameters
import calibrate
from pcraster.framework import *
#### Script to read in the metrics saved as the result of the LU_urb.py script.
#### Metrics are transformed into an array
# Work directory:
work_dir = parameters.getWorkDir()
# Get metrics
metricNames = parameters.getSumStats()
# Get the number of parameter iterations and number of time step defined in the parameter.py script
nrOfTimesteps=parameters.getNrTimesteps()
numberOfIterations = parameters.getNumberofIterations()
iterations = range(1, numberOfIterations+1, 1)
timeSteps=range(1,nrOfTimesteps+1,1)
# Get the observed time steps. Time steps relate to the year of the CLC data, where 1990 was time step 0.
obsSampleNumbers = [1] #range(1,20+1,1) <- for stochastic model
obsTimeSteps = parameters.getObsTimesteps()
# Read the reference files
refArray = parameters.getColFiles()
# Path to the folder with the metrics stored
country = parameters.getCountryName()
resultFolder = os.path.join(work_dir,'results',country)
output_mainfolder = os.path.join(resultFolder, "metrics")
#################
### FUNCTIONS ###
#################
def openPickledSamplesAndTimestepsAsNumpyArray(basename,iterations,timesteps, \
obs=False):
output=[]
for timestep in timesteps:
allIterations=[]
for i in iterations: # Loop parameters
# Read in the parameters
pName = 'parameters_iteration_' + str(i) + '.obj'
pFileName = os.path.join(resultFolder, str(i), pName)
filehandler = open(pFileName, 'rb')
pData = pickle.load(filehandler)
pArray = np.array(pData, ndmin=1)
# If we are working with the observed data (CLC data):
if obs:
name = generateNameT(basename, timestep)
fileName = os.path.join('observations', country, 'realizations', str(i), name)
data = metrics.map2Array(fileName, os.path.join('input_data', country, refArray[basename]))
# If we are working with the modelled values:
else:
theName = basename + str(timestep) + '.obj'
fileName = os.path.join(resultFolder, str(i), theName)
filehandler = open(fileName, 'rb')
data = pickle.load(filehandler)
'''# Keep these lines for the later use:
if it is a dictionary, get the sugar cane parameters (lu type 6)
if type(data) == dict:
data = data.get(1)
filehandler.close()'''
# if the loaded data was not yet an array, make it into one
# minimum number of dimensions is one, to prevent a zero-dimension array
# (not an array at all)
array = np.array(data, ndmin=1)
# add an extra dimension that would normally be y, if the data was a map
# so that Derek's plot functions can be used
array = array.reshape(len(array),1)
allIterations.append([pArray,array])
output.append(allIterations)
outputAsArray = np.array(output)
return outputAsArray
def setNameClearSave(basename, output, obs=False):
# Check if the directory exists. If not, create.
if not os.path.isdir(output_mainfolder):
os.mkdir(output_mainfolder)
# Set the name of the file
if obs:
fileName = os.path.join(output_mainfolder, basename + '_obs')
else:
fileName = os.path.join(output_mainfolder, basename)
# Clear the directory if needed
if os.path.exists(fileName + '.npy'):
os.remove(fileName + '.npy')
# Save the data
np.save(fileName, output)
#################################
### SAVE OUTPUTS OF THE MODEL ###
#################################
########### Save the OBSERVED metrics and urban areas
# Metrics:
for aVariable in metricNames:
output_obs = openPickledSamplesAndTimestepsAsNumpyArray(aVariable, obsSampleNumbers,obsTimeSteps, True)
setNameClearSave(aVariable, output_obs,obs=True)
# Urban areas
output_urb_obs = openPickledSamplesAndTimestepsAsNumpyArray('urb', obsSampleNumbers,obsTimeSteps, True)
setNameClearSave('urb', output_urb_obs,obs=True)
########### Save the MODELLED metrics and urban areas
# Metrics:
for aVariable in metricNames:
output_mod = openPickledSamplesAndTimestepsAsNumpyArray(aVariable, iterations, timeSteps, False)
setNameClearSave(aVariable, output_mod,obs=False)
# Modellled urban areas only for the observation years:
for a_step in obsTimeSteps:
subset_urb_mod = openPickledSamplesAndTimestepsAsNumpyArray('urb', iterations, [a_step], False)
setNameClearSave('urb_subset_'+str(a_step), subset_urb_mod,obs=False)
# Parameter sets
parameter_sets = subset_urb_mod[0,:,0]
setNameClearSave('parameter_sets', parameter_sets, obs=False)
print("Modelled and observed metrics and urban areas saved as npy files")
'''
########### Delete all number folders
files = os.listdir(resultFolder)
for f in files:
if f not in ['metrics']:
shutil.rmtree(os.path.join(resultFolder, f))
print("All number folders deleted.")
'''
##################################
### CALCULATE KAPPA STATISTICS ###
##################################
########### Calculate Kappa statistics
calibrate.calculateKappa()
calibrate.calculateKappaSimulation()