Skip to content

Commit 7727114

Browse files
jeandetjeandet
committed
Few minor fixes plus CDA masters analysis notebook
Signed-off-by: Alexis Jeandet <[email protected]>
1 parent c2bc8d5 commit 7727114

File tree

3 files changed

+357
-13
lines changed

3 files changed

+357
-13
lines changed

‎notebooks/CDAWeb_masters.ipynb

+338
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,338 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "code",
5+
"execution_count": 7,
6+
"metadata": {
7+
"collapsed": true,
8+
"pycharm": {
9+
"name": "#%%\n"
10+
}
11+
},
12+
"outputs": [],
13+
"source": [
14+
"import os\n",
15+
"\n",
16+
"os.environ[\"PYISTP_CDFLIB\"] = 'pycdfpp'\n",
17+
"#os.environ[\"PYISTP_CDFLIB\"]='spacepy'\n",
18+
"import pyistp\n",
19+
"from pyistp import _impl as pyistp_impl\n",
20+
"import requests\n",
21+
"from glob import glob\n",
22+
"import zstd\n",
23+
"import tempfile\n",
24+
"import tarfile\n",
25+
"import io\n",
26+
"import logging\n",
27+
"from tqdm import tqdm\n",
28+
"\n",
29+
"def clean_logs():\n",
30+
" lines=open('output/result.log').readlines()\n",
31+
" index = 1\n",
32+
" while index < len(lines):\n",
33+
" if lines[index].startswith('Loading') and lines[index-1].startswith('Loading'):\n",
34+
" lines.pop(index-1)\n",
35+
" else:\n",
36+
" index+=1\n",
37+
" open('output/result_clean.log','w').writelines(lines)\n",
38+
"\n",
39+
"def compare_master_and_sample(master, sample_url_part, variable_name):\n",
40+
" pyistp_impl.log.handlers.clear()\n",
41+
" pyistp_impl.log.addHandler(logging.StreamHandler())\n",
42+
" master = pyistp.load(buffer=requests.get(f\"https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/{master}\").content)\n",
43+
" sample_file = pyistp.load(buffer=requests.get(f\"https://cdaweb.gsfc.nasa.gov/pub/data/{sample_url_part}\").content)\n",
44+
" print(\"master:\", flush=True)\n",
45+
" v=master.data_variable(variable_name)\n",
46+
" print(\"sample_file:\", flush=True)\n",
47+
" v=sample_file.data_variable(variable_name)"
48+
]
49+
},
50+
{
51+
"cell_type": "markdown",
52+
"source": [
53+
"# Prepare output folder"
54+
],
55+
"metadata": {
56+
"collapsed": false,
57+
"pycharm": {
58+
"name": "#%% md\n"
59+
}
60+
}
61+
},
62+
{
63+
"cell_type": "code",
64+
"execution_count": 2,
65+
"outputs": [],
66+
"source": [
67+
"if not os.path.exists('output'):\n",
68+
" os.mkdir('output')\n",
69+
"\n",
70+
"if os.path.exists('output/result.log'):\n",
71+
" os.remove('output/result.log')\n",
72+
"\n",
73+
"pyistp_impl.log.handlers.clear()\n",
74+
"pyistp_impl.log.addHandler(logging.FileHandler(\"output/result.log\"))\n",
75+
"pyistp_impl.log.setLevel(logging.DEBUG)"
76+
],
77+
"metadata": {
78+
"collapsed": false,
79+
"pycharm": {
80+
"name": "#%%\n"
81+
}
82+
}
83+
},
84+
{
85+
"cell_type": "markdown",
86+
"source": [
87+
"# Download and a load all masters as ISTP files"
88+
],
89+
"metadata": {
90+
"collapsed": false,
91+
"pycharm": {
92+
"name": "#%% md\n"
93+
}
94+
}
95+
},
96+
{
97+
"cell_type": "code",
98+
"execution_count": 3,
99+
"outputs": [
100+
{
101+
"name": "stderr",
102+
"output_type": "stream",
103+
"text": [
104+
"100%|██████████| 3324/3324 [00:21<00:00, 152.98it/s]\n"
105+
]
106+
}
107+
],
108+
"source": [
109+
"\n",
110+
"with tempfile.TemporaryDirectory() as tmp_dir:\n",
111+
" tar = tarfile.open(fileobj=io.BytesIO(\n",
112+
" zstd.decompress(requests.get('https://hephaistos.lpp.polytechnique.fr/data/jeandet/master.tar.zst').content)))\n",
113+
" tar.extractall(tmp_dir)\n",
114+
" del tar\n",
115+
" cdf_masters = glob(f\"{tmp_dir}/*.cdf\")\n",
116+
" for master in tqdm(cdf_masters):\n",
117+
" istp_file = pyistp.load(master)\n",
118+
" for vname in istp_file.data_variables():\n",
119+
" v = istp_file.data_variable(vname)\n",
120+
"\n",
121+
"clean_logs()"
122+
],
123+
"metadata": {
124+
"collapsed": false,
125+
"pycharm": {
126+
"name": "#%%\n"
127+
}
128+
}
129+
},
130+
{
131+
"cell_type": "markdown",
132+
"source": [
133+
"# let's manually check few files"
134+
],
135+
"metadata": {
136+
"collapsed": false,
137+
"pycharm": {
138+
"name": "#%% md\n"
139+
}
140+
}
141+
},
142+
{
143+
"cell_type": "markdown",
144+
"source": [
145+
"## c3_cp_efw_l3_e3d_inert\n",
146+
"Both master and sample file have the same issue"
147+
],
148+
"metadata": {
149+
"collapsed": false,
150+
"pycharm": {
151+
"name": "#%% md\n"
152+
}
153+
}
154+
},
155+
{
156+
"cell_type": "code",
157+
"execution_count": 9,
158+
"outputs": [
159+
{
160+
"name": "stdout",
161+
"output_type": "stream",
162+
"text": [
163+
"master:\n"
164+
]
165+
},
166+
{
167+
"name": "stderr",
168+
"output_type": "stream",
169+
"text": [
170+
"Non compliant ISTP file: variable delta_Ez_ISR2__C3_CP_EFW_L3_E3D_INERT has LABL_PTR_1 attribute which points to variable dEz which does not exist\n"
171+
]
172+
},
173+
{
174+
"name": "stdout",
175+
"output_type": "stream",
176+
"text": [
177+
"sample_file:\n"
178+
]
179+
},
180+
{
181+
"name": "stderr",
182+
"output_type": "stream",
183+
"text": [
184+
"Non compliant ISTP file: variable delta_Ez_ISR2__C3_CP_EFW_L3_E3D_INERT has LABL_PTR_1 attribute which points to variable dEz which does not exist\n"
185+
]
186+
}
187+
],
188+
"source": [
189+
"compare_master_and_sample(master=\"c3_cp_efw_l3_e3d_inert_00000000_v01.cdf\",\n",
190+
" sample_url_part=\"cluster/c3/efw/efield_3dvect_spinreso_l3_inertialframe/2003/c3_cp_efw_l3_e3d_inert_20030120_v20130803.cdf\",\n",
191+
" variable_name=\"delta_Ez_ISR2__C3_CP_EFW_L3_E3D_INERT\")"
192+
],
193+
"metadata": {
194+
"collapsed": false,
195+
"pycharm": {
196+
"name": "#%%\n"
197+
}
198+
}
199+
},
200+
{
201+
"cell_type": "markdown",
202+
"source": [
203+
"## mms4_fpi_fast_l2_dis-momsaux\n",
204+
"Both master and sample file have the same issue"
205+
],
206+
"metadata": {
207+
"collapsed": false,
208+
"pycharm": {
209+
"name": "#%% md\n"
210+
}
211+
}
212+
},
213+
{
214+
"cell_type": "code",
215+
"execution_count": 10,
216+
"outputs": [
217+
{
218+
"name": "stdout",
219+
"output_type": "stream",
220+
"text": [
221+
"master:\n"
222+
]
223+
},
224+
{
225+
"name": "stderr",
226+
"output_type": "stream",
227+
"text": [
228+
"Non compliant ISTP file: mms4_dis_compressionloss_fast was marked as data variable but it has 0 support variable\n"
229+
]
230+
},
231+
{
232+
"name": "stdout",
233+
"output_type": "stream",
234+
"text": [
235+
"sample_file:\n"
236+
]
237+
},
238+
{
239+
"name": "stderr",
240+
"output_type": "stream",
241+
"text": [
242+
"Non compliant ISTP file: mms4_dis_compressionloss_fast was marked as data variable but it has 0 support variable\n"
243+
]
244+
}
245+
],
246+
"source": [
247+
"compare_master_and_sample(master=\"mms4_fpi_fast_l2_dis-momsaux_00000000_v01.cdf\",\n",
248+
" sample_url_part=\"mms/mms4/fpi/fast/l2/dis-momsaux/2021/10/mms4_fpi_fast_l2_dis-momsaux_20211003140000_v3.4.0.cdf\",\n",
249+
" variable_name=\"mms4_dis_compressionloss_fast\")"
250+
],
251+
"metadata": {
252+
"collapsed": false,
253+
"pycharm": {
254+
"name": "#%%\n"
255+
}
256+
}
257+
},
258+
{
259+
"cell_type": "markdown",
260+
"source": [
261+
"## mvn_swi_l2_finearc3d\n",
262+
"Both master and sample file have the same issue"
263+
],
264+
"metadata": {
265+
"collapsed": false,
266+
"pycharm": {
267+
"name": "#%% md\n"
268+
}
269+
}
270+
},
271+
{
272+
"cell_type": "code",
273+
"execution_count": 11,
274+
"outputs": [
275+
{
276+
"name": "stdout",
277+
"output_type": "stream",
278+
"text": [
279+
"master:\n"
280+
]
281+
},
282+
{
283+
"name": "stderr",
284+
"output_type": "stream",
285+
"text": [
286+
"Non compliant ISTP file: eindex was marked as data variable but it has 0 support variable\n"
287+
]
288+
},
289+
{
290+
"name": "stdout",
291+
"output_type": "stream",
292+
"text": [
293+
"sample_file:\n"
294+
]
295+
},
296+
{
297+
"name": "stderr",
298+
"output_type": "stream",
299+
"text": [
300+
"Non compliant ISTP file: eindex was marked as data variable but it has 0 support variable\n"
301+
]
302+
}
303+
],
304+
"source": [
305+
"compare_master_and_sample(master=\"mvn_swi_l2_finearc3d_00000000_v01.cdf\",\n",
306+
" sample_url_part=\"maven/swia/l2/finearc3d/2020/11/mvn_swi_l2_finearc3d_20201125_v02_r00.cdf\",\n",
307+
" variable_name=\"eindex\")"
308+
],
309+
"metadata": {
310+
"collapsed": false,
311+
"pycharm": {
312+
"name": "#%%\n"
313+
}
314+
}
315+
}
316+
],
317+
"metadata": {
318+
"kernelspec": {
319+
"display_name": "Python 3",
320+
"language": "python",
321+
"name": "python3"
322+
},
323+
"language_info": {
324+
"codemirror_mode": {
325+
"name": "ipython",
326+
"version": 2
327+
},
328+
"file_extension": ".py",
329+
"mimetype": "text/x-python",
330+
"name": "python",
331+
"nbconvert_exporter": "python",
332+
"pygments_lexer": "ipython2",
333+
"version": "2.7.6"
334+
}
335+
},
336+
"nbformat": 4,
337+
"nbformat_minor": 0
338+
}

0 commit comments

Comments
 (0)