From 65a2f50cdfb5d5369af2be0f17a9d713e25d98c5 Mon Sep 17 00:00:00 2001 From: Alexa Huerta Date: Fri, 26 Oct 2018 19:22:36 -0700 Subject: [PATCH 1/5] automation of downloading data from AVRIS-NG --- RetrieveAVRISNGData.py | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 RetrieveAVRISNGData.py diff --git a/RetrieveAVRISNGData.py b/RetrieveAVRISNGData.py new file mode 100644 index 0000000..6fe4627 --- /dev/null +++ b/RetrieveAVRISNGData.py @@ -0,0 +1,40 @@ +from ftplib import FTP +import os, sys, os.path + +def handleDownload(block): + file.write(block) + print ".", + +ddir='/Users' # set to local path where you want files downloaded to +os.chdir(ddir) +ftp = FTP('avng.jpl.nasa.gov') + +print 'Logging in.' +ftp.login() +directory = '/AVNG_2015_data_distribution/L2/' +print 'Changing to ' + directory +ftp.cwd(directory) + +dirnames = ftp.nlst() # get list of directories +print dirnames + +#for each directory, download the directory and its contents to local computer +for dirname in dirnames: + print 'Changing to ' + dirname + ftp.cwd(dirname) + filenames = ftp.nlst() # get list of filenames in directory + print filenames + count = 0 + localpath = ddir + '/' + dirname + os.mkdir(localpath) # create this directory locally + #download each file to the newly created local directory + for filename in filenames: + local_filename = os.path.join(localpath, filename) + file = open(local_filename, 'wb') + ftp.retrbinary('RETR '+ filename, file.write) + file.close() + + ftp.cwd('..') + +ftp.quit() + From c639e738f1dfc52431bcf6531e60d0521a93c45b Mon Sep 17 00:00:00 2001 From: Alexa Huerta Date: Thu, 1 Nov 2018 15:34:53 -0700 Subject: [PATCH 2/5] responding to PR comments --- .../main/resources/bin/RetrieveAVRISNGData.py | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 crawler/src/main/resources/bin/RetrieveAVRISNGData.py diff --git a/crawler/src/main/resources/bin/RetrieveAVRISNGData.py b/crawler/src/main/resources/bin/RetrieveAVRISNGData.py new file mode 100644 index 0000000..bb061b0 --- /dev/null +++ b/crawler/src/main/resources/bin/RetrieveAVRISNGData.py @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ftplib import FTP +import os, sys, os.path + +def handleDownload(block): + file.write(block) + print ".", + +ddir='/Users/alexahuerta/Documents' # set to local path where you want files downloaded to +os.chdir(ddir) +ftp = FTP('avng.jpl.nasa.gov') + +print 'Logging in.' +ftp.login() +directory = '/' + +flight_line = raw_input("Enter name of flight line you wish to download: ") +data2014 = '/AVNG_2014_data_distribution/L2/' +data2015 = '/AVNG_2015_data_distribution/L2/' + +ftp.cwd(data2015) +dirnames2015 = ftp.nlst() +for dirname in dirnames2015: + if dirname == flight_line: + print 'Changing to ' + dirname + ftp.cwd(dirname) + filenames = ftp.nlst() # get list of filenames in directory + print filenames + localpath = ddir + '/' + dirname + print localpath + os.mkdir(localpath) # create this directory locally + #download each file to the newly created local directory + for filename in filenames: + local_filename = os.path.join(localpath, filename) + file = open(local_filename, 'wb') + ftp.retrbinary('RETR '+ filename, file.write) + file.close() + ftp.cwd('../..') + +ftp.cwd(data2014) +dirnames2014 = ftp.nlst() +for dirname in dirnames2014: + if dirname == flight_line: + print 'Changing to ' + dirname + ftp.cwd(dirname) + filenames = ftp.nlst() # get list of filenames in directory + print filenames + localpath = ddir + '/' + dirname + print localpath + os.mkdir(localpath) # create this directory locally + #download each file to the newly created local directory + for filename in filenames: + local_filename = os.path.join(localpath, filename) + file = open(local_filename, 'wb') + ftp.retrbinary('RETR '+ filename, file.write) + file.close() + ftp.cwd('../..') + +ftp.quit() From 1d01c5f9cf1f7aa4c74fc82ba61649f7bb1a9724 Mon Sep 17 00:00:00 2001 From: Alex Yi Date: Thu, 1 Nov 2018 19:12:25 -0700 Subject: [PATCH 3/5] configured for Python3. Added multiple ways to add filename. Added print to notify user if flight name was incorrect. --- .../main/resources/bin/RetrieveAVRISNGData.py | 31 +++++++++++++------ 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/crawler/src/main/resources/bin/RetrieveAVRISNGData.py b/crawler/src/main/resources/bin/RetrieveAVRISNGData.py index bb061b0..3cd4491 100644 --- a/crawler/src/main/resources/bin/RetrieveAVRISNGData.py +++ b/crawler/src/main/resources/bin/RetrieveAVRISNGData.py @@ -18,30 +18,38 @@ def handleDownload(block): file.write(block) - print ".", + print (".") -ddir='/Users/alexahuerta/Documents' # set to local path where you want files downloaded to + +ddir='/Users/Alex/Documents' # set to local path where you want files downloaded to os.chdir(ddir) ftp = FTP('avng.jpl.nasa.gov') -print 'Logging in.' +print ('Logging in.') ftp.login() directory = '/' -flight_line = raw_input("Enter name of flight line you wish to download: ") +#if flight name is not included in the command, prompt it +if len(sys.argv) < 2: + flight_line = input("Enter name of flight line you wish to download: ") +else: + flight_line = sys.argv[1] + data2014 = '/AVNG_2014_data_distribution/L2/' data2015 = '/AVNG_2015_data_distribution/L2/' +exists = False ftp.cwd(data2015) dirnames2015 = ftp.nlst() for dirname in dirnames2015: if dirname == flight_line: - print 'Changing to ' + dirname + exists = True + print ('Changing to ' + dirname) ftp.cwd(dirname) filenames = ftp.nlst() # get list of filenames in directory - print filenames + print (filenames) localpath = ddir + '/' + dirname - print localpath + print (localpath) os.mkdir(localpath) # create this directory locally #download each file to the newly created local directory for filename in filenames: @@ -55,12 +63,13 @@ def handleDownload(block): dirnames2014 = ftp.nlst() for dirname in dirnames2014: if dirname == flight_line: - print 'Changing to ' + dirname + exists = True + print ('Changing to ' + dirname) ftp.cwd(dirname) filenames = ftp.nlst() # get list of filenames in directory - print filenames + print (filenames) localpath = ddir + '/' + dirname - print localpath + print (localpath) os.mkdir(localpath) # create this directory locally #download each file to the newly created local directory for filename in filenames: @@ -70,4 +79,6 @@ def handleDownload(block): file.close() ftp.cwd('../..') +if exists == False: + print('Could not locate file: ' + flight_line) ftp.quit() From 083e3f5108b6577ef62879424a47a6182e47b858 Mon Sep 17 00:00:00 2001 From: Alexa Huerta Date: Fri, 2 Nov 2018 12:35:39 -0700 Subject: [PATCH 4/5] Delete RetrieveAVRISNGData.py removing unnecessary file --- RetrieveAVRISNGData.py | 40 ---------------------------------------- 1 file changed, 40 deletions(-) delete mode 100644 RetrieveAVRISNGData.py diff --git a/RetrieveAVRISNGData.py b/RetrieveAVRISNGData.py deleted file mode 100644 index 6fe4627..0000000 --- a/RetrieveAVRISNGData.py +++ /dev/null @@ -1,40 +0,0 @@ -from ftplib import FTP -import os, sys, os.path - -def handleDownload(block): - file.write(block) - print ".", - -ddir='/Users' # set to local path where you want files downloaded to -os.chdir(ddir) -ftp = FTP('avng.jpl.nasa.gov') - -print 'Logging in.' -ftp.login() -directory = '/AVNG_2015_data_distribution/L2/' -print 'Changing to ' + directory -ftp.cwd(directory) - -dirnames = ftp.nlst() # get list of directories -print dirnames - -#for each directory, download the directory and its contents to local computer -for dirname in dirnames: - print 'Changing to ' + dirname - ftp.cwd(dirname) - filenames = ftp.nlst() # get list of filenames in directory - print filenames - count = 0 - localpath = ddir + '/' + dirname - os.mkdir(localpath) # create this directory locally - #download each file to the newly created local directory - for filename in filenames: - local_filename = os.path.join(localpath, filename) - file = open(local_filename, 'wb') - ftp.retrbinary('RETR '+ filename, file.write) - file.close() - - ftp.cwd('..') - -ftp.quit() - From 837b0b3c0052091a5abb23da626364fa6cbdf446 Mon Sep 17 00:00:00 2001 From: Alexa Huerta Date: Fri, 2 Nov 2018 12:42:29 -0700 Subject: [PATCH 5/5] respond to PR comments --- crawler/src/main/resources/bin/RetrieveAVRISNGData.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crawler/src/main/resources/bin/RetrieveAVRISNGData.py b/crawler/src/main/resources/bin/RetrieveAVRISNGData.py index 3cd4491..392f25e 100644 --- a/crawler/src/main/resources/bin/RetrieveAVRISNGData.py +++ b/crawler/src/main/resources/bin/RetrieveAVRISNGData.py @@ -20,8 +20,7 @@ def handleDownload(block): file.write(block) print (".") - -ddir='/Users/Alex/Documents' # set to local path where you want files downloaded to +ddir='/usr/local/coal-sds-deploy/data/staging' # set to local path where you want files downloaded to os.chdir(ddir) ftp = FTP('avng.jpl.nasa.gov')