uploading is functional

README.md

@@ -18,12 +18,9 @@ All data in the data lake has the following metadata:
 
 ## Commands
 ### Uploading
-#### Upload a single file
+#### Upload a file
 `datateer upload orders_feed ./my_exported_data/orders.csv` will upload the file at `./my_exported_data/orders.csv` using the feed key `orders_feed`
 
-#### Upload a directory of multiple files
-`datateer upload orders_feed ./my_exported_data/orders/` will upload the files found in `./myexported_data/orders/` using the feed key `orders_feed`
-
 ### Configuring
 #### Configure the upload agent
 `datateer config upload-agent` will ask you a series of questions to configure your agent

datateer/upload_agent/config.py

@@ -31,13 +31,16 @@ def save_feed(key, feed, path: str=DEFAULT_PATH):
     config = load_config(path)
     if 'upload-agent' not in config:
         raise click.ClickException('Could not find configuration. Run "datateer config upload-agent" before configuring any feeds"')
-    if 'feeds' not in config:
+    if 'feeds' not in config['upload-agent']:
         config['upload-agent']['feeds'] = {}
     config['upload-agent']['feeds'][key] = feed
 
     save_config(config)
 
 def get_feed(key, path: str=DEFAULT_PATH):
     config = load_config(path)
-    return config.get('upload-agent', {}).get('feeds', {}).get(key)
+    feed = config.get('upload-agent', {}).get('feeds', {}).get(key)
+    if not feed:
+        raise click.ClickException(f'Feed with key {key} does not exist')
+    return feed
 

datateer/upload_agent/constants.py

@@ -16,3 +16,4 @@
     },
 }
 
+KEY_TEMPLATE = '{provider}/{source}/{feed}/export_date={export_date}/{file}'

datateer/upload_agent/main.py

@@ -4,6 +4,8 @@
 
 # import datateer.upload_agent.config as config
 from .config import load_config, get_feed, save_config, save_feed, DEFAULT_PATH as default_config_path
+from .upload import upload as upload_file
+
 config = load_config()
 
 @click.group()
@@ -111,6 +113,6 @@ def feed(feed, feed_key, provider, source):
 @click.argument('feed-key')
 @click.argument('path')
 def upload(feed_key, path):
-    pass
+    upload_file(feed_key, path)
 
 

datateer/upload_agent/upload.py

@@ -0,0 +1,39 @@
+from datetime import date
+from pathlib import Path
+
+from botocore.exceptions import ClientError
+import boto3
+import click
+
+from .config import get_feed, load_config
+from .constants import KEY_TEMPLATE
+
+def format_s3_key(feed_key, file_name):
+    feed = get_feed(feed_key)
+    return KEY_TEMPLATE.format(
+        provider=feed['provider'],
+        source=feed['source'],
+        feed=feed['feed'],
+        export_date=date.today().strftime('%Y-%d-%m'),
+        file=file_name
+    )
+
+def get_resource(config):
+    return boto3.resource(
+        's3',
+        aws_access_key_id=config['upload-agent']['access-key'],
+        aws_secret_access_key=config['upload-agent']['access-secret']
+    )
+
+def upload(feed_key: str, path: str) -> None:
+    path = Path(path)
+    key = format_s3_key(feed_key, path.name)
+
+    config = load_config()    
+    s3 = get_resource(config)
+
+    bucket = config['upload-agent']['raw-bucket']
+
+    click.echo(f'Uploading {str(path.absolute())} to s3://{bucket}/{key}')
+    s3.Bucket(bucket).upload_file(str(path.absolute()), key)
+

test.txt

@@ -0,0 +1 @@
+this is a test file

test/test_cli.py

@@ -41,18 +41,10 @@ def test_command_config_feed_show_option_errors_if_not_exist(mock_load_config, c
 
 def test_command_upload_handles_feed_key_and_path_arguments(runner):
     result = runner.invoke(cli, ['upload', 'FEED-KEY', 'PATH'])
+
     assert result.exit_code == 0
 
 
-def test_command_upload_handles_individual_file_argument(runner):
-    result = runner.invoke(cli, ['upload', 'FEED-KEY', 'PATH.csv'])
-    assert result.exit_code == 0
-
-
-def test_command_upload_handles_directory_argument(runner):
-    result = runner.invoke(cli, ['upload', 'FEED-KEY', 'PATH/'])
-    assert result.exit_code == 0
-
 
 
 @patch.dict('datateer.upload_agent.main.config', constants.SAMPLE_CONFIG, clear=True)