Skip to content

Commit 06e3662

Browse files
authored
Pilot 6700: update -s option in file upload command (#198)
* update source id into list * update payload to None if not presented * update file upload with starge source zone * update error message when invalid source file * fixup test cases * update test cases for validating source id * unrelease the constraint for source lineage operation. Allowing folder id to link ONLY one source item * unrelease the constraint for source lineage operation. Allowing folder id to link ONLY one source item * unrelease the constraint for source lineage operation. Allowing folder id to link ONLY one source item * add new check when duplicated source items are provided * update source_file help message
1 parent 3bf3199 commit 06e3662

File tree

9 files changed

+156
-25
lines changed

9 files changed

+156
-25
lines changed

app/commands/file.py

+19-4
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
#
33
# Contact Indoc Systems for any questions regarding the use of this source code.
44

5+
import ast
56
import json
67
import os
78
from sys import exit
@@ -83,6 +84,14 @@ def cli():
8384
default=None,
8485
required=False,
8586
help=file_help.file_help_page(file_help.FileHELP.FILE_UPLOAD_S),
87+
type=click.File('rb'),
88+
show_default=True,
89+
)
90+
@click.option(
91+
'--source-zone',
92+
default=AppConfig.Env.green_zone,
93+
required=False,
94+
help=file_help.file_help_page(file_help.FileHELP.FILE_Z),
8695
show_default=True,
8796
)
8897
@click.option(
@@ -120,6 +129,7 @@ def file_put(**kwargs): # noqa: C901
120129
tag_files = kwargs.get('tag')
121130
zone = kwargs.get('zone')
122131
source_file = kwargs.get('source_file')
132+
source_zone = kwargs.get('source_zone')
123133
zipping = kwargs.get('zip')
124134
attribute_file = kwargs.get('attribute')
125135
thread = kwargs.get('thread')
@@ -136,6 +146,10 @@ def file_put(**kwargs): # noqa: C901
136146
attribute = json.load(attribute_file) if attribute_file else None
137147
except Exception:
138148
SrvErrorHandler.customized_handle(ECustomizedError.INVALID_TEMPLATE, True)
149+
try:
150+
source_files = ast.literal_eval(source_file.read().decode('utf-8')) if source_file else None
151+
except Exception:
152+
SrvErrorHandler.customized_handle(ECustomizedError.INVALID_SOURCE_FILE, True)
139153

140154
# Check zone and upload-message
141155
zone = get_zone(zone) if zone else AppConfig.Env.green_zone.lower()
@@ -165,7 +179,8 @@ def file_put(**kwargs): # noqa: C901
165179
srv_manifest = SrvFileManifests()
166180
upload_val_event = {
167181
'zone': zone,
168-
'source': source_file,
182+
'source': source_files,
183+
'source_zone': source_zone,
169184
'project_code': project_code,
170185
'attribute': attribute,
171186
'tag': tag,
@@ -217,7 +232,7 @@ def file_put(**kwargs): # noqa: C901
217232
'attribute': attribute,
218233
}
219234
if source_file:
220-
upload_event['source_id'] = src_file_info.get('id', '')
235+
upload_event['source_id'] = src_file_info
221236

222237
item_ids = simple_upload(upload_event, num_of_thread=thread, output_path=output_path)
223238

@@ -284,12 +299,12 @@ def file_resume(**kwargs): # noqa: C901
284299

285300
def validate_upload_event(event):
286301
"""validate upload request, raise error when filed."""
287-
zone = event.get('zone')
288302
source = event.get('source')
303+
source_zone = event.get('source_zone')
289304
project_code = event.get('project_code')
290305
attribute = event.get('attribute')
291306
tag = event.get('tag')
292-
validator = UploadEventValidator(project_code, zone, source, attribute, tag)
307+
validator = UploadEventValidator(project_code, source_zone, source, attribute, tag)
293308
converted_content = validator.validate_upload_event()
294309
return converted_content
295310

app/resources/custom_error.py

+5-1
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ class Error:
2424
),
2525
'INVALID_TEMPLATE': 'Attribute validation failed. Please correct JSON format and try again.',
2626
'INVALID_TAG_FILE': 'Tag files validation failed. Please correct JSON format and try again.',
27+
'INVALID_SOURCE_FILE': 'Source file validation failed. Please verify and try again.',
2728
'LIMIT_TAG_ERROR': 'Tag limit has been reached. A maximum of 10 tags are allowed per file.',
2829
'INVALID_TAG_ERROR': (
2930
'Invalid tag format. Tags must be between 1 and 32 characters long '
@@ -36,7 +37,7 @@ class Error:
3637
'MANIFEST_NOT_EXIST': "Attribute '%s' not found in Project. Please verify and try again.",
3738
'INVALID_ATTRIBUTE': "Invalid attribute '%s'. Please verify and try again.",
3839
'INVALID_UPLOAD_REQUEST': 'Invalid upload request: %s',
39-
'INVALID_SOURCE_FILE': 'File does not exist or source file provided is invalid: %s',
40+
'INVALID_SOURCE_ITEM': 'File does not exist or source file provided is invalid: %s in (%s) zone',
4041
'INVALID_PIPELINENAME': (
4142
'Invalid pipeline name. Pipeline names must be between 1 and 20 characters long and '
4243
'may only contain lowercase letters, numbers, and/or special characters of -_, .'
@@ -77,6 +78,9 @@ class Error:
7778
'Please to double check the file content.'
7879
),
7980
'UNSUPPORT_TAG_MANIFEST': 'Tagging and manifest attaching are not supported for folder type.',
81+
'UNSUPPORT_SOURCE_MANIFEST': (
82+
'Multiple source files attaching are not supported for folder type. Please only use one source item.'
83+
),
8084
'INVALID_INPUT': 'Invalid input. Please try again.',
8185
'UNSUPPORTED_PROJECT': 'This function is not supported in the given Project %s',
8286
'CREATE_FOLDER_IF_NOT_EXIST': 'Target folder does not exist. Would you like to create a new folder?',

app/resources/custom_help.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,8 @@ class HelpPage:
4646
'FILE_UPLOAD_A': 'Add attributes to the file using a File Attribute Template.',
4747
'FILE_UPLOAD_T': 'Add tags to the file using a Tag file.',
4848
'FILE_UPLOAD_S': (
49-
'Project file path for identifying a source file when creating an upstream '
50-
'file lineage node. Source files must exist in the Core zone.'
49+
'The location of a json that contains the project file path(s) for identifying source file(s) '
50+
'when creating an upstream file lineage node. '
5151
),
5252
'FILE_UPLOAD_PIPELINE': (
5353
"The processed pipeline of your processed files. [only used with '--source' option]"

app/services/file_manager/file_upload/file_upload.py

+6-1
Original file line numberDiff line numberDiff line change
@@ -145,8 +145,13 @@ def simple_upload( # noqa: C901
145145
if job_type == UploadType.AS_FILE:
146146
upload_file_path = [input_path.rstrip('/').lstrip() + '.zip']
147147
compress_folder_to_zip(input_path)
148-
elif tags or attribute or source_id:
148+
# currently not support tag and attribute for a folder upload
149+
elif tags or attribute:
149150
SrvErrorHandler.customized_handle(ECustomizedError.UNSUPPORT_TAG_MANIFEST, True)
151+
# currently not support n-to-n relationship in lineage, meaning
152+
# can ONLY specify one source id for a folder upload
153+
elif len(source_id) > 1 and job_type == UploadType.AS_FOLDER:
154+
SrvErrorHandler.customized_handle(ECustomizedError.UNSUPPORT_SOURCE_MANIFEST, True)
150155
else:
151156
upload_file_path = get_file_in_folder(input_path)
152157
else:

app/services/file_manager/file_upload/upload_client.py

+4-2
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
from typing import Dict
1616
from typing import List
1717
from typing import Tuple
18+
from uuid import UUID
1819

1920
import httpx
2021
from httpx import HTTPStatusError
@@ -58,7 +59,7 @@ def __init__(
5859
current_folder_node: str = '',
5960
regular_file: str = True,
6061
tags: list = None,
61-
source_id: str = '',
62+
source_id: list[UUID] = '',
6263
attributes: dict = None,
6364
):
6465
super().__init__('', timeout=60)
@@ -226,11 +227,12 @@ def pre_upload(self, file_objects: List[FileObject], output_path: str) -> List[F
226227
'current_folder_node': self.current_folder_node,
227228
'parent_folder_id': self.parent_folder_id,
228229
'folder_tags': self.tags,
229-
'source_id': self.source_id,
230230
'data': [
231231
{'resumable_filename': x.file_name, 'resumable_relative_path': x.parent_path} for x in file_objects
232232
],
233233
}
234+
if self.source_id:
235+
payload.update({'source_id': self.source_id})
234236

235237
try:
236238
self.endpoint = AppConfig.Connections.url_bff + '/v1'

app/services/file_manager/file_upload/upload_validator.py

+23-13
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
from typing import Dict
77
from typing import List
88

9-
from app.configs.app_config import AppConfig
109
from app.services.file_manager.file_manifests import SrvFileManifests
1110
from app.services.file_manager.file_tag import SrvFileTag
1211
from app.services.output_manager.error_handler import ECustomizedError
@@ -15,21 +14,33 @@
1514

1615

1716
class UploadEventValidator:
18-
def __init__(self, project_code: str, zone: str, source: str, attribute: Dict[str, Any], tag: List[str]):
17+
def __init__(self, project_code: str, source_zone: str, source: str, attribute: Dict[str, Any], tag: List[str]):
1918
self.project_code = project_code
20-
self.zone = zone
19+
self.source_zone = source_zone
2120
self.source = source
2221
self.attribute = attribute
2322
self.tag = tag
2423

2524
def validate_zone(self):
26-
source_file_info = {}
25+
source_ids = []
2726
if self.source:
28-
source_file_info = search_item(self.project_code, AppConfig.Env.core_zone.lower(), self.source)
29-
source_file_info = source_file_info['result']
30-
if not source_file_info:
31-
SrvErrorHandler.customized_handle(ECustomizedError.INVALID_SOURCE_FILE, True, value=self.source)
32-
return source_file_info
27+
for source in self.source:
28+
_, source_path = source.split('/', 1)
29+
source_file_info = search_item(self.project_code, self.source_zone, source_path)
30+
if not source_file_info['result']:
31+
SrvErrorHandler.customized_handle(
32+
ECustomizedError.INVALID_SOURCE_ITEM, True, value=(source, self.source_zone)
33+
)
34+
source_ids.append(source_file_info['result'].get('id'))
35+
36+
# check if there is any duplication source id
37+
if len(source_ids) != len(set(source_ids)):
38+
SrvErrorHandler.customized_handle(
39+
ECustomizedError.INVALID_UPLOAD_REQUEST,
40+
value=('Source file list contains duplication',),
41+
if_exit=True,
42+
)
43+
return source_ids
3344

3445
def validate_attribute(self):
3546
srv_manifest = SrvFileManifests()
@@ -45,12 +56,11 @@ def validate_tag(self):
4556
srv_tag.validate_taglist(self.tag)
4657

4758
def validate_upload_event(self):
48-
source_file_info, loaded_attribute = {}, {}
59+
loaded_attribute = {}
4960
if self.attribute:
5061
loaded_attribute = self.validate_attribute()
5162
if self.tag:
5263
self.validate_tag()
53-
if self.zone == AppConfig.Env.core_zone.lower():
54-
source_file_info = self.validate_zone()
55-
converted_content = {'source_file': source_file_info, 'attribute': loaded_attribute}
64+
source_ids = self.validate_zone()
65+
converted_content = {'source_file': source_ids, 'attribute': loaded_attribute}
5666
return converted_content

app/services/output_manager/error_handler.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -23,13 +23,14 @@ class ECustomizedError(enum.Enum):
2323
FIELD_REQUIRED = 'FIELD_REQUIRED'
2424
INVALID_TEMPLATE = 'INVALID_TEMPLATE'
2525
INVALID_TAG_FILE = 'INVALID_TAG_FILE'
26+
INVALID_SOURCE_FILE = 'INVALID_SOURCE_FILE'
2627
LIMIT_TAG_ERROR = 'LIMIT_TAG_ERROR'
2728
INVALID_TAG_ERROR = 'INVALID_TAG_ERROR'
2829
RESERVED_TAG = 'RESERVED_TAG'
2930
INVALID_ATTRIBUTE = 'INVALID_ATTRIBUTE'
3031
MISSING_REQUIRED_ATTRIBUTE = 'MISSING_REQUIRED_ATTRIBUTE'
3132
INVALID_UPLOAD_REQUEST = 'INVALID_UPLOAD_REQUEST'
32-
INVALID_SOURCE_FILE = 'INVALID_SOURCE_FILE'
33+
INVALID_SOURCE_ITEM = 'INVALID_SOURCE_ITEM'
3334
INVALID_PIPELINENAME = 'INVALID_PIPELINENAME'
3435
INVALID_PATHS = 'INVALID_PATHS'
3536
INVALID_RESUMABLE_FILE = 'INVALID_RESUMABLE_FILE'
@@ -46,6 +47,7 @@ class ECustomizedError(enum.Enum):
4647
# the error when chunk md5 is not match
4748
INVALID_CHUNK_UPLOAD = 'INVALID_CHUNK_UPLOAD'
4849
UNSUPPORT_TAG_MANIFEST = 'UNSUPPORT_TAG_MANIFEST'
50+
UNSUPPORT_SOURCE_MANIFEST = 'UNSUPPORT_SOURCE_MANIFEST'
4951
MANIFEST_NOT_FOUND = 'MANIFEST_NOT_FOUND'
5052
INVALID_INPUT = 'INVALID_INPUT'
5153
UNSUPPORTED_PROJECT = 'UNSUPPORTED_PROJECT'

pyproject.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[tool.poetry]
22
name = "app"
3-
version = "3.12.3"
3+
version = "3.13.0"
44
description = "This service is designed to support pilot platform"
55
authors = ["Indoc Systems"]
66

tests/app/commands/test_file.py

+93
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
from app.commands.file import file_put
1919
from app.commands.file import file_resume
2020
from app.commands.file import file_trash
21+
from app.configs.app_config import AppConfig
2122
from app.models.item import ItemType
2223
from app.services.file_manager.file_metadata.file_metadata_client import FileMetaClient
2324
from app.services.file_manager.file_upload.models import FileObject
@@ -91,6 +92,98 @@ def test_file_upload_failed_with_invalid_attribute_file(cli_runner):
9192
assert result.output == customized_error_msg(ECustomizedError.INVALID_TEMPLATE) + '\n'
9293

9394

95+
def test_file_upload_failed_with_invalid_source_file(cli_runner):
96+
# create invalid source file with wrong format
97+
runner = click.testing.CliRunner()
98+
with runner.isolated_filesystem():
99+
with open('wrong_source.json', 'w') as f:
100+
f.write('wrong_source.json')
101+
102+
result = cli_runner.invoke(
103+
file_put,
104+
['test', '--thread', 1, '--source-file', 'wrong_source.json', 'wrong_source.json'],
105+
)
106+
assert result.exit_code == 1
107+
assert result.output == customized_error_msg(ECustomizedError.INVALID_SOURCE_FILE) + '\n'
108+
109+
110+
def test_file_upload_failed_with_duplicated_source_items(cli_runner, mocker):
111+
runner = click.testing.CliRunner()
112+
with runner.isolated_filesystem():
113+
114+
with open('source.txt', 'w') as f:
115+
f.write('[\'test_project/users/test\', \'test_project/users/test\']')
116+
117+
mocker.patch(
118+
'app.commands.file.identify_target_folder', return_value=('test_project', ItemType.FOLDER, 'users/admin')
119+
)
120+
mocker.patch(
121+
'app.services.file_manager.file_upload.upload_validator.search_item',
122+
return_value={'code': 200, 'result': {'id': 'id'}},
123+
)
124+
125+
result = cli_runner.invoke(
126+
file_put,
127+
['test', '--thread', 1, '--source-file', 'source.txt', 'source.txt'],
128+
)
129+
assert result.exit_code == 1
130+
assert (
131+
customized_error_msg(ECustomizedError.INVALID_UPLOAD_REQUEST) % 'Source file list contains duplication'
132+
in result.output
133+
)
134+
135+
136+
def test_file_upload_failed_with_n2n_relationship_in_source_lineage(cli_runner, mocker):
137+
runner = click.testing.CliRunner()
138+
with runner.isolated_filesystem():
139+
140+
test_folder = 'test_folder'
141+
makedirs(test_folder, exist_ok=True)
142+
143+
with open('source.txt', 'w') as f:
144+
f.write('[\'test1\', \'test2\']')
145+
146+
mocker.patch(
147+
'app.commands.file.identify_target_folder', return_value=('test_project', ItemType.FOLDER, 'users/admin')
148+
)
149+
mocker.patch(
150+
'app.commands.file.validate_upload_event',
151+
return_value={'source_file': ['test1', 'test2'], 'attribute': None},
152+
)
153+
mocker.patch('app.commands.file.assemble_path', return_value=('test', {'id': 'id'}, False, 'test'))
154+
155+
result = cli_runner.invoke(
156+
file_put,
157+
['test', '--thread', 1, '--source-file', 'source.txt', test_folder],
158+
)
159+
assert result.exit_code == 1
160+
assert customized_error_msg(ECustomizedError.UNSUPPORT_SOURCE_MANIFEST) in result.output
161+
162+
163+
@pytest.mark.parametrize('source_zone', [AppConfig.Env.green_zone, AppConfig.Env.core_zone])
164+
def test_file_upload_failed_with_invalid_source_items(cli_runner, mocker, source_zone):
165+
runner = click.testing.CliRunner()
166+
with runner.isolated_filesystem():
167+
file_path = 'test_project/users/admin/test.txt'
168+
with open('source.txt', 'w') as f:
169+
f.write(f'[\'{file_path}\']')
170+
171+
mocker.patch(
172+
'app.commands.file.identify_target_folder', return_value=('test_project', ItemType.FOLDER, 'users/admin')
173+
)
174+
mocker.patch(
175+
'app.services.file_manager.file_upload.upload_validator.search_item',
176+
return_value={'code': 404, 'result': {}},
177+
)
178+
179+
result = cli_runner.invoke(
180+
file_put,
181+
['test', '--thread', 1, '--source-file', 'source.txt', '--source-zone', source_zone, 'source.txt'],
182+
)
183+
assert result.exit_code == 1
184+
assert result.output == customized_error_msg(ECustomizedError.INVALID_SOURCE_ITEM) % (file_path, source_zone) + '\n'
185+
186+
94187
def test_resumable_upload_command_success(mocker, cli_runner):
95188
runner = click.testing.CliRunner()
96189
with runner.isolated_filesystem():

0 commit comments

Comments
 (0)