From 1337755f357318f3213e3a9789c905c193026212 Mon Sep 17 00:00:00 2001 From: ayn leslie Date: Tue, 21 Oct 2014 21:23:55 -0700 Subject: [PATCH] ISSUE-84: the check for column agreement between the dataset and csv was neglecting the synthetically generated columns --- .../datasync/validation/IntegrationJobValidity.java | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/socrata/datasync/validation/IntegrationJobValidity.java b/src/main/java/com/socrata/datasync/validation/IntegrationJobValidity.java index 0d7606bb..f2ccb5a9 100644 --- a/src/main/java/com/socrata/datasync/validation/IntegrationJobValidity.java +++ b/src/main/java/com/socrata/datasync/validation/IntegrationJobValidity.java @@ -14,6 +14,7 @@ import com.socrata.datasync.job.JobStatus; import com.socrata.model.importer.Dataset; import org.apache.commons.cli.CommandLine; +import org.apache.commons.lang3.ArrayUtils; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.entity.ContentType; import org.codehaus.jackson.map.ObjectMapper; @@ -112,7 +113,8 @@ public static JobStatus validateJobParams(SocrataConnectionInfo connectionInfo, return noHeaders; } - JobStatus csvDatasetAgreement = checkColumnAgreement(schema, headers, fileControl.ignoreColumns, publishFile.getName()); + Set synthetics = fileControl.syntheticLocations.keySet(); + JobStatus csvDatasetAgreement = checkColumnAgreement(schema, headers, synthetics, publishFile.getName()); if (csvDatasetAgreement.isError()) return csvDatasetAgreement; @@ -554,10 +556,11 @@ private static String getUnsupportedType(String component, String[] supportedTyp * @param schema the soda-java Dataset that provides dataset info * @param headers the columns that will be uploaded; taken from either the file or the control * file and minus the ignoredColumns + * @param synthetics the synthetic columns that will be generated from the csv * @param csvFilename the file name, for printing purposes. * @return */ - private static JobStatus checkColumnAgreement(Dataset schema, String[] headers, String[] ignoredColumns, String csvFilename) { + private static JobStatus checkColumnAgreement(Dataset schema, String[] headers, Set synthetics, String csvFilename) { if (schema == null) return JobStatus.VALID; @@ -581,9 +584,9 @@ private static JobStatus checkColumnAgreement(Dataset schema, String[] headers, columnNames.remove(field); } } - for (String ignored : ignoredColumns) { - if (columnNames.contains(ignored)) - columnNames.remove(ignored); + for (String synth : synthetics) { + if (columnNames.contains(synth)) + columnNames.remove(synth); } if (columnNames.size() > 0) { if (rowIdentifier == null) {