Skip to content

Commit c5b2718

Browse files
MrRahulSharmaMrRahulSharma
authored
Added hook for treating all fields as nullable (#256)
1 parent 735cdf6 commit c5b2718

File tree

3 files changed

+22
-17
lines changed

3 files changed

+22
-17
lines changed

‎src/main/java/io/cdap/plugin/salesforce/SalesforceSchemaUtil.java

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -107,18 +107,18 @@ public static Schema getSchema(AuthenticatorCredentials credentials, SObjectDesc
107107
*
108108
* @param credentials connection credentials
109109
* @param sObjectDescriptor sObject descriptor
110-
* @param setAllCustomFieldsNullable set all custom fields nullable by default
110+
* @param setAllFieldsNullable set all fields nullable by default
111111
* @return CDAP schema
112112
* @throws ConnectionException if unable to connect to Salesforce
113113
*/
114114
public static Schema getSchema(AuthenticatorCredentials credentials, SObjectDescriptor sObjectDescriptor,
115-
boolean setAllCustomFieldsNullable)
115+
boolean setAllFieldsNullable)
116116
throws ConnectionException {
117117
PartnerConnection partnerConnection = SalesforceConnectionUtil.getPartnerConnection(credentials);
118118
SObjectsDescribeResult describeResult = SObjectsDescribeResult.of(partnerConnection,
119119
sObjectDescriptor.getName(), sObjectDescriptor.getFeaturedSObjects());
120120

121-
return getSchemaWithFields(sObjectDescriptor, describeResult, setAllCustomFieldsNullable);
121+
return getSchemaWithFields(sObjectDescriptor, describeResult, setAllFieldsNullable);
122122
}
123123

124124
/**
@@ -211,15 +211,15 @@ public static Schema getSchemaWithFields(SObjectDescriptor sObjectDescriptor,
211211

212212
public static Schema getSchemaWithFields(SObjectDescriptor sObjectDescriptor,
213213
SObjectsDescribeResult describeResult,
214-
boolean setAllCustomFieldsNullable) {
214+
boolean setAllFieldsNullable) {
215215
return getSchemaWithFields(sObjectDescriptor, describeResult,
216-
Collections.emptyList(), setAllCustomFieldsNullable);
216+
Collections.emptyList(), setAllFieldsNullable);
217217
}
218218

219219
public static Schema getSchemaWithFields(SObjectDescriptor sObjectDescriptor,
220220
SObjectsDescribeResult describeResult,
221221
List<String> topLevelParents,
222-
boolean setAllCustomFieldsNullable) {
222+
boolean setAllFieldsNullable) {
223223
List<Schema.Field> schemaFields = new ArrayList<>();
224224

225225
for (SObjectDescriptor.FieldDescriptor fieldDescriptor : sObjectDescriptor.getFields()) {
@@ -238,7 +238,7 @@ public static Schema getSchemaWithFields(SObjectDescriptor sObjectDescriptor,
238238
fieldDescriptor.getFullName(), parentsPath));
239239
}
240240

241-
fieldSchema = createFieldSchema(field, fieldDescriptor.hasParents(), setAllCustomFieldsNullable);
241+
fieldSchema = createFieldSchema(field, fieldDescriptor.hasParents(), setAllFieldsNullable);
242242
}
243243

244244
Schema queryFieldSchema = functionType.getSchema(fieldSchema);
@@ -274,7 +274,7 @@ public static Schema getSchemaWithFields(SObjectDescriptor sObjectDescriptor,
274274
*/
275275
for (SObjectDescriptor childSObject : sObjectDescriptor.getChildSObjects()) {
276276
Schema childSchema = getSchemaWithFields(childSObject, describeResult,
277-
Collections.singletonList(sObjectDescriptor.getName()), setAllCustomFieldsNullable);
277+
Collections.singletonList(sObjectDescriptor.getName()), setAllFieldsNullable);
278278

279279
String childName = normalizeAvroName(childSObject.getName());
280280
Schema.Field childField = Schema.Field.of(childName,
@@ -288,9 +288,9 @@ public static Schema getSchemaWithFields(SObjectDescriptor sObjectDescriptor,
288288

289289
// Setting all the child columns as Nullable as in child object these fields can be mandatory but its reference
290290
// object in parent class can be null.
291-
private static Schema createFieldSchema(Field field, boolean isChild, boolean setAllCustomFieldsNullable) {
291+
private static Schema createFieldSchema(Field field, boolean isChild, boolean setAllFieldsNullable) {
292292
Schema fieldSchema = SALESFORCE_TYPE_TO_CDAP_SCHEMA.getOrDefault(field.getType(), DEFAULT_SCHEMA);
293-
return field.isNillable() || isChild || (setAllCustomFieldsNullable && field.isCustom()) ?
293+
return field.isNillable() || isChild || setAllFieldsNullable ?
294294
Schema.nullableOf(fieldSchema) : fieldSchema;
295295
}
296296
}

‎src/main/java/io/cdap/plugin/salesforce/plugin/source/batch/SalesforceBatchSource.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -203,19 +203,19 @@ public static Schema getSchema(SalesforceSourceConfig config, OAuthInfo oAuthInf
203203
* Get Salesforce schema by query, with the option to allow null values in non-nullable custom fields
204204
*
205205
* @param config Salesforce Source Batch config
206-
* @param setAllCustomFieldsNullable set all custom fields nullable by default
206+
* @param setAllFieldsNullable set all custom fields nullable by default
207207
* @return schema calculated from query
208208
*/
209209
public static Schema getSchema(SalesforceSourceConfig config, OAuthInfo oAuthInfo,
210-
boolean setAllCustomFieldsNullable) {
210+
boolean setAllFieldsNullable) {
211211
String query = config.getQuery(System.currentTimeMillis(), oAuthInfo);
212212
SObjectDescriptor sObjectDescriptor = SObjectDescriptor.fromQuery(query);
213213
try {
214214
AuthenticatorCredentials credentials = AuthenticatorCredentials.fromParameters(oAuthInfo,
215215
config.getConnection().getConnectTimeout(),
216216
config.getConnection().getReadTimeout(),
217217
config.getConnection().getProxyUrl());
218-
return SalesforceSchemaUtil.getSchema(credentials, sObjectDescriptor, setAllCustomFieldsNullable);
218+
return SalesforceSchemaUtil.getSchema(credentials, sObjectDescriptor, setAllFieldsNullable);
219219
} catch (ConnectionException e) {
220220
String errorMessage = SalesforceConnectionUtil.getSalesforceErrorMessageFromException(e);
221221
throw new RuntimeException(

‎src/test/java/io/cdap/plugin/salesforce/SalesforceSchemaUtilTest.java

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -169,11 +169,11 @@ public void testGetSchemaWithFields() {
169169
}
170170

171171
@Test
172-
public void testSchemaWithSetAllCustomFieldsNullable() {
172+
public void testSchemaWithSetAllFieldsNullable() {
173173
String objectName = "CustomTable";
174174

175175
List<SObjectDescriptor.FieldDescriptor> fieldDescriptors = Stream
176-
.of("Name", "Value", "CreatedDate")
176+
.of("Name", "Value", "CreatedDate", "UpdatedDate")
177177
.map(name -> getFieldWithType(name, FieldType.anyType, false))
178178
.map(SObjectDescriptor.FieldDescriptor::new)
179179
.collect(Collectors.toList());
@@ -183,19 +183,23 @@ public void testSchemaWithSetAllCustomFieldsNullable() {
183183
Collections.singletonList("Value"), null, SalesforceFunctionType.NONE));
184184
fieldDescriptors.add(new SObjectDescriptor.FieldDescriptor(
185185
Collections.singletonList("CreatedDate"), null, SalesforceFunctionType.NONE));
186+
fieldDescriptors.add(new SObjectDescriptor.FieldDescriptor(
187+
Collections.singletonList("UpdatedDate"), null, SalesforceFunctionType.NONE));
186188
SObjectDescriptor sObjectDescriptor = new SObjectDescriptor(objectName, fieldDescriptors, ImmutableList.of());
187189

188190
Map<String, Field> objectFields = new LinkedHashMap<>();
189191
objectFields.put("Name", getCustomFieldWithType("Name", FieldType.string, false));
190192
objectFields.put("Value", getCustomFieldWithType("Value", FieldType.currency, false));
191193
objectFields.put("CreatedDate", getCustomFieldWithType("CreatedDate", FieldType.date, false));
194+
objectFields.put("UpdatedDate", getCustomFieldWithType("UpdatedDate", FieldType.date, false));
192195
SObjectsDescribeResult describeResult = SObjectsDescribeResult.of(ImmutableMap.of(objectName, objectFields));
193196

194197
// Testing case with flag setAllCustomFieldsNullable = true
195198
Schema expectedSchema = Schema.recordOf("output",
196199
Schema.Field.of("Name", Schema.nullableOf(Schema.of(Schema.Type.STRING))),
197200
Schema.Field.of("Value", Schema.nullableOf(Schema.of(Schema.Type.DOUBLE))),
198-
Schema.Field.of("CreatedDate", Schema.nullableOf(Schema.of(Schema.LogicalType.DATE)))
201+
Schema.Field.of("CreatedDate", Schema.nullableOf(Schema.of(Schema.LogicalType.DATE))),
202+
Schema.Field.of("UpdatedDate", Schema.nullableOf(Schema.of(Schema.LogicalType.DATE)))
199203
);
200204

201205
Schema actualSchema = SalesforceSchemaUtil.getSchemaWithFields(sObjectDescriptor, describeResult, true);
@@ -206,7 +210,8 @@ public void testSchemaWithSetAllCustomFieldsNullable() {
206210
expectedSchema = Schema.recordOf("output",
207211
Schema.Field.of("Name", Schema.of(Schema.Type.STRING)),
208212
Schema.Field.of("Value", Schema.of(Schema.Type.DOUBLE)),
209-
Schema.Field.of("CreatedDate", Schema.of(Schema.LogicalType.DATE))
213+
Schema.Field.of("CreatedDate", Schema.of(Schema.LogicalType.DATE)),
214+
Schema.Field.of("UpdatedDate", Schema.of(Schema.LogicalType.DATE))
210215
);
211216

212217
actualSchema = SalesforceSchemaUtil.getSchemaWithFields(sObjectDescriptor, describeResult, false);

0 commit comments

Comments
 (0)