Skip to content

Commit f2e9775

Browse files
yruslanyruslan
committed
#324 Drop fillers from AST if requested when invoking 'parseSimple()'
1 parent d346f0c commit f2e9775

File tree

4 files changed

+177
-11
lines changed

4 files changed

+177
-11
lines changed

‎README.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -306,7 +306,7 @@ import za.co.absa.cobrix.cobol.parser.CopybookParser
306306
import za.co.absa.cobrix.cobol.reader.policies.SchemaRetentionPolicy
307307
import za.co.absa.cobrix.spark.cobol.schema.CobolSchema
308308

309-
val parsedSchema = CopybookParser.parseTree(copyBookContents)
309+
val parsedSchema = CopybookParser.parseSimple(copyBookContents)
310310
val cobolSchema = new CobolSchema(parsedSchema, SchemaRetentionPolicy.CollapseRoot, inputFileNameField = "", generateRecordId = false)
311311
val sparkSchema = cobolSchema.getSparkSchema.toString()
312312

@@ -318,7 +318,7 @@ If you want to check the layout of the copybook:
318318
```scala
319319
import za.co.absa.cobrix.cobol.parser.CopybookParser
320320

321-
val copyBook = CopybookParser.parseTree(copyBookContents)
321+
val copyBook = CopybookParser.parseSimple(copyBookContents)
322322
println(copyBook.generateRecordLayoutPositions())
323323
```
324324

‎cobol-parser/src/main/scala/za/co/absa/cobrix/cobol/parser/Copybook.scala‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -268,6 +268,34 @@ class Copybook(val ast: CopybookAST) extends Serializable {
268268
new Copybook(CopybookParser.calculateBinaryProperties(newRoot))
269269
}
270270

271+
def dropFillers(dropValueFillers: Boolean, dropGroupFillers: Boolean): Copybook = {
272+
def dropFillersAst(group: Group): Option[Group] = {
273+
if (dropGroupFillers && group.isFiller) {
274+
None
275+
} else {
276+
val newChildren: ArrayBuffer[Statement] = group.children.flatMap {
277+
case g: Group => dropFillersAst(g)
278+
case p: Primitive =>
279+
if (dropValueFillers && p.isFiller) {
280+
None
281+
} else {
282+
Some(p)
283+
}
284+
}
285+
if (newChildren.isEmpty) {
286+
None
287+
} else {
288+
Some(group.withUpdatedChildren(newChildren))
289+
}
290+
}
291+
}
292+
293+
dropFillersAst(ast) match {
294+
case Some(newAst) => new Copybook(newAst)
295+
case None => throw new IllegalArgumentException("Removing of fillers made the copybook empty.")
296+
}
297+
}
298+
271299
def restrictTo(fieldName: String): Copybook = {
272300
val stmt = getFieldByName(fieldName)
273301
if (stmt.isInstanceOf[Primitive])

‎cobol-parser/src/main/scala/za/co/absa/cobrix/cobol/parser/CopybookParser.scala‎

Lines changed: 21 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -59,20 +59,32 @@ object CopybookParser {
5959
*
6060
* This method accepts arguments that affect only structure of the output AST.
6161
*
62-
* @param copyBookContents A string containing all lines of a copybook
63-
* @param dropGroupFillers Drop groups marked as fillers from the output AST
64-
* @param dropValueFillers Drop primitive fields marked as fillers from the output AST
65-
* @param commentPolicy Specifies a policy for comments truncation inside a copybook
62+
* FILLER fields renaming/removal scenarios can be seen in `ParseCopybookFeaturesSpec.scala`
63+
*
64+
* @param copyBookContents A string containing all lines of a copybook
65+
* @param giveUniqueNameToGroupFillers If true each FILLER GROUP field will have a unique name that helps retaining it in
66+
* target schemas where column names must be unique. FILLERs become FILLER_1, FILLER_2, etc.
67+
* @param giveUniqueNameToValueFillers If true each FILLER value field will have a unique name that helps retaining it in
68+
* target schemas where column names must be unique. FILLERs become FILLER_P1, FILLER_P2, etc.
69+
* @param dropFillersFromAst If true FILLER fields that are not renamed will be removed from the AST.
70+
* @param commentPolicy Specifies a policy for comments truncation inside a copybook
6671
* @return Seq[Group] where a group is a record inside the copybook
6772
*/
6873
def parseSimple(copyBookContents: String,
69-
dropGroupFillers: Boolean = false,
70-
dropValueFillers: Boolean = true,
74+
giveUniqueNameToGroupFillers: Boolean = true,
75+
giveUniqueNameToValueFillers: Boolean = false,
76+
dropFillersFromAst: Boolean = false,
7177
commentPolicy: CommentPolicy = CommentPolicy()): Copybook = {
72-
parse(copyBookContents = copyBookContents,
73-
dropGroupFillers = dropGroupFillers,
74-
dropValueFillers = dropValueFillers,
78+
val copybook = parse(copyBookContents = copyBookContents,
79+
dropGroupFillers = !giveUniqueNameToGroupFillers,
80+
dropValueFillers = !giveUniqueNameToValueFillers,
7581
commentPolicy = commentPolicy)
82+
83+
if (dropFillersFromAst && (!giveUniqueNameToGroupFillers || !giveUniqueNameToValueFillers)) {
84+
copybook.dropFillers(!giveUniqueNameToValueFillers, !giveUniqueNameToGroupFillers)
85+
} else {
86+
copybook
87+
}
7688
}
7789

7890
/**

‎cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/copybooks/ParseCopybookFeaturesSpec.scala‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,29 @@ import za.co.absa.cobrix.cobol.testutils.SimpleComparisonBase
2424
class ParseCopybookFeaturesSpec extends FunSuite with SimpleComparisonBase {
2525
private implicit val logger: Logger = LoggerFactory.getLogger(this.getClass)
2626

27+
private val copybookFillers =
28+
""" 01 RECORD.
29+
| 05 FILLER PIC X(1).
30+
| 05 COMPANY_PREFIX PIC X(3).
31+
| 05 FILLER PIC X(1).
32+
| 05 FILLER PIC X(1).
33+
| 05 COMPANY_NAME PIC X(9).
34+
| 05 FILLER REDEFINES COMPANY_NAME.
35+
| 10 STR1 PIC X(5).
36+
| 10 STR2 PIC X(2).
37+
| 10 FILLER PIC X(1).
38+
| 05 ADDRESS PIC X(25).
39+
| 05 FILLER REDEFINES ADDRESS.
40+
| 10 STR4 PIC X(10).
41+
| 10 FILLER PIC X(20).
42+
| 05 FILL_FIELD.
43+
| 10 FILLER PIC X(5).
44+
| 10 FILLER PIC X(2).
45+
| 05 CONTACT_PERSON REDEFINES FILL_FIELD.
46+
| 10 FIRST_NAME PIC X(6).
47+
| 05 AMOUNT PIC S9(09)V99 BINARY.
48+
|""".stripMargin
49+
2750
test("Test copybooks with indexed by clauses with multiple indexes separated by comma") {
2851
val copybookContents =
2952
"""
@@ -91,4 +114,107 @@ class ParseCopybookFeaturesSpec extends FunSuite with SimpleComparisonBase {
91114

92115
assertEqualsMultiline(layout, expectedLayout)
93116
}
117+
118+
test("Test parseSimple() not dropping fillers") {
119+
val copybook = CopybookParser.parseSimple(copybookFillers, giveUniqueNameToValueFillers = true, giveUniqueNameToGroupFillers = true)
120+
val layout = copybook.generateRecordLayoutPositions()
121+
122+
val expectedLayout =
123+
"""-------- FIELD LEVEL/NAME --------- --ATTRIBS-- FLD START END LENGTH
124+
|
125+
|1 RECORD 1 1 60 60
126+
| 5 FILLER_P1 2 1 1 1
127+
| 5 COMPANY_PREFIX 3 2 4 3
128+
| 5 FILLER_P2 4 5 5 1
129+
| 5 FILLER_P3 5 6 6 1
130+
| 5 COMPANY_NAME r 6 7 15 9
131+
| 5 FILLER_1 R 7 7 15 9
132+
| 10 STR1 8 7 11 5
133+
| 10 STR2 9 12 13 2
134+
| 10 FILLER_P4 10 14 14 1
135+
| 5 ADDRESS r 11 16 45 30
136+
| 5 FILLER_2 R 12 16 45 30
137+
| 10 STR4 13 16 25 10
138+
| 10 FILLER_P5 14 26 45 20
139+
| 5 FILL_FIELD r 15 46 52 7
140+
| 10 FILLER_P6 16 46 50 5
141+
| 10 FILLER_P7 17 51 52 2
142+
| 5 CONTACT_PERSON R 18 46 52 7
143+
| 10 FIRST_NAME 19 46 51 6
144+
| 5 AMOUNT 20 53 60 8
145+
|"""
146+
.stripMargin.replace("\r\n", "\n")
147+
148+
assertEqualsMultiline(layout, expectedLayout)
149+
}
150+
test("Test parseSimple() drop value fillers") {
151+
val copybook = CopybookParser.parseSimple(copybookFillers, giveUniqueNameToValueFillers = false, giveUniqueNameToGroupFillers = true, dropFillersFromAst = true)
152+
val layout = copybook.generateRecordLayoutPositions()
153+
154+
val expectedLayout =
155+
"""-------- FIELD LEVEL/NAME --------- --ATTRIBS-- FLD START END LENGTH
156+
|
157+
|1 RECORD 1 1 60 60
158+
| 5 COMPANY_PREFIX 2 2 4 3
159+
| 5 COMPANY_NAME r 3 7 15 9
160+
| 5 FILLER_1 R 4 7 15 9
161+
| 10 STR1 5 7 11 5
162+
| 10 STR2 6 12 13 2
163+
| 5 ADDRESS r 7 16 45 30
164+
| 5 FILLER_2 R 8 16 45 30
165+
| 10 STR4 9 16 25 10
166+
| 5 CONTACT_PERSON R 10 46 52 7
167+
| 10 FIRST_NAME 11 46 51 6
168+
| 5 AMOUNT 12 53 60 8
169+
|"""
170+
.stripMargin.replace("\r\n", "\n")
171+
172+
assertEqualsMultiline(layout, expectedLayout)
173+
}
174+
175+
test("Test parseSimple() drop group fillers") {
176+
val copybook = CopybookParser.parseSimple(copybookFillers, giveUniqueNameToValueFillers = true, giveUniqueNameToGroupFillers = false, dropFillersFromAst = true)
177+
val layout = copybook.generateRecordLayoutPositions()
178+
179+
val expectedLayout =
180+
"""-------- FIELD LEVEL/NAME --------- --ATTRIBS-- FLD START END LENGTH
181+
|
182+
|1 RECORD 1 1 60 60
183+
| 5 FILLER_P1 2 1 1 1
184+
| 5 COMPANY_PREFIX 3 2 4 3
185+
| 5 FILLER_P2 4 5 5 1
186+
| 5 FILLER_P3 5 6 6 1
187+
| 5 COMPANY_NAME r 6 7 15 9
188+
| 5 ADDRESS r 7 16 45 30
189+
| 5 FILL_FIELD r 8 46 52 7
190+
| 10 FILLER_P6 9 46 50 5
191+
| 10 FILLER_P7 10 51 52 2
192+
| 5 CONTACT_PERSON R 11 46 52 7
193+
| 10 FIRST_NAME 12 46 51 6
194+
| 5 AMOUNT 13 53 60 8
195+
|"""
196+
.stripMargin.replace("\r\n", "\n")
197+
198+
assertEqualsMultiline(layout, expectedLayout)
199+
}
200+
201+
test("Test parseSimple() drop all fillers") {
202+
val copybook = CopybookParser.parseSimple(copybookFillers, giveUniqueNameToValueFillers = false, giveUniqueNameToGroupFillers = false, dropFillersFromAst = true)
203+
val layout = copybook.generateRecordLayoutPositions()
204+
205+
val expectedLayout =
206+
"""-------- FIELD LEVEL/NAME --------- --ATTRIBS-- FLD START END LENGTH
207+
|
208+
|1 RECORD 1 1 60 60
209+
| 5 COMPANY_PREFIX 2 2 4 3
210+
| 5 COMPANY_NAME r 3 7 15 9
211+
| 5 ADDRESS r 4 16 45 30
212+
| 5 CONTACT_PERSON R 5 46 52 7
213+
| 10 FIRST_NAME 6 46 51 6
214+
| 5 AMOUNT 7 53 60 8
215+
|"""
216+
.stripMargin.replace("\r\n", "\n")
217+
218+
assertEqualsMultiline(layout, expectedLayout)
219+
}
94220
}

0 commit comments

Comments
 (0)