Merge pull request #70 from AbsaOSS/feature/69-allow-infinity-values-with-different-pattern-than-input-values

InfinitySupport changed from mix-in train to class

Author: MatloaItumeleng

.github/workflows/build.yml

@@ -32,10 +32,10 @@ jobs:
     name: Scala ${{matrix.scala}}
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
-      - uses: coursier/cache-action@v5
+        uses: actions/checkout@v4
+      - uses: coursier/cache-action@v6
       - name: Setup Scala
-        uses: olafurpg/setup-scala@v10
+        uses: olafurpg/setup-scala@v14
         with:
           java-version: "adopt@1.8"
       - name: Build and run tests

.github/workflows/format_check.yml

@@ -26,13 +26,13 @@ jobs:
     name: Scalafmt Check
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
           ref: ${{ github.event.pull_request.head.ref }}
-
+      - uses: coursier/cache-action@v6
       - name: Setup Scala
-        uses: olafurpg/setup-scala@v10
+        uses: olafurpg/setup-scala@v14
         with:
           java-version: "adopt@1.8"
 

.github/workflows/jacoco_check.yml

@@ -37,9 +37,10 @@ jobs:
             changed: 80.0
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
+      - uses: coursier/cache-action@v6
       - name: Setup Scala
-        uses: olafurpg/setup-scala@v10
+        uses: olafurpg/setup-scala@v14
         with:
           java-version: "adopt@1.8"
       - name: Build and run tests

README.md

@@ -2,6 +2,7 @@
 
 [![License](http://img.shields.io/:license-apache-blue.svg)](http://www.apache.org/licenses/LICENSE-2.0.html)
 [![Release](https://github.com/AbsaOSS/spark-data-standardization/actions/workflows/release.yml/badge.svg)](https://github.com/AbsaOSS/spark-data-standardization/actions/workflows/release.yml)
+![Java 8](https://img.shields.io/badge/Java_1.8-ED8B00?style=flat&logo=openjdk&logoColor=black)
 
 - Dataframe in 
 - Standardized Dataframe out

build.sbt

@@ -21,8 +21,8 @@ ThisBuild / name := "spark-data-standardization"
 ThisBuild / organization := "za.co.absa"
 
 lazy val scala211 = "2.11.12"
-lazy val scala212 = "2.12.18"
-lazy val scala213 = "2.13.11"
+lazy val scala212 = "2.12.20"
+lazy val scala213 = "2.13.16"
 
 ThisBuild / crossScalaVersions := Seq(scala211, scala212, scala213)
 ThisBuild / scalaVersion := scala211
@@ -39,6 +39,8 @@ ThisBuild / printSparkScalaVersion := {
 }
 
 Test / parallelExecution := false
+Test / logBuffered := false
+Test / fork := true
 
 // Only apply scalafmt to files that differ from master (i.e. files changed in the feature branch or so; n/a on Windows)
 lazy val fmtFilterExpression: String = System.getProperty("os.name").toLowerCase match {

src/main/scala/za/co/absa/standardization/SchemaValidator.scala

@@ -18,7 +18,6 @@ package za.co.absa.standardization
 
 import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.types._
-import za.co.absa.standardization.ErrorMessage
 import za.co.absa.standardization.types.{TypeDefaults, TypedStructField}
 import za.co.absa.standardization.validation.field.FieldValidationIssue
 
@@ -116,7 +115,7 @@ object SchemaValidator {
             fields += prefixedField
         }
       }
-      fields.toSeq
+      fields.toSeq //has to be here for Scala 2.13 compatibility
     }
 
     def flattenArray(field: StructField, arr: ArrayType, structPath: String): Seq[FlatField] = {
@@ -128,7 +127,7 @@ object SchemaValidator {
           val prefixedField = FlatField(structPath, field)
           arrayFields += prefixedField
       }
-      arrayFields.toSeq
+      arrayFields.toSeq //has to be here for Scala 2.13 compatibility
     }
 
     flattenStruct(schema, "")

src/main/scala/za/co/absa/standardization/adr/001-infinity-support-iso-pattern-defaults/Infinity-support-iso.md

@@ -0,0 +1,22 @@
+# Context
+
+The current support for infinity values in the Standardization library, needs the infinity values to have the same pattern as the input values. For example, if the input values have pattern yyyy-MM-dd, then the infinity values need to be specified as e.g. 1000-01-01 or 9999-01-01.
+
+This results in a problem when patterns like yyMMdd are used, due to truncation dates such as 2010/01/01 or 2000/01/01  because of the leading zeros they can't be interpreted leading to standardizaton error.
+
+
+# Solution design
+
+As  TypeParser (it's inner classes and methods), first replaces the infinity symbol with infinity value and only then converts to timestamp/date using the provided pattern. The solution will  defer the replacement based on what pattern the infinity value follows.
+
+TypeParser is to provide the InfinitySupport with the column and the converting function  where InfinitySupport now decides if infinity value is encoded by provided pattern or ISO pattern for both positive and negative infinity , where in case no infinity defined there is no transformation else if input column === infinity symbol use the infinity_value converted using ISO pattern and  return the converting function result with original column.
+  
+# Outcome
+
+InfinitySupport replaced with class providing unified `replaceInfinitySymbols` method with a conversion function, which is extended by InfinitySupportIso which implements the date/time algorithm to parse infinity values matching the input pattern using `defaultInfinityValueInjection` else use `isoCast` for ISO-formatted infinity values using `isOfISODateFormat` and `isOfISOTimestampFormat` to detect SO formats
+
+TypeParser continues orchestrating standardisation and delegating Parsers for source to target types.
+
+
+![Architecture diagram!](InfinitySupport-ISO-fallback.png)
+

src/main/scala/za/co/absa/standardization/adr/001-infinity-support-iso-pattern-defaults/InfinitySupport-ISO-fallback.drawio

@@ -21,7 +21,7 @@ import java.security.InvalidParameterException
 import scala.annotation.tailrec
 
 object StringImplicits {
-  implicit class StringEnhancements(string: String) {
+  implicit class StringEnhancements(val string: String) extends AnyVal {
 
     /**
       * Replaces all occurrences of the provided characters with their mapped values