Clean code

Author: Christian Herrera

src/main/com/codely/lesson_05_monitoring_and_optimizations/video_02_broadcast_join/JoinOptimizationApp.scala

@@ -4,10 +4,6 @@ import org.apache.spark.sql.functions.broadcast
 
 object JoinOptimizationApp extends SparkApp {
 
-  // ./bin/spark-shell --master spark://spark-master:7077 --driver-memory 3g --conf spark.sql.adaptive.enabled=false
-
-  spark.sparkContext.setLogLevel("WARN")
-
   import spark.implicits._
 
   spark.sparkContext.setJobGroup("join without optimization", "join without optimization")
@@ -39,6 +35,4 @@ object JoinOptimizationApp extends SparkApp {
   veryLargeDF.join(broadcast(largeDF), "id").count()
 
   spark.sparkContext.clearJobGroup()
-
-  Thread.sleep(1000000)
 }

src/main/com/codely/lesson_05_monitoring_and_optimizations/video_03_skew_join/SkewJoinApp.scala

@@ -4,10 +4,6 @@ import org.apache.spark.sql.functions.when
 
 object SkewJoinApp extends SparkApp {
 
-  // ./bin/spark-shell --master spark://spark-master:7077  --driver-memory 3g --executor-memory 1024mb --conf spark.sql.autoBroadcastJoinThreshold=-1 --conf spark.sql.adaptive.enabled=false
-
-  spark.sparkContext.setLogLevel("WARN")
-
   import spark.implicits._
 
   spark.sparkContext.setJobGroup("skewed data", "skewed data")
@@ -40,6 +36,4 @@ object SkewJoinApp extends SparkApp {
   joinedAQE.count()
 
   spark.sparkContext.clearJobGroup()
-
-  Thread.sleep(1000000)
 }

src/main/com/codely/lesson_06__deploy_to_cloud/video_01__infra_aws/scripts/1-setup-resources.sh

@@ -63,9 +63,4 @@ aws iam create-service-linked-role --aws-service-name elasticmapreduce.amazonaws
 echo "Creating Step Functions state machine for EMR cluster"
 aws stepfunctions create-state-machine --name "EMR_StepFunctions_Machine" --definition file://policies/state_machine_definition.json --role-arn arn:aws:iam::$AWS_ACCOUNT_ID:role/$EMR_STEP_FUNCTIONS_ROLE --profile $AWS_PROFILE
 
-# Start execution of Step Functions (Uncomment to enable execution)
-# echo "Starting execution of Step Functions state machine"
-
-aws stepfunctions start-execution --state-machine-arn arn:aws:states:eu-west-1:010928190667:stateMachine:EMR_StepFunctions_Machine --profile emr-user
-
 echo "Setup complete."