Skip to content

Commit 0c88301

Browse files
Rahul-4480Rahul-4480
committed
feat: Add Glue catalog table for ALB connection logs
1 parent a96ba79 commit 0c88301

File tree

3 files changed

+96
-172
lines changed

3 files changed

+96
-172
lines changed

‎locals.tf‎

Lines changed: 85 additions & 152 deletions
Original file line numberDiff line numberDiff line change
@@ -1,156 +1,89 @@
11
locals {
2-
query_results_bucket_location = "s3://${var.query_results_bucket}/output/"
3-
alb_logs_bucket_location = "s3://${var.s3_bucket_name}/${var.s3_log_prefix}"
4-
input_format = "org.apache.hadoop.mapred.TextInputFormat"
5-
output_format = "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"
6-
ser_de_name = "alb-logs-serde"
7-
ser_de_library = "org.apache.hadoop.hive.serde2.RegexSerDe"
8-
ser_de_parameters = {
9-
"serialization.format" = "1"
10-
"input.regex" = "([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*):([0-9]*) ([^ ]*)[:-]([0-9]*) ([-.0-9]*) ([-.0-9]*) ([-.0-9]*) (|[-0-9]*) (-|[-0-9]*) ([-0-9]*) ([-0-9]*) \"([^ ]*) (.*) (- |[^ ]*)\" \"([^\"]*)\" ([A-Z0-9-_]+) ([A-Za-z0-9.-]*) ([^ ]*) \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" ([-.0-9]*) ([^ ]*) \"([^\"]*)\" \"([^\"]*)\" \"([^ ]*)\" \"([^\\s]+?)\" \"([^\\s]+)\" \"([^ ]*)\" \"([^ ]*)\" ?([^ ]*)?"
11-
}
12-
table_type = "EXTERNAL_TABLE"
13-
parameters = {
14-
"classification" = "log"
15-
"typeOfData" = "awsALBLogs"
16-
}
2+
query_results_bucket_location = "s3://${var.query_results_bucket}/output/"
3+
force_destroy_value = true
4+
publish_cloudwatch_metrics_enabled_value = false
5+
6+
tables = {
7+
alb_access_logs = {
8+
location = "s3://${var.s3_bucket_name}/alb-access-logs/"
9+
ser_de_name = "alb-access-logs-serde"
10+
ser_de_serialization_library = "org.apache.hadoop.hive.serde2.RegexSerDe"
11+
ser_de_params = {
12+
"serialization.format" = "1"
13+
"input.regex" = "([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*):([0-9]*) ([^ ]*)[:-]([0-9]*) ([-.0-9]*) ([-.0-9]*) ([-.0-9]*) (|[-0-9]*) (-|[-0-9]*) ([-0-9]*) ([-0-9]*) \"([^ ]*) (.*) (- |[^ ]*)\" \"([^\"]*)\" ([A-Z0-9-_]+) ([A-Za-z0-9.-]*) ([^ ]*) \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" ([-.0-9]*) ([^ ]*) \"([^\"]*)\" \"([^\"]*)\" \"([^ ]*)\" \"([^\\s]+?)\" \"([^\\s]+)\" \"([^ ]*)\" \"([^ ]*)\" ?([^ ]*)?"
14+
}
15+
columns = [
16+
{ name = "type", type = "string" },
17+
{ name = "time", type = "string" },
18+
{ name = "elb", type = "string" },
19+
{ name = "client_ip", type = "string" },
20+
{ name = "client_port", type = "int" },
21+
{ name = "target_ip", type = "string" },
22+
{ name = "target_port", type = "int" },
23+
{ name = "request_processing_time", type = "double" },
24+
{ name = "target_processing_time", type = "double" },
25+
{ name = "response_processing_time", type = "double" },
26+
{ name = "elb_status_code", type = "int" },
27+
{ name = "target_status_code", type = "string" },
28+
{ name = "received_bytes", type = "bigint" },
29+
{ name = "sent_bytes", type = "bigint" },
30+
{ name = "request_verb", type = "string" },
31+
{ name = "request_url", type = "string" },
32+
{ name = "request_proto", type = "string" },
33+
{ name = "user_agent", type = "string" },
34+
{ name = "ssl_cipher", type = "string" },
35+
{ name = "ssl_protocol", type = "string" },
36+
{ name = "target_group_arn", type = "string" },
37+
{ name = "trace_id", type = "string" },
38+
{ name = "domain_name", type = "string" },
39+
{ name = "chosen_cert_arn", type = "string" },
40+
{ name = "matched_rule_priority", type = "string" },
41+
{ name = "request_creation_time", type = "string" },
42+
{ name = "actions_executed", type = "string" },
43+
{ name = "redirect_url", type = "string" },
44+
{ name = "lambda_error_reason", type = "string" },
45+
{ name = "target_port_list", type = "string" },
46+
{ name = "target_status_code_list", type = "string" },
47+
{ name = "classification", type = "string" },
48+
{ name = "classification_reason", type = "string" },
49+
{ name = "conn_trace_id", type = "string" }
50+
]
51+
parameters = {
52+
"classification" = "log"
53+
"typeOfData" = "awsALBAccessLogs"
54+
}
55+
}
1756

18-
table_columns = [
19-
{
20-
name = "type"
21-
type = "string"
22-
},
23-
{
24-
name = "time"
25-
type = "string"
26-
},
27-
{
28-
name = "elb"
29-
type = "string"
30-
},
31-
{
32-
name = "client_ip"
33-
type = "string"
34-
},
35-
{
36-
name = "client_port"
37-
type = "int"
38-
},
39-
{
40-
name = "target_ip"
41-
type = "string"
42-
},
43-
{
44-
name = "target_port"
45-
type = "int"
46-
},
47-
{
48-
name = "request_processing_time"
49-
type = "double"
50-
},
51-
{
52-
name = "target_processing_time"
53-
type = "double"
54-
},
55-
{
56-
name = "response_processing_time"
57-
type = "double"
58-
},
59-
{
60-
name = "elb_status_code"
61-
type = "int"
62-
},
63-
{
64-
name = "target_status_code"
65-
type = "string"
66-
},
67-
{
68-
name = "received_bytes"
69-
type = "bigint"
70-
},
71-
{
72-
name = "sent_bytes"
73-
type = "bigint"
74-
},
75-
{
76-
name = "request_verb"
77-
type = "string"
78-
},
79-
{
80-
name = "request_url"
81-
type = "string"
82-
},
83-
{
84-
name = "request_proto"
85-
type = "string"
86-
},
87-
{
88-
name = "user_agent"
89-
type = "string"
90-
},
91-
{
92-
name = "ssl_cipher"
93-
type = "string"
94-
},
95-
{
96-
name = "ssl_protocol"
97-
type = "string"
98-
},
99-
{
100-
name = "target_group_arn"
101-
type = "string"
102-
},
103-
{
104-
name = "trace_id"
105-
type = "string"
106-
},
107-
{
108-
name = "domain_name"
109-
type = "string"
110-
},
111-
{
112-
name = "chosen_cert_arn"
113-
type = "string"
114-
},
115-
{
116-
name = "matched_rule_priority"
117-
type = "string"
118-
},
119-
{
120-
name = "request_creation_time"
121-
type = "string"
122-
},
123-
{
124-
name = "actions_executed"
125-
type = "string"
126-
},
127-
{
128-
name = "redirect_url"
129-
type = "string"
130-
},
131-
{
132-
name = "lambda_error_reason"
133-
type = "string"
134-
},
135-
{
136-
name = "target_port_list"
137-
type = "string"
138-
},
139-
{
140-
name = "target_status_code_list"
141-
type = "string"
142-
},
143-
{
144-
name = "classification"
145-
type = "string"
146-
},
147-
{
148-
name = "classification_reason"
149-
type = "string"
150-
},
151-
{
152-
name = "conn_trace_id"
153-
type = "string"
57+
alb_connection_logs = {
58+
location = "s3://${var.s3_bucket_name}/alb-connection-logs/"
59+
ser_de_name = "alb-connection-logs-serde"
60+
ser_de_serialization_library = "org.apache.hadoop.hive.serde2.RegexSerDe"
61+
ser_de_params = {
62+
"serialization.format" = "1"
63+
"input.regex" = "([^ ]*) ([^ ]*) ([0-9]*) ([0-9]*) ([A-Za-z0-9.-]*) ([^ ]*) ([-.0-9]*) \"([^\"]*)\" ([^ ]*) ([^ ]*) ([^ ]*) ?([^ ]*)?( .*)?"
64+
}
65+
columns = [
66+
{ name = "time", type = "string" },
67+
{ name = "client_ip", type = "string" },
68+
{ name = "client_port", type = "int" },
69+
{ name = "listener_port", type = "int" },
70+
{ name = "tls_protocol", type = "string" },
71+
{ name = "tls_cipher", type = "string" },
72+
{ name = "tls_handshake_latency", type = "double" },
73+
{ name = "leaf_client_cert_subject", type = "string" },
74+
{ name = "leaf_client_cert_validity", type = "string" },
75+
{ name = "leaf_client_cert_serial_number", type = "string" },
76+
{ name = "tls_verify_status", type = "string" },
77+
{ name = "conn_trace_id", type = "string" }
78+
]
79+
parameters = {
80+
"classification" = "log"
81+
"typeOfData" = "awsALBConnectionLogs"
82+
}
15483
}
155-
]
84+
}
85+
86+
input_format = "org.apache.hadoop.mapred.TextInputFormat"
87+
output_format = "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"
88+
table_type = "EXTERNAL_TABLE"
15689
}

‎main.tf‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
resource "aws_athena_workgroup" "this" {
22
name = var.workgroup_name
3-
force_destroy = true
3+
force_destroy = local.force_destroy_value
44

55
configuration {
6-
publish_cloudwatch_metrics_enabled = false
6+
publish_cloudwatch_metrics_enabled = local.publish_cloudwatch_metrics_enabled_value
77

88
result_configuration {
99
output_location = local.query_results_bucket_location
@@ -15,26 +15,27 @@ resource "aws_glue_catalog_database" "this" {
1515
name = var.database_name
1616
}
1717

18-
resource "aws_glue_catalog_table" "this" {
19-
name = var.table_name
18+
resource "aws_glue_catalog_table" "alb_logs" {
19+
for_each = local.tables
20+
name = each.key
2021
database_name = aws_glue_catalog_database.this.name
2122
table_type = local.table_type
22-
parameters = local.parameters
23+
parameters = each.value.parameters
2324

2425
storage_descriptor {
25-
location = local.alb_logs_bucket_location
26+
location = each.value.location
2627
input_format = local.input_format
2728
output_format = local.output_format
2829

2930
ser_de_info {
30-
name = local.ser_de_name
31-
serialization_library = local.ser_de_library
31+
name = each.value.ser_de_name
32+
serialization_library = each.value.ser_de_serialization_library
3233

33-
parameters = local.ser_de_parameters
34+
parameters = each.value.ser_de_params
3435
}
3536

3637
dynamic "columns" {
37-
for_each = local.table_columns
38+
for_each = each.value.columns
3839
content {
3940
name = columns.value.name
4041
type = columns.value.type

‎variables.tf‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -13,17 +13,7 @@ variable "database_name" {
1313
type = string
1414
}
1515

16-
variable "table_name" {
17-
description = "The name of the Athena table to query ALB logs."
18-
type = string
19-
}
20-
2116
variable "s3_bucket_name" {
2217
description = "The name of the S3 bucket where ALB logs are stored."
2318
type = string
2419
}
25-
26-
variable "s3_log_prefix" {
27-
description = "The prefix within the S3 bucket where ALB logs are stored (e.g., 'AWSLogs/')."
28-
type = string
29-
}

0 commit comments

Comments
 (0)