Skip to content

Commit ee27ad9

Browse files
d10xad10xa
authored
format-in csv #18 (#29)
closes #29
1 parent 4da644f commit ee27ad9

File tree

13 files changed

+423
-77
lines changed

13 files changed

+423
-77
lines changed

‎frontend-laminar/src/main/scala/ru/d10xa/jsonlogviewer/App.scala

+17-1
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import ru.d10xa.jsonlogviewer.Router0.ViewPage
1717
import ru.d10xa.jsonlogviewer.Router0.navigateTo
1818
import ru.d10xa.jsonlogviewer.decline.Config
1919
import ru.d10xa.jsonlogviewer.decline.Config.FormatIn
20+
import ru.d10xa.jsonlogviewer.decline.Config.FormatIn.Csv
2021
import ru.d10xa.jsonlogviewer.decline.Config.FormatIn.Json
2122
import ru.d10xa.jsonlogviewer.decline.Config.FormatIn.Logfmt
2223
import ru.d10xa.jsonlogviewer.decline.Config.FormatOut
@@ -44,6 +45,16 @@ object App {
4445
|@timestamp=2023-09-18T19:10:10.123456Z second line {"level":"INFO"}
4546
|""".stripMargin
4647

48+
val csvSample: String =
49+
"""@timestamp,level,logger_name,thread_name,message
50+
|2023-09-18T19:10:10.123456Z,INFO,MakeLogs,main,"first line, with comma"
51+
|2023-09-18T19:11:20.132318Z,INFO,MakeLogs,main,test
52+
|2023-09-18T19:12:30.132319Z,DEBUG,MakeLogs,main,debug msg
53+
|2023-09-18T19:13:42.132321Z,WARN,MakeLogs,main,warn msg
54+
|2023-09-18T19:14:42.137207Z,ERROR,MakeLogs,main,"error message,error details"
55+
|2023-09-18T19:15:42.137207Z,INFO,MakeLogs,main,last line
56+
|""".stripMargin
57+
4758
val textVar: Var[String] = Var("")
4859

4960
val cliVar: Var[String] = Var(
@@ -132,13 +143,16 @@ object App {
132143
value <-- formatInVar.signal.map {
133144
case FormatIn.Json => "json"
134145
case FormatIn.Logfmt => "logfmt"
146+
case FormatIn.Csv => "csv"
135147
},
136148
onChange.mapToValue.map {
137149
case "json" => FormatIn.Json
138150
case "logfmt" => FormatIn.Logfmt
151+
case "csv" => FormatIn.Csv
139152
} --> formatInVar,
140153
option(value := "json", "json"),
141-
option(value := "logfmt", "logfmt")
154+
option(value := "logfmt", "logfmt"),
155+
option(value := "csv", "csv")
142156
)
143157
)
144158
def formatOutDiv: ReactiveHtmlElement[HTMLDivElement] = div(
@@ -196,11 +210,13 @@ object App {
196210
child.text <-- formatInVar.signal.map {
197211
case Logfmt => "Generate logfmt logs"
198212
case Json => "Generate json logs"
213+
case Csv => "Generate csv logs"
199214
},
200215
onClick --> { _ =>
201216
formatInVar.now() match
202217
case Config.FormatIn.Json => textVar.set(jsonLogSample)
203218
case Config.FormatIn.Logfmt => textVar.set(logfmtSample)
219+
case Config.FormatIn.Csv => textVar.set(csvSample)
204220
}
205221
)
206222
private def renderLivePage(): HtmlElement = {

‎frontend-laminar/src/main/scala/ru/d10xa/jsonlogviewer/ViewElement.scala

+2-1
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,8 @@ object ViewElement {
3535
filter = config.filter,
3636
formatIn = config.formatIn,
3737
rawInclude = None,
38-
rawExclude = None
38+
rawExclude = None,
39+
excludeFields = None
3940
)
4041
)
4142
)

‎json-log-viewer/jvm/src/main/scala/ru/d10xa/jsonlogviewer/decline/yaml/ConfigYamlLoaderImpl.scala

+7-2
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ import cats.data.Validated
55
import cats.data.ValidatedNel
66
import cats.syntax.all.*
77
import io.circe.*
8-
import io.circe.generic.auto.*
98
import io.circe.yaml.scalayaml.parser
109
import ru.d10xa.jsonlogviewer.decline.Config.FormatIn
1110
import ru.d10xa.jsonlogviewer.decline.FormatInValidator
@@ -148,14 +147,20 @@ class ConfigYamlLoaderImpl extends ConfigYamlLoader {
148147
parseOptionalListString(feedFields, "rawInclude")
149148
val rawExcludeValidated =
150149
parseOptionalListString(feedFields, "rawExclude")
150+
val excludeFieldsValidated =
151+
parseOptionalListString(
152+
feedFields,
153+
"excludeFields"
154+
)
151155
(
152156
nameValidated,
153157
commandsValidated,
154158
inlineInputValidated,
155159
filterValidated,
156160
formatInValidated,
157161
rawIncludeValidated,
158-
rawExcludeValidated
162+
rawExcludeValidated,
163+
excludeFieldsValidated
159164
)
160165
.mapN(Feed.apply)
161166
}

‎json-log-viewer/jvm/src/test/scala/ru/d10xa/jsonlogviewer/decline/yaml/ConfigYamlLoaderTest.scala

+37
Original file line numberDiff line numberDiff line change
@@ -70,4 +70,41 @@ class ConfigYamlLoaderTest extends FunSuite {
7070
val errors = result.swap.toOption.get
7171
assert(errors.exists(_.contains("Invalid 'feeds' field format, should be a list")))
7272
}
73+
74+
test("parse valid yaml with excludeFields") {
75+
val yaml =
76+
"""|feeds:
77+
| - name: "pod-logs"
78+
| commands:
79+
| - "./mock-logs.sh pod1"
80+
| excludeFields:
81+
| - "level"
82+
| - "logger_name"
83+
| - "thread_name"
84+
| - name: "service-logs"
85+
| commands:
86+
| - "./mock-logs.sh service1"
87+
| excludeFields:
88+
| - "@timestamp"
89+
|""".stripMargin
90+
91+
val result = configYamlLoader.parseYamlFile(yaml)
92+
assert(result.isValid, s"Result should be valid: $result")
93+
94+
val config = result.toOption.get
95+
96+
val feeds = config.feeds.get
97+
assertEquals(feeds.size, 2)
98+
99+
val feed1 = feeds.head
100+
assertEquals(feed1.name, Some("pod-logs"))
101+
assertEquals(
102+
feed1.excludeFields,
103+
Some(List("level", "logger_name", "thread_name"))
104+
)
105+
106+
val feed2 = feeds(1)
107+
assertEquals(feed2.name, Some("service-logs"))
108+
assertEquals(feed2.excludeFields, Some(List("@timestamp")))
109+
}
73110
}

‎json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/LogViewerStream.scala

+128-57
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@ package ru.d10xa.jsonlogviewer
33
import cats.effect.IO
44
import cats.effect.Ref
55
import fs2.*
6+
import fs2.Pull
7+
import ru.d10xa.jsonlogviewer.csv.CsvLogLineParser
68
import ru.d10xa.jsonlogviewer.decline.yaml.ConfigYaml
79
import ru.d10xa.jsonlogviewer.decline.yaml.Feed
810
import ru.d10xa.jsonlogviewer.decline.Config
@@ -35,81 +37,97 @@ object LogViewerStream {
3537
val feedStreams = feeds.zipWithIndex.map { (feed, index) =>
3638
val feedStream: Stream[IO, String] =
3739
commandsAndInlineInputToStream(feed.commands, feed.inlineInput)
38-
processStream(
39-
config,
40-
feedStream,
41-
configYamlRef,
42-
index
40+
41+
createProcessStream(
42+
config = config,
43+
lines = feedStream,
44+
configYamlRef = configYamlRef,
45+
index = index,
46+
initialFormatIn = feed.formatIn.orElse(config.formatIn)
4347
)
4448
}
4549
Stream.emits(feedStreams).parJoin(feedStreams.size)
4650
case None =>
47-
processStream(config, stdinLinesStream, configYamlRef, -1)
51+
createProcessStream(
52+
config = config,
53+
lines = stdinLinesStream,
54+
configYamlRef = configYamlRef,
55+
index = -1,
56+
initialFormatIn = config.formatIn
57+
)
4858
}
4959

5060
finalStream
5161
.intersperse("\n")
5262
.append(Stream.emit("\n"))
5363
}
5464

55-
private def commandsAndInlineInputToStream(
56-
commands: List[String],
57-
inlineInput: Option[String]
58-
): Stream[IO, String] =
59-
new ShellImpl().mergeCommandsAndInlineInput(commands, inlineInput)
60-
61-
def makeLogLineParser(
65+
private def createProcessStream(
6266
config: Config,
63-
optFormatIn: Option[FormatIn]
64-
): LogLineParser = {
65-
val jsonPrefixPostfix = JsonPrefixPostfix(JsonDetector())
66-
optFormatIn match {
67-
case Some(FormatIn.Logfmt) => LogfmtLogLineParser(config)
68-
case _ => JsonLogLineParser(config, jsonPrefixPostfix)
67+
lines: Stream[IO, String],
68+
configYamlRef: Ref[IO, Option[ConfigYaml]],
69+
index: Int,
70+
initialFormatIn: Option[FormatIn]
71+
): Stream[IO, String] =
72+
if (initialFormatIn.contains(FormatIn.Csv)) {
73+
lines.pull.uncons1.flatMap {
74+
case Some((headerLine, rest)) =>
75+
val csvHeaderParser = CsvLogLineParser(config, headerLine)
76+
processStreamWithEffectiveConfig(
77+
config = config,
78+
lines = rest,
79+
configYamlRef = configYamlRef,
80+
index = index,
81+
parser = Some(csvHeaderParser)
82+
).pull.echo
83+
case None =>
84+
Pull.done
85+
}.stream
86+
} else {
87+
processStreamWithEffectiveConfig(
88+
config = config,
89+
lines = lines,
90+
configYamlRef = configYamlRef,
91+
index = index,
92+
parser = None
93+
)
6994
}
70-
}
7195

72-
private def processStream(
73-
baseConfig: Config,
96+
private def processStreamWithEffectiveConfig(
97+
config: Config,
7498
lines: Stream[IO, String],
7599
configYamlRef: Ref[IO, Option[ConfigYaml]],
76-
index: Int
100+
index: Int,
101+
parser: Option[LogLineParser]
77102
): Stream[IO, String] =
78103
for {
79104
line <- lines
80105
optConfigYaml <- Stream.eval(configYamlRef.get)
81-
formatIn = optConfigYaml
82-
.flatMap(_.feeds)
83-
.flatMap(_.lift(index).flatMap(_.formatIn))
84-
.orElse(baseConfig.formatIn)
85-
filter = optConfigYaml
86-
.flatMap(_.feeds)
87-
.flatMap(_.lift(index).flatMap(_.filter))
88-
.orElse(baseConfig.filter)
89-
rawInclude = optConfigYaml
90-
.flatMap(_.feeds)
91-
.flatMap(_.lift(index).flatMap(_.rawInclude))
92-
rawExclude = optConfigYaml
93-
.flatMap(_.feeds)
94-
.flatMap(_.lift(index).flatMap(_.rawExclude))
95-
feedName = optConfigYaml
96-
.flatMap(_.feeds)
97-
.flatMap(_.lift(index).flatMap(_.name))
98-
effectiveConfig = baseConfig.copy(
99-
filter = filter,
100-
formatIn = formatIn
106+
107+
feedConfig = extractFeedConfig(optConfigYaml, index)
108+
109+
effectiveConfig = config.copy(
110+
filter = feedConfig.filter.orElse(config.filter),
111+
formatIn = feedConfig.formatIn.orElse(config.formatIn)
101112
)
113+
102114
timestampFilter = TimestampFilter()
103115
parseResultKeys = ParseResultKeys(effectiveConfig)
104116
logLineFilter = LogLineFilter(effectiveConfig, parseResultKeys)
105-
logLineParser = makeLogLineParser(effectiveConfig, formatIn)
106-
outputLineFormatter = effectiveConfig.formatOut match
117+
118+
logLineParser = parser.getOrElse(
119+
makeNonCsvLogLineParser(effectiveConfig, feedConfig.formatIn)
120+
)
121+
122+
outputLineFormatter = effectiveConfig.formatOut match {
107123
case Some(Config.FormatOut.Raw) => RawFormatter()
108124
case Some(Config.FormatOut.Pretty) | None =>
109-
ColorLineFormatter(effectiveConfig, feedName)
125+
ColorLineFormatter(effectiveConfig, feedConfig.feedName, feedConfig.excludeFields)
126+
}
127+
110128
evaluatedLine <- Stream
111129
.emit(line)
112-
.filter(rawFilter(_, rawInclude, rawExclude))
130+
.filter(rawFilter(_, feedConfig.rawInclude, feedConfig.rawExclude))
113131
.map(logLineParser.parse)
114132
.filter(logLineFilter.grep)
115133
.filter(logLineFilter.logLineQueryPredicate)
@@ -121,27 +139,80 @@ object LogViewerStream {
121139
effectiveConfig.timestamp.before
122140
)
123141
)
124-
.map(pr =>
125-
Try(outputLineFormatter.formatLine(pr)) match {
126-
case Success(formatted) => formatted.toString
127-
case Failure(_) => pr.raw
128-
}
129-
)
130-
.map(_.toString)
142+
.map(formatWithSafety(_, outputLineFormatter))
131143
} yield evaluatedLine
132144

145+
private def formatWithSafety(
146+
parseResult: ParseResult,
147+
formatter: OutputLineFormatter
148+
): String =
149+
Try(formatter.formatLine(parseResult)) match {
150+
case Success(formatted) => formatted.toString
151+
case Failure(_) => parseResult.raw
152+
}
153+
154+
// TODO
155+
private case class FeedConfig(
156+
feedName: Option[String],
157+
filter: Option[ru.d10xa.jsonlogviewer.query.QueryAST],
158+
formatIn: Option[FormatIn],
159+
rawInclude: Option[List[String]],
160+
rawExclude: Option[List[String]],
161+
excludeFields: Option[List[String]]
162+
)
163+
164+
private def extractFeedConfig(
165+
optConfigYaml: Option[ConfigYaml],
166+
index: Int
167+
): FeedConfig = {
168+
val feedOpt = optConfigYaml
169+
.flatMap(_.feeds)
170+
.flatMap(_.lift(index))
171+
172+
FeedConfig(
173+
feedName = feedOpt.flatMap(_.name),
174+
filter = feedOpt.flatMap(_.filter),
175+
formatIn = feedOpt.flatMap(_.formatIn),
176+
rawInclude = feedOpt.flatMap(_.rawInclude),
177+
rawExclude = feedOpt.flatMap(_.rawExclude),
178+
excludeFields = feedOpt.flatMap(_.excludeFields)
179+
)
180+
}
181+
182+
private def commandsAndInlineInputToStream(
183+
commands: List[String],
184+
inlineInput: Option[String]
185+
): Stream[IO, String] =
186+
new ShellImpl().mergeCommandsAndInlineInput(commands, inlineInput)
187+
188+
def makeNonCsvLogLineParser(
189+
config: Config,
190+
optFormatIn: Option[FormatIn]
191+
): LogLineParser = {
192+
val jsonPrefixPostfix = JsonPrefixPostfix(JsonDetector())
193+
optFormatIn match {
194+
case Some(FormatIn.Logfmt) => LogfmtLogLineParser(config)
195+
case Some(FormatIn.Csv) =>
196+
throw new IllegalStateException(
197+
"method makeNonCsvLogLineParser does not support csv"
198+
)
199+
case _ => JsonLogLineParser(config, jsonPrefixPostfix)
200+
}
201+
}
202+
133203
def rawFilter(
134204
str: String,
135205
include: Option[List[String]],
136206
exclude: Option[List[String]]
137207
): Boolean = {
138-
val includeRegexes: List[Regex] = include.getOrElse(Nil).map(_.r)
139-
val excludeRegexes: List[Regex] = exclude.getOrElse(Nil).map(_.r)
208+
val includeRegexes: List[Regex] =
209+
include.getOrElse(Nil).map(_.r)
210+
val excludeRegexes: List[Regex] =
211+
exclude.getOrElse(Nil).map(_.r)
140212
val includeMatches = includeRegexes.isEmpty || includeRegexes.exists(
141213
_.findFirstIn(str).isDefined
142214
)
143215
val excludeMatches = excludeRegexes.forall(_.findFirstIn(str).isEmpty)
144216
includeMatches && excludeMatches
145217
}
146-
147218
}

0 commit comments

Comments
 (0)