Skip to content

Commit 2a7448d

Browse files
nices96nices96
committed
Add Elaspsed Time, GC Time and Thread Count thresholds for create
alert(s).
1 parent 5dac969 commit 2a7448d

File tree

2 files changed

+153
-2
lines changed

2 files changed

+153
-2
lines changed

‎README.md

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,9 @@
99
- 신규 Agent 연결
1010
- Agent의 연결 해제
1111
- Agent의 재접속
12+
- 응답시간의 임계치 초과
13+
- GC Time의 임계치 초과
14+
- Thread 갯수의 임계치 초과
1215

1316
### Properties (스카우터 서버 설치 경로 하위의 conf/scouter.conf)
1417
* **_ext\_plugin\_email\_send_alert_** : Email 발송 여부 (true / false) - 기본 값은 false
@@ -22,6 +25,11 @@
2225
* **_ext\_plugin\_email\_from_address_** : Email 발신자 계정
2326
* **_ext\_plugin\_email\_to_address_** : Email 수신 계정(다중 사용자 지정 시 ',' 구분자 사용)
2427
* **_ext\_plugin\_email\_cc_address_** : Email 참조 수신 계정(다중 사용자 지정 시 ',' 구분자 사용)
28+
* **_ext\_plugin\_elapsed\_time_threshold_** : 응답시간의 임계치 (ms) - 기본 값은 0으로, 0일때 응답시간의 임계치 초과 여부를 확인하지 않는다.
29+
* **_ext\_plugin\_gc\_time_threshold_** : GC Time의 임계치 (ms) - 기본 값은 0으로, 0일때 GC Time의 임계치 초과 여부를 확인하지 않는다.
30+
* **_ext\_plugin\_thread\_count_threshold_** : Thread Count의 임계치 (ms) - 기본 값은 0으로, 0일때 Thread Count의 임계치 초과 여부를 확인하지 않는다.
31+
32+
2533

2634
* Example
2735
```
@@ -37,6 +45,10 @@ ext_plugin_email_tls_enabled=true
3745
ext_plugin_email_from_address=noreply@scouter.com
3846
ext_plugin_email_to_address=receiver1@scouter.com,receiver2@scouter.com
3947
ext_plugin_email_cc_address=ccreceiver@yopmail.com
48+
49+
ext_plugin_elapsed_time_threshold=5000
50+
ext_plugin_gc_time_threshold=5000
51+
ext_plugin_thread_count_threshold=300
4052
```
4153

4254
### Dependencies

‎src/scouter/plugin/server/alert/email/EmailPlugin.java

Lines changed: 141 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,18 +17,37 @@
1717
*/
1818
package scouter.plugin.server.alert.email;
1919

20+
import java.util.ArrayList;
21+
import java.util.List;
22+
import java.util.concurrent.Executors;
23+
import java.util.concurrent.ScheduledExecutorService;
24+
import java.util.concurrent.TimeUnit;
25+
import java.util.concurrent.atomic.AtomicInteger;
26+
2027
import org.apache.commons.mail.DefaultAuthenticator;
2128
import org.apache.commons.mail.Email;
2229
import org.apache.commons.mail.SimpleEmail;
2330

2431
import scouter.lang.AlertLevel;
32+
import scouter.lang.TextTypes;
33+
import scouter.lang.TimeTypeEnum;
34+
import scouter.lang.counters.CounterConstants;
2535
import scouter.lang.pack.AlertPack;
36+
import scouter.lang.pack.MapPack;
2637
import scouter.lang.pack.ObjectPack;
38+
import scouter.lang.pack.PerfCounterPack;
39+
import scouter.lang.pack.XLogPack;
2740
import scouter.lang.plugin.PluginConstants;
2841
import scouter.lang.plugin.annotation.ServerPlugin;
42+
import scouter.net.RequestCmd;
2943
import scouter.server.Configure;
44+
import scouter.server.CounterManager;
3045
import scouter.server.Logger;
3146
import scouter.server.core.AgentManager;
47+
import scouter.server.db.TextRD;
48+
import scouter.server.netio.AgentCall;
49+
import scouter.util.DateUtil;
50+
import scouter.util.HashUtil;
3251

3352
/**
3453
* Scouter server plugin to send alert via email
@@ -39,6 +58,48 @@ public class EmailPlugin {
3958

4059
// Get singleton Configure instance from server
4160
final Configure conf = Configure.getInstance();
61+
62+
private static AtomicInteger ai = new AtomicInteger(0);
63+
private static List<Integer> javaeeObjHashList = new ArrayList<Integer>();
64+
65+
public EmailPlugin() {
66+
if (ai.incrementAndGet() == 1) {
67+
ScheduledExecutorService executor = Executors.newScheduledThreadPool(1);
68+
69+
// thread count check
70+
executor.scheduleAtFixedRate(new Runnable() {
71+
@Override
72+
public void run() {
73+
for (int objHash : javaeeObjHashList) {
74+
if (AgentManager.isActive(objHash)) {
75+
ObjectPack objectPack = AgentManager.getAgent(objHash);
76+
MapPack mapPack = new MapPack();
77+
mapPack.put("objHash", objHash);
78+
79+
mapPack = AgentCall.call(objectPack, RequestCmd.OBJECT_THREAD_LIST, mapPack);
80+
81+
int threadCountThreshold = conf.getInt("ext_plugin_thread_count_threshold", 0);
82+
int threadCount = mapPack.getList("name").size();
83+
84+
if (threadCountThreshold != 0 && threadCount > threadCountThreshold) {
85+
AlertPack ap = new AlertPack();
86+
87+
ap.level = AlertLevel.WARN;
88+
ap.objHash = objHash;
89+
ap.title = "Thread count exceed a threahold.";
90+
ap.message = objectPack.objName + "'s Thread count(" + threadCount + ") exceed a threshold.";
91+
ap.time = System.currentTimeMillis();
92+
ap.objType = objectPack.objType;
93+
94+
alert(ap);
95+
}
96+
}
97+
}
98+
}
99+
},
100+
0, 5, TimeUnit.SECONDS);
101+
}
102+
}
42103

43104
@ServerPlugin(PluginConstants.PLUGIN_SERVER_ALERT)
44105
public void alert(final AlertPack pack) {
@@ -150,7 +211,12 @@ public void object(ObjectPack pack) {
150211
ap.title = "An object has been activated.";
151212
ap.message = pack.objName + " is connected.";
152213
ap.time = System.currentTimeMillis();
153-
ap.objType = "scouter";
214+
215+
if (AgentManager.getAgent(pack.objHash) != null) {
216+
ap.objType = AgentManager.getAgent(pack.objHash).objType;
217+
} else {
218+
ap.objType = "scouter";
219+
}
154220

155221
alert(ap);
156222
} else if (op.alive == false) {
@@ -161,13 +227,86 @@ public void object(ObjectPack pack) {
161227
ap.title = "An object has been activated.";
162228
ap.message = pack.objName + " is reconnected.";
163229
ap.time = System.currentTimeMillis();
164-
ap.objType = "scouter";
230+
ap.objType = AgentManager.getAgent(pack.objHash).objType;
165231

166232
alert(ap);
167233
}
168234
// inactive state can be handled in alert() method.
169235
}
170236
}
237+
238+
@ServerPlugin(PluginConstants.PLUGIN_SERVER_XLOG)
239+
public void xlog(XLogPack pack) {
240+
try {
241+
int elapsedThreshold = conf.getInt("ext_plugin_elapsed_time_threshold", 0);
242+
243+
if (elapsedThreshold != 0 && pack.elapsed > elapsedThreshold) {
244+
String serviceName = TextRD.getString(DateUtil.yyyymmdd(pack.endTime), TextTypes.SERVICE, pack.service);
245+
246+
AlertPack ap = new AlertPack();
247+
248+
ap.level = AlertLevel.WARN;
249+
ap.objHash = pack.objHash;
250+
ap.title = "Elapsed time exceed a threahold.";
251+
ap.message = "[" + AgentManager.getAgentName(pack.objHash) + "] "
252+
+ pack.service + "(" + serviceName + ") "
253+
+ "elapsed time(" + pack.elapsed + " ms) exceed a threshold.";
254+
ap.time = System.currentTimeMillis();
255+
ap.objType = AgentManager.getAgent(pack.objHash).objType;
256+
257+
alert(ap);
258+
}
259+
260+
} catch (Exception e) {
261+
Logger.printStackTrace(e);
262+
}
263+
}
264+
265+
@ServerPlugin(PluginConstants.PLUGIN_SERVER_COUNTER)
266+
public void counter(PerfCounterPack pack) {
267+
String objName = pack.objName;
268+
int objHash = HashUtil.hash(objName);
269+
String objType = null;
270+
String objFamily = null;
271+
272+
if (AgentManager.getAgent(objHash) != null) {
273+
objType = AgentManager.getAgent(objHash).objType;
274+
}
275+
276+
if (objType != null) {
277+
objFamily = CounterManager.getInstance().getCounterEngine().getObjectType(objType).getFamily().getName();
278+
}
279+
280+
try {
281+
// in case of objFamily is javaee
282+
if (CounterConstants.FAMILY_JAVAEE.equals(objFamily)) {
283+
// save javaee type's objHash
284+
if (!javaeeObjHashList.contains(objHash)) {
285+
javaeeObjHashList.add(objHash);
286+
}
287+
288+
if (pack.timetype == TimeTypeEnum.REALTIME) {
289+
long gcTimeThreshold = conf.getLong("ext_plugin_gc_time_threshold", 0);
290+
long gcTime = pack.data.getLong(CounterConstants.JAVA_GC_TIME);
291+
292+
if (gcTimeThreshold != 0 && gcTime > gcTimeThreshold) {
293+
AlertPack ap = new AlertPack();
294+
295+
ap.level = AlertLevel.WARN;
296+
ap.objHash = objHash;
297+
ap.title = "Elapsed time exceed a threahold.";
298+
ap.message = objName + "'s GC time(" + gcTime + " ms) exceed a threshold.";
299+
ap.time = System.currentTimeMillis();
300+
ap.objType = objType;
301+
302+
alert(ap);
303+
}
304+
}
305+
}
306+
} catch (Exception e) {
307+
Logger.printStackTrace(e);
308+
}
309+
}
171310

172311
private void println(Object o) {
173312
if (conf.getBoolean("ext_plugin_email_debug", false)) {

0 commit comments

Comments
 (0)