Skip to content

Commit 0d66f4f

Browse files
jclausenjclausen
committed
Appender service tests and docs
1 parent 2d3e62a commit 0d66f4f

File tree

4 files changed

+297
-3
lines changed

4 files changed

+297
-3
lines changed

‎changelog.md

+1
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
1212

1313
* Add `labels` property to Logstash Appender properties to allow for custom labels
1414
* Add `labels` convention to Logstash Appender UserInfo UDF to allow keyword label filtering
15+
* Add `AppenderService` object which allows for easier creation of detached appenders and index-specific logging
1516
* Improved error handling to include the status code in the error message if a "reason" could not be extracted from the response: `Elasticsearch server responded with [504 Gateway Timeout]. The response received was not JSON.`.
1617

1718
## [3.3.0] - 04-18-2024

‎docs/Logging.md

+139-1
Original file line numberDiff line numberDiff line change
@@ -61,4 +61,142 @@ logBox = {
6161
};
6262
```
6363

64-
For more information on configuring log appenders for your application, see the [Coldbox documentation](https://coldbox.ortusbooks.com/getting-started/configuration/coldbox.cfc/configuration-directives/logbox)
64+
For more information on configuring LogBox log appenders for your application, see the [Coldbox documentation](https://coldbox.ortusbooks.com/getting-started/configuration/coldbox.cfc/configuration-directives/logbox)
65+
66+
67+
# Detached Appenders
68+
69+
The logging capbilities of the Elasticsearch module extend beyond the framework LogBox appenders. In a era of big data an analytics, developers also have the ability to create custom appenders appenders for ad-hoc use in storing messages, collecting metrics, or for use in aggregations. Simple messages can be logged or even raw messages which adhere to the [Elastic Common Schema](https://www.elastic.co/guide/en/ecs/current/index.html). By using detached appenders, you can capture custom information for later reference.
70+
71+
## Creating a Detached Appender
72+
73+
To create a detached appender, use the `AppenderService` method `createDetachedAppender( string name, struct properties )`. The properties passed can be any of the above, or you can omit those and the default properties will be used:
74+
```
75+
getInstance( "AppenderService@cbelasticsearch" )
76+
.createDetachedAppender(
77+
"myCustomAppender",
78+
{
79+
80+
"retentionDays" : 30,
81+
"applicationName" : "Custom Appender Logs",
82+
"rolloverSize" : "1gb"
83+
}
84+
);
85+
```
86+
87+
Now we can log messages to this appender on an ad-hoc basis by calling the methods in the Appender service.
88+
89+
### Logging a single message
90+
91+
We can log a traditional single message by using the `logToAppender` method of the Appender service. This is familiar to many Coldbox developers:
92+
93+
```java
94+
getInstance( "AppenderService@cbelasticsearch" )
95+
.logToAppender(
96+
"myCustomAppender",
97+
"This is my custom log message which contains information I need to search later",
98+
"info",
99+
{
100+
// labels are stored as exact match keywords which allow you aggregate and filter the log messages
101+
// These are promoted to the root labels object
102+
"labels" : {
103+
"manager" : "Jim Leyland",
104+
"team" : "Detroit Tigers"
105+
},
106+
// Any other key value pairs become part of the log entry extra info, which is searchable but not filterable
107+
"person" : {
108+
"firstName" : "Jim",
109+
"lastName" : "Leyland",
110+
"teams" : [
111+
"Detroit Tigers",
112+
"Pittsburg Pirates",
113+
"Colorado Rockies"
114+
],
115+
"hallOfFamer" : true,
116+
"inductionYear" : 2024
117+
}
118+
}
119+
);
120+
```
121+
122+
123+
### Logging one or more raw formatted messages
124+
125+
If you are comfortable assembling your own JSON and want to ship those log entries to elasticsearch raw, you can do so by usin the `logRawToAppender` method of the Appender service. This functionality can also allow you to assemble a series of logs and ship them all off in one bulk operation. The `messages` argument to the function may be a single entry struct, or it may be an array of multiple message which adhere to the [Elastic Common Schema](https://www.elastic.co/guide/en/ecs/current/index.html).
126+
127+
```java
128+
getInstance( "AppenderService@cbelasticsearch" )
129+
.logRawToAppender(
130+
"myCustomAppender",
131+
[
132+
{
133+
"@timestamp" : dateTimeFormat( now(), "yyyy-mm-dd'T'HH:nn:ssZZ" ),
134+
"log" : {
135+
"level" : "info",
136+
"logger" : "myCustomLogger",
137+
"category" : "CustomEvents"
138+
},
139+
"message" : "This is my custom log message which contains information I need to search later",
140+
"event" : {
141+
"action" : event.getCurrentAction(),
142+
"duration" : myProcessingDurationNanos,
143+
"created" : dateTimeFormat( now(), "yyyy-mm-dd'T'HH:nn:ssZZ" ),
144+
"severity" : 4,
145+
"category" : "myCustomLogger",
146+
"dataset" : "cfml",
147+
"timezone" : createObject( "java", "java.util.TimeZone" ).getDefault().getId()
148+
},
149+
"file" : { "path" : CGI.CF_TEMPLATE_PATH },
150+
"url" : {
151+
"domain" : CGI.SERVER_NAME,
152+
"path" : CGI.PATH_INFO,
153+
"port" : CGI.SERVER_PORT,
154+
"query" : CGI.QUERY_STRING,
155+
"scheme" : lCase( listFirst( CGI.SERVER_PROTOCOL, "/" ) )
156+
},
157+
"http" : {
158+
"request" : { "referer" : CGI.HTTP_REFERER },
159+
},
160+
"labels" : {
161+
"manager" : "Jim Leyland",
162+
"team" : "Detroit Tigers",
163+
"hallOfFameYear" : "2024"
164+
},
165+
"package" : {
166+
"name" : getProperty( "applicationName" ),
167+
"version" : "1.1.0",
168+
"type" : "cfml",
169+
"path" : expandPath( "/" )
170+
},
171+
"host" : { "name" : CGI.HTTP_HOST, "hostname" : CGI.SERVER_NAME },
172+
"client" : { "ip" : CGI.REMOTE_ADDR },
173+
"user" : {},
174+
"user_agent" : { "original" : CGI.HTTP_USER_AGENT },
175+
"error" : {
176+
"type" : "message",
177+
"level" : level,
178+
"message" : loge.getMessage(),
179+
"extrainfo" : serializeJSON(
180+
{
181+
"person" : {
182+
"firstName" : "Jim",
183+
"lastName" : "Leyland",
184+
"teams" : [
185+
"Detroit Tigers",
186+
"Pittsburg Pirates",
187+
"Colorado Rockies"
188+
],
189+
"hallOfFamer" : true,
190+
"inductionYear" : 2024
191+
}
192+
}
193+
)
194+
}
195+
},
196+
... and so on ...
197+
]
198+
);
199+
```
200+
201+
See our docs [on search](https://cbelasticsearch.ortusbooks.com/searching/search) and [aggregations](https://cbelasticsearch.ortusbooks.com/searching/aggregations) for more information on how to assemble custom reports and aggregations of your logged data.
202+

‎models/logging/AppenderService.cfc

+11-2
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ component accessors="true" singleton{
44
property name="util" inject="Util@cbelasticsearch";
55
property name="detachedAppenders";
66

7+
this.logLevels = new coldbox.system.logging.LogLevels();
8+
79
function init(){
810
variables.detachedAppenders = [];
911
return this;
@@ -31,7 +33,7 @@ component accessors="true" singleton{
3133
variables.logBox.registerAppender(
3234
name = arguments.name,
3335
class = arguments.class,
34-
// Turn this appender off for all other logging, as it is intended to use ad-hoc
36+
// Turn this appender off for all other logging, as it is intended to be used ad-hoc
3537
levelMin = -1,
3638
levelMax = -1,
3739
properties = arguments.properties
@@ -71,10 +73,17 @@ component accessors="true" singleton{
7173
public function logToAppender(
7274
required string appenderName,
7375
required string message,
74-
required numeric severity,
76+
required any severity,
7577
struct extraInfo = {},
7678
string category,
7779
){
80+
if( !isNumeric( arguments.severity ) ){
81+
if( !this.logLevels.keyExists( arguments.severity ) ){
82+
throw( type = "cbelasticsearch.logging.InvalidSeverity", message = "The severity [#arguments.severity#] provided is not valid. Please provide a valid numeric serverity or one of the following levels [#this.logLevels.keyArray().toList()#]." );
83+
}
84+
arguments.severity = this.logLevels[ arguments.severity ];
85+
}
86+
7887
var appender = getAppender( appenderName );
7988
if ( !isNull( appender ) ) {
8089
appender.logMessage(

‎test-harness/tests/specs/unit/AppenderServiceTest.cfc

+146
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
component extends="coldbox.system.testing.BaseTestCase" {
2+
3+
function beforeAll(){
4+
this.loadColdbox = true;
5+
6+
super.beforeAll();
7+
variables.esClient = getWirebox().getInstance( "Client@cbelasticsearch" );
8+
variables.model = getMockBox().createMock( className = "cbelasticsearch.models.logging.AppenderService" );
9+
variables.model.init();
10+
getWirebox().autowire( variables.model );
11+
12+
variables.testEntries = mockData(
13+
$num = 10,
14+
"log.level" : "oneof:info:warn:error",
15+
"message" : "sentence"
16+
);
17+
18+
}
19+
20+
function afterAll(){
21+
var detachedAppenders = variables.model.getDetachedAppenders();
22+
23+
detachedAppenders.each( function( appenderName ){
24+
var appender = variables.model.getAppender( appenderName );
25+
if( !isNull( appender ) ){
26+
if( esClient.dataStreamExists( appender.getProperty( "dataStream" ) ) ){
27+
esClient.deleteDataStream( appender.getProperty( "dataStream" ) );
28+
}
29+
if( esClient.indexTemplateExists( appender.getProperty( "indexTemplateName" ) ) ){
30+
esClient.deleteIndexTemplate( appender.getProperty( "indexTemplateName" ) );
31+
}
32+
33+
if( esClient.componentTemplateExists( appender.getProperty( "componentTemplateName" ) ) ){
34+
esClient.deleteComponentTemplate( appender.getProperty( "componentTemplateName" ) );
35+
}
36+
37+
if( esClient.ILMPolicyExists( appender.getProperty( "ILMPolicyName" ) ) ){
38+
esClient.deleteILMPolicy( appender.getProperty( "ILMPolicyName" ) );
39+
}
40+
}
41+
} );
42+
43+
super.afterAll();
44+
}
45+
46+
function run(){
47+
describe( "Tests detached appenders", function(){
48+
it( "Tests the ability to create a detached appender", function(){
49+
var appenderName = "detachedAppenderTest";
50+
variables.model.createDetachedAppender(
51+
appenderName,
52+
{
53+
// The data stream name to use for this appenders logs
54+
"dataStreamPattern" : "logs-coldbox-#lcase( appenderName )#*",
55+
"dataStream" : "logs-coldbox-#lcase( appenderName )#",
56+
"ILMPolicyName" : "cbelasticsearch-logs-#lcase( appenderName )#",
57+
"indexTemplateName" : "cbelasticsearch-logs-#lcase( appenderName )#",
58+
"componentTemplateName" : "cbelasticsearch-logs-#lcase( appenderName )#",
59+
"pipelineName" : "cbelasticsearch-logs-#lcase( appenderName )#",
60+
"indexTemplatePriority" : 151,
61+
"retentionDays" : 1,
62+
// The name of the application which will be transmitted with the log data and used for grouping
63+
"applicationName" : "Detached Test Appender Logs",
64+
// The max shard size at which the hot phase will rollover data
65+
"rolloverSize" : "1gb"
66+
}
67+
);
68+
var createdAppender = variables.model.getAppender( appenderName );
69+
expect( isNull( createdAppender) ).toBeFalse();
70+
71+
expect( createdAppender.getProperty( "dataStream" ) ).toBe( "logs-coldbox-#lcase( appenderName )#" );
72+
73+
} );
74+
75+
describe( "Perform actions on detached appender", function(){
76+
var appenderName = "detachedAppenderTest";
77+
beforeEach( function(){
78+
var appender = variables.model.getAppender( appenderName );
79+
if( isNull( appender ) ){
80+
variables.model.createDetachedAppender(
81+
appenderName,
82+
{
83+
// The data stream name to use for this appenders logs
84+
"dataStreamPattern" : "logs-coldbox-#lcase( appenderName )#*",
85+
"dataStream" : "logs-coldbox-#lcase( appenderName )#",
86+
"ILMPolicyName" : "cbelasticsearch-logs-#lcase( appenderName )#",
87+
"indexTemplateName" : "cbelasticsearch-logs-#lcase( appenderName )#",
88+
"componentTemplateName" : "cbelasticsearch-logs-#lcase( appenderName )#",
89+
"pipelineName" : "cbelasticsearch-logs-#lcase( appenderName )#",
90+
"indexTemplatePriority" : 151,
91+
"retentionDays" : 1,
92+
// The name of the application which will be transmitted with the log data and used for grouping
93+
"applicationName" : "Detached Test Appender Logs",
94+
// The max shard size at which the hot phase will rollover data
95+
"rolloverSize" : "1gb"
96+
}
97+
);
98+
}
99+
});
100+
101+
it( "Tests the ability to log a single message to the appender", function(){
102+
var appender = variables.model.getAppender( appenderName );
103+
var dataStreamCount = getDataStreamCount( appender.getProperty( "dataStreamPattern" ) );
104+
variables.model.logToAppender(
105+
appenderName,
106+
"Test message",
107+
4
108+
);
109+
sleep( 1000 );
110+
expect( getDataStreamCount( appender.getProperty( "dataStreamPattern" ) ) ).toBe( dataStreamCount + 1 );
111+
} );
112+
113+
it( "Tests the ability to log a single raw message to the appender", function(){
114+
var appender = variables.model.getAppender( appenderName );
115+
var dataStreamCount = getDataStreamCount( appender.getProperty( "dataStreamPattern" ) );
116+
variables.model.logRawToAppender(
117+
appenderName,
118+
variables.testEntries.first(),
119+
true
120+
);
121+
expect( getDataStreamCount( appender.getProperty( "dataStreamPattern" ) ) ).toBe( dataStreamCount + 1 );
122+
} );
123+
124+
125+
126+
it( "Tests the ability to log a multiple raw message to the appender", function(){
127+
var appender = variables.model.getAppender( appenderName );
128+
var dataStreamCount = getDataStreamCount( appender.getProperty( "dataStreamPattern" ) );
129+
variables.model.logRawToAppender(
130+
appenderName,
131+
variables.testEntries,
132+
true
133+
);
134+
expect( getDataStreamCount( appender.getProperty( "dataStreamPattern" ) ) ).toBe( dataStreamCount + variables.testEntries.len() );
135+
} );
136+
137+
138+
} );
139+
} );
140+
}
141+
142+
function getDataStreamCount( required string dataStreamPattern ){
143+
return getWirebox().getInstance( "SearchBuilder@cbelasticsearch" ).setIndex( dataStreamPattern ).setQuery( { "match_all" : {} } ).count();
144+
}
145+
146+
}

0 commit comments

Comments
 (0)