Skip to content

Commit 9f8f1eb

Browse files
To-omTo-om
committed
Fix MISP modules integration
Enable MISP modules in docker package Send config to MISP modules Catch exception in MISP modules loader Update documentation
1 parent 7d676b2 commit 9f8f1eb

File tree

11 files changed

+278
-88
lines changed

11 files changed

+278
-88
lines changed

‎app/models/JsonFormat.scala

+2-1
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,8 @@ object JsonFormat {
1818
implicit val fileArtifactWrites: OWrites[FileArtifact] = OWrites[FileArtifact](fileArtifact ⇒ Json.obj(
1919
"attributes" → fileArtifact.attributes))
2020

21-
implicit val dataArtifactWrites: OWrites[DataArtifact] = Json.writes[DataArtifact]
21+
implicit val dataArtifactWrites: OWrites[DataArtifact] = OWrites[DataArtifact](artifact ⇒
22+
artifact.attributes + ("data" → JsString(artifact.data)))
2223
implicit val dataActifactReads: Reads[DataArtifact] = Json.reads[DataArtifact]
2324

2425
val artifactWrites: OWrites[Artifact] = OWrites[Artifact] {

‎app/models/MispModule.scala

+3-1
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,15 @@
11
package models
22

3+
import play.api.libs.json.JsObject
4+
35
case class MispModule(
46
name: String,
57
version: String,
68
description: String,
79
author: String,
810
dataTypeList: Seq[String],
911
inputAttributes: Seq[String],
10-
config: Seq[String],
12+
config: JsObject,
1113
loaderCommand: String) extends Analyzer {
1214
val license = "AGPL-3.0"
1315
val url = "https://github.com/MISP/misp-modules"

‎app/services/MispSrv.scala

+74-51
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,13 @@
11
package services
22

33
import java.io.{ ByteArrayInputStream, FileInputStream, InputStream, SequenceInputStream }
4-
import javax.inject.Inject
4+
import javax.inject.{ Inject, Singleton }
55

66
import akka.actor.ActorSystem
77
import models.JsonFormat._
88
import models._
99
import org.apache.commons.codec.binary.{ Base64, Base64InputStream }
10+
import util.JsonConfig
1011
import play.api.libs.json.{ Json, _ }
1112
import play.api.{ Configuration, Logger }
1213

@@ -15,8 +16,11 @@ import scala.concurrent.{ ExecutionContext, Future }
1516
import scala.sys.process._
1617
import scala.util.{ Failure, Success, Try }
1718

19+
@Singleton
1820
class MispSrv(
19-
loaderCommandOption: Option[String],
21+
mispModulesEnabled: Boolean,
22+
loaderCommand: String,
23+
mispModuleConfig: JsObject,
2024
externalAnalyzerSrv: ExternalAnalyzerSrv,
2125
jobSrv: JobSrv,
2226
akkaSystem: ActorSystem) {
@@ -26,33 +30,37 @@ class MispSrv(
2630
externalAnalyzerSrv: ExternalAnalyzerSrv,
2731
jobSrv: JobSrv,
2832
akkaSystem: ActorSystem) = this(
29-
configuration.getString("misp.modules.loader"),
33+
configuration.getBoolean("misp.modules.enabled").getOrElse(false),
34+
configuration.getString("misp.modules.loader").get,
35+
JsonConfig.configWrites.writes(configuration.getConfig("misp.modules.config").getOrElse(Configuration.empty)),
3036
externalAnalyzerSrv,
3137
jobSrv,
3238
akkaSystem)
3339

3440
private[MispSrv] lazy val logger = Logger(getClass)
3541
private[MispSrv] lazy val analyzeExecutionContext: ExecutionContext = akkaSystem.dispatchers.lookup("analyzer")
3642

37-
lazy val list: Seq[MispModule] =
38-
loaderCommandOption.fold(Seq.empty[MispModule]) { loaderCommand ⇒
39-
Json.parse(s"$loaderCommand --list".!!)
40-
.as[Seq[String]]
41-
.map { moduleName ⇒
42-
moduleName → (for {
43-
moduleInfo ← Try(Json.parse(s"$loaderCommand --info $moduleName".!!))
44-
module ← Try(moduleInfo.as[MispModule](reads(loaderCommand)))
45-
} yield module)
46-
}
47-
.flatMap {
48-
case (moduleName, Failure(error)) ⇒
49-
logger.warn(s"Load MISP module $moduleName fails", error)
50-
Nil
51-
case (_, Success(module)) ⇒
52-
logger.info(s"Register MISP module ${module.name} ${module.version}")
53-
Seq(module)
54-
}
55-
}
43+
logger.info(s"MISP modules is ${if (mispModulesEnabled) "enabled" else "disabled"}, loader is $loaderCommand")
44+
45+
lazy val list: Seq[MispModule] = if (mispModulesEnabled) {
46+
Json.parse(s"$loaderCommand --list".!!)
47+
.as[Seq[String]]
48+
.map { moduleName ⇒
49+
moduleName → (for {
50+
moduleInfo ← Try(Json.parse(s"$loaderCommand --info $moduleName".!!))
51+
module ← Try(moduleInfo.as[MispModule](reads(loaderCommand, mispModuleConfig)))
52+
} yield module)
53+
}
54+
.flatMap {
55+
case (moduleName, Failure(error)) ⇒
56+
logger.warn(s"Load MISP module $moduleName fails", error)
57+
Nil
58+
case (_, Success(module)) ⇒
59+
logger.info(s"Register MISP module ${module.name} ${module.version}")
60+
Seq(module)
61+
}
62+
}
63+
else Nil
5664

5765
def get(moduleName: String): Option[MispModule] = list.find(_.name == moduleName)
5866

@@ -89,30 +97,31 @@ class MispSrv(
8997
}
9098

9199
def query(module: String, mispType: String, data: String)(implicit ec: ExecutionContext): Future[JsObject] = {
92-
loaderCommandOption
93-
.flatMap { loaderCommand ⇒
94-
val artifact = toArtifact(mispType, data)
95-
get(module)
96-
.map { mispModule ⇒
97-
val mispReport = Future {
98-
val input = Json.obj(mispType → data)
99-
val output = (s"$loaderCommand --run $module" #< input.toString).!!
100-
Json.parse(output).as[JsObject]
101-
}
102-
jobSrv.create(mispModule, artifact, mispReport.map(toReport))
103-
mispReport
104-
100+
val artifact = toArtifact(mispType, data)
101+
val mispModule = if (mispModulesEnabled) {
102+
get(module)
103+
.map { mispModule ⇒
104+
val mispReport = Future {
105+
val input = Json.obj("config" → mispModule.config, mispType → data)
106+
val output = (s"$loaderCommand --run $module" #< input.toString).!!
107+
Json.parse(output).as[JsObject]
105108
}
106-
.orElse {
107-
externalAnalyzerSrv
108-
.get(module)
109-
.map { analyzer ⇒
110-
externalAnalyzerSrv.analyze(analyzer, artifact)
111-
.map { report ⇒ toMispOutput(report) }
112-
}
109+
jobSrv.create(mispModule, artifact, mispReport.map(toReport))
110+
mispReport
111+
112+
}
113+
}
114+
else None
115+
mispModule
116+
.orElse {
117+
externalAnalyzerSrv
118+
.get(module)
119+
.map { analyzer ⇒
120+
externalAnalyzerSrv.analyze(analyzer, artifact)
121+
.map { report ⇒ toMispOutput(report) }
113122
}
114123
}
115-
.getOrElse(Future.failed(new Exception(s"Module $module not found")))
124+
.getOrElse(Future.failed(new Exception(s"Module $module not found"))) // TODO add appropriate exception
116125
}
117126

118127
def analyze(module: MispModule, artifact: Artifact): Future[Report] = {
@@ -121,10 +130,13 @@ class MispSrv(
121130

122131
val input = artifact match {
123132
case DataArtifact(data, _) ⇒
124-
stringStream(Json.obj(dataType2mispType(artifact.dataType).head → data).toString)
133+
val mispType = dataType2mispType(artifact.dataType)
134+
.filter(module.inputAttributes.contains)
135+
.head
136+
stringStream((Json.obj("config" → module.config) + (mispType → JsString(data))).toString)
125137
case FileArtifact(data, _) ⇒
126138
new SequenceInputStream(Iterator(
127-
stringStream("""{"attachment":""""),
139+
stringStream(Json.obj("config" → module.config).toString.replaceFirst("}$", ""","attachment":"""")),
128140
new Base64InputStream(new FileInputStream(data), true),
129141
stringStream("\"}")).asJavaEnumeration)
130142
}
@@ -207,15 +219,26 @@ class MispSrv(
207219
else mispTypes
208220
}
209221

210-
private def reads(loaderCommand: String): Reads[MispModule] =
222+
private def reads(loaderCommand: String, mispModuleConfig: JsObject): Reads[MispModule] =
211223
for {
212224
name ← (__ \ "name").read[String]
213-
version ← (__ \ "meta" \ "version").read[String]
214-
description ← (__ \ "meta" \ "description").read[String]
215-
author ← (__ \ "meta" \ "author").read[String]
216-
config ← (__ \ "meta" \ "config").read[Seq[String]]
225+
version ← (__ \ "moduleinfo" \ "version").read[String]
226+
description ← (__ \ "moduleinfo" \ "description").read[String]
227+
author ← (__ \ "moduleinfo" \ "author").read[String]
228+
config = (mispModuleConfig \ name).asOpt[JsObject].getOrElse(JsObject(Nil))
229+
requiredConfig ← (__ \ "config").read[Set[String]]
230+
missingConfig = requiredConfig -- config.keys
231+
_ ← if (missingConfig.nonEmpty) {
232+
val message = s"MISP module $name is disabled because the following configuration " +
233+
s"item${if (missingConfig.size > 1) "s are" else " is"} missing: ${missingConfig.mkString(", ")}"
234+
logger.warn(message)
235+
Reads[Unit](_ ⇒ JsError(message))
236+
}
237+
else {
238+
Reads[Unit](_ ⇒ JsSuccess(()))
239+
}
217240
input ← (__ \ "mispattributes" \ "input").read[Seq[String]]
218-
dataTypes = input.map(mispType2dataType)
241+
dataTypes = input.map(mispType2dataType).distinct
219242
} yield MispModule(name, version, description, author, dataTypes, input, config, loaderCommand)
220243

221244
private val typeLookup = Map(

‎build.sbt

+17-6
Original file line numberDiff line numberDiff line change
@@ -43,14 +43,21 @@ mappings in Universal ~= {
4343
file("package/cortex.service") -> "package/cortex.service",
4444
file("package/cortex.conf") -> "package/cortex.conf",
4545
file("package/cortex") -> "package/cortex",
46-
file("package/logback.xml") -> "conf/logback.xml"
46+
file("package/logback.xml") -> "conf/logback.xml",
47+
file("contrib/misp-modules-loader.py") -> "contrib/misp-modules-loader.py"
4748
)
4849
}
4950

5051
// Package //
51-
maintainer := "Thomas Franco <[email protected]"
52-
packageSummary := "-"
53-
packageDescription := """--""".stripMargin
52+
maintainer := "TheHive Project <[email protected]>"
53+
packageSummary := "Powerful Observable Analysis Engine"
54+
packageDescription := """Cortex tries to solve a common problem frequently encountered by SOCs, CSIRTs and security
55+
| researchers in the course of threat intelligence, digital forensics and incident response: how to analyze
56+
| observables they have collected, at scale, by querying a single tool instead of several?
57+
|
58+
| Cortex, an open source and free software, has been created by TheHive Project for this very purpose. Observables,
59+
| such as IP and email addresses, URLs, domain names, files or hashes, can be analyzed one by one or in bulk mode
60+
| using a Web interface. Analysts can also automate these operations thanks to the Cortex REST API. """.stripMargin
5461
defaultLinuxInstallLocation := "/opt"
5562
linuxPackageMappings ~= { _.map { pm =>
5663
val mappings = pm.mappings.filterNot {
@@ -62,7 +69,7 @@ linuxPackageMappings ~= { _.map { pm =>
6269
file("package/cortex.conf") -> "/etc/init/cortex.conf",
6370
file("package/cortex") -> "/etc/init.d/cortex",
6471
file("conf/application.sample") -> "/etc/cortex/application.conf",
65-
file("conf/logback.xml") -> "/etc/cortex/logback.xml"
72+
file("package/logback.xml") -> "/etc/cortex/logback.xml"
6673
).withConfig()
6774
}
6875

@@ -125,7 +132,11 @@ dockerCommands ~= { dc =>
125132
"apt-get install -y --no-install-recommends python-pip python2.7-dev ssdeep libfuzzy-dev libfuzzy2 libimage-exiftool-perl libmagic1 build-essential git && " +
126133
"cd /opt && " +
127134
"git clone https://github.com/CERT-BDF/Cortex-Analyzers.git && " +
128-
"pip install $(sort -u Cortex-Analyzers/analyzers/*/requirements.txt)"),
135+
"pip install $(sort -u Cortex-Analyzers/analyzers/*/requirements.txt) && " +
136+
"apt-get install -y --no-install-recommends python3-setuptools python3-dev zlib1g-dev libxslt1-dev libxml2-dev libpq5 libjpeg-dev && git clone https://github.com/MISP/misp-modules.git && " +
137+
"easy_install3 pip && " +
138+
"(cd misp-modules && pip3 install -I -r REQUIREMENTS && pip3 install -I .) && " +
139+
"rm -rf misp_modules /var/lib/apt/lists/*"),
129140
Cmd("ADD", "var", "/var"),
130141
Cmd("ADD", "etc", "/etc"),
131142
ExecCmd("RUN", "chown", "-R", "daemon:daemon", "/var/log/cortex")) ++

‎conf/application.sample

+78-7
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,6 @@
77
analyzer {
88
path = "path/to/Cortex-Analyzers/analyzers"
99
config {
10-
global {
11-
proxy {
12-
#http="http://PROXY_ADDRESS:PORT",
13-
#https="http://PROXY_ADDRESS:PORT"
14-
}
15-
}
1610
CIRCLPassiveDNS {
1711
#user= "..."
1812
#password= "..."
@@ -79,4 +73,81 @@ analyzer {
7973
# Max number of threads available for analyze
8074
parallelism-max = 4
8175
}
82-
}
76+
}
77+
78+
misp.modules {
79+
enabled = true
80+
81+
config {
82+
shodan {
83+
#apikey = ""
84+
}
85+
eupi {
86+
#apikey = ""
87+
#url = ""
88+
}
89+
passivetotal {
90+
#username = ""
91+
#api_key = ""
92+
}
93+
dns {
94+
#nameserver = ""
95+
}
96+
whois {
97+
#server = ""
98+
#port = ""
99+
}
100+
sourcecache {
101+
#archivepath = ""
102+
}
103+
geoip_country {
104+
}
105+
circl_passivessl {
106+
#username = ""
107+
#password = ""
108+
}
109+
iprep {
110+
#apikey = ""
111+
}
112+
countrycode {
113+
}
114+
cve {
115+
}
116+
virustotal {
117+
#apikey = ""
118+
#event_limit = ""
119+
}
120+
ipasn {
121+
#host = ""
122+
#port = ""
123+
#db = ""
124+
}
125+
circl_passivedns {
126+
#username = ""
127+
#password = ""
128+
}
129+
vmray_submit {
130+
#apikey = ""
131+
#url = ""
132+
#shareable = ""
133+
#do_not_reanalyze = ""
134+
#do_not_include_vmrayjobids = ""
135+
}
136+
wiki {
137+
}
138+
domaintools {
139+
#username = ""
140+
#api_key = ""
141+
}
142+
reversedns {
143+
#nameserver = ""
144+
}
145+
threatminer {
146+
}
147+
asn_history {
148+
#host = ""
149+
#port = ""
150+
#db = ""
151+
}
152+
}
153+
}

‎conf/reference.conf

+7-1
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,20 @@
11
# handler for errors (transform exception to related http status code
22
play.http.errorHandler = services.ErrorHandler
33

4+
# MISP modules loader location
5+
misp.modules {
6+
enabled = false
7+
loader = ${play.server.dir}/"contrib/misp-modules-loader.py"
8+
}
9+
410
analyzer {
511
# Directory that holds analyzers
612
path = analyzers
713
# Analyzer configuration
814
config {
915
dummy = dummy
1016
}
11-
17+
1218
fork-join-executor {
1319
# Min number of threads available for analyze
1420
parallelism-min = 2

0 commit comments

Comments
 (0)