remove log4j references. change logging policy, 1GB max logs.

davidlgr · Aug 15, 2012 · 5b3bec2 · 5b3bec2
1 parent 78ff4a5
commit 5b3bec2
Show file tree

Hide file tree

Showing 5 changed files with 47 additions and 27 deletions.
diff --git a/bin/storm b/bin/storm
@@ -239,10 +239,10 @@ def nimbus(klass="backtype.storm.daemon.nimbus"):
     See Setting up a Storm cluster for more information.
     (https://github.com/nathanmarz/storm/wiki/Setting-up-a-Storm-cluster)
     """
-    cppaths = [STORM_DIR + "/log4j", STORM_DIR + "/conf"]
+    cppaths = [STORM_DIR + "/conf"]
     jvmopts = parse_args(confvalue("nimbus.childopts", cppaths)) + [
-        "-Dlogfile.name=nimbus.log",
-        "-Dlogback.configurationFile=logback/production.xml",
+        "-Dlogfile.name=nimbus",
+        "-Dlogback.configurationFile=" + STORM_DIR + "/logback/production.xml",
     ]
     exec_storm_class(
         klass, 
@@ -259,10 +259,10 @@ def supervisor(klass="backtype.storm.daemon.supervisor"):
     See Setting up a Storm cluster for more information.
     (https://github.com/nathanmarz/storm/wiki/Setting-up-a-Storm-cluster)
     """
-    cppaths = [STORM_DIR + "/log4j", STORM_DIR + "/conf"]
+    cppaths = [STORM_DIR + "/conf"]
     jvmopts = parse_args(confvalue("supervisor.childopts", cppaths)) + [
-        "-Dlogfile.name=supervisor.log",
-        "-Dlogback.configurationFile=logback/production.xml",
+        "-Dlogfile.name=supervisor",
+        "-Dlogback.configurationFile=" + STORM_DIR + "/logback/production.xml",
     ]
     exec_storm_class(
         klass, 
@@ -280,16 +280,16 @@ def ui():
     See Setting up a Storm cluster for more information.
     (https://github.com/nathanmarz/storm/wiki/Setting-up-a-Storm-cluster)
     """
-    cppaths = [STORM_DIR + "/log4j", STORM_DIR + "/conf"]
+    cppaths = [STORM_DIR + "/conf"]
     jvmopts = parse_args(confvalue("ui.childopts", cppaths)) + [
-        "-Dlogfile.name=ui.log",
-        "-Dlogback.configurationFile=logback/production.xml",
+        "-Dlogfile.name=ui",
+        "-Dlogback.configurationFile=" + STORM_DIR + "/logback/production.xml",
     ]
     exec_storm_class(
         "backtype.storm.ui.core", 
         jvmtype="-server", 
         jvmopts=jvmopts, 
-        extrajars=[STORM_DIR + "/log4j", STORM_DIR, STORM_DIR + "/conf"])
+        extrajars=[STORM_DIR, STORM_DIR + "/conf"])
 
 def drpc():
     """Syntax: [storm drpc]
@@ -300,12 +300,15 @@ def drpc():
     See Distributed RPC for more information.
     (https://github.com/nathanmarz/storm/wiki/Distributed-RPC)
     """
-    jvmopts = ["-Xmx768m", "-Dlogfile.name=drpc.log", "-Dlogback.configurationFile=logback/production.xml"]
+    jvmopts = ["-Xmx768m", 
+        "-Dlogfile.name=drpc", 
+        "-Dlogback.configurationFile=" + STORM_DIR + "/logback/production.xml"
+    ]
     exec_storm_class(
         "backtype.storm.daemon.drpc", 
         jvmtype="-server", 
         jvmopts=jvmopts, 
-        extrajars=[STORM_DIR + "/log4j", STORM_DIR + "/conf"])
+        extrajars=[STORM_DIR + "/conf"])
 
 def dev_zookeeper():
     """Syntax: [storm dev-zookeeper]
@@ -314,11 +317,11 @@ def dev_zookeeper():
     "storm.zookeeper.port" as its port. This is only intended for development/testing, the
     Zookeeper instance launched is not configured to be used in production.
     """
-    cppaths = [STORM_DIR + "/log4j", STORM_DIR + "/conf"]
+    cppaths = [STORM_DIR + "/conf"]
     exec_storm_class(
         "backtype.storm.command.dev_zookeeper", 
         jvmtype="-server", 
-        extrajars=[STORM_DIR + "/log4j", STORM_DIR + "/conf"])
+        extrajars=[STORM_DIR + "/conf"])
 
 def version():
   """Syntax: [storm version]

diff --git a/bin/to_maven.sh b/bin/to_maven.sh
@@ -1,22 +1,36 @@
-#!/bin/bash
+#!/bin/bash 
+function quit {
+    exit 1
+}
+trap quit 1 2 3 15  #Ctrl+C exits.
 
 RELEASE=`head -1 project.clj | awk '{print $3}' | sed -e 's/\"//' | sed -e 's/\"//'`
 
 rm -rf classes
-rm *jar
-rm *xml
+rm -f *jar
+rm -f *xml
 lein jar
 lein pom
 scp storm*jar pom.xml [email protected]:
 
 rm *jar
 rm -rf classes
-rm conf/log4j.properties
+rm conf/logback.xml
 lein jar
+
+cp project.clj orig-project.clj
+sed -i '' -e 's/\[.*logback[^]]*\]//g' project.clj
+sed -i '' -e 's/\[.*log4j-over-slf4j[^]]*\]//g' project.clj
+
+lein pom
+mv orig-project.clj project.clj
+
 mv pom.xml old-pom.xml
 sed 's/artifactId\>storm/artifactId\>storm-lib/g' old-pom.xml > pom.xml
 mv storm-$RELEASE.jar storm-lib-$RELEASE.jar
 scp storm*jar pom.xml [email protected]:
 rm *xml
 rm *jar
-git checkout conf/log4j.properties
+git checkout conf/logback.xml
+
+
diff --git a/logback/production.xml b/logback/production.xml
@@ -2,14 +2,17 @@
 
 <configuration scan="true" scanPeriod="60 seconds">
  <appender name="A1" class="ch.qos.logback.core.rolling.RollingFileAppender">
-    <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
-      <!-- daily rollover -->
-      <fileNamePattern>${storm.home}/logs/${logfile.name}.%d{yyyy-MM-dd}.log</fileNamePattern>
-
-      <!-- keep 7 days' worth of history -->
-      <maxHistory>7</maxHistory>
+    <file>${storm.home}/logs/${logfile.name}.log</file>
+    <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
+      <fileNamePattern>${storm.home}/logs/${logfile.name}.%i.log</fileNamePattern>
+      <minIndex>1</minIndex>
+      <maxIndex>9</maxIndex>
     </rollingPolicy>
 
+    <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
+      <maxFileSize>100MB</maxFileSize>
+    </triggeringPolicy>
+
     <encoder>
       <pattern>%d{yyyy-MM-dd HH:mm:ss} %c{1} [%p] %m%n</pattern>
     </encoder>

diff --git a/src/clj/backtype/storm/daemon/worker.clj b/src/clj/backtype/storm/daemon/worker.clj
@@ -323,7 +323,7 @@
   (log-message "Launching worker for " storm-id " on " supervisor-id ":" port " with id " worker-id
                " and conf " conf)
   (if-not (local-mode? conf)
-    (redirect-stdio-to-log4j!))
+    (redirect-stdio-to-slf4j!))
   ;; because in local mode, its not a separate
   ;; process. supervisor will register it in this case
   (when (= :distributed (cluster-mode conf))

diff --git a/src/clj/backtype/storm/util.clj b/src/clj/backtype/storm/util.clj
@@ -674,7 +674,7 @@
 (defn throw-runtime [& strs]
   (throw (RuntimeException. (apply str strs))))
 
-(defn redirect-stdio-to-log4j! []
+(defn redirect-stdio-to-slf4j! []
   ;; set-var-root doesn't work with *out* and *err*, so digging much deeper here
   ;; Unfortunately, this code seems to work at the REPL but not when spawned as worker processes
   ;; it might have something to do with being a child process