From 5b3bec2362ab1df8316a9bdd2b4a6b05b1ed1fe1 Mon Sep 17 00:00:00 2001 From: Jason Jackson Date: Wed, 15 Aug 2012 12:25:02 -0700 Subject: [PATCH] remove log4j references. change logging policy, 1GB max logs. --- bin/storm | 31 +++++++++++++----------- bin/to_maven.sh | 24 ++++++++++++++---- logback/production.xml | 15 +++++++----- src/clj/backtype/storm/daemon/worker.clj | 2 +- src/clj/backtype/storm/util.clj | 2 +- 5 files changed, 47 insertions(+), 27 deletions(-) diff --git a/bin/storm b/bin/storm index c779ebc8291..baf39bd5870 100755 --- a/bin/storm +++ b/bin/storm @@ -239,10 +239,10 @@ def nimbus(klass="backtype.storm.daemon.nimbus"): See Setting up a Storm cluster for more information. (https://github.com/nathanmarz/storm/wiki/Setting-up-a-Storm-cluster) """ - cppaths = [STORM_DIR + "/log4j", STORM_DIR + "/conf"] + cppaths = [STORM_DIR + "/conf"] jvmopts = parse_args(confvalue("nimbus.childopts", cppaths)) + [ - "-Dlogfile.name=nimbus.log", - "-Dlogback.configurationFile=logback/production.xml", + "-Dlogfile.name=nimbus", + "-Dlogback.configurationFile=" + STORM_DIR + "/logback/production.xml", ] exec_storm_class( klass, @@ -259,10 +259,10 @@ def supervisor(klass="backtype.storm.daemon.supervisor"): See Setting up a Storm cluster for more information. (https://github.com/nathanmarz/storm/wiki/Setting-up-a-Storm-cluster) """ - cppaths = [STORM_DIR + "/log4j", STORM_DIR + "/conf"] + cppaths = [STORM_DIR + "/conf"] jvmopts = parse_args(confvalue("supervisor.childopts", cppaths)) + [ - "-Dlogfile.name=supervisor.log", - "-Dlogback.configurationFile=logback/production.xml", + "-Dlogfile.name=supervisor", + "-Dlogback.configurationFile=" + STORM_DIR + "/logback/production.xml", ] exec_storm_class( klass, @@ -280,16 +280,16 @@ def ui(): See Setting up a Storm cluster for more information. (https://github.com/nathanmarz/storm/wiki/Setting-up-a-Storm-cluster) """ - cppaths = [STORM_DIR + "/log4j", STORM_DIR + "/conf"] + cppaths = [STORM_DIR + "/conf"] jvmopts = parse_args(confvalue("ui.childopts", cppaths)) + [ - "-Dlogfile.name=ui.log", - "-Dlogback.configurationFile=logback/production.xml", + "-Dlogfile.name=ui", + "-Dlogback.configurationFile=" + STORM_DIR + "/logback/production.xml", ] exec_storm_class( "backtype.storm.ui.core", jvmtype="-server", jvmopts=jvmopts, - extrajars=[STORM_DIR + "/log4j", STORM_DIR, STORM_DIR + "/conf"]) + extrajars=[STORM_DIR, STORM_DIR + "/conf"]) def drpc(): """Syntax: [storm drpc] @@ -300,12 +300,15 @@ def drpc(): See Distributed RPC for more information. (https://github.com/nathanmarz/storm/wiki/Distributed-RPC) """ - jvmopts = ["-Xmx768m", "-Dlogfile.name=drpc.log", "-Dlogback.configurationFile=logback/production.xml"] + jvmopts = ["-Xmx768m", + "-Dlogfile.name=drpc", + "-Dlogback.configurationFile=" + STORM_DIR + "/logback/production.xml" + ] exec_storm_class( "backtype.storm.daemon.drpc", jvmtype="-server", jvmopts=jvmopts, - extrajars=[STORM_DIR + "/log4j", STORM_DIR + "/conf"]) + extrajars=[STORM_DIR + "/conf"]) def dev_zookeeper(): """Syntax: [storm dev-zookeeper] @@ -314,11 +317,11 @@ def dev_zookeeper(): "storm.zookeeper.port" as its port. This is only intended for development/testing, the Zookeeper instance launched is not configured to be used in production. """ - cppaths = [STORM_DIR + "/log4j", STORM_DIR + "/conf"] + cppaths = [STORM_DIR + "/conf"] exec_storm_class( "backtype.storm.command.dev_zookeeper", jvmtype="-server", - extrajars=[STORM_DIR + "/log4j", STORM_DIR + "/conf"]) + extrajars=[STORM_DIR + "/conf"]) def version(): """Syntax: [storm version] diff --git a/bin/to_maven.sh b/bin/to_maven.sh index 7013829f192..659d4c48e06 100644 --- a/bin/to_maven.sh +++ b/bin/to_maven.sh @@ -1,22 +1,36 @@ -#!/bin/bash +#!/bin/bash +function quit { + exit 1 +} +trap quit 1 2 3 15 #Ctrl+C exits. RELEASE=`head -1 project.clj | awk '{print $3}' | sed -e 's/\"//' | sed -e 's/\"//'` rm -rf classes -rm *jar -rm *xml +rm -f *jar +rm -f *xml lein jar lein pom scp storm*jar pom.xml clojars@clojars.org: rm *jar rm -rf classes -rm conf/log4j.properties +rm conf/logback.xml lein jar + +cp project.clj orig-project.clj +sed -i '' -e 's/\[.*logback[^]]*\]//g' project.clj +sed -i '' -e 's/\[.*log4j-over-slf4j[^]]*\]//g' project.clj + +lein pom +mv orig-project.clj project.clj + mv pom.xml old-pom.xml sed 's/artifactId\>storm/artifactId\>storm-lib/g' old-pom.xml > pom.xml mv storm-$RELEASE.jar storm-lib-$RELEASE.jar scp storm*jar pom.xml clojars@clojars.org: rm *xml rm *jar -git checkout conf/log4j.properties +git checkout conf/logback.xml + + diff --git a/logback/production.xml b/logback/production.xml index e643ab0cb37..792721333d6 100644 --- a/logback/production.xml +++ b/logback/production.xml @@ -2,14 +2,17 @@ - - - ${storm.home}/logs/${logfile.name}.%d{yyyy-MM-dd}.log - - - 7 + ${storm.home}/logs/${logfile.name}.log + + ${storm.home}/logs/${logfile.name}.%i.log + 1 + 9 + + 100MB + + %d{yyyy-MM-dd HH:mm:ss} %c{1} [%p] %m%n diff --git a/src/clj/backtype/storm/daemon/worker.clj b/src/clj/backtype/storm/daemon/worker.clj index 7e6162e3d9b..2219ff44e28 100644 --- a/src/clj/backtype/storm/daemon/worker.clj +++ b/src/clj/backtype/storm/daemon/worker.clj @@ -323,7 +323,7 @@ (log-message "Launching worker for " storm-id " on " supervisor-id ":" port " with id " worker-id " and conf " conf) (if-not (local-mode? conf) - (redirect-stdio-to-log4j!)) + (redirect-stdio-to-slf4j!)) ;; because in local mode, its not a separate ;; process. supervisor will register it in this case (when (= :distributed (cluster-mode conf)) diff --git a/src/clj/backtype/storm/util.clj b/src/clj/backtype/storm/util.clj index 6e6a18ed189..02c413128df 100644 --- a/src/clj/backtype/storm/util.clj +++ b/src/clj/backtype/storm/util.clj @@ -674,7 +674,7 @@ (defn throw-runtime [& strs] (throw (RuntimeException. (apply str strs)))) -(defn redirect-stdio-to-log4j! [] +(defn redirect-stdio-to-slf4j! [] ;; set-var-root doesn't work with *out* and *err*, so digging much deeper here ;; Unfortunately, this code seems to work at the REPL but not when spawned as worker processes ;; it might have something to do with being a child process