forked from standardebooks/web
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Anonymize web logs after rotating and disable explicit download loggi…
…ng in favor of grepping the regular web logs
- Loading branch information
Showing
4 changed files
with
76 additions
and
16 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,72 @@ | ||
| #!/bin/bash | ||
|
|
||
| usage(){ | ||
| fmt <<EOF | ||
| DESCRIPTION | ||
| Moves Apache access log files into a by-month subdirectory, and gzip them. | ||
| This script must be run as root, and is generally run by the Apache rotatelogs subprocess as such. | ||
| Log files are moved to <LOG-DIR>/apache/YYYY-MM/ | ||
| USAGE | ||
| rotate-www-logs NEW-LOG-FILENAME | ||
| EOF | ||
| exit | ||
| } | ||
| die(){ printf "\033[0;7;31mError:\033[0m %s\n" "${1}" 1>&2; exit 1; } | ||
| if [ $# -eq 1 ]; then if [ "$1" = "--help" ] || [ "$1" = "-h" ]; then usage; fi fi | ||
| # End boilerplate | ||
|
|
||
| if [ $# -eq 0 ]; then | ||
| usage | ||
| fi | ||
|
|
||
| # Apache has a habit of starting this script twice, which can stomp on its own files | ||
| for pid in $(pidof -x rotate-www-logs); do | ||
| if [ "${pid}" != $$ ]; then | ||
| # We echo and exit instead of die() because Apache prints stderr to the log, but not stdout. We don't need this logged. | ||
| echo "rotate-www-logs is already running with PID ${pid}" | ||
| exit 1 | ||
| fi | ||
| done | ||
|
|
||
| # Prevent the loop from entering if no matches are found for the pattern | ||
| shopt -s nullglob | ||
|
|
||
| filenameBase=$(basename "$1" | sed --regexp-extended "s/\.[0-9]+$//") | ||
| directory=$(dirname "$1") | ||
|
|
||
| for filename in ${directory}/${filenameBase}.*; do | ||
| # When Apache calls this script, it passes the filename of the new log file it created. | ||
| # Thus, we check here to make sure we don't process and then delete the brand-new log file! | ||
| if [ "${filename}" != "$1" ]; then | ||
| # Apache log files can have data for more than one day. Here we pull out entries for different days into different files. | ||
| dates=$(grep --extended-regexp --only-matching "\[[0-9]{1,2}\/[a-zA-Z]{3}\/20[0-9]{2}" "${filename}" | sort -u) | ||
|
|
||
| while read -r line; do | ||
| logRawDate=$(echo "${line}" | sed "s/\[//g" | sed "s/\// /g") | ||
| logDate=$(date -d"${logRawDate}" "+%Y-%m-%d") | ||
| logMonth=$(date -d"${logRawDate}" "+%Y-%m") | ||
| grepString=${line//\[/} | ||
| logFilename="www-access-${logDate}.log" | ||
|
|
||
| mkdir -p "${directory}/${logMonth}" | ||
|
|
||
| # Is the log file already existing and gzipped? | ||
| if [ -f "${directory}/${logMonth}/${logFilename}.gz" ]; then | ||
| gunzip "${directory}/${logMonth}/${logFilename}.gz" | ||
| fi | ||
|
|
||
| # ipv6loganon is provided by the `ipv6calc` package | ||
| grep --extended-regexp "\[${grepString}" "${filename}" | ipv6loganon --anonymize-paranoid >> "${directory}/${logMonth}/${logFilename}" | ||
|
|
||
| gzip --best "${directory}/${logMonth}/${logFilename}" | ||
|
|
||
| chown --preserve-root --recursive www-data:adm "${directory}/${logMonth}" | ||
| chmod --preserve-root --recursive g+w "${directory}/${logMonth}" | ||
| done <<< "${dates}" | ||
|
|
||
| rm "${filename}" | ||
| fi | ||
| done |