diff --git a/README.md b/README.md index 7e2c0c3..353d97b 100644 --- a/README.md +++ b/README.md @@ -29,19 +29,21 @@ The plugin also includes other custom REST APIs. * v1.0.0 * v1.0.1 * v1.0.2 + * v1.0.3 * Branches Available: * master * v0.0.2-branch * v1.0.0-branch * v1.0.1-branch * v1.0.2-branch + * v1.0.3-branch * ULR to Download From: https://github.com/teamclairvoyant/airflow-rest-api-plugin/archive/{RELEASE_VERSION_OR_BRANCH_NAME}.zip + + * Note: Each release/branch has its own README.md file that describes the specific options and steps you should take to deploy and configure. Verify the options available in each release/branch after you download it. 3. Unzip the file and move the contents of the plugins folder into your Airflow plugins directory - - mkdir -p {AIRFLOW_PLUGINS_FOLDER} unzip airflow-rest-api-plugin-{RELEASE_VERSION_OR_BRANCH_NAME}.zip @@ -55,6 +57,10 @@ The plugin also includes other custom REST APIs. [rest_api_plugin] + # Logs global variables used in the REST API plugin when the plugin is loaded. Set to False by default to avoid too many logging messages. + # DEFAULT: False + log_loading = False + # Filters out loading messages from the standard out # DEFAULT: True filter_loading_messages_in_cli_response = True diff --git a/RELEASE_PROCEDURE.md b/RELEASE_PROCEDURE.md index c43f8f1..2c5ed6a 100644 --- a/RELEASE_PROCEDURE.md +++ b/RELEASE_PROCEDURE.md @@ -7,6 +7,20 @@ This document describes the branching procedure and how to create new releases o 1. (if not already done) Create a new branch for all the changes you want in the release * Branch naming convention: v{VERSION}-branch * Example: v1.0.0-branch + * Steps to Create Branch: + 1. Checkout the master + * $ git checkout master + 2. Make sure your local copy of the master branch is up to date + * $ git pull + 3. Create the branch: + * $ git branch {BRANCH_NAME} + 4. Switch to the new branch + * $ git checkout {BRANCH_NAME} + 5. Make any desired first commit changes + 6. Commit branch + * $ git commit -m " Creating new branch {BRANCH_NAME} and updating documents" + 7. Push Branch + * $ git push --set-upstream origin {BRANCH_NAME} 2. Push all your changes into the Branch * For Pull Requests from External Contributors, those changes will also go into the Release Branch and not the Master 3. Create a Pull Request to merge the changes from the Branch into Master diff --git a/plugins/rest_api_plugin.py b/plugins/rest_api_plugin.py index 0bb77c1..036652c 100644 --- a/plugins/rest_api_plugin.py +++ b/plugins/rest_api_plugin.py @@ -1,5 +1,5 @@ __author__ = 'robertsanders' -__version__ = "1.0.2" +__version__ = "1.0.3" from airflow.models import DagBag, DagModel from airflow.plugins_manager import AirflowPlugin @@ -35,20 +35,26 @@ airflow_webserver_base_url = configuration.get('webserver', 'BASE_URL') airflow_base_log_folder = configuration.get('core', 'BASE_LOG_FOLDER') airflow_dags_folder = configuration.get('core', 'DAGS_FOLDER') +log_loading = configuration.getboolean("rest_api_plugin", "LOG_LOADING") if configuration.has_option("rest_api_plugin", "LOG_LOADING") else False +filter_loading_messages_in_cli_response = configuration.getboolean("rest_api_plugin", "FILTER_LOADING_MESSAGES_IN_CLI_RESPONSE") if configuration.has_option("rest_api_plugin", "FILTER_LOADING_MESSAGES_IN_CLI_RESPONSE") else True airflow_rest_api_plugin_http_token_header_name = configuration.get("rest_api_plugin", "REST_API_PLUGIN_HTTP_TOKEN_HEADER_NAME") if configuration.has_option("rest_api_plugin", "REST_API_PLUGIN_HTTP_TOKEN_HEADER_NAME") else "rest_api_plugin_http_token" airflow_expected_http_token = configuration.get("rest_api_plugin", "REST_API_PLUGIN_EXPECTED_HTTP_TOKEN") if configuration.has_option("rest_api_plugin", "REST_API_PLUGIN_EXPECTED_HTTP_TOKEN") else None -filter_loading_messages_in_cli_response = configuration.get("rest_api_plugin", "FILTER_LOADING_MESSAGES_IN_CLI_RESPONSE") if configuration.has_option("rest_api_plugin", "FILTER_LOADING_MESSAGES_IN_CLI_RESPONSE") else "True" # Using UTF-8 Encoding so that response messages don't have any characters in them that can't be handled os.environ['PYTHONIOENCODING'] = 'utf-8' -logging.info("Initializing Airflow REST API Plugin with configs:") -logging.info("\tairflow_webserver_base_url: " + str(airflow_webserver_base_url)) -logging.info("\tairflow_base_log_folder: " + str(airflow_base_log_folder)) -logging.info("\tairflow_dags_folder: " + str(airflow_dags_folder)) -logging.info("\tairflow_rest_api_plugin_http_token_header_name: " + str(airflow_rest_api_plugin_http_token_header_name)) -logging.info("\tairflow_expected_http_token: OMITTED FOR SECURITY") -logging.info("\tfilter_loading_messages_in_cli_response: " + str(filter_loading_messages_in_cli_response)) +if log_loading: + logging.info("Initializing Airflow REST API Plugin with configs:") + logging.info("\trest_api_endpoint: " + str(rest_api_endpoint)) + logging.info("\thostname: " + str(hostname)) + logging.info("\tairflow_version: " + str(airflow_version)) + logging.info("\trest_api_plugin_version: " + str(rest_api_plugin_version)) + logging.info("\tairflow_webserver_base_url: " + str(airflow_webserver_base_url)) + logging.info("\tairflow_base_log_folder: " + str(airflow_base_log_folder)) + logging.info("\tairflow_dags_folder: " + str(airflow_dags_folder)) + logging.info("\tairflow_rest_api_plugin_http_token_header_name: " + str(airflow_rest_api_plugin_http_token_header_name)) + logging.info("\tairflow_expected_http_token: OMITTED_FOR_SECURITY") + logging.info("\tfilter_loading_messages_in_cli_response: " + str(filter_loading_messages_in_cli_response)) """ Metadata that defines a single API: @@ -669,7 +675,7 @@ def execute_cli(self, base_response, api_metadata): output = self.execute_cli_command(airflow_cmd_split) # if desired, filter out the loading messages to reduce the noise in the output - if filter_loading_messages_in_cli_response and filter_loading_messages_in_cli_response.lower() == "true": + if filter_loading_messages_in_cli_response: logging.info("Filtering Loading Messages from the CLI Response") output = self.filter_loading_messages(output) @@ -736,7 +742,7 @@ def deploy_dag(self, base_response): if unpause: airflow_cmd_split = ["airflow", "unpause", dag_id] cli_output = self.execute_cli_command(airflow_cmd_split) - except Exception, e: + except Exception as e: warning = "Failed to set the state (pause, unpause) of the DAG: " + str(e) logging.warning(warning) else: @@ -761,7 +767,7 @@ def refresh_dag(self, base_response): # NOTE: The request argument 'dag_id' is required for the refresh() function to get the dag_id refresh_result = Airflow().refresh() logging.info("Refresh Result: " + str(refresh_result)) - except Exception, e: + except Exception as e: error_message = "An error occurred while trying to Refresh the DAG '" + str(dag_id) + "': " + str(e) logging.error(error_message) return REST_API_Response_Util.get_500_error_response(base_response, error_message)