From b076d629abca99dc6d0000752e3255499c86a07b Mon Sep 17 00:00:00 2001 From: rssanders3 Date: Wed, 17 May 2017 12:15:19 -0700 Subject: [PATCH 1/5] Creating new branch v1.0.3-branch and updating documents --- README.md | 2 ++ RELEASE_PROCEDURE.md | 14 ++++++++++++++ plugins/rest_api_plugin.py | 2 +- 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7e2c0c3..fb2580b 100644 --- a/README.md +++ b/README.md @@ -29,12 +29,14 @@ The plugin also includes other custom REST APIs. * v1.0.0 * v1.0.1 * v1.0.2 + * v1.0.3 * Branches Available: * master * v0.0.2-branch * v1.0.0-branch * v1.0.1-branch * v1.0.2-branch + * v1.0.3-branch * ULR to Download From: https://github.com/teamclairvoyant/airflow-rest-api-plugin/archive/{RELEASE_VERSION_OR_BRANCH_NAME}.zip diff --git a/RELEASE_PROCEDURE.md b/RELEASE_PROCEDURE.md index c43f8f1..17ec30c 100644 --- a/RELEASE_PROCEDURE.md +++ b/RELEASE_PROCEDURE.md @@ -7,6 +7,20 @@ This document describes the branching procedure and how to create new releases o 1. (if not already done) Create a new branch for all the changes you want in the release * Branch naming convention: v{VERSION}-branch * Example: v1.0.0-branch + * Steps to Create Branch: + 1. Checkout the master + * $ git checkout master + 2. Make sure your local copy of the master branch is up to date + * $ git pull + 3. Create the branch: + * $ git branch {BRANCH_NAME} + 4. Switch to the new branch + * $ git checkout {BRANCH_NAME} + 5. Make any desired first commit changes + 6. Commit branch + * $ git commit -m " Creating new branch {BRANCH_NAME} and updating documents" + 7. Push Branch + * $ git push 2. Push all your changes into the Branch * For Pull Requests from External Contributors, those changes will also go into the Release Branch and not the Master 3. Create a Pull Request to merge the changes from the Branch into Master diff --git a/plugins/rest_api_plugin.py b/plugins/rest_api_plugin.py index 0bb77c1..1a9919d 100644 --- a/plugins/rest_api_plugin.py +++ b/plugins/rest_api_plugin.py @@ -1,5 +1,5 @@ __author__ = 'robertsanders' -__version__ = "1.0.2" +__version__ = "1.0.3" from airflow.models import DagBag, DagModel from airflow.plugins_manager import AirflowPlugin From 239e8390b320bdb255cde805d6f7d5b441d9ab4e Mon Sep 17 00:00:00 2001 From: rssanders3 Date: Wed, 17 May 2017 12:16:32 -0700 Subject: [PATCH 2/5] Updating RELEASE_PROCEDURE.md to add more details about creating the branch --- RELEASE_PROCEDURE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_PROCEDURE.md b/RELEASE_PROCEDURE.md index 17ec30c..2c5ed6a 100644 --- a/RELEASE_PROCEDURE.md +++ b/RELEASE_PROCEDURE.md @@ -20,7 +20,7 @@ This document describes the branching procedure and how to create new releases o 6. Commit branch * $ git commit -m " Creating new branch {BRANCH_NAME} and updating documents" 7. Push Branch - * $ git push + * $ git push --set-upstream origin {BRANCH_NAME} 2. Push all your changes into the Branch * For Pull Requests from External Contributors, those changes will also go into the Release Branch and not the Master 3. Create a Pull Request to merge the changes from the Branch into Master From ebad6a472183f884281e0eb8f50c7c67b50d693d Mon Sep 17 00:00:00 2001 From: Ivan Danov Date: Mon, 22 May 2017 10:46:14 +0100 Subject: [PATCH 3/5] Catch exceptions in python3 compatible way The syntax for exception handling in python3 has changed: http://python3porting.com/noconv.html#handling-exceptions --- plugins/rest_api_plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/rest_api_plugin.py b/plugins/rest_api_plugin.py index 0bb77c1..e81f421 100644 --- a/plugins/rest_api_plugin.py +++ b/plugins/rest_api_plugin.py @@ -736,7 +736,7 @@ def deploy_dag(self, base_response): if unpause: airflow_cmd_split = ["airflow", "unpause", dag_id] cli_output = self.execute_cli_command(airflow_cmd_split) - except Exception, e: + except Exception as e: warning = "Failed to set the state (pause, unpause) of the DAG: " + str(e) logging.warning(warning) else: @@ -761,7 +761,7 @@ def refresh_dag(self, base_response): # NOTE: The request argument 'dag_id' is required for the refresh() function to get the dag_id refresh_result = Airflow().refresh() logging.info("Refresh Result: " + str(refresh_result)) - except Exception, e: + except Exception as e: error_message = "An error occurred while trying to Refresh the DAG '" + str(dag_id) + "': " + str(e) logging.error(error_message) return REST_API_Response_Util.get_500_error_response(base_response, error_message) From 85a491aa148f2d38df6942838d11df931344a179 Mon Sep 17 00:00:00 2001 From: rssanders3 Date: Mon, 22 May 2017 11:55:55 -0700 Subject: [PATCH 4/5] Providing an option for people to skip the initialization/loading message of the rest api plugin code. Also adding more information to the loading message. Updating readme. --- README.md | 6 ++++-- plugins/rest_api_plugin.py | 24 +++++++++++++++--------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index fb2580b..44ce6e7 100644 --- a/README.md +++ b/README.md @@ -42,8 +42,6 @@ The plugin also includes other custom REST APIs. https://github.com/teamclairvoyant/airflow-rest-api-plugin/archive/{RELEASE_VERSION_OR_BRANCH_NAME}.zip 3. Unzip the file and move the contents of the plugins folder into your Airflow plugins directory - - mkdir -p {AIRFLOW_PLUGINS_FOLDER} unzip airflow-rest-api-plugin-{RELEASE_VERSION_OR_BRANCH_NAME}.zip @@ -57,6 +55,10 @@ The plugin also includes other custom REST APIs. [rest_api_plugin] + # Logs global variables used in the REST API plugin when the plugin is loaded. Set to False by default to avoid too many logging messages. + # DEFAULT: False + log_loading = False + # Filters out loading messages from the standard out # DEFAULT: True filter_loading_messages_in_cli_response = True diff --git a/plugins/rest_api_plugin.py b/plugins/rest_api_plugin.py index 1750ce2..036652c 100644 --- a/plugins/rest_api_plugin.py +++ b/plugins/rest_api_plugin.py @@ -35,20 +35,26 @@ airflow_webserver_base_url = configuration.get('webserver', 'BASE_URL') airflow_base_log_folder = configuration.get('core', 'BASE_LOG_FOLDER') airflow_dags_folder = configuration.get('core', 'DAGS_FOLDER') +log_loading = configuration.getboolean("rest_api_plugin", "LOG_LOADING") if configuration.has_option("rest_api_plugin", "LOG_LOADING") else False +filter_loading_messages_in_cli_response = configuration.getboolean("rest_api_plugin", "FILTER_LOADING_MESSAGES_IN_CLI_RESPONSE") if configuration.has_option("rest_api_plugin", "FILTER_LOADING_MESSAGES_IN_CLI_RESPONSE") else True airflow_rest_api_plugin_http_token_header_name = configuration.get("rest_api_plugin", "REST_API_PLUGIN_HTTP_TOKEN_HEADER_NAME") if configuration.has_option("rest_api_plugin", "REST_API_PLUGIN_HTTP_TOKEN_HEADER_NAME") else "rest_api_plugin_http_token" airflow_expected_http_token = configuration.get("rest_api_plugin", "REST_API_PLUGIN_EXPECTED_HTTP_TOKEN") if configuration.has_option("rest_api_plugin", "REST_API_PLUGIN_EXPECTED_HTTP_TOKEN") else None -filter_loading_messages_in_cli_response = configuration.get("rest_api_plugin", "FILTER_LOADING_MESSAGES_IN_CLI_RESPONSE") if configuration.has_option("rest_api_plugin", "FILTER_LOADING_MESSAGES_IN_CLI_RESPONSE") else "True" # Using UTF-8 Encoding so that response messages don't have any characters in them that can't be handled os.environ['PYTHONIOENCODING'] = 'utf-8' -logging.info("Initializing Airflow REST API Plugin with configs:") -logging.info("\tairflow_webserver_base_url: " + str(airflow_webserver_base_url)) -logging.info("\tairflow_base_log_folder: " + str(airflow_base_log_folder)) -logging.info("\tairflow_dags_folder: " + str(airflow_dags_folder)) -logging.info("\tairflow_rest_api_plugin_http_token_header_name: " + str(airflow_rest_api_plugin_http_token_header_name)) -logging.info("\tairflow_expected_http_token: OMITTED FOR SECURITY") -logging.info("\tfilter_loading_messages_in_cli_response: " + str(filter_loading_messages_in_cli_response)) +if log_loading: + logging.info("Initializing Airflow REST API Plugin with configs:") + logging.info("\trest_api_endpoint: " + str(rest_api_endpoint)) + logging.info("\thostname: " + str(hostname)) + logging.info("\tairflow_version: " + str(airflow_version)) + logging.info("\trest_api_plugin_version: " + str(rest_api_plugin_version)) + logging.info("\tairflow_webserver_base_url: " + str(airflow_webserver_base_url)) + logging.info("\tairflow_base_log_folder: " + str(airflow_base_log_folder)) + logging.info("\tairflow_dags_folder: " + str(airflow_dags_folder)) + logging.info("\tairflow_rest_api_plugin_http_token_header_name: " + str(airflow_rest_api_plugin_http_token_header_name)) + logging.info("\tairflow_expected_http_token: OMITTED_FOR_SECURITY") + logging.info("\tfilter_loading_messages_in_cli_response: " + str(filter_loading_messages_in_cli_response)) """ Metadata that defines a single API: @@ -669,7 +675,7 @@ def execute_cli(self, base_response, api_metadata): output = self.execute_cli_command(airflow_cmd_split) # if desired, filter out the loading messages to reduce the noise in the output - if filter_loading_messages_in_cli_response and filter_loading_messages_in_cli_response.lower() == "true": + if filter_loading_messages_in_cli_response: logging.info("Filtering Loading Messages from the CLI Response") output = self.filter_loading_messages(output) From 3a9722407c31a6c2975692c9ab71a2f308b69a1d Mon Sep 17 00:00:00 2001 From: rssanders3 Date: Mon, 22 May 2017 11:59:44 -0700 Subject: [PATCH 5/5] Updating the README file to add a disclaimer that the documentation might be different accross versions. --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 44ce6e7..353d97b 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,8 @@ The plugin also includes other custom REST APIs. * ULR to Download From: https://github.com/teamclairvoyant/airflow-rest-api-plugin/archive/{RELEASE_VERSION_OR_BRANCH_NAME}.zip + + * Note: Each release/branch has its own README.md file that describes the specific options and steps you should take to deploy and configure. Verify the options available in each release/branch after you download it. 3. Unzip the file and move the contents of the plugins folder into your Airflow plugins directory