Improve diff

dump_dags.py

@@ -1,30 +1,30 @@
 from airflow.models import DagBag
 from collections import defaultdict
-import sys
+import os, sys
 
 
-def dump_dags(outfile):
+def dump_dags(output_base_path):
+    os.mkdirs(output_base_path, exist_ok=True)
     bag = DagBag()
     for dag_id in sorted(bag.dag_ids):
         dag = bag.get_dag(dag_id)
-        outfile.write(f'== {dag_id} ==\n\n')
 
-        adj = defaultdict(list)  # Adjacency list of DAG.
-        for task in dag.tasks:
-            for upstream_task_id in task.upstream_task_ids:
-                adj[upstream_task_id].append(task.task_id)
+        with open(os.path.join(output_base_path, dag_id), 'w') as outfile:
+            adj = defaultdict(list)  # Adjacency list of DAG.
+            for task in dag.tasks:
+                for upstream_task_id in task.upstream_task_ids:
+                    adj[upstream_task_id].append(task.task_id)
 
-        for task_id in sorted(dag.task_ids):
-            task = dag.get_task(task_id)
-            outfile.write(f'{task_id}\n')
-            for child_task_id in sorted(adj[task_id]):
-                outfile.write(f' -> {child_task_id}\n')
-            outfile.write('\n')
+            for task_id in sorted(dag.task_ids):
+                task = dag.get_task(task_id)
+                outfile.write(f'{task_id}\n')
+                for child_task_id in sorted(adj[task_id]):
+                    outfile.write(f' -> {child_task_id}\n')
+                outfile.write('\n')
 
 
 if __name__ == '__main__':
     if len(sys.argv) != 2:
-        print('Usage: dump_dags.py <output filename>')
+        print('Usage: dump_dags.py <output base path>')
         sys.exit(1)
-    with open(sys.argv[1], 'w') as outfile:
-        dump_dags(outfile)
+    dump_dags(sys.argv[1])

entrypoint.sh

@@ -8,12 +8,26 @@ export AIRFLOW__CORE__LOGS_FOLDER="/tmp/logs/"
 export FERNET_KEY=$(openssl rand -base64 32)
 echo Base ref is $GITHUB_BASE_REF
 airflow initdb
-python /dump_dags.py /tmp/current.txt
+python /dump_dags.py /tmp/current
 git checkout $GITHUB_BASE_REF
-python /dump_dags.py /tmp/base.txt
-DIFF=$(diff -u /tmp/base.txt /tmp/current.txt)
-DIFF="${DIFF//'%'/'%25'}"
-DIFF="${DIFF//$'\n'/'%0A'}"
-DIFF="${DIFF//$'\r'/'%0D'}"
-echo "::set-output name=diff::$DIFF"
+python /dump_dags.py /tmp/base
+DAG_IDS=$(basename -a /tmp/base/* /tmp/current/* | sort | uniq)
+SUMMARY=""
+for dag_id in DAG_IDS; do
+    if [[ ! -f "/tmp/base/$dag_id" ]]; then
+        SUMMARY+="DAG added: $dag_id\n\n"
+    elif [[ ! -f "/tmp/current/$dag_id" ]]; then
+        SUMMARY+="DAG deleted: $dag_id\n\n"
+    else
+        DIFF=$(diff -u /tmp/base.txt /tmp/current.txt)
+        retVal=$?
+        if [ $retVal -ne 0 ]; then
+            SUMMARY+="DAG modified: $dag_id\n\n$DIFF\n\n"
+        fi
+    fi
+done
+SUMMARY="${SUMMARY//'%'/'%25'}"
+SUMMARY="${SUMMARY//$'\n'/'%0A'}"
+SUMMARY="${SUMMARY//$'\r'/'%0D'}"
+echo "::set-output name=diff::$SUMMARY"
 exit 0