diff --git a/.gitignore b/.gitignore
index c01f70b4b4bcf..3208b1e56bf63 100644
--- a/.gitignore
+++ b/.gitignore
@@ -54,6 +54,7 @@ sql/docs
sql/site
lib_managed/
lint-r-report.log
+lint-js-report.log
log/
logs/
out/
@@ -105,3 +106,6 @@ spark-warehouse/
# For SBT
.jvmopts
+
+# For Node.js
+node_modules
diff --git a/core/src/main/resources/org/apache/spark/ui/static/executorspage.js b/core/src/main/resources/org/apache/spark/ui/static/executorspage.js
index 9133ef87a8a8e..2055c8ff11882 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/executorspage.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/executorspage.js
@@ -15,114 +15,120 @@
* limitations under the License.
*/
+/* global $, Mustache, createRESTEndPointForExecutorsPage, createRESTEndPointForMiscellaneousProcess, */
+/* global createTemplateURI, formatBytes, formatDuration, formatLogsCells, getStandAloneAppId, */
+/* global jQuery, setDataTableDefaults */
+
var threadDumpEnabled = false;
+/* eslint-disable no-unused-vars */
function setThreadDumpEnabled(val) {
- threadDumpEnabled = val;
+ threadDumpEnabled = val;
}
+/* eslint-enable no-unused-vars */
function getThreadDumpEnabled() {
- return threadDumpEnabled;
+ return threadDumpEnabled;
}
function formatStatus(status, type, row) {
- if (row.isExcluded) {
- return "Excluded";
- }
+ if (row.isExcluded) {
+ return "Excluded";
+ }
- if (status) {
- if (row.excludedInStages.length == 0) {
- return "Active"
- }
- return "Active (Excluded in Stages: [" + row.excludedInStages.join(", ") + "])";
+ if (status) {
+ if (row.excludedInStages.length == 0) {
+ return "Active"
}
- return "Dead"
+ return "Active (Excluded in Stages: [" + row.excludedInStages.join(", ") + "])";
+ }
+ return "Dead"
}
function formatProcessStatus(activeStatus) {
- if (activeStatus) {
- return "Active"
- }
- return "Dead"
+ if (activeStatus) {
+ return "Active"
+ }
+ return "Dead"
}
function formatResourceCells(resources) {
- var result = ""
- var count = 0
- $.each(resources, function (name, resInfo) {
- if (count > 0) {
- result += ", "
- }
- result += name + ': [' + resInfo.addresses.join(", ") + ']'
- count += 1
- });
- return result
+ var result = ""
+ var count = 0
+ $.each(resources, function (name, resInfo) {
+ if (count > 0) {
+ result += ", "
+ }
+ result += name + ': [' + resInfo.addresses.join(", ") + ']';
+ count += 1
+ });
+ return result
}
jQuery.extend(jQuery.fn.dataTableExt.oSort, {
- "title-numeric-pre": function (a) {
- var x = a.match(/title="*(-?[0-9\.]+)/)[1];
- return parseFloat(x);
- },
-
- "title-numeric-asc": function (a, b) {
- return ((a < b) ? -1 : ((a > b) ? 1 : 0));
- },
-
- "title-numeric-desc": function (a, b) {
- return ((a < b) ? 1 : ((a > b) ? -1 : 0));
- }
+ "title-numeric-pre": function (a) {
+ var x = a.match(/title="*(-?[0-9.]+)/)[1];
+ return parseFloat(x);
+ },
+
+ "title-numeric-asc": function (a, b) {
+ return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+ },
+
+ "title-numeric-desc": function (a, b) {
+ return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+ }
});
$(document).ajaxStop($.unblockUI);
$(document).ajaxStart(function () {
- $.blockUI({message: 'Loading Executors Page...
'});
+ $.blockUI({message: 'Loading Executors Page...
'});
});
function logsExist(execs) {
- return execs.some(function(exec) {
- return !($.isEmptyObject(exec["executorLogs"]));
- });
+ return execs.some(function(exec) {
+ return !($.isEmptyObject(exec["executorLogs"]));
+ });
}
// Determine Color Opacity from 0.5-1
// activeTasks range from 0 to maxTasks
function activeTasksAlpha(activeTasks, maxTasks) {
- return maxTasks > 0 ? ((activeTasks / maxTasks) * 0.5 + 0.5) : 1;
+ return maxTasks > 0 ? ((activeTasks / maxTasks) * 0.5 + 0.5) : 1;
}
function activeTasksStyle(activeTasks, maxTasks) {
- return activeTasks > 0 ? ("hsla(240, 100%, 50%, " + activeTasksAlpha(activeTasks, maxTasks) + ")") : "";
+ return activeTasks > 0 ? ("hsla(240, 100%, 50%, " + activeTasksAlpha(activeTasks, maxTasks) + ")") : "";
}
// failedTasks range max at 10% failure, alpha max = 1
function failedTasksAlpha(failedTasks, totalTasks) {
- return totalTasks > 0 ?
- (Math.min(10 * failedTasks / totalTasks, 1) * 0.5 + 0.5) : 1;
+ return totalTasks > 0 ?
+ (Math.min(10 * failedTasks / totalTasks, 1) * 0.5 + 0.5) : 1;
}
function failedTasksStyle(failedTasks, totalTasks) {
- return failedTasks > 0 ?
- ("hsla(0, 100%, 50%, " + failedTasksAlpha(failedTasks, totalTasks) + ")") : "";
+ return failedTasks > 0 ?
+ ("hsla(0, 100%, 50%, " + failedTasksAlpha(failedTasks, totalTasks) + ")") : "";
}
// totalDuration range from 0 to 50% GC time, alpha max = 1
function totalDurationAlpha(totalGCTime, totalDuration) {
- return totalDuration > 0 ?
- (Math.min(totalGCTime / totalDuration + 0.5, 1)) : 1;
+ return totalDuration > 0 ?
+ (Math.min(totalGCTime / totalDuration + 0.5, 1)) : 1;
}
// When GCTimePercent is edited change ToolTips.TASK_TIME to match
var GCTimePercent = 0.1;
function totalDurationStyle(totalGCTime, totalDuration) {
- // Red if GC time over GCTimePercent of total time
- return (totalGCTime > GCTimePercent * totalDuration) ?
- ("hsla(0, 100%, 50%, " + totalDurationAlpha(totalGCTime, totalDuration) + ")") : "";
+ // Red if GC time over GCTimePercent of total time
+ return (totalGCTime > GCTimePercent * totalDuration) ?
+ ("hsla(0, 100%, 50%, " + totalDurationAlpha(totalGCTime, totalDuration) + ")") : "";
}
function totalDurationColor(totalGCTime, totalDuration) {
- return (totalGCTime > GCTimePercent * totalDuration) ? "white" : "black";
+ return (totalGCTime > GCTimePercent * totalDuration) ? "white" : "black";
}
var sumOptionalColumns = [3, 4];
@@ -131,622 +137,624 @@ var execDataTable;
var sumDataTable;
function reselectCheckboxesBasedOnTaskTableState() {
- var allChecked = true;
- if (typeof execDataTable !== "undefined") {
- for (var k = 0; k < execOptionalColumns.length; k++) {
- if (execDataTable.column(execOptionalColumns[k]).visible()) {
- $("[data-exec-col-idx=" + execOptionalColumns[k] + "]").prop("checked", true);
- } else {
- allChecked = false;
- }
- }
- }
- if (allChecked) {
- $("#select-all-box").prop("checked", true);
+ var allChecked = true;
+ if (typeof execDataTable !== "undefined") {
+ for (var k = 0; k < execOptionalColumns.length; k++) {
+ if (execDataTable.column(execOptionalColumns[k]).visible()) {
+ $("[data-exec-col-idx=" + execOptionalColumns[k] + "]").prop("checked", true);
+ } else {
+ allChecked = false;
+ }
}
+ }
+ if (allChecked) {
+ $("#select-all-box").prop("checked", true);
+ }
}
$(document).ready(function () {
- setDataTableDefaults();
-
- var executorsSummary = $("#active-executors");
-
- getStandAloneAppId(function (appId) {
-
- var endPoint = createRESTEndPointForExecutorsPage(appId);
- $.getJSON(endPoint, function (response, status, jqXHR) {
- var allExecCnt = 0;
- var allRDDBlocks = 0;
- var allMemoryUsed = 0;
- var allMaxMemory = 0;
- var allOnHeapMemoryUsed = 0;
- var allOnHeapMaxMemory = 0;
- var allOffHeapMemoryUsed = 0;
- var allOffHeapMaxMemory = 0;
- var allDiskUsed = 0;
- var allTotalCores = 0;
- var allMaxTasks = 0;
- var allActiveTasks = 0;
- var allFailedTasks = 0;
- var allCompletedTasks = 0;
- var allTotalTasks = 0;
- var allTotalDuration = 0;
- var allTotalGCTime = 0;
- var allTotalInputBytes = 0;
- var allTotalShuffleRead = 0;
- var allTotalShuffleWrite = 0;
- var allTotalExcluded = 0;
-
- var activeExecCnt = 0;
- var activeRDDBlocks = 0;
- var activeMemoryUsed = 0;
- var activeMaxMemory = 0;
- var activeOnHeapMemoryUsed = 0;
- var activeOnHeapMaxMemory = 0;
- var activeOffHeapMemoryUsed = 0;
- var activeOffHeapMaxMemory = 0;
- var activeDiskUsed = 0;
- var activeTotalCores = 0;
- var activeMaxTasks = 0;
- var activeActiveTasks = 0;
- var activeFailedTasks = 0;
- var activeCompletedTasks = 0;
- var activeTotalTasks = 0;
- var activeTotalDuration = 0;
- var activeTotalGCTime = 0;
- var activeTotalInputBytes = 0;
- var activeTotalShuffleRead = 0;
- var activeTotalShuffleWrite = 0;
- var activeTotalExcluded = 0;
-
- var deadExecCnt = 0;
- var deadRDDBlocks = 0;
- var deadMemoryUsed = 0;
- var deadMaxMemory = 0;
- var deadOnHeapMemoryUsed = 0;
- var deadOnHeapMaxMemory = 0;
- var deadOffHeapMemoryUsed = 0;
- var deadOffHeapMaxMemory = 0;
- var deadDiskUsed = 0;
- var deadTotalCores = 0;
- var deadMaxTasks = 0;
- var deadActiveTasks = 0;
- var deadFailedTasks = 0;
- var deadCompletedTasks = 0;
- var deadTotalTasks = 0;
- var deadTotalDuration = 0;
- var deadTotalGCTime = 0;
- var deadTotalInputBytes = 0;
- var deadTotalShuffleRead = 0;
- var deadTotalShuffleWrite = 0;
- var deadTotalExcluded = 0;
-
- response.forEach(function (exec) {
- var memoryMetrics = {
- usedOnHeapStorageMemory: 0,
- usedOffHeapStorageMemory: 0,
- totalOnHeapStorageMemory: 0,
- totalOffHeapStorageMemory: 0
- };
-
- exec.memoryMetrics = exec.hasOwnProperty('memoryMetrics') ? exec.memoryMetrics : memoryMetrics;
- });
-
- response.forEach(function (exec) {
- allExecCnt += 1;
- allRDDBlocks += exec.rddBlocks;
- allMemoryUsed += exec.memoryUsed;
- allMaxMemory += exec.maxMemory;
- allOnHeapMemoryUsed += exec.memoryMetrics.usedOnHeapStorageMemory;
- allOnHeapMaxMemory += exec.memoryMetrics.totalOnHeapStorageMemory;
- allOffHeapMemoryUsed += exec.memoryMetrics.usedOffHeapStorageMemory;
- allOffHeapMaxMemory += exec.memoryMetrics.totalOffHeapStorageMemory;
- allDiskUsed += exec.diskUsed;
- allTotalCores += exec.totalCores;
- allMaxTasks += exec.maxTasks;
- allActiveTasks += exec.activeTasks;
- allFailedTasks += exec.failedTasks;
- allCompletedTasks += exec.completedTasks;
- allTotalTasks += exec.totalTasks;
- allTotalDuration += exec.totalDuration;
- allTotalGCTime += exec.totalGCTime;
- allTotalInputBytes += exec.totalInputBytes;
- allTotalShuffleRead += exec.totalShuffleRead;
- allTotalShuffleWrite += exec.totalShuffleWrite;
- allTotalExcluded += exec.isExcluded ? 1 : 0;
- if (exec.isActive) {
- activeExecCnt += 1;
- activeRDDBlocks += exec.rddBlocks;
- activeMemoryUsed += exec.memoryUsed;
- activeMaxMemory += exec.maxMemory;
- activeOnHeapMemoryUsed += exec.memoryMetrics.usedOnHeapStorageMemory;
- activeOnHeapMaxMemory += exec.memoryMetrics.totalOnHeapStorageMemory;
- activeOffHeapMemoryUsed += exec.memoryMetrics.usedOffHeapStorageMemory;
- activeOffHeapMaxMemory += exec.memoryMetrics.totalOffHeapStorageMemory;
- activeDiskUsed += exec.diskUsed;
- activeTotalCores += exec.totalCores;
- activeMaxTasks += exec.maxTasks;
- activeActiveTasks += exec.activeTasks;
- activeFailedTasks += exec.failedTasks;
- activeCompletedTasks += exec.completedTasks;
- activeTotalTasks += exec.totalTasks;
- activeTotalDuration += exec.totalDuration;
- activeTotalGCTime += exec.totalGCTime;
- activeTotalInputBytes += exec.totalInputBytes;
- activeTotalShuffleRead += exec.totalShuffleRead;
- activeTotalShuffleWrite += exec.totalShuffleWrite;
- activeTotalExcluded += exec.isExcluded ? 1 : 0;
+ setDataTableDefaults();
+
+ var executorsSummary = $("#active-executors");
+
+ getStandAloneAppId(function (appId) {
+
+ var endPoint = createRESTEndPointForExecutorsPage(appId);
+ $.getJSON(endPoint, function (response, _ignored_status, _ignored_jqXHR) {
+ var allExecCnt = 0;
+ var allRDDBlocks = 0;
+ var allMemoryUsed = 0;
+ var allMaxMemory = 0;
+ var allOnHeapMemoryUsed = 0;
+ var allOnHeapMaxMemory = 0;
+ var allOffHeapMemoryUsed = 0;
+ var allOffHeapMaxMemory = 0;
+ var allDiskUsed = 0;
+ var allTotalCores = 0;
+ var allMaxTasks = 0;
+ var allActiveTasks = 0;
+ var allFailedTasks = 0;
+ var allCompletedTasks = 0;
+ var allTotalTasks = 0;
+ var allTotalDuration = 0;
+ var allTotalGCTime = 0;
+ var allTotalInputBytes = 0;
+ var allTotalShuffleRead = 0;
+ var allTotalShuffleWrite = 0;
+ var allTotalExcluded = 0;
+
+ var activeExecCnt = 0;
+ var activeRDDBlocks = 0;
+ var activeMemoryUsed = 0;
+ var activeMaxMemory = 0;
+ var activeOnHeapMemoryUsed = 0;
+ var activeOnHeapMaxMemory = 0;
+ var activeOffHeapMemoryUsed = 0;
+ var activeOffHeapMaxMemory = 0;
+ var activeDiskUsed = 0;
+ var activeTotalCores = 0;
+ var activeMaxTasks = 0;
+ var activeActiveTasks = 0;
+ var activeFailedTasks = 0;
+ var activeCompletedTasks = 0;
+ var activeTotalTasks = 0;
+ var activeTotalDuration = 0;
+ var activeTotalGCTime = 0;
+ var activeTotalInputBytes = 0;
+ var activeTotalShuffleRead = 0;
+ var activeTotalShuffleWrite = 0;
+ var activeTotalExcluded = 0;
+
+ var deadExecCnt = 0;
+ var deadRDDBlocks = 0;
+ var deadMemoryUsed = 0;
+ var deadMaxMemory = 0;
+ var deadOnHeapMemoryUsed = 0;
+ var deadOnHeapMaxMemory = 0;
+ var deadOffHeapMemoryUsed = 0;
+ var deadOffHeapMaxMemory = 0;
+ var deadDiskUsed = 0;
+ var deadTotalCores = 0;
+ var deadMaxTasks = 0;
+ var deadActiveTasks = 0;
+ var deadFailedTasks = 0;
+ var deadCompletedTasks = 0;
+ var deadTotalTasks = 0;
+ var deadTotalDuration = 0;
+ var deadTotalGCTime = 0;
+ var deadTotalInputBytes = 0;
+ var deadTotalShuffleRead = 0;
+ var deadTotalShuffleWrite = 0;
+ var deadTotalExcluded = 0;
+
+ response.forEach(function (exec) {
+ var memoryMetrics = {
+ usedOnHeapStorageMemory: 0,
+ usedOffHeapStorageMemory: 0,
+ totalOnHeapStorageMemory: 0,
+ totalOffHeapStorageMemory: 0
+ };
+
+ // TODO: Replace hasOwnProperty with prototype.hasOwnProperty after we find it's safe to do.
+ /* eslint-disable no-prototype-builtins */
+ exec.memoryMetrics = exec.hasOwnProperty('memoryMetrics') ? exec.memoryMetrics : memoryMetrics;
+ });
+
+ response.forEach(function (exec) {
+ allExecCnt += 1;
+ allRDDBlocks += exec.rddBlocks;
+ allMemoryUsed += exec.memoryUsed;
+ allMaxMemory += exec.maxMemory;
+ allOnHeapMemoryUsed += exec.memoryMetrics.usedOnHeapStorageMemory;
+ allOnHeapMaxMemory += exec.memoryMetrics.totalOnHeapStorageMemory;
+ allOffHeapMemoryUsed += exec.memoryMetrics.usedOffHeapStorageMemory;
+ allOffHeapMaxMemory += exec.memoryMetrics.totalOffHeapStorageMemory;
+ allDiskUsed += exec.diskUsed;
+ allTotalCores += exec.totalCores;
+ allMaxTasks += exec.maxTasks;
+ allActiveTasks += exec.activeTasks;
+ allFailedTasks += exec.failedTasks;
+ allCompletedTasks += exec.completedTasks;
+ allTotalTasks += exec.totalTasks;
+ allTotalDuration += exec.totalDuration;
+ allTotalGCTime += exec.totalGCTime;
+ allTotalInputBytes += exec.totalInputBytes;
+ allTotalShuffleRead += exec.totalShuffleRead;
+ allTotalShuffleWrite += exec.totalShuffleWrite;
+ allTotalExcluded += exec.isExcluded ? 1 : 0;
+ if (exec.isActive) {
+ activeExecCnt += 1;
+ activeRDDBlocks += exec.rddBlocks;
+ activeMemoryUsed += exec.memoryUsed;
+ activeMaxMemory += exec.maxMemory;
+ activeOnHeapMemoryUsed += exec.memoryMetrics.usedOnHeapStorageMemory;
+ activeOnHeapMaxMemory += exec.memoryMetrics.totalOnHeapStorageMemory;
+ activeOffHeapMemoryUsed += exec.memoryMetrics.usedOffHeapStorageMemory;
+ activeOffHeapMaxMemory += exec.memoryMetrics.totalOffHeapStorageMemory;
+ activeDiskUsed += exec.diskUsed;
+ activeTotalCores += exec.totalCores;
+ activeMaxTasks += exec.maxTasks;
+ activeActiveTasks += exec.activeTasks;
+ activeFailedTasks += exec.failedTasks;
+ activeCompletedTasks += exec.completedTasks;
+ activeTotalTasks += exec.totalTasks;
+ activeTotalDuration += exec.totalDuration;
+ activeTotalGCTime += exec.totalGCTime;
+ activeTotalInputBytes += exec.totalInputBytes;
+ activeTotalShuffleRead += exec.totalShuffleRead;
+ activeTotalShuffleWrite += exec.totalShuffleWrite;
+ activeTotalExcluded += exec.isExcluded ? 1 : 0;
+ } else {
+ deadExecCnt += 1;
+ deadRDDBlocks += exec.rddBlocks;
+ deadMemoryUsed += exec.memoryUsed;
+ deadMaxMemory += exec.maxMemory;
+ deadOnHeapMemoryUsed += exec.memoryMetrics.usedOnHeapStorageMemory;
+ deadOnHeapMaxMemory += exec.memoryMetrics.totalOnHeapStorageMemory;
+ deadOffHeapMemoryUsed += exec.memoryMetrics.usedOffHeapStorageMemory;
+ deadOffHeapMaxMemory += exec.memoryMetrics.totalOffHeapStorageMemory;
+ deadDiskUsed += exec.diskUsed;
+ deadTotalCores += exec.totalCores;
+ deadMaxTasks += exec.maxTasks;
+ deadActiveTasks += exec.activeTasks;
+ deadFailedTasks += exec.failedTasks;
+ deadCompletedTasks += exec.completedTasks;
+ deadTotalTasks += exec.totalTasks;
+ deadTotalDuration += exec.totalDuration;
+ deadTotalGCTime += exec.totalGCTime;
+ deadTotalInputBytes += exec.totalInputBytes;
+ deadTotalShuffleRead += exec.totalShuffleRead;
+ deadTotalShuffleWrite += exec.totalShuffleWrite;
+ deadTotalExcluded += exec.isExcluded ? 1 : 0; // todo - TEST BACKWARDS compatibility history?
+ }
+ });
+
+ var totalSummary = {
+ "execCnt": ( "Total(" + allExecCnt + ")"),
+ "allRDDBlocks": allRDDBlocks,
+ "allMemoryUsed": allMemoryUsed,
+ "allMaxMemory": allMaxMemory,
+ "allOnHeapMemoryUsed": allOnHeapMemoryUsed,
+ "allOnHeapMaxMemory": allOnHeapMaxMemory,
+ "allOffHeapMemoryUsed": allOffHeapMemoryUsed,
+ "allOffHeapMaxMemory": allOffHeapMaxMemory,
+ "allDiskUsed": allDiskUsed,
+ "allTotalCores": allTotalCores,
+ "allMaxTasks": allMaxTasks,
+ "allActiveTasks": allActiveTasks,
+ "allFailedTasks": allFailedTasks,
+ "allCompletedTasks": allCompletedTasks,
+ "allTotalTasks": allTotalTasks,
+ "allTotalDuration": allTotalDuration,
+ "allTotalGCTime": allTotalGCTime,
+ "allTotalInputBytes": allTotalInputBytes,
+ "allTotalShuffleRead": allTotalShuffleRead,
+ "allTotalShuffleWrite": allTotalShuffleWrite,
+ "allTotalExcluded": allTotalExcluded
+ };
+ var activeSummary = {
+ "execCnt": ( "Active(" + activeExecCnt + ")"),
+ "allRDDBlocks": activeRDDBlocks,
+ "allMemoryUsed": activeMemoryUsed,
+ "allMaxMemory": activeMaxMemory,
+ "allOnHeapMemoryUsed": activeOnHeapMemoryUsed,
+ "allOnHeapMaxMemory": activeOnHeapMaxMemory,
+ "allOffHeapMemoryUsed": activeOffHeapMemoryUsed,
+ "allOffHeapMaxMemory": activeOffHeapMaxMemory,
+ "allDiskUsed": activeDiskUsed,
+ "allTotalCores": activeTotalCores,
+ "allMaxTasks": activeMaxTasks,
+ "allActiveTasks": activeActiveTasks,
+ "allFailedTasks": activeFailedTasks,
+ "allCompletedTasks": activeCompletedTasks,
+ "allTotalTasks": activeTotalTasks,
+ "allTotalDuration": activeTotalDuration,
+ "allTotalGCTime": activeTotalGCTime,
+ "allTotalInputBytes": activeTotalInputBytes,
+ "allTotalShuffleRead": activeTotalShuffleRead,
+ "allTotalShuffleWrite": activeTotalShuffleWrite,
+ "allTotalExcluded": activeTotalExcluded
+ };
+ var deadSummary = {
+ "execCnt": ( "Dead(" + deadExecCnt + ")" ),
+ "allRDDBlocks": deadRDDBlocks,
+ "allMemoryUsed": deadMemoryUsed,
+ "allMaxMemory": deadMaxMemory,
+ "allOnHeapMemoryUsed": deadOnHeapMemoryUsed,
+ "allOnHeapMaxMemory": deadOnHeapMaxMemory,
+ "allOffHeapMemoryUsed": deadOffHeapMemoryUsed,
+ "allOffHeapMaxMemory": deadOffHeapMaxMemory,
+ "allDiskUsed": deadDiskUsed,
+ "allTotalCores": deadTotalCores,
+ "allMaxTasks": deadMaxTasks,
+ "allActiveTasks": deadActiveTasks,
+ "allFailedTasks": deadFailedTasks,
+ "allCompletedTasks": deadCompletedTasks,
+ "allTotalTasks": deadTotalTasks,
+ "allTotalDuration": deadTotalDuration,
+ "allTotalGCTime": deadTotalGCTime,
+ "allTotalInputBytes": deadTotalInputBytes,
+ "allTotalShuffleRead": deadTotalShuffleRead,
+ "allTotalShuffleWrite": deadTotalShuffleWrite,
+ "allTotalExcluded": deadTotalExcluded
+ };
+
+ var data = {executors: response, "execSummary": [activeSummary, deadSummary, totalSummary]};
+ $.get(createTemplateURI(appId, "executorspage"), function (template) {
+
+ executorsSummary.append(Mustache.render($(template).filter("#executors-summary-template").html(), data));
+ var selector = "#active-executors-table";
+ var conf = {
+ "data": response,
+ "columns": [
+ {
+ data: function (row, type) {
+ return type !== 'display' ? (isNaN(row.id) ? 0 : row.id ) : row.id;
+ }
+ },
+ {data: 'hostPort'},
+ {
+ data: 'isActive',
+ render: function (data, type, row) {
+ return formatStatus (data, type, row);
+ }
+ },
+ {data: 'rddBlocks'},
+ {
+ data: function (row, type) {
+ if (type !== 'display')
+ return row.memoryUsed;
+ else
+ return (formatBytes(row.memoryUsed, type) + ' / ' +
+ formatBytes(row.maxMemory, type));
+ }
+ },
+ {
+ data: function (row, type) {
+ if (type !== 'display')
+ return row.memoryMetrics.usedOnHeapStorageMemory;
+ else
+ return (formatBytes(row.memoryMetrics.usedOnHeapStorageMemory, type) + ' / ' +
+ formatBytes(row.memoryMetrics.totalOnHeapStorageMemory, type));
+ }
+ },
+ {
+ data: function (row, type) {
+ if (type !== 'display')
+ return row.memoryMetrics.usedOffHeapStorageMemory;
+ else
+ return (formatBytes(row.memoryMetrics.usedOffHeapStorageMemory, type) + ' / ' +
+ formatBytes(row.memoryMetrics.totalOffHeapStorageMemory, type));
+ }
+ },
+ {
+ data: function (row, type) {
+ var peakMemoryMetrics = row.peakMemoryMetrics;
+ if (typeof peakMemoryMetrics !== 'undefined') {
+ if (type !== 'display')
+ return peakMemoryMetrics.JVMHeapMemory;
+ else
+ return (formatBytes(peakMemoryMetrics.JVMHeapMemory, type) + ' / ' +
+ formatBytes(peakMemoryMetrics.JVMOffHeapMemory, type));
} else {
- deadExecCnt += 1;
- deadRDDBlocks += exec.rddBlocks;
- deadMemoryUsed += exec.memoryUsed;
- deadMaxMemory += exec.maxMemory;
- deadOnHeapMemoryUsed += exec.memoryMetrics.usedOnHeapStorageMemory;
- deadOnHeapMaxMemory += exec.memoryMetrics.totalOnHeapStorageMemory;
- deadOffHeapMemoryUsed += exec.memoryMetrics.usedOffHeapStorageMemory;
- deadOffHeapMaxMemory += exec.memoryMetrics.totalOffHeapStorageMemory;
- deadDiskUsed += exec.diskUsed;
- deadTotalCores += exec.totalCores;
- deadMaxTasks += exec.maxTasks;
- deadActiveTasks += exec.activeTasks;
- deadFailedTasks += exec.failedTasks;
- deadCompletedTasks += exec.completedTasks;
- deadTotalTasks += exec.totalTasks;
- deadTotalDuration += exec.totalDuration;
- deadTotalGCTime += exec.totalGCTime;
- deadTotalInputBytes += exec.totalInputBytes;
- deadTotalShuffleRead += exec.totalShuffleRead;
- deadTotalShuffleWrite += exec.totalShuffleWrite;
- deadTotalExcluded += exec.isExcluded ? 1 : 0; // todo - TEST BACKWARDS compatibility history?
+ if (type !== 'display') {
+ return 0;
+ } else {
+ return '0.0 B / 0.0 B';
+ }
}
- });
-
- var totalSummary = {
- "execCnt": ( "Total(" + allExecCnt + ")"),
- "allRDDBlocks": allRDDBlocks,
- "allMemoryUsed": allMemoryUsed,
- "allMaxMemory": allMaxMemory,
- "allOnHeapMemoryUsed": allOnHeapMemoryUsed,
- "allOnHeapMaxMemory": allOnHeapMaxMemory,
- "allOffHeapMemoryUsed": allOffHeapMemoryUsed,
- "allOffHeapMaxMemory": allOffHeapMaxMemory,
- "allDiskUsed": allDiskUsed,
- "allTotalCores": allTotalCores,
- "allMaxTasks": allMaxTasks,
- "allActiveTasks": allActiveTasks,
- "allFailedTasks": allFailedTasks,
- "allCompletedTasks": allCompletedTasks,
- "allTotalTasks": allTotalTasks,
- "allTotalDuration": allTotalDuration,
- "allTotalGCTime": allTotalGCTime,
- "allTotalInputBytes": allTotalInputBytes,
- "allTotalShuffleRead": allTotalShuffleRead,
- "allTotalShuffleWrite": allTotalShuffleWrite,
- "allTotalExcluded": allTotalExcluded
- };
- var activeSummary = {
- "execCnt": ( "Active(" + activeExecCnt + ")"),
- "allRDDBlocks": activeRDDBlocks,
- "allMemoryUsed": activeMemoryUsed,
- "allMaxMemory": activeMaxMemory,
- "allOnHeapMemoryUsed": activeOnHeapMemoryUsed,
- "allOnHeapMaxMemory": activeOnHeapMaxMemory,
- "allOffHeapMemoryUsed": activeOffHeapMemoryUsed,
- "allOffHeapMaxMemory": activeOffHeapMaxMemory,
- "allDiskUsed": activeDiskUsed,
- "allTotalCores": activeTotalCores,
- "allMaxTasks": activeMaxTasks,
- "allActiveTasks": activeActiveTasks,
- "allFailedTasks": activeFailedTasks,
- "allCompletedTasks": activeCompletedTasks,
- "allTotalTasks": activeTotalTasks,
- "allTotalDuration": activeTotalDuration,
- "allTotalGCTime": activeTotalGCTime,
- "allTotalInputBytes": activeTotalInputBytes,
- "allTotalShuffleRead": activeTotalShuffleRead,
- "allTotalShuffleWrite": activeTotalShuffleWrite,
- "allTotalExcluded": activeTotalExcluded
- };
- var deadSummary = {
- "execCnt": ( "Dead(" + deadExecCnt + ")" ),
- "allRDDBlocks": deadRDDBlocks,
- "allMemoryUsed": deadMemoryUsed,
- "allMaxMemory": deadMaxMemory,
- "allOnHeapMemoryUsed": deadOnHeapMemoryUsed,
- "allOnHeapMaxMemory": deadOnHeapMaxMemory,
- "allOffHeapMemoryUsed": deadOffHeapMemoryUsed,
- "allOffHeapMaxMemory": deadOffHeapMaxMemory,
- "allDiskUsed": deadDiskUsed,
- "allTotalCores": deadTotalCores,
- "allMaxTasks": deadMaxTasks,
- "allActiveTasks": deadActiveTasks,
- "allFailedTasks": deadFailedTasks,
- "allCompletedTasks": deadCompletedTasks,
- "allTotalTasks": deadTotalTasks,
- "allTotalDuration": deadTotalDuration,
- "allTotalGCTime": deadTotalGCTime,
- "allTotalInputBytes": deadTotalInputBytes,
- "allTotalShuffleRead": deadTotalShuffleRead,
- "allTotalShuffleWrite": deadTotalShuffleWrite,
- "allTotalExcluded": deadTotalExcluded
+ }
+ },
+ {
+ data: function (row, type) {
+ var peakMemoryMetrics = row.peakMemoryMetrics;
+ if (typeof peakMemoryMetrics !== 'undefined') {
+ if (type !== 'display')
+ return peakMemoryMetrics.OnHeapExecutionMemory;
+ else
+ return (formatBytes(peakMemoryMetrics.OnHeapExecutionMemory, type) + ' / ' +
+ formatBytes(peakMemoryMetrics.OffHeapExecutionMemory, type));
+ } else {
+ if (type !== 'display') {
+ return 0;
+ } else {
+ return '0.0 B / 0.0 B';
+ }
+ }
+ }
+ },
+ {
+ data: function (row, type) {
+ var peakMemoryMetrics = row.peakMemoryMetrics;
+ if (typeof peakMemoryMetrics !== 'undefined') {
+ if (type !== 'display')
+ return peakMemoryMetrics.OnHeapStorageMemory;
+ else
+ return (formatBytes(peakMemoryMetrics.OnHeapStorageMemory, type) + ' / ' +
+ formatBytes(peakMemoryMetrics.OffHeapStorageMemory, type));
+ } else {
+ if (type !== 'display') {
+ return 0;
+ } else {
+ return '0.0 B / 0.0 B';
+ }
+ }
+ }
+ },
+ {
+ data: function (row, type) {
+ var peakMemoryMetrics = row.peakMemoryMetrics;
+ if (typeof peakMemoryMetrics !== 'undefined') {
+ if (type !== 'display')
+ return peakMemoryMetrics.DirectPoolMemory;
+ else
+ return (formatBytes(peakMemoryMetrics.DirectPoolMemory, type) + ' / ' +
+ formatBytes(peakMemoryMetrics.MappedPoolMemory, type));
+ } else {
+ if (type !== 'display') {
+ return 0;
+ } else {
+ return '0.0 B / 0.0 B';
+ }
+ }
+ }
+ },
+ {data: 'diskUsed', render: formatBytes},
+ {data: 'totalCores'},
+ {name: 'resourcesCol', data: 'resources', render: formatResourceCells, orderable: false},
+ {name: 'resourceProfileIdCol', data: 'resourceProfileId'},
+ {
+ data: 'activeTasks',
+ "fnCreatedCell": function (nTd, sData, oData, _ignored_iRow, _ignored_iCol) {
+ if (sData > 0) {
+ $(nTd).css('color', 'white');
+ $(nTd).css('background', activeTasksStyle(oData.activeTasks, oData.maxTasks));
+ }
+ }
+ },
+ {
+ data: 'failedTasks',
+ "fnCreatedCell": function (nTd, sData, oData, _ignored_iRow, _ignored_iCol) {
+ if (sData > 0) {
+ $(nTd).css('color', 'white');
+ $(nTd).css('background', failedTasksStyle(oData.failedTasks, oData.totalTasks));
+ }
+ }
+ },
+ {data: 'completedTasks'},
+ {data: 'totalTasks'},
+ {
+ data: function (row, type) {
+ return type === 'display' ? (formatDuration(row.totalDuration) + ' (' + formatDuration(row.totalGCTime) + ')') : row.totalDuration
+ },
+ "fnCreatedCell": function (nTd, sData, oData, _ignored_iRow, _ignored_iCol) {
+ if (oData.totalDuration > 0) {
+ $(nTd).css('color', totalDurationColor(oData.totalGCTime, oData.totalDuration));
+ $(nTd).css('background', totalDurationStyle(oData.totalGCTime, oData.totalDuration));
+ }
+ }
+ },
+ {data: 'totalInputBytes', render: formatBytes},
+ {data: 'totalShuffleRead', render: formatBytes},
+ {data: 'totalShuffleWrite', render: formatBytes},
+ {name: 'executorLogsCol', data: 'executorLogs', render: formatLogsCells},
+ {
+ name: 'threadDumpCol',
+ data: 'id', render: function (data, type) {
+ return type === 'display' ? ("Thread Dump" ) : data;
+ }
+ }
+ ],
+ "order": [[0, "asc"]],
+ "columnDefs": [
+ {"visible": false, "targets": 5},
+ {"visible": false, "targets": 6},
+ {"visible": false, "targets": 7},
+ {"visible": false, "targets": 8},
+ {"visible": false, "targets": 9},
+ {"visible": false, "targets": 10},
+ {"visible": false, "targets": 13},
+ {"visible": false, "targets": 14}
+ ],
+ "deferRender": true
+ };
+
+ execDataTable = $(selector).DataTable(conf);
+ execDataTable.column('executorLogsCol:name').visible(logsExist(response));
+ execDataTable.column('threadDumpCol:name').visible(getThreadDumpEnabled());
+ $('#active-executors [data-toggle="tooltip"]').tooltip();
+
+ // This section should be visible once API gives the response.
+ $('.active-process-container').hide();
+ var endPoint = createRESTEndPointForMiscellaneousProcess(appId);
+ $.getJSON(endPoint, function( response, _ignored_status, _ignored_jqXHR ) {
+ if (response.length) {
+ var processSummaryResponse = response;
+ var processSummaryConf = {
+ "data": processSummaryResponse,
+ "columns": [{
+ data: "id"
+ },
+ {
+ data: "hostPort"
+ },
+ {
+ data: function(row) {
+ return formatProcessStatus(row.isActive);
+ }
+ },
+ {
+ data: "totalCores"
+ },
+ {
+ data: "processLogs",
+ render: formatLogsCells
+ },
+ ],
+ "deferRender": true,
+ "order": [
+ [0, "asc"]
+ ],
+ "bAutoWidth": false,
+ "oLanguage": {
+ "sEmptyTable": "No data to show yet"
+ }
};
+ $("#active-process-table").DataTable(processSummaryConf);
+ $('.active-process-container').show()
+ }
+ });
- var data = {executors: response, "execSummary": [activeSummary, deadSummary, totalSummary]};
- $.get(createTemplateURI(appId, "executorspage"), function (template) {
-
- executorsSummary.append(Mustache.render($(template).filter("#executors-summary-template").html(), data));
- var selector = "#active-executors-table";
- var conf = {
- "data": response,
- "columns": [
- {
- data: function (row, type) {
- return type !== 'display' ? (isNaN(row.id) ? 0 : row.id ) : row.id;
- }
- },
- {data: 'hostPort'},
- {
- data: 'isActive',
- render: function (data, type, row) {
- return formatStatus (data, type, row);
- }
- },
- {data: 'rddBlocks'},
- {
- data: function (row, type) {
- if (type !== 'display')
- return row.memoryUsed;
- else
- return (formatBytes(row.memoryUsed, type) + ' / ' +
- formatBytes(row.maxMemory, type));
- }
- },
- {
- data: function (row, type) {
- if (type !== 'display')
- return row.memoryMetrics.usedOnHeapStorageMemory;
- else
- return (formatBytes(row.memoryMetrics.usedOnHeapStorageMemory, type) + ' / ' +
- formatBytes(row.memoryMetrics.totalOnHeapStorageMemory, type));
- }
- },
- {
- data: function (row, type) {
- if (type !== 'display')
- return row.memoryMetrics.usedOffHeapStorageMemory;
- else
- return (formatBytes(row.memoryMetrics.usedOffHeapStorageMemory, type) + ' / ' +
- formatBytes(row.memoryMetrics.totalOffHeapStorageMemory, type));
- }
- },
- {
- data: function (row, type) {
- var peakMemoryMetrics = row.peakMemoryMetrics;
- if (typeof peakMemoryMetrics !== 'undefined') {
- if (type !== 'display')
- return peakMemoryMetrics.JVMHeapMemory;
- else
- return (formatBytes(peakMemoryMetrics.JVMHeapMemory, type) + ' / ' +
- formatBytes(peakMemoryMetrics.JVMOffHeapMemory, type));
- } else {
- if (type !== 'display') {
- return 0;
- } else {
- return '0.0 B / 0.0 B';
- }
- }
- }
- },
- {
- data: function (row, type) {
- var peakMemoryMetrics = row.peakMemoryMetrics;
- if (typeof peakMemoryMetrics !== 'undefined') {
- if (type !== 'display')
- return peakMemoryMetrics.OnHeapExecutionMemory;
- else
- return (formatBytes(peakMemoryMetrics.OnHeapExecutionMemory, type) + ' / ' +
- formatBytes(peakMemoryMetrics.OffHeapExecutionMemory, type));
- } else {
- if (type !== 'display') {
- return 0;
- } else {
- return '0.0 B / 0.0 B';
- }
- }
- }
- },
- {
- data: function (row, type) {
- var peakMemoryMetrics = row.peakMemoryMetrics;
- if (typeof peakMemoryMetrics !== 'undefined') {
- if (type !== 'display')
- return peakMemoryMetrics.OnHeapStorageMemory;
- else
- return (formatBytes(peakMemoryMetrics.OnHeapStorageMemory, type) + ' / ' +
- formatBytes(peakMemoryMetrics.OffHeapStorageMemory, type));
- } else {
- if (type !== 'display') {
- return 0;
- } else {
- return '0.0 B / 0.0 B';
- }
- }
- }
- },
- {
- data: function (row, type) {
- var peakMemoryMetrics = row.peakMemoryMetrics;
- if (typeof peakMemoryMetrics !== 'undefined') {
- if (type !== 'display')
- return peakMemoryMetrics.DirectPoolMemory;
- else
- return (formatBytes(peakMemoryMetrics.DirectPoolMemory, type) + ' / ' +
- formatBytes(peakMemoryMetrics.MappedPoolMemory, type));
- } else {
- if (type !== 'display') {
- return 0;
- } else {
- return '0.0 B / 0.0 B';
- }
- }
- }
- },
- {data: 'diskUsed', render: formatBytes},
- {data: 'totalCores'},
- {name: 'resourcesCol', data: 'resources', render: formatResourceCells, orderable: false},
- {name: 'resourceProfileIdCol', data: 'resourceProfileId'},
- {
- data: 'activeTasks',
- "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) {
- if (sData > 0) {
- $(nTd).css('color', 'white');
- $(nTd).css('background', activeTasksStyle(oData.activeTasks, oData.maxTasks));
- }
- }
- },
- {
- data: 'failedTasks',
- "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) {
- if (sData > 0) {
- $(nTd).css('color', 'white');
- $(nTd).css('background', failedTasksStyle(oData.failedTasks, oData.totalTasks));
- }
- }
- },
- {data: 'completedTasks'},
- {data: 'totalTasks'},
- {
- data: function (row, type) {
- return type === 'display' ? (formatDuration(row.totalDuration) + ' (' + formatDuration(row.totalGCTime) + ')') : row.totalDuration
- },
- "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) {
- if (oData.totalDuration > 0) {
- $(nTd).css('color', totalDurationColor(oData.totalGCTime, oData.totalDuration));
- $(nTd).css('background', totalDurationStyle(oData.totalGCTime, oData.totalDuration));
- }
- }
- },
- {data: 'totalInputBytes', render: formatBytes},
- {data: 'totalShuffleRead', render: formatBytes},
- {data: 'totalShuffleWrite', render: formatBytes},
- {name: 'executorLogsCol', data: 'executorLogs', render: formatLogsCells},
- {
- name: 'threadDumpCol',
- data: 'id', render: function (data, type) {
- return type === 'display' ? ("Thread Dump" ) : data;
- }
- }
- ],
- "order": [[0, "asc"]],
- "columnDefs": [
- {"visible": false, "targets": 5},
- {"visible": false, "targets": 6},
- {"visible": false, "targets": 7},
- {"visible": false, "targets": 8},
- {"visible": false, "targets": 9},
- {"visible": false, "targets": 10},
- {"visible": false, "targets": 13},
- {"visible": false, "targets": 14}
- ],
- "deferRender": true
- };
-
- execDataTable = $(selector).DataTable(conf);
- execDataTable.column('executorLogsCol:name').visible(logsExist(response));
- execDataTable.column('threadDumpCol:name').visible(getThreadDumpEnabled());
- $('#active-executors [data-toggle="tooltip"]').tooltip();
-
- // This section should be visible once API gives the response.
- $('.active-process-container').hide()
- var endPoint = createRESTEndPointForMiscellaneousProcess(appId);
- $.getJSON(endPoint, function( response, status, jqXHR ) {
- if (response.length) {
- var processSummaryResponse = response;
- var processSummaryConf = {
- "data": processSummaryResponse,
- "columns": [{
- data: "id"
- },
- {
- data: "hostPort"
- },
- {
- data: function(row) {
- return formatProcessStatus(row.isActive);
- }
- },
- {
- data: "totalCores"
- },
- {
- data: "processLogs",
- render: formatLogsCells
- },
- ],
- "deferRender": true,
- "order": [
- [0, "asc"]
- ],
- "bAutoWidth": false,
- "oLanguage": {
- "sEmptyTable": "No data to show yet"
- }
- };
- $("#active-process-table").DataTable(processSummaryConf);
- $('.active-process-container').show()
- }
- });
-
- var sumSelector = "#summary-execs-table";
- var sumConf = {
- "data": [activeSummary, deadSummary, totalSummary],
- "columns": [
- {
- data: 'execCnt',
- "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) {
- $(nTd).css('font-weight', 'bold');
- }
- },
- {data: 'allRDDBlocks'},
- {
- data: function (row, type) {
- if (type !== 'display')
- return row.allMemoryUsed
- else
- return (formatBytes(row.allMemoryUsed, type) + ' / ' +
- formatBytes(row.allMaxMemory, type));
- }
- },
- {
- data: function (row, type) {
- if (type !== 'display')
- return row.allOnHeapMemoryUsed;
- else
- return (formatBytes(row.allOnHeapMemoryUsed, type) + ' / ' +
- formatBytes(row.allOnHeapMaxMemory, type));
- }
- },
- {
- data: function (row, type) {
- if (type !== 'display')
- return row.allOffHeapMemoryUsed;
- else
- return (formatBytes(row.allOffHeapMemoryUsed, type) + ' / ' +
- formatBytes(row.allOffHeapMaxMemory, type));
- }
- },
- {data: 'allDiskUsed', render: formatBytes},
- {data: 'allTotalCores'},
- {
- data: 'allActiveTasks',
- "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) {
- if (sData > 0) {
- $(nTd).css('color', 'white');
- $(nTd).css('background', activeTasksStyle(oData.allActiveTasks, oData.allMaxTasks));
- }
- }
- },
- {
- data: 'allFailedTasks',
- "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) {
- if (sData > 0) {
- $(nTd).css('color', 'white');
- $(nTd).css('background', failedTasksStyle(oData.allFailedTasks, oData.allTotalTasks));
- }
- }
- },
- {data: 'allCompletedTasks'},
- {data: 'allTotalTasks'},
- {
- data: function (row, type) {
- return type === 'display' ? (formatDuration(row.allTotalDuration) + ' (' + formatDuration(row.allTotalGCTime) + ')') : row.allTotalDuration
- },
- "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) {
- if (oData.allTotalDuration > 0) {
- $(nTd).css('color', totalDurationColor(oData.allTotalGCTime, oData.allTotalDuration));
- $(nTd).css('background', totalDurationStyle(oData.allTotalGCTime, oData.allTotalDuration));
- }
- }
- },
- {data: 'allTotalInputBytes', render: formatBytes},
- {data: 'allTotalShuffleRead', render: formatBytes},
- {data: 'allTotalShuffleWrite', render: formatBytes},
- {data: 'allTotalExcluded'}
- ],
- "paging": false,
- "searching": false,
- "info": false,
- "columnDefs": [
- {"visible": false, "targets": 3},
- {"visible": false, "targets": 4}
- ]
-
- };
-
- sumDataTable = $(sumSelector).DataTable(sumConf);
- $('#execSummary [data-toggle="tooltip"]').tooltip();
-
- $("#showAdditionalMetrics").append(
- "" +
- "");
-
- reselectCheckboxesBasedOnTaskTableState();
-
- $("#additionalMetrics").click(function() {
- $("#arrowtoggle-optional-metrics").toggleClass("arrow-open arrow-closed");
- $("#toggle-metrics").toggleClass("d-none");
- if (window.localStorage) {
- window.localStorage.setItem("arrowtoggle-optional-metrics-class", $("#arrowtoggle-optional-metrics").attr('class'));
- }
- });
-
- $(".toggle-vis").on("click", function() {
- var thisBox = $(this);
- if (thisBox.is("#select-all-box")) {
- var sumColumn = sumDataTable.columns(sumOptionalColumns);
- var execColumn = execDataTable.columns(execOptionalColumns);
- if (thisBox.is(":checked")) {
- $(".toggle-vis").prop("checked", true);
- sumColumn.visible(true);
- execColumn.visible(true);
- } else {
- $(".toggle-vis").prop("checked", false);
- sumColumn.visible(false);
- execColumn.visible(false);
- }
- } else {
- var execColIdx = thisBox.attr("data-exec-col-idx");
- var execCol = execDataTable.column(execColIdx);
- execCol.visible(!execCol.visible());
- var sumColIdx = thisBox.attr("data-sum-col-idx");
- if (sumColIdx) {
- var sumCol = sumDataTable.column(sumColIdx);
- sumCol.visible(!sumCol.visible());
- }
- }
- });
-
- if (window.localStorage) {
- if (window.localStorage.getItem("arrowtoggle-optional-metrics-class") != null &&
- window.localStorage.getItem("arrowtoggle-optional-metrics-class").includes("arrow-open")) {
- $("#arrowtoggle-optional-metrics").toggleClass("arrow-open arrow-closed");
- $("#toggle-metrics").toggleClass("d-none");
- }
+ var sumSelector = "#summary-execs-table";
+ var sumConf = {
+ "data": [activeSummary, deadSummary, totalSummary],
+ "columns": [
+ {
+ data: 'execCnt',
+ "fnCreatedCell": function (nTd, _ignored_sData, _ignored_oData, _ignored_iRow, _ignored_iCol) {
+ $(nTd).css('font-weight', 'bold');
+ }
+ },
+ {data: 'allRDDBlocks'},
+ {
+ data: function (row, type) {
+ if (type !== 'display')
+ return row.allMemoryUsed;
+ else
+ return (formatBytes(row.allMemoryUsed, type) + ' / ' +
+ formatBytes(row.allMaxMemory, type));
+ }
+ },
+ {
+ data: function (row, type) {
+ if (type !== 'display')
+ return row.allOnHeapMemoryUsed;
+ else
+ return (formatBytes(row.allOnHeapMemoryUsed, type) + ' / ' +
+ formatBytes(row.allOnHeapMaxMemory, type));
+ }
+ },
+ {
+ data: function (row, type) {
+ if (type !== 'display')
+ return row.allOffHeapMemoryUsed;
+ else
+ return (formatBytes(row.allOffHeapMemoryUsed, type) + ' / ' +
+ formatBytes(row.allOffHeapMaxMemory, type));
+ }
+ },
+ {data: 'allDiskUsed', render: formatBytes},
+ {data: 'allTotalCores'},
+ {
+ data: 'allActiveTasks',
+ "fnCreatedCell": function (nTd, sData, oData, _ignored_iRow, _ignored_iCol) {
+ if (sData > 0) {
+ $(nTd).css('color', 'white');
+ $(nTd).css('background', activeTasksStyle(oData.allActiveTasks, oData.allMaxTasks));
+ }
+ }
+ },
+ {
+ data: 'allFailedTasks',
+ "fnCreatedCell": function (nTd, sData, oData, _ignored_iRow, _ignored_iCol) {
+ if (sData > 0) {
+ $(nTd).css('color', 'white');
+ $(nTd).css('background', failedTasksStyle(oData.allFailedTasks, oData.allTotalTasks));
+ }
+ }
+ },
+ {data: 'allCompletedTasks'},
+ {data: 'allTotalTasks'},
+ {
+ data: function (row, type) {
+ return type === 'display' ? (formatDuration(row.allTotalDuration) + ' (' + formatDuration(row.allTotalGCTime) + ')') : row.allTotalDuration
+ },
+ "fnCreatedCell": function (nTd, sData, oData, _ignored_iRow, _ignored_iCol) {
+ if (oData.allTotalDuration > 0) {
+ $(nTd).css('color', totalDurationColor(oData.allTotalGCTime, oData.allTotalDuration));
+ $(nTd).css('background', totalDurationStyle(oData.allTotalGCTime, oData.allTotalDuration));
}
- });
+ }
+ },
+ {data: 'allTotalInputBytes', render: formatBytes},
+ {data: 'allTotalShuffleRead', render: formatBytes},
+ {data: 'allTotalShuffleWrite', render: formatBytes},
+ {data: 'allTotalExcluded'}
+ ],
+ "paging": false,
+ "searching": false,
+ "info": false,
+ "columnDefs": [
+ {"visible": false, "targets": 3},
+ {"visible": false, "targets": 4}
+ ]
+
+ };
+
+ sumDataTable = $(sumSelector).DataTable(sumConf);
+ $('#execSummary [data-toggle="tooltip"]').tooltip();
+
+ $("#showAdditionalMetrics").append(
+ "" +
+ "");
+
+ reselectCheckboxesBasedOnTaskTableState();
+
+ $("#additionalMetrics").click(function() {
+ $("#arrowtoggle-optional-metrics").toggleClass("arrow-open arrow-closed");
+ $("#toggle-metrics").toggleClass("d-none");
+ if (window.localStorage) {
+ window.localStorage.setItem("arrowtoggle-optional-metrics-class", $("#arrowtoggle-optional-metrics").attr('class'));
+ }
+ });
+
+ $(".toggle-vis").on("click", function() {
+ var thisBox = $(this);
+ if (thisBox.is("#select-all-box")) {
+ var sumColumn = sumDataTable.columns(sumOptionalColumns);
+ var execColumn = execDataTable.columns(execOptionalColumns);
+ if (thisBox.is(":checked")) {
+ $(".toggle-vis").prop("checked", true);
+ sumColumn.visible(true);
+ execColumn.visible(true);
+ } else {
+ $(".toggle-vis").prop("checked", false);
+ sumColumn.visible(false);
+ execColumn.visible(false);
+ }
+ } else {
+ var execColIdx = thisBox.attr("data-exec-col-idx");
+ var execCol = execDataTable.column(execColIdx);
+ execCol.visible(!execCol.visible());
+ var sumColIdx = thisBox.attr("data-sum-col-idx");
+ if (sumColIdx) {
+ var sumCol = sumDataTable.column(sumColIdx);
+ sumCol.visible(!sumCol.visible());
+ }
+ }
});
+
+ if (window.localStorage) {
+ if (window.localStorage.getItem("arrowtoggle-optional-metrics-class") != null &&
+ window.localStorage.getItem("arrowtoggle-optional-metrics-class").includes("arrow-open")) {
+ $("#arrowtoggle-optional-metrics").toggleClass("arrow-open arrow-closed");
+ $("#toggle-metrics").toggleClass("d-none");
+ }
+ }
+ });
});
+ });
});
diff --git a/core/src/main/resources/org/apache/spark/ui/static/historypage-common.js b/core/src/main/resources/org/apache/spark/ui/static/historypage-common.js
index 4cfe46ec914ae..cd8cf098ef1c0 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/historypage-common.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/historypage-common.js
@@ -15,6 +15,8 @@
* limitations under the License.
*/
+/* global $, formatTimeMillis, getTimeZone */
+
$(document).ready(function() {
if ($('#last-updated').length) {
var lastUpdatedMillis = Number($('#last-updated').text());
diff --git a/core/src/main/resources/org/apache/spark/ui/static/historypage.js b/core/src/main/resources/org/apache/spark/ui/static/historypage.js
index aa542a733fe75..b334bceb5a039 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/historypage.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/historypage.js
@@ -15,11 +15,15 @@
* limitations under the License.
*/
+/* global $, Mustache, formatDuration, formatTimeMillis, jQuery, uiRoot */
+
var appLimit = -1;
+/* eslint-disable no-unused-vars */
function setAppLimit(val) {
- appLimit = val;
+ appLimit = val;
}
+/* eslint-enable no-unused-vars*/
function makeIdNumeric(id) {
var strs = id.split("_");
@@ -30,8 +34,8 @@ function makeIdNumeric(id) {
var resl = strs[0] + "_" + strs[1] + "_";
var diff = 10 - appSeqNum.length;
while (diff > 0) {
- resl += "0"; // padding 0 before the app sequence number to make sure it has 10 characters
- diff--;
+ resl += "0"; // padding 0 before the app sequence number to make sure it has 10 characters
+ diff--;
}
resl += appSeqNum;
return resl;
@@ -39,7 +43,7 @@ function makeIdNumeric(id) {
function getParameterByName(name, searchString) {
var regex = new RegExp("[\\?&]" + name + "=([^&#]*)"),
- results = regex.exec(searchString);
+ results = regex.exec(searchString);
return results === null ? "" : decodeURIComponent(results[1].replace(/\+/g, " "));
}
@@ -56,183 +60,185 @@ function getColumnIndex(columns, columnName) {
}
jQuery.extend( jQuery.fn.dataTableExt.oSort, {
- "title-numeric-pre": function ( a ) {
- var x = a.match(/title="*(-?[0-9\.]+)/)[1];
- return parseFloat( x );
- },
+ "title-numeric-pre": function ( a ) {
+ var x = a.match(/title="*(-?[0-9.]+)/)[1];
+ return parseFloat( x );
+ },
- "title-numeric-asc": function ( a, b ) {
- return ((a < b) ? -1 : ((a > b) ? 1 : 0));
- },
+ "title-numeric-asc": function ( a, b ) {
+ return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+ },
- "title-numeric-desc": function ( a, b ) {
- return ((a < b) ? 1 : ((a > b) ? -1 : 0));
- }
-} );
+ "title-numeric-desc": function ( a, b ) {
+ return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+ }
+});
jQuery.extend( jQuery.fn.dataTableExt.oSort, {
- "appid-numeric-pre": function ( a ) {
- var x = a.match(/title="*(-?[0-9a-zA-Z\-\_]+)/)[1];
- return makeIdNumeric(x);
- },
+ "appid-numeric-pre": function ( a ) {
+ var x = a.match(/title="*(-?[0-9a-zA-Z\-_]+)/)[1];
+ return makeIdNumeric(x);
+ },
- "appid-numeric-asc": function ( a, b ) {
- return ((a < b) ? -1 : ((a > b) ? 1 : 0));
- },
+ "appid-numeric-asc": function ( a, b ) {
+ return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+ },
- "appid-numeric-desc": function ( a, b ) {
- return ((a < b) ? 1 : ((a > b) ? -1 : 0));
- }
-} );
+ "appid-numeric-desc": function ( a, b ) {
+ return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+ }
+});
jQuery.extend( jQuery.fn.dataTableExt.ofnSearch, {
- "appid-numeric": function ( a ) {
- return a.replace(/[\r\n]/g, " ").replace(/<.*?>/g, "");
- }
-} );
+ "appid-numeric": function ( a ) {
+ return a.replace(/[\r\n]/g, " ").replace(/<.*?>/g, "");
+ }
+});
$(document).ajaxStop($.unblockUI);
$(document).ajaxStart(function(){
- $.blockUI({ message: 'Loading history summary...
'});
+ $.blockUI({ message: 'Loading history summary...
'});
});
$(document).ready(function() {
- $.extend( $.fn.dataTable.defaults, {
- stateSave: true,
- lengthMenu: [[20,40,60,100,-1], [20, 40, 60, 100, "All"]],
- pageLength: 20
- });
+ $.extend( $.fn.dataTable.defaults, {
+ stateSave: true,
+ lengthMenu: [[20,40,60,100,-1], [20, 40, 60, 100, "All"]],
+ pageLength: 20
+ });
+
+ var historySummary = $("#history-summary");
+ var searchString = window.location.search;
+ var requestedIncomplete = getParameterByName("showIncomplete", searchString);
+ requestedIncomplete = (requestedIncomplete == "true" ? true : false);
+
+ var appParams = {
+ limit: appLimit,
+ status: (requestedIncomplete ? "running" : "completed")
+ };
+
+ $.getJSON(uiRoot + "/api/v1/applications", appParams, function(response, _ignored_status, _ignored_jqXHR) {
+ var array = [];
+ var hasMultipleAttempts = false;
+ for (var i in response) {
+ var app = response[i];
+ if (app["attempts"][0]["completed"] == requestedIncomplete) {
+ continue; // if we want to show for Incomplete, we skip the completed apps; otherwise skip incomplete ones.
+ }
+ var version = "Unknown"
+ if (app["attempts"].length > 0) {
+ version = app["attempts"][0]["appSparkVersion"]
+ }
+ var id = app["id"];
+ var name = app["name"];
+ if (app["attempts"].length > 1) {
+ hasMultipleAttempts = true;
+ }
- var historySummary = $("#history-summary");
- var searchString = window.location.search;
- var requestedIncomplete = getParameterByName("showIncomplete", searchString);
- requestedIncomplete = (requestedIncomplete == "true" ? true : false);
+ // TODO: Replace hasOwnProperty with prototype.hasOwnProperty after we find it's safe to do.
+ /* eslint-disable no-prototype-builtins */
+ for (var j in app["attempts"]) {
+ var attempt = app["attempts"][j];
+ attempt["startTime"] = formatTimeMillis(attempt["startTimeEpoch"]);
+ attempt["endTime"] = formatTimeMillis(attempt["endTimeEpoch"]);
+ attempt["lastUpdated"] = formatTimeMillis(attempt["lastUpdatedEpoch"]);
+ attempt["log"] = uiRoot + "/api/v1/applications/" + id + "/" +
+ (attempt.hasOwnProperty("attemptId") ? attempt["attemptId"] + "/" : "") + "logs";
+ attempt["durationMillisec"] = attempt["duration"];
+ attempt["duration"] = formatDuration(attempt["duration"]);
+ attempt["id"] = id;
+ attempt["name"] = name;
+ attempt["version"] = version;
+ attempt["attemptUrl"] = uiRoot + "/history/" + id + "/" +
+ (attempt.hasOwnProperty("attemptId") ? attempt["attemptId"] + "/" : "") + "jobs/";
+ array.push(attempt);
+ }
+ /* eslint-enable no-prototype-builtins */
+ }
+ if(array.length < 20) {
+ $.fn.dataTable.defaults.paging = false;
+ }
- var appParams = {
- limit: appLimit,
- status: (requestedIncomplete ? "running" : "completed")
+ var data = {
+ "uiroot": uiRoot,
+ "applications": array,
+ "hasMultipleAttempts": hasMultipleAttempts,
+ "showCompletedColumns": !requestedIncomplete,
};
- $.getJSON(uiRoot + "/api/v1/applications", appParams, function(response,status,jqXHR) {
- var array = [];
- var hasMultipleAttempts = false;
- for (var i in response) {
- var app = response[i];
- if (app["attempts"][0]["completed"] == requestedIncomplete) {
- continue; // if we want to show for Incomplete, we skip the completed apps; otherwise skip incomplete ones.
- }
- var version = "Unknown"
- if (app["attempts"].length > 0) {
- version = app["attempts"][0]["appSparkVersion"]
- }
- var id = app["id"];
- var name = app["name"];
- if (app["attempts"].length > 1) {
- hasMultipleAttempts = true;
- }
-
- for (var j in app["attempts"]) {
- var attempt = app["attempts"][j];
- attempt["startTime"] = formatTimeMillis(attempt["startTimeEpoch"]);
- attempt["endTime"] = formatTimeMillis(attempt["endTimeEpoch"]);
- attempt["lastUpdated"] = formatTimeMillis(attempt["lastUpdatedEpoch"]);
- attempt["log"] = uiRoot + "/api/v1/applications/" + id + "/" +
- (attempt.hasOwnProperty("attemptId") ? attempt["attemptId"] + "/" : "") + "logs";
- attempt["durationMillisec"] = attempt["duration"];
- attempt["duration"] = formatDuration(attempt["duration"]);
- attempt["id"] = id;
- attempt["name"] = name;
- attempt["version"] = version;
- attempt["attemptUrl"] = uiRoot + "/history/" + id + "/" +
- (attempt.hasOwnProperty("attemptId") ? attempt["attemptId"] + "/" : "") + "jobs/";
-
- array.push(attempt);
- }
- }
- if(array.length < 20) {
- $.fn.dataTable.defaults.paging = false;
- }
-
- var data = {
- "uiroot": uiRoot,
- "applications": array,
- "hasMultipleAttempts": hasMultipleAttempts,
- "showCompletedColumns": !requestedIncomplete,
+ $.get(uiRoot + "/static/historypage-template.html", function(template) {
+ var sibling = historySummary.prev();
+ historySummary.detach();
+ var apps = $(Mustache.render($(template).filter("#history-summary-template").html(),data));
+ var attemptIdColumnName = 'attemptId';
+ var startedColumnName = 'started';
+ var completedColumnName = 'completed';
+ var durationColumnName = 'duration';
+ var conf = {
+ "data": array,
+ "columns": [
+ {name: 'version', data: 'version' },
+ {
+ name: 'appId',
+ type: "appid-numeric",
+ data: 'id',
+ render: (id, type, row) => `${id}`
+ },
+ {name: 'appName', data: 'name' },
+ {
+ name: attemptIdColumnName,
+ data: 'attemptId',
+ render: (attemptId, type, row) => (attemptId ? `${attemptId}` : '')
+ },
+ {name: startedColumnName, data: 'startTime' },
+ {name: completedColumnName, data: 'endTime' },
+ {name: durationColumnName, type: "title-numeric", data: 'duration' },
+ {name: 'user', data: 'sparkUser' },
+ {name: 'lastUpdated', data: 'lastUpdated' },
+ {
+ name: 'eventLog',
+ data: 'log',
+ render: (log, _ignored_type, _ignored_row) => `Download`
+ },
+ ],
+ "aoColumnDefs": [
+ {
+ aTargets: [0, 1, 2],
+ fnCreatedCell: (nTd, _ignored_sData, _ignored_oData, _ignored_iRow, _ignored_iCol) => {
+ if (hasMultipleAttempts) {
+ $(nTd).css('background-color', '#fff');
+ }
+ }
+ },
+ ],
+ "autoWidth": false,
+ "deferRender": true
};
- $.get(uiRoot + "/static/historypage-template.html", function(template) {
- var sibling = historySummary.prev();
- historySummary.detach();
- var apps = $(Mustache.render($(template).filter("#history-summary-template").html(),data));
- var attemptIdColumnName = 'attemptId';
- var startedColumnName = 'started';
- var completedColumnName = 'completed';
- var durationColumnName = 'duration';
- var conf = {
- "data": array,
- "columns": [
- {name: 'version', data: 'version' },
- {
- name: 'appId',
- type: "appid-numeric",
- data: 'id',
- render: (id, type, row) => `${id}`
- },
- {name: 'appName', data: 'name' },
- {
- name: attemptIdColumnName,
- data: 'attemptId',
- render: (attemptId, type, row) => (attemptId ? `${attemptId}` : '')
- },
- {name: startedColumnName, data: 'startTime' },
- {name: completedColumnName, data: 'endTime' },
- {name: durationColumnName, type: "title-numeric", data: 'duration' },
- {name: 'user', data: 'sparkUser' },
- {name: 'lastUpdated', data: 'lastUpdated' },
- {
- name: 'eventLog',
- data: 'log',
- render: (log, type, row) => `Download`
- },
- ],
- "aoColumnDefs": [
- {
- aTargets: [0, 1, 2],
- fnCreatedCell: (nTd, sData, oData, iRow, iCol) => {
- if (hasMultipleAttempts) {
- $(nTd).css('background-color', '#fff');
- }
- }
- },
- ],
- "autoWidth": false,
- "deferRender": true
- };
-
- if (hasMultipleAttempts) {
- conf.rowsGroup = [
- 'appId:name',
- 'version:name',
- 'appName:name'
- ];
- } else {
- conf.columns = removeColumnByName(conf.columns, attemptIdColumnName);
- }
-
- var defaultSortColumn = completedColumnName;
- if (requestedIncomplete) {
- defaultSortColumn = startedColumnName;
- conf.columns = removeColumnByName(conf.columns, completedColumnName);
- conf.columns = removeColumnByName(conf.columns, durationColumnName);
- }
- conf.order = [[ getColumnIndex(conf.columns, defaultSortColumn), "desc" ]];
- conf.columnDefs = [
- {"searchable": false, "targets": [getColumnIndex(conf.columns, durationColumnName)]}
+ if (hasMultipleAttempts) {
+ conf.rowsGroup = [
+ 'appId:name',
+ 'version:name',
+ 'appName:name'
];
- historySummary.append(apps);
- apps.DataTable(conf);
- sibling.after(historySummary);
- $('#history-summary [data-toggle="tooltip"]').tooltip();
- });
+ } else {
+ conf.columns = removeColumnByName(conf.columns, attemptIdColumnName);
+ }
+
+ var defaultSortColumn = completedColumnName;
+ if (requestedIncomplete) {
+ defaultSortColumn = startedColumnName;
+ conf.columns = removeColumnByName(conf.columns, completedColumnName);
+ conf.columns = removeColumnByName(conf.columns, durationColumnName);
+ }
+ conf.order = [[ getColumnIndex(conf.columns, defaultSortColumn), "desc" ]];
+ conf.columnDefs = [
+ {"searchable": false, "targets": [getColumnIndex(conf.columns, durationColumnName)]}
+ ];
+ historySummary.append(apps);
+ apps.DataTable(conf);
+ sibling.after(historySummary);
+ $('#history-summary [data-toggle="tooltip"]').tooltip();
});
+ });
});
diff --git a/core/src/main/resources/org/apache/spark/ui/static/initialize-tooltips.js b/core/src/main/resources/org/apache/spark/ui/static/initialize-tooltips.js
index 70f355dfb49cb..b273a19783f8f 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/initialize-tooltips.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/initialize-tooltips.js
@@ -15,7 +15,9 @@
* limitations under the License.
*/
+/* global $ */
+
$(document).ready(function(){
- $("[data-toggle=tooltip]").tooltip({container: 'body'});
+ $("[data-toggle=tooltip]").tooltip({container: 'body'});
});
diff --git a/core/src/main/resources/org/apache/spark/ui/static/log-view.js b/core/src/main/resources/org/apache/spark/ui/static/log-view.js
index b5c43e5788bc3..1a4f9254742ff 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/log-view.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/log-view.js
@@ -15,6 +15,8 @@
* limitations under the License.
*/
+/* global $ */
+
var baseParams;
var curLogLength;
@@ -59,11 +61,12 @@ function getRESTEndPoint() {
var words = document.baseURI.split('/');
var ind = words.indexOf("proxy");
if (ind > 0) {
- return words.slice(0, ind + 2).join('/') + "/log";
+ return words.slice(0, ind + 2).join('/') + "/log";
}
return "/log"
}
+/* eslint-disable no-unused-vars */
function loadMore() {
var offset = Math.max(startByte - byteLength, 0);
var moreByteLength = Math.min(byteLength, startByte);
@@ -139,4 +142,5 @@ function initLogPage(params, logLen, start, end, totLogLen, defaultLen) {
if (startByte == 0) {
disableMoreButton();
}
-}
\ No newline at end of file
+}
+/* eslint-enable no-unused-vars */
\ No newline at end of file
diff --git a/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.js b/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.js
index 48a3c93cce914..6a0cd012146ec 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.js
@@ -51,6 +51,8 @@
* since it was forked (commit 101503833a8ce5fe369547f6addf3e71172ce10b).
*/
+/* global $, appBasePath, d3, dagreD3, graphlibDot, uiRoot */
+
var VizConstants = {
svgMarginX: 16,
svgMarginY: 16,
@@ -166,7 +168,7 @@ function renderDagViz(forJob) {
}
// Find cached RDDs and mark them as such
- metadataContainer().selectAll(".cached-rdd").each(function(v) {
+ metadataContainer().selectAll(".cached-rdd").each(function(_ignored_v) {
var rddId = d3.select(this).text().trim();
var nodeId = VizConstants.nodePrefix + rddId;
svg.selectAll("g." + nodeId).classed("cached", true);
@@ -180,7 +182,7 @@ function renderDagViz(forJob) {
svg.selectAll("g[id=" + stageClusterId + "] g." + opClusterId).classed("barrier", true)
});
- metadataContainer().selectAll(".indeterminate-rdd").each(function(v) {
+ metadataContainer().selectAll(".indeterminate-rdd").each(function(_ignored_v) {
var rddId = d3.select(this).text().trim();
var nodeId = VizConstants.nodePrefix + rddId;
svg.selectAll("g." + nodeId).classed("indeterminate", true);
@@ -275,7 +277,7 @@ function renderDagVizForJob(svgContainer) {
// If there are any incoming edges into this graph, keep track of them to render
// them separately later. Note that we cannot draw them now because we need to
// put these edges in a separate container that is on top of all stage graphs.
- metadata.selectAll(".incoming-edge").each(function(v) {
+ metadata.selectAll(".incoming-edge").each(function(_ignored_v) {
var edge = d3.select(this).text().trim().split(","); // e.g. 3,4 => [3, 4]
crossStageEdges.push(edge);
});
diff --git a/core/src/main/resources/org/apache/spark/ui/static/stagepage.js b/core/src/main/resources/org/apache/spark/ui/static/stagepage.js
index 8b32fe7d3e20d..fea595d0a6216 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/stagepage.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/stagepage.js
@@ -15,284 +15,291 @@
* limitations under the License.
*/
+/* global $, ConvertDurationString, Mustache, createRESTEndPointForExecutorsPage */
+/* global createTemplateURI, formatBytes, formatDate, formatDuration, formatLogsCells */
+/* global getStandAloneAppId, setDataTableDefaults, uiRoot */
+
var shouldBlockUI = true;
$(document).ajaxStop(function () {
- if (shouldBlockUI) {
- $.unblockUI();
- shouldBlockUI = false;
- }
+ if (shouldBlockUI) {
+ $.unblockUI();
+ shouldBlockUI = false;
+ }
});
$(document).ajaxStart(function () {
- if (shouldBlockUI) {
- $.blockUI({message: 'Loading Stage Page...
'});
- }
+ if (shouldBlockUI) {
+ $.blockUI({message: 'Loading Stage Page...
'});
+ }
});
$.extend( $.fn.dataTable.ext.type.order, {
- "duration-pre": ConvertDurationString,
-
- "duration-asc": function ( a, b ) {
- a = ConvertDurationString( a );
- b = ConvertDurationString( b );
- return ((a < b) ? -1 : ((a > b) ? 1 : 0));
- },
-
- "duration-desc": function ( a, b ) {
- a = ConvertDurationString( a );
- b = ConvertDurationString( b );
- return ((a < b) ? 1 : ((a > b) ? -1 : 0));
- },
-
- "size-pre": function (data) {
- var floatValue = parseFloat(data)
- return isNaN(floatValue) ? 0 : floatValue;
- },
-
- "size-asc": function (a, b) {
- a = parseFloat(a);
- b = parseFloat(b);
- return ((a < b) ? -1 : ((a > b) ? 1 : 0));
- },
-
- "size-desc": function (a, b) {
- a = parseFloat(a);
- b = parseFloat(b);
- return ((a < b) ? 1 : ((a > b) ? -1 : 0));
- }
-} );
+ "duration-pre": ConvertDurationString,
+
+ "duration-asc": function ( a, b ) {
+ a = ConvertDurationString( a );
+ b = ConvertDurationString( b );
+ return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+ },
+
+ "duration-desc": function ( a, b ) {
+ a = ConvertDurationString( a );
+ b = ConvertDurationString( b );
+ return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+ },
+
+ "size-pre": function (data) {
+ var floatValue = parseFloat(data)
+ return isNaN(floatValue) ? 0 : floatValue;
+ },
+
+ "size-asc": function (a, b) {
+ a = parseFloat(a);
+ b = parseFloat(b);
+ return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+ },
+
+ "size-desc": function (a, b) {
+ a = parseFloat(a);
+ b = parseFloat(b);
+ return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+ }
+});
// This function will only parse the URL under certain format
// e.g. (history) https://domain:50509/history/application_1536254569791_3806251/1/stages/stage/?id=4&attempt=1
// e.g. (proxy) https://domain:50505/proxy/application_1502220952225_59143/stages/stage?id=4&attempt=1
function stageEndPoint(appId) {
- var queryString = document.baseURI.split('?');
- var words = document.baseURI.split('/');
- var indexOfProxy = words.indexOf("proxy");
- var stageId = queryString[1].split("&").filter(word => word.includes("id="))[0].split("=")[1];
- if (indexOfProxy > 0) {
- var appId = words[indexOfProxy + 1];
- var newBaseURI = words.slice(0, words.indexOf("proxy") + 2).join('/');
- return newBaseURI + "/api/v1/applications/" + appId + "/stages/" + stageId;
- }
- var indexOfHistory = words.indexOf("history");
- if (indexOfHistory > 0) {
- var appId = words[indexOfHistory + 1];
- var appAttemptId = words[indexOfHistory + 2];
- var newBaseURI = words.slice(0, words.indexOf("history")).join('/');
- if (isNaN(appAttemptId) || appAttemptId == "0") {
- return newBaseURI + "/api/v1/applications/" + appId + "/stages/" + stageId;
- } else {
- return newBaseURI + "/api/v1/applications/" + appId + "/" + appAttemptId + "/stages/" + stageId;
- }
+ var queryString = document.baseURI.split('?');
+ var words = document.baseURI.split('/');
+ var indexOfProxy = words.indexOf("proxy");
+ var stageId = queryString[1].split("&").filter(word => word.includes("id="))[0].split("=")[1];
+ var newBaseURI;
+ if (indexOfProxy > 0) {
+ appId = words[indexOfProxy + 1];
+ newBaseURI = words.slice(0, words.indexOf("proxy") + 2).join('/');
+ return newBaseURI + "/api/v1/applications/" + appId + "/stages/" + stageId;
+ }
+ var indexOfHistory = words.indexOf("history");
+ if (indexOfHistory > 0) {
+ appId = words[indexOfHistory + 1];
+ var appAttemptId = words[indexOfHistory + 2];
+ newBaseURI = words.slice(0, words.indexOf("history")).join('/');
+ if (isNaN(appAttemptId) || appAttemptId == "0") {
+ return newBaseURI + "/api/v1/applications/" + appId + "/stages/" + stageId;
+ } else {
+ return newBaseURI + "/api/v1/applications/" + appId + "/" + appAttemptId + "/stages/" + stageId;
}
- return uiRoot + "/api/v1/applications/" + appId + "/stages/" + stageId;
+ }
+ return uiRoot + "/api/v1/applications/" + appId + "/stages/" + stageId;
}
function getColumnNameForTaskMetricSummary(columnKey) {
- switch(columnKey) {
- case "executorRunTime":
- return "Duration";
+ switch(columnKey) {
+ case "executorRunTime":
+ return "Duration";
- case "jvmGcTime":
- return "GC Time";
+ case "jvmGcTime":
+ return "GC Time";
- case "gettingResultTime":
- return "Getting Result Time";
+ case "gettingResultTime":
+ return "Getting Result Time";
- case "inputMetrics":
- return "Input Size / Records";
+ case "inputMetrics":
+ return "Input Size / Records";
- case "outputMetrics":
- return "Output Size / Records";
+ case "outputMetrics":
+ return "Output Size / Records";
- case "peakExecutionMemory":
- return "Peak Execution Memory";
+ case "peakExecutionMemory":
+ return "Peak Execution Memory";
- case "resultSerializationTime":
- return "Result Serialization Time";
+ case "resultSerializationTime":
+ return "Result Serialization Time";
- case "schedulerDelay":
- return "Scheduler Delay";
+ case "schedulerDelay":
+ return "Scheduler Delay";
- case "diskBytesSpilled":
- return "Spill (disk)";
+ case "diskBytesSpilled":
+ return "Spill (disk)";
- case "memoryBytesSpilled":
- return "Spill (memory)";
+ case "memoryBytesSpilled":
+ return "Spill (memory)";
- case "shuffleReadMetrics":
- return "Shuffle Read Size / Records";
+ case "shuffleReadMetrics":
+ return "Shuffle Read Size / Records";
- case "shuffleWriteMetrics":
- return "Shuffle Write Size / Records";
+ case "shuffleWriteMetrics":
+ return "Shuffle Write Size / Records";
- case "executorDeserializeTime":
- return "Task Deserialization Time";
+ case "executorDeserializeTime":
+ return "Task Deserialization Time";
- case "shuffleReadBlockedTime":
- return "Shuffle Read Blocked Time";
+ case "shuffleReadBlockedTime":
+ return "Shuffle Read Blocked Time";
- case "shuffleRemoteReads":
- return "Shuffle Remote Reads";
+ case "shuffleRemoteReads":
+ return "Shuffle Remote Reads";
- case "shuffleWriteTime":
- return "Shuffle Write Time";
+ case "shuffleWriteTime":
+ return "Shuffle Write Time";
- default:
- return "NA";
- }
+ default:
+ return "NA";
+ }
}
function displayRowsForSummaryMetricsTable(row, type, columnIndex) {
- switch(row.columnKey) {
- case 'inputMetrics':
- var str = formatBytes(row.data.bytesRead[columnIndex], type) + " / " +
- row.data.recordsRead[columnIndex];
- return str;
-
- case 'outputMetrics':
- var str = formatBytes(row.data.bytesWritten[columnIndex], type) + " / " +
- row.data.recordsWritten[columnIndex];
- return str;
-
- case 'shuffleReadMetrics':
- var str = formatBytes(row.data.readBytes[columnIndex], type) + " / " +
- row.data.readRecords[columnIndex];
- return str;
-
- case 'shuffleReadBlockedTime':
- var str = formatDuration(row.data.fetchWaitTime[columnIndex]);
- return str;
-
- case 'shuffleRemoteReads':
- var str = formatBytes(row.data.remoteBytesRead[columnIndex], type);
- return str;
-
- case 'shuffleWriteMetrics':
- var str = formatBytes(row.data.writeBytes[columnIndex], type) + " / " +
- row.data.writeRecords[columnIndex];
- return str;
-
- case 'shuffleWriteTime':
- var str = formatDuration(row.data.writeTime[columnIndex] / 1000000.0);
- return str;
-
- default:
- return (row.columnKey == 'peakExecutionMemory' || row.columnKey == 'memoryBytesSpilled'
- || row.columnKey == 'diskBytesSpilled') ? formatBytes(
- row.data[columnIndex], type) : (formatDuration(row.data[columnIndex]));
-
- }
+ var str;
+ switch(row.columnKey) {
+ case 'inputMetrics':
+ str = formatBytes(row.data.bytesRead[columnIndex], type) + " / " +
+ row.data.recordsRead[columnIndex];
+ return str;
+
+ case 'outputMetrics':
+ str = formatBytes(row.data.bytesWritten[columnIndex], type) + " / " +
+ row.data.recordsWritten[columnIndex];
+ return str;
+
+ case 'shuffleReadMetrics':
+ str = formatBytes(row.data.readBytes[columnIndex], type) + " / " +
+ row.data.readRecords[columnIndex];
+ return str;
+
+ case 'shuffleReadBlockedTime':
+ str = formatDuration(row.data.fetchWaitTime[columnIndex]);
+ return str;
+
+ case 'shuffleRemoteReads':
+ str = formatBytes(row.data.remoteBytesRead[columnIndex], type);
+ return str;
+
+ case 'shuffleWriteMetrics':
+ str = formatBytes(row.data.writeBytes[columnIndex], type) + " / " +
+ row.data.writeRecords[columnIndex];
+ return str;
+
+ case 'shuffleWriteTime':
+ str = formatDuration(row.data.writeTime[columnIndex] / 1000000.0);
+ return str;
+
+ default:
+ return (row.columnKey == 'peakExecutionMemory' || row.columnKey == 'memoryBytesSpilled'
+ || row.columnKey == 'diskBytesSpilled') ? formatBytes(
+ row.data[columnIndex], type) : (formatDuration(row.data[columnIndex]));
+
+ }
}
function createDataTableForTaskSummaryMetricsTable(taskSummaryMetricsTable) {
- var taskMetricsTable = "#summary-metrics-table";
- if ($.fn.dataTable.isDataTable(taskMetricsTable)) {
- taskSummaryMetricsDataTable.clear().draw();
- taskSummaryMetricsDataTable.rows.add(taskSummaryMetricsTable).draw();
- } else {
- var taskConf = {
- "data": taskSummaryMetricsTable,
- "columns": [
- {data : 'metric'},
- // Min
- {
- data: function (row, type) {
- return displayRowsForSummaryMetricsTable(row, type, 0);
- }
- },
- // 25th percentile
- {
- data: function (row, type) {
- return displayRowsForSummaryMetricsTable(row, type, 1);
- }
- },
- // Median
- {
- data: function (row, type) {
- return displayRowsForSummaryMetricsTable(row, type, 2);
- }
- },
- // 75th percentile
- {
- data: function (row, type) {
- return displayRowsForSummaryMetricsTable(row, type, 3);
- }
- },
- // Max
- {
- data: function (row, type) {
- return displayRowsForSummaryMetricsTable(row, type, 4);
- }
- }
- ],
- "columnDefs": [
- { "type": "duration", "targets": 1 },
- { "type": "duration", "targets": 2 },
- { "type": "duration", "targets": 3 },
- { "type": "duration", "targets": 4 },
- { "type": "duration", "targets": 5 }
- ],
- "paging": false,
- "searching": false,
- "order": [[0, "asc"]],
- "bSort": false,
- "bAutoWidth": false,
- "oLanguage": {
- "sEmptyTable": "No tasks have reported metrics yet"
- }
- };
- taskSummaryMetricsDataTable = $(taskMetricsTable).DataTable(taskConf);
- }
- taskSummaryMetricsTableCurrentStateArray = taskSummaryMetricsTable.slice();
+ var taskMetricsTable = "#summary-metrics-table";
+ if ($.fn.dataTable.isDataTable(taskMetricsTable)) {
+ taskSummaryMetricsDataTable.clear().draw();
+ taskSummaryMetricsDataTable.rows.add(taskSummaryMetricsTable).draw();
+ } else {
+ var taskConf = {
+ "data": taskSummaryMetricsTable,
+ "columns": [
+ {data : 'metric'},
+ // Min
+ {
+ data: function (row, type) {
+ return displayRowsForSummaryMetricsTable(row, type, 0);
+ }
+ },
+ // 25th percentile
+ {
+ data: function (row, type) {
+ return displayRowsForSummaryMetricsTable(row, type, 1);
+ }
+ },
+ // Median
+ {
+ data: function (row, type) {
+ return displayRowsForSummaryMetricsTable(row, type, 2);
+ }
+ },
+ // 75th percentile
+ {
+ data: function (row, type) {
+ return displayRowsForSummaryMetricsTable(row, type, 3);
+ }
+ },
+ // Max
+ {
+ data: function (row, type) {
+ return displayRowsForSummaryMetricsTable(row, type, 4);
+ }
+ }
+ ],
+ "columnDefs": [
+ { "type": "duration", "targets": 1 },
+ { "type": "duration", "targets": 2 },
+ { "type": "duration", "targets": 3 },
+ { "type": "duration", "targets": 4 },
+ { "type": "duration", "targets": 5 }
+ ],
+ "paging": false,
+ "searching": false,
+ "order": [[0, "asc"]],
+ "bSort": false,
+ "bAutoWidth": false,
+ "oLanguage": {
+ "sEmptyTable": "No tasks have reported metrics yet"
+ }
+ };
+ taskSummaryMetricsDataTable = $(taskMetricsTable).DataTable(taskConf);
+ }
+ taskSummaryMetricsTableCurrentStateArray = taskSummaryMetricsTable.slice();
}
function createRowMetadataForColumn(colKey, data, checkboxId) {
var row = {
- "metric": getColumnNameForTaskMetricSummary(colKey),
- "data": data,
- "checkboxId": checkboxId,
- "columnKey": colKey
+ "metric": getColumnNameForTaskMetricSummary(colKey),
+ "data": data,
+ "checkboxId": checkboxId,
+ "columnKey": colKey
};
return row;
}
function reselectCheckboxesBasedOnTaskTableState() {
- var taskSummaryHasSelected = false;
- var executorSummaryHasSelected = false;
- var allTaskSummaryChecked = true;
- var allExecutorSummaryChecked = true;
- var taskSummaryMetricsTableCurrentFilteredArray = taskSummaryMetricsTableCurrentStateArray.slice();
- if (typeof taskTableSelector !== 'undefined' && taskSummaryMetricsTableCurrentStateArray.length > 0) {
- for (var k = 0; k < optionalColumns.length; k++) {
- if (taskTableSelector.column(optionalColumns[k]).visible()) {
- taskSummaryHasSelected = true;
- $("#box-"+optionalColumns[k]).prop('checked', true);
- taskSummaryMetricsTableCurrentStateArray.push(taskSummaryMetricsTableArray.filter(row => (row.checkboxId).toString() == optionalColumns[k])[0]);
- taskSummaryMetricsTableCurrentFilteredArray = taskSummaryMetricsTableCurrentStateArray.slice();
- } else {
- allTaskSummaryChecked = false;
- }
- }
- createDataTableForTaskSummaryMetricsTable(taskSummaryMetricsTableCurrentFilteredArray);
+ var taskSummaryHasSelected = false;
+ var executorSummaryHasSelected = false;
+ var allTaskSummaryChecked = true;
+ var allExecutorSummaryChecked = true;
+ var taskSummaryMetricsTableCurrentFilteredArray = taskSummaryMetricsTableCurrentStateArray.slice();
+ var k;
+ if (typeof taskTableSelector !== 'undefined' && taskSummaryMetricsTableCurrentStateArray.length > 0) {
+ for (k = 0; k < optionalColumns.length; k++) {
+ if (taskTableSelector.column(optionalColumns[k]).visible()) {
+ taskSummaryHasSelected = true;
+ $("#box-"+optionalColumns[k]).prop('checked', true);
+ taskSummaryMetricsTableCurrentStateArray.push(taskSummaryMetricsTableArray.filter(row => (row.checkboxId).toString() == optionalColumns[k])[0]);
+ taskSummaryMetricsTableCurrentFilteredArray = taskSummaryMetricsTableCurrentStateArray.slice();
+ } else {
+ allTaskSummaryChecked = false;
+ }
}
-
- if (typeof executorSummaryTableSelector !== 'undefined') {
- for (var k = 0; k < executorOptionalColumns.length; k++) {
- if (executorSummaryTableSelector.column(executorOptionalColumns[k]).visible()) {
- executorSummaryHasSelected = true;
- $("#executor-box-"+executorOptionalColumns[k]).prop('checked', true);
- } else {
- allExecutorSummaryChecked = false;
- }
- }
+ createDataTableForTaskSummaryMetricsTable(taskSummaryMetricsTableCurrentFilteredArray);
+ }
+
+ if (typeof executorSummaryTableSelector !== 'undefined') {
+ for (k = 0; k < executorOptionalColumns.length; k++) {
+ if (executorSummaryTableSelector.column(executorOptionalColumns[k]).visible()) {
+ executorSummaryHasSelected = true;
+ $("#executor-box-"+executorOptionalColumns[k]).prop('checked', true);
+ } else {
+ allExecutorSummaryChecked = false;
+ }
}
+ }
- if ((taskSummaryHasSelected || executorSummaryHasSelected) && allTaskSummaryChecked && allExecutorSummaryChecked) {
- $("#box-0").prop('checked', true);
- }
+ if ((taskSummaryHasSelected || executorSummaryHasSelected) && allTaskSummaryChecked && allExecutorSummaryChecked) {
+ $("#box-0").prop('checked', true);
+ }
}
function getStageAttemptId() {
@@ -301,7 +308,7 @@ function getStageAttemptId() {
// We are using regex here to extract the stage attempt id as there might be certain url's with format
// like /proxy/application_1539986433979_27115/stages/stage/?id=0&attempt=0#tasksTitle
var stgAttemptId = words[1].split("&").filter(
- word => word.includes("attempt="))[0].split("=")[1].match(digitsRegex);
+ word => word.includes("attempt="))[0].split("=")[1].match(digitsRegex);
return stgAttemptId;
}
@@ -315,826 +322,833 @@ var executorOptionalColumns = [15, 16, 17, 18];
var executorSummaryTableSelector;
$(document).ready(function () {
- setDataTableDefaults();
-
- $("#showAdditionalMetrics").append(
- "" +
- "");
-
- $('#scheduler_delay').attr("data-toggle", "tooltip")
- .attr("data-placement", "top")
- .attr("title", "Scheduler delay includes time to ship the task from the scheduler to the executor, and time to send " +
- "the task result from the executor to the scheduler. If scheduler delay is large, consider decreasing the size of tasks or decreasing the size of task results.");
- $('#task_deserialization_time').attr("data-toggle", "tooltip")
- .attr("data-placement", "top")
- .attr("title", "Time spent deserializing the task closure on the executor, including the time to read the broadcasted task.");
- $('#shuffle_read_blocked_time').attr("data-toggle", "tooltip")
- .attr("data-placement", "top")
- .attr("title", "Time that the task spent blocked waiting for shuffle data to be read from remote machines.");
- $('#shuffle_remote_reads').attr("data-toggle", "tooltip")
- .attr("data-placement", "top")
- .attr("title", "Total shuffle bytes read from remote executors. This is a subset of the shuffle read bytes; the remaining shuffle data is read locally. ");
- $('#shuffle_write_time').attr("data-toggle", "tooltip")
- .attr("data-placement", "top")
- .attr("title", "Time that the task spent writing shuffle data.");
- $('#result_serialization_time').attr("data-toggle", "tooltip")
- .attr("data-placement", "top")
- .attr("title", "Time spent serializing the task result on the executor before sending it back to the driver.");
- $('#getting_result_time').attr("data-toggle", "tooltip")
- .attr("data-placement", "top")
- .attr("title", "Time that the driver spends fetching task results from workers. If this is large, consider decreasing the amount of data returned from each task.");
- $('#peak_execution_memory').attr("data-toggle", "tooltip")
- .attr("data-placement", "top")
- .attr("title", "Execution memory refers to the memory used by internal data structures created during " +
- "shuffles, aggregations and joins when Tungsten is enabled. The value of this accumulator " +
- "should be approximately the sum of the peak sizes across all such data structures created " +
- "in this task. For SQL jobs, this only tracks all unsafe operators, broadcast joins, and " +
- "external sort.");
- $('[data-toggle="tooltip"]').tooltip();
- var tasksSummary = $("#parent-container");
- getStandAloneAppId(function (appId) {
- // rendering the UI page
- $.get(createTemplateURI(appId, "stagespage"), function(template) {
- tasksSummary.append(Mustache.render($(template).filter("#stages-summary-template").html()));
-
- $("#additionalMetrics").click(function(){
- $("#arrowtoggle1").toggleClass("arrow-open arrow-closed");
- $("#toggle-metrics").toggleClass("d-none");
- if (window.localStorage) {
- window.localStorage.setItem("arrowtoggle1class", $("#arrowtoggle1").attr('class'));
- }
+ setDataTableDefaults();
+
+ $("#showAdditionalMetrics").append(
+ "" +
+ "");
+
+ $('#scheduler_delay').attr("data-toggle", "tooltip")
+ .attr("data-placement", "top")
+ .attr("title", "Scheduler delay includes time to ship the task from the scheduler to the executor, and time to send " +
+ "the task result from the executor to the scheduler. If scheduler delay is large, consider decreasing the size of tasks or decreasing the size of task results.");
+ $('#task_deserialization_time').attr("data-toggle", "tooltip")
+ .attr("data-placement", "top")
+ .attr("title", "Time spent deserializing the task closure on the executor, including the time to read the broadcasted task.");
+ $('#shuffle_read_blocked_time').attr("data-toggle", "tooltip")
+ .attr("data-placement", "top")
+ .attr("title", "Time that the task spent blocked waiting for shuffle data to be read from remote machines.");
+ $('#shuffle_remote_reads').attr("data-toggle", "tooltip")
+ .attr("data-placement", "top")
+ .attr("title", "Total shuffle bytes read from remote executors. This is a subset of the shuffle read bytes; the remaining shuffle data is read locally. ");
+ $('#shuffle_write_time').attr("data-toggle", "tooltip")
+ .attr("data-placement", "top")
+ .attr("title", "Time that the task spent writing shuffle data.");
+ $('#result_serialization_time').attr("data-toggle", "tooltip")
+ .attr("data-placement", "top")
+ .attr("title", "Time spent serializing the task result on the executor before sending it back to the driver.");
+ $('#getting_result_time').attr("data-toggle", "tooltip")
+ .attr("data-placement", "top")
+ .attr("title", "Time that the driver spends fetching task results from workers. If this is large, consider decreasing the amount of data returned from each task.");
+ $('#peak_execution_memory').attr("data-toggle", "tooltip")
+ .attr("data-placement", "top")
+ .attr("title", "Execution memory refers to the memory used by internal data structures created during " +
+ "shuffles, aggregations and joins when Tungsten is enabled. The value of this accumulator " +
+ "should be approximately the sum of the peak sizes across all such data structures created " +
+ "in this task. For SQL jobs, this only tracks all unsafe operators, broadcast joins, and " +
+ "external sort.");
+ $('[data-toggle="tooltip"]').tooltip();
+ var tasksSummary = $("#parent-container");
+ getStandAloneAppId(function (appId) {
+ // rendering the UI page
+ $.get(createTemplateURI(appId, "stagespage"), function(template) {
+ tasksSummary.append(Mustache.render($(template).filter("#stages-summary-template").html()));
+
+ $("#additionalMetrics").click(function(){
+ $("#arrowtoggle1").toggleClass("arrow-open arrow-closed");
+ $("#toggle-metrics").toggleClass("d-none");
+ if (window.localStorage) {
+ window.localStorage.setItem("arrowtoggle1class", $("#arrowtoggle1").attr('class'));
+ }
+ });
+
+ $("#aggregatedMetrics").click(function(){
+ $("#arrowtoggle2").toggleClass("arrow-open arrow-closed");
+ $("#toggle-aggregatedMetrics").toggleClass("d-none");
+ if (window.localStorage) {
+ window.localStorage.setItem("arrowtoggle2class", $("#arrowtoggle2").attr('class'));
+ }
+ });
+
+ var endPoint = stageEndPoint(appId);
+ var stageAttemptId = getStageAttemptId();
+ $.getJSON(endPoint + "/" + stageAttemptId, function(response, _ignored_status, _ignored_jqXHR) {
+
+ var responseBody = response;
+ var dataToShow = {};
+ dataToShow.showInputData = responseBody.inputBytes > 0;
+ dataToShow.showOutputData = responseBody.outputBytes > 0;
+ dataToShow.showShuffleReadData = responseBody.shuffleReadBytes > 0;
+ dataToShow.showShuffleWriteData = responseBody.shuffleWriteBytes > 0;
+ dataToShow.showBytesSpilledData =
+ (responseBody.diskBytesSpilled > 0 || responseBody.memoryBytesSpilled > 0);
+
+ var columnIndicesToRemove = [];
+ if (!dataToShow.showShuffleReadData) {
+ $('#shuffle_read_blocked_time').remove();
+ $('#shuffle_remote_reads').remove();
+ columnIndicesToRemove.push(2);
+ columnIndicesToRemove.push(3);
+ }
+
+ if (!dataToShow.showShuffleWriteData) {
+ $('#shuffle_write_time').remove();
+ columnIndicesToRemove.push(7);
+ }
+
+ if (columnIndicesToRemove.length > 0) {
+ columnIndicesToRemove.sort(function(a, b) { return b - a; });
+ columnIndicesToRemove.forEach(function(idx) {
+ optionalColumns.splice(idx, 1);
});
+ }
+
+ // prepare data for executor summary table
+ var stageExecutorSummaryInfoKeys = Object.keys(responseBody.executorSummary);
+ $.getJSON(createRESTEndPointForExecutorsPage(appId),
+ function(executorSummaryResponse, _ignored_status, _ignored_jqXHR) {
+ var executorDetailsMap = {};
+ executorSummaryResponse.forEach(function (executorDetail) {
+ executorDetailsMap[executorDetail.id] = executorDetail;
+ });
+
+ var executorSummaryTable = [];
+ stageExecutorSummaryInfoKeys.forEach(function (columnKeyIndex) {
+ var executorSummary = responseBody.executorSummary[columnKeyIndex];
+ var executorDetail = executorDetailsMap[columnKeyIndex.toString()];
+ executorSummary.id = columnKeyIndex;
+ executorSummary.executorLogs = {};
+ executorSummary.hostPort = "CANNOT FIND ADDRESS";
+
+ if (executorDetail) {
+ if (executorDetail["executorLogs"]) {
+ responseBody.executorSummary[columnKeyIndex].executorLogs =
+ executorDetail["executorLogs"];
+ }
+ if (executorDetail["hostPort"]) {
+ responseBody.executorSummary[columnKeyIndex].hostPort =
+ executorDetail["hostPort"];
+ }
+ }
+ executorSummaryTable.push(responseBody.executorSummary[columnKeyIndex]);
+ });
+ // building task aggregated metrics by executor table
+ var executorSummaryConf = {
+ "data": executorSummaryTable,
+ "columns": [
+ {data : "id"},
+ {data : "executorLogs", render: formatLogsCells},
+ {data : "hostPort"},
+ {
+ data : function (row, type) {
+ return type === 'display' ? formatDuration(row.taskTime) : row.taskTime;
+ }
+ },
+ {
+ data : function (row, type) {
+ var totaltasks = row.succeededTasks + row.failedTasks + row.killedTasks;
+ return type === 'display' ? totaltasks : totaltasks.toString();
+ }
+ },
+ {data : "failedTasks"},
+ {data : "killedTasks"},
+ {data : "succeededTasks"},
+ {data : "isExcludedForStage"},
+ {
+ data : function (row, type) {
+ return row.inputRecords != 0 ? formatBytes(row.inputBytes, type) + " / " + row.inputRecords : "";
+ }
+ },
+ {
+ data : function (row, type) {
+ return row.outputRecords != 0 ? formatBytes(row.outputBytes, type) + " / " + row.outputRecords : "";
+ }
+ },
+ {
+ data : function (row, type) {
+ return row.shuffleReadRecords != 0 ? formatBytes(row.shuffleRead, type) + " / " + row.shuffleReadRecords : "";
+ }
+ },
+ {
+ data : function (row, type) {
+ return row.shuffleWriteRecords != 0 ? formatBytes(row.shuffleWrite, type) + " / " + row.shuffleWriteRecords : "";
+ }
+ },
+ {
+ data : function (row, type) {
+ return typeof row.memoryBytesSpilled != 'undefined' ? formatBytes(row.memoryBytesSpilled, type) : "";
+ }
+ },
+ {
+ data : function (row, type) {
+ return typeof row.diskBytesSpilled != 'undefined' ? formatBytes(row.diskBytesSpilled, type) : "";
+ }
+ },
+ {
+ data : function (row, type) {
+ var peakMemoryMetrics = row.peakMemoryMetrics;
+ if (typeof peakMemoryMetrics !== 'undefined') {
+ if (type !== 'display')
+ return peakMemoryMetrics.JVMHeapMemory;
+ else
+ return (formatBytes(peakMemoryMetrics.JVMHeapMemory, type) + ' / ' +
+ formatBytes(peakMemoryMetrics.JVMOffHeapMemory, type));
+ } else {
+ if (type !== 'display') {
+ return 0;
+ } else {
+ return '0.0 B / 0.0 B';
+ }
+ }
- $("#aggregatedMetrics").click(function(){
- $("#arrowtoggle2").toggleClass("arrow-open arrow-closed");
- $("#toggle-aggregatedMetrics").toggleClass("d-none");
- if (window.localStorage) {
- window.localStorage.setItem("arrowtoggle2class", $("#arrowtoggle2").attr('class'));
+ }
+ },
+ {
+ data : function (row, type) {
+ var peakMemoryMetrics = row.peakMemoryMetrics
+ if (typeof peakMemoryMetrics !== 'undefined') {
+ if (type !== 'display')
+ return peakMemoryMetrics.OnHeapExecutionMemory;
+ else
+ return (formatBytes(peakMemoryMetrics.OnHeapExecutionMemory, type) + ' / ' +
+ formatBytes(peakMemoryMetrics.OffHeapExecutionMemory, type));
+ } else {
+ if (type !== 'display') {
+ return 0;
+ } else {
+ return '0.0 B / 0.0 B';
+ }
+ }
+ }
+ },
+ {
+ data : function (row, type) {
+ var peakMemoryMetrics = row.peakMemoryMetrics
+ if (typeof peakMemoryMetrics !== 'undefined') {
+ if (type !== 'display')
+ return peakMemoryMetrics.OnHeapStorageMemory;
+ else
+ return (formatBytes(peakMemoryMetrics.OnHeapStorageMemory, type) + ' / ' +
+ formatBytes(peakMemoryMetrics.OffHeapStorageMemory, type));
+ } else {
+ if (type !== 'display') {
+ return 0;
+ } else {
+ return '0.0 B / 0.0 B';
+ }
+ }
+ }
+ },
+ {
+ data : function (row, type) {
+ var peakMemoryMetrics = row.peakMemoryMetrics
+ if (typeof peakMemoryMetrics !== 'undefined') {
+ if (type !== 'display')
+ return peakMemoryMetrics.DirectPoolMemory;
+ else
+ return (formatBytes(peakMemoryMetrics.DirectPoolMemory, type) + ' / ' +
+ formatBytes(peakMemoryMetrics.MappedPoolMemory, type));
+ } else {
+ if (type !== 'display') {
+ return 0;
+ } else {
+ return '0.0 B / 0.0 B';
+ }
+ }
+ }
+ }
+ ],
+ "columnDefs": [
+ // SPARK-35087 [type:size] means String with structures like : 'size / records',
+ // they should be sorted as numerical-order instead of lexicographical-order by default.
+ // The targets: $id represents column id which comes from stagespage-template.html
+ // #summary-executor-table.If the relative position of the columns in the table
+ // #summary-executor-table has changed,please be careful to adjust the column index here
+ // Input Size / Records
+ {"type": "size", "targets": 9},
+ // Output Size / Records
+ {"type": "size", "targets": 10},
+ // Shuffle Read Size / Records
+ {"type": "size", "targets": 11},
+ // Shuffle Write Size / Records
+ {"type": "size", "targets": 12},
+ // Peak JVM Memory OnHeap / OffHeap
+ {"visible": false, "targets": 15},
+ // Peak Execution Memory OnHeap / OffHeap
+ {"visible": false, "targets": 16},
+ // Peak Storage Memory OnHeap / OffHeap
+ {"visible": false, "targets": 17},
+ // Peak Pool Memory Direct / Mapped
+ {"visible": false, "targets": 18}
+ ],
+ "deferRender": true,
+ "order": [[0, "asc"]],
+ "bAutoWidth": false,
+ "oLanguage": {
+ "sEmptyTable": "No data to show yet"
}
+ };
+ executorSummaryTableSelector =
+ $("#summary-executor-table").DataTable(executorSummaryConf);
+ $('#parent-container [data-toggle="tooltip"]').tooltip();
+
+ executorSummaryTableSelector.column(9).visible(dataToShow.showInputData);
+ if (dataToShow.showInputData) {
+ $('#executor-summary-input').attr("data-toggle", "tooltip")
+ .attr("data-placement", "top")
+ .attr("title", "Bytes and records read from Hadoop or from Spark storage.");
+ $('#executor-summary-input').tooltip(true);
+ }
+ executorSummaryTableSelector.column(10).visible(dataToShow.showOutputData);
+ if (dataToShow.showOutputData) {
+ $('#executor-summary-output').attr("data-toggle", "tooltip")
+ .attr("data-placement", "top")
+ .attr("title", "Bytes and records written to Hadoop.");
+ $('#executor-summary-output').tooltip(true);
+ }
+ executorSummaryTableSelector.column(11).visible(dataToShow.showShuffleReadData);
+ if (dataToShow.showShuffleReadData) {
+ $('#executor-summary-shuffle-read').attr("data-toggle", "tooltip")
+ .attr("data-placement", "top")
+ .attr("title", "Total shuffle bytes and records read (includes both data read locally and data read from remote executors).");
+ $('#executor-summary-shuffle-read').tooltip(true);
+ }
+ executorSummaryTableSelector.column(12).visible(dataToShow.showShuffleWriteData);
+ if (dataToShow.showShuffleWriteData) {
+ $('#executor-summary-shuffle-write').attr("data-toggle", "tooltip")
+ .attr("data-placement", "top")
+ .attr("title", "Bytes and records written to disk in order to be read by a shuffle in a future stage.");
+ $('#executor-summary-shuffle-write').tooltip(true);
+ }
+ executorSummaryTableSelector.column(13).visible(dataToShow.showBytesSpilledData);
+ executorSummaryTableSelector.column(14).visible(dataToShow.showBytesSpilledData);
});
- var endPoint = stageEndPoint(appId);
- var stageAttemptId = getStageAttemptId();
- $.getJSON(endPoint + "/" + stageAttemptId, function(response, status, jqXHR) {
-
- var responseBody = response;
- var dataToShow = {};
- dataToShow.showInputData = responseBody.inputBytes > 0;
- dataToShow.showOutputData = responseBody.outputBytes > 0;
- dataToShow.showShuffleReadData = responseBody.shuffleReadBytes > 0;
- dataToShow.showShuffleWriteData = responseBody.shuffleWriteBytes > 0;
- dataToShow.showBytesSpilledData =
- (responseBody.diskBytesSpilled > 0 || responseBody.memoryBytesSpilled > 0);
-
- var columnIndicesToRemove = [];
- if (!dataToShow.showShuffleReadData) {
- $('#shuffle_read_blocked_time').remove();
- $('#shuffle_remote_reads').remove();
- columnIndicesToRemove.push(2);
- columnIndicesToRemove.push(3);
- }
+ // prepare data for accumulatorUpdates
+ var accumulatorTable = responseBody.accumulatorUpdates.filter(accumUpdate =>
+ !(accumUpdate.name).toString().includes("internal."));
+
+ var quantiles = "0,0.25,0.5,0.75,1.0";
+ $.getJSON(endPoint + "/" + stageAttemptId + "/taskSummary?quantiles=" + quantiles,
+ function(taskMetricsResponse, _ignored_status, _ignored_jqXHR) {
+ var taskMetricKeys = Object.keys(taskMetricsResponse);
+ taskMetricKeys.forEach(function (columnKey) {
+ var row;
+ var row1;
+ var row2;
+ var row3;
+ switch(columnKey) {
+ case "shuffleReadMetrics":
+ row1 = createRowMetadataForColumn(
+ columnKey, taskMetricsResponse[columnKey], 3);
+ row2 = createRowMetadataForColumn(
+ "shuffleReadBlockedTime", taskMetricsResponse[columnKey], 13);
+ row3 = createRowMetadataForColumn(
+ "shuffleRemoteReads", taskMetricsResponse[columnKey], 14);
+ if (dataToShow.showShuffleReadData) {
+ taskSummaryMetricsTableArray.push(row1);
+ taskSummaryMetricsTableArray.push(row2);
+ taskSummaryMetricsTableArray.push(row3);
+ }
+ break;
+
+ case "schedulerDelay":
+ row = createRowMetadataForColumn(
+ columnKey, taskMetricsResponse[columnKey], 11);
+ taskSummaryMetricsTableArray.push(row);
+ break;
+
+ case "executorDeserializeTime":
+ row = createRowMetadataForColumn(
+ columnKey, taskMetricsResponse[columnKey], 12);
+ taskSummaryMetricsTableArray.push(row);
+ break;
+
+ case "resultSerializationTime":
+ row = createRowMetadataForColumn(
+ columnKey, taskMetricsResponse[columnKey], 15);
+ taskSummaryMetricsTableArray.push(row);
+ break;
+
+ case "gettingResultTime":
+ row = createRowMetadataForColumn(
+ columnKey, taskMetricsResponse[columnKey], 16);
+ taskSummaryMetricsTableArray.push(row);
+ break;
+
+ case "peakExecutionMemory":
+ row = createRowMetadataForColumn(
+ columnKey, taskMetricsResponse[columnKey], 17);
+ taskSummaryMetricsTableArray.push(row);
+ break;
+
+ case "inputMetrics":
+ row = createRowMetadataForColumn(
+ columnKey, taskMetricsResponse[columnKey], 1);
+ if (dataToShow.showInputData) {
+ taskSummaryMetricsTableArray.push(row);
+ }
+ break;
- if (!dataToShow.showShuffleWriteData) {
- $('#shuffle_write_time').remove();
- columnIndicesToRemove.push(7);
- }
+ case "outputMetrics":
+ row = createRowMetadataForColumn(
+ columnKey, taskMetricsResponse[columnKey], 2);
+ if (dataToShow.showOutputData) {
+ taskSummaryMetricsTableArray.push(row);
+ }
+ break;
+
+ case "shuffleWriteMetrics":
+ row1 = createRowMetadataForColumn(
+ columnKey, taskMetricsResponse[columnKey], 4);
+ row2 = createRowMetadataForColumn(
+ "shuffleWriteTime", taskMetricsResponse[columnKey], 21);
+ if (dataToShow.showShuffleWriteData) {
+ taskSummaryMetricsTableArray.push(row1);
+ taskSummaryMetricsTableArray.push(row2);
+ }
+ break;
- if (columnIndicesToRemove.length > 0) {
- columnIndicesToRemove.sort(function(a, b) { return b - a; });
- columnIndicesToRemove.forEach(function(idx) {
- optionalColumns.splice(idx, 1);
- });
- }
+ case "diskBytesSpilled":
+ row = createRowMetadataForColumn(
+ columnKey, taskMetricsResponse[columnKey], 5);
+ if (dataToShow.showBytesSpilledData) {
+ taskSummaryMetricsTableArray.push(row);
+ }
+ break;
- // prepare data for executor summary table
- var stageExecutorSummaryInfoKeys = Object.keys(responseBody.executorSummary);
- $.getJSON(createRESTEndPointForExecutorsPage(appId),
- function(executorSummaryResponse, status, jqXHR) {
- var executorDetailsMap = {};
- executorSummaryResponse.forEach(function (executorDetail) {
- executorDetailsMap[executorDetail.id] = executorDetail;
- });
-
- var executorSummaryTable = [];
- stageExecutorSummaryInfoKeys.forEach(function (columnKeyIndex) {
- var executorSummary = responseBody.executorSummary[columnKeyIndex];
- var executorDetail = executorDetailsMap[columnKeyIndex.toString()];
- executorSummary.id = columnKeyIndex;
- executorSummary.executorLogs = {};
- executorSummary.hostPort = "CANNOT FIND ADDRESS";
-
- if (executorDetail) {
- if (executorDetail["executorLogs"]) {
- responseBody.executorSummary[columnKeyIndex].executorLogs =
- executorDetail["executorLogs"];
- }
- if (executorDetail["hostPort"]) {
- responseBody.executorSummary[columnKeyIndex].hostPort =
- executorDetail["hostPort"];
- }
- }
- executorSummaryTable.push(responseBody.executorSummary[columnKeyIndex]);
- });
- // building task aggregated metrics by executor table
- var executorSummaryConf = {
- "data": executorSummaryTable,
- "columns": [
- {data : "id"},
- {data : "executorLogs", render: formatLogsCells},
- {data : "hostPort"},
- {
- data : function (row, type) {
- return type === 'display' ? formatDuration(row.taskTime) : row.taskTime;
- }
- },
- {
- data : function (row, type) {
- var totaltasks = row.succeededTasks + row.failedTasks + row.killedTasks;
- return type === 'display' ? totaltasks : totaltasks.toString();
- }
- },
- {data : "failedTasks"},
- {data : "killedTasks"},
- {data : "succeededTasks"},
- {data : "isExcludedForStage"},
- {
- data : function (row, type) {
- return row.inputRecords != 0 ? formatBytes(row.inputBytes, type) + " / " + row.inputRecords : "";
- }
- },
- {
- data : function (row, type) {
- return row.outputRecords != 0 ? formatBytes(row.outputBytes, type) + " / " + row.outputRecords : "";
- }
- },
- {
- data : function (row, type) {
- return row.shuffleReadRecords != 0 ? formatBytes(row.shuffleRead, type) + " / " + row.shuffleReadRecords : "";
- }
- },
- {
- data : function (row, type) {
- return row.shuffleWriteRecords != 0 ? formatBytes(row.shuffleWrite, type) + " / " + row.shuffleWriteRecords : "";
- }
- },
- {
- data : function (row, type) {
- return typeof row.memoryBytesSpilled != 'undefined' ? formatBytes(row.memoryBytesSpilled, type) : "";
- }
- },
- {
- data : function (row, type) {
- return typeof row.diskBytesSpilled != 'undefined' ? formatBytes(row.diskBytesSpilled, type) : "";
- }
- },
- {
- data : function (row, type) {
- var peakMemoryMetrics = row.peakMemoryMetrics;
- if (typeof peakMemoryMetrics !== 'undefined') {
- if (type !== 'display')
- return peakMemoryMetrics.JVMHeapMemory;
- else
- return (formatBytes(peakMemoryMetrics.JVMHeapMemory, type) + ' / ' +
- formatBytes(peakMemoryMetrics.JVMOffHeapMemory, type));
- } else {
- if (type !== 'display') {
- return 0;
- } else {
- return '0.0 B / 0.0 B';
- }
- }
-
- }
- },
- {
- data : function (row, type) {
- var peakMemoryMetrics = row.peakMemoryMetrics
- if (typeof peakMemoryMetrics !== 'undefined') {
- if (type !== 'display')
- return peakMemoryMetrics.OnHeapExecutionMemory;
- else
- return (formatBytes(peakMemoryMetrics.OnHeapExecutionMemory, type) + ' / ' +
- formatBytes(peakMemoryMetrics.OffHeapExecutionMemory, type));
- } else {
- if (type !== 'display') {
- return 0;
- } else {
- return '0.0 B / 0.0 B';
- }
- }
- }
- },
- {
- data : function (row, type) {
- var peakMemoryMetrics = row.peakMemoryMetrics
- if (typeof peakMemoryMetrics !== 'undefined') {
- if (type !== 'display')
- return peakMemoryMetrics.OnHeapStorageMemory;
- else
- return (formatBytes(peakMemoryMetrics.OnHeapStorageMemory, type) + ' / ' +
- formatBytes(peakMemoryMetrics.OffHeapStorageMemory, type));
- } else {
- if (type !== 'display') {
- return 0;
- } else {
- return '0.0 B / 0.0 B';
- }
- }
- }
- },
- {
- data : function (row, type) {
- var peakMemoryMetrics = row.peakMemoryMetrics
- if (typeof peakMemoryMetrics !== 'undefined') {
- if (type !== 'display')
- return peakMemoryMetrics.DirectPoolMemory;
- else
- return (formatBytes(peakMemoryMetrics.DirectPoolMemory, type) + ' / ' +
- formatBytes(peakMemoryMetrics.MappedPoolMemory, type));
- } else {
- if (type !== 'display') {
- return 0;
- } else {
- return '0.0 B / 0.0 B';
- }
- }
- }
- }
- ],
- "columnDefs": [
- // SPARK-35087 [type:size] means String with structures like : 'size / records',
- // they should be sorted as numerical-order instead of lexicographical-order by default.
- // The targets: $id represents column id which comes from stagespage-template.html
- // #summary-executor-table.If the relative position of the columns in the table
- // #summary-executor-table has changed,please be careful to adjust the column index here
- // Input Size / Records
- {"type": "size", "targets": 9},
- // Output Size / Records
- {"type": "size", "targets": 10},
- // Shuffle Read Size / Records
- {"type": "size", "targets": 11},
- // Shuffle Write Size / Records
- {"type": "size", "targets": 12},
- // Peak JVM Memory OnHeap / OffHeap
- {"visible": false, "targets": 15},
- // Peak Execution Memory OnHeap / OffHeap
- {"visible": false, "targets": 16},
- // Peak Storage Memory OnHeap / OffHeap
- {"visible": false, "targets": 17},
- // Peak Pool Memory Direct / Mapped
- {"visible": false, "targets": 18}
- ],
- "deferRender": true,
- "order": [[0, "asc"]],
- "bAutoWidth": false,
- "oLanguage": {
- "sEmptyTable": "No data to show yet"
- }
- };
- executorSummaryTableSelector =
- $("#summary-executor-table").DataTable(executorSummaryConf);
- $('#parent-container [data-toggle="tooltip"]').tooltip();
-
- executorSummaryTableSelector.column(9).visible(dataToShow.showInputData);
- if (dataToShow.showInputData) {
- $('#executor-summary-input').attr("data-toggle", "tooltip")
- .attr("data-placement", "top")
- .attr("title", "Bytes and records read from Hadoop or from Spark storage.");
- $('#executor-summary-input').tooltip(true);
+ case "memoryBytesSpilled":
+ row = createRowMetadataForColumn(
+ columnKey, taskMetricsResponse[columnKey], 6);
+ if (dataToShow.showBytesSpilledData) {
+ taskSummaryMetricsTableArray.push(row);
+ }
+ break;
+
+ default:
+ if (getColumnNameForTaskMetricSummary(columnKey) != "NA") {
+ row = createRowMetadataForColumn(
+ columnKey, taskMetricsResponse[columnKey], 0);
+ taskSummaryMetricsTableArray.push(row);
+ }
+ break;
+ }
+ });
+ var taskSummaryMetricsTableFilteredArray =
+ taskSummaryMetricsTableArray.filter(row => row.checkboxId < 11);
+ taskSummaryMetricsTableCurrentStateArray = taskSummaryMetricsTableFilteredArray.slice();
+ reselectCheckboxesBasedOnTaskTableState();
+ });
+
+ // building accumulator update table
+ var accumulatorConf = {
+ "data": accumulatorTable,
+ "columns": [
+ {data : "id"},
+ {data : "name"},
+ {data : "value"}
+ ],
+ "paging": false,
+ "searching": false,
+ "order": [[0, "asc"]],
+ "bAutoWidth": false
+ };
+ $("#accumulator-table").DataTable(accumulatorConf);
+
+ // building tasks table that uses server side functionality
+ var totalTasksToShow = responseBody.numCompleteTasks + responseBody.numActiveTasks +
+ responseBody.numKilledTasks + responseBody.numFailedTasks;
+ var taskTable = "#active-tasks-table";
+ var taskConf = {
+ "serverSide": true,
+ "paging": true,
+ "info": true,
+ "processing": true,
+ "lengthMenu": [[20, 40, 60, 100, -1], [20, 40, 60, 100, "All"]],
+ "orderMulti": false,
+ "bAutoWidth": false,
+ "ajax": {
+ "url": endPoint + "/" + stageAttemptId + "/taskTable",
+ "data": function (data) {
+ var columnIndexToSort = 0;
+ var columnNameToSort = "Index";
+ if (data.order[0].column && data.order[0].column != "") {
+ columnIndexToSort = parseInt(data.order[0].column);
+ columnNameToSort = data.columns[columnIndexToSort].name;
+ }
+ delete data.columns;
+ data.numTasks = totalTasksToShow;
+ data.columnIndexToSort = columnIndexToSort;
+ data.columnNameToSort = columnNameToSort;
+ if (data.length === -1) {
+ data.length = totalTasksToShow;
+ }
+ },
+ "dataSrc": function (jsons) {
+ var jsonStr = JSON.stringify(jsons);
+ var tasksToShow = JSON.parse(jsonStr);
+ return tasksToShow.aaData;
+ },
+ "error": function (_ignored_jqXHR, _ignored_textStatus, _ignored_errorThrown) {
+ alert("Unable to connect to the server. Looks like the Spark " +
+ "application must have ended. Please Switch to the history UI.");
+ $("#active-tasks-table_processing").css("display","none");
+ }
+ },
+ "columns": [
+ {
+ data: function (row, type) {
+ return type !== 'display' ? (isNaN(row.index) ? 0 : row.index ) : row.index;
+ },
+ name: "Index"
+ },
+ {data : "taskId", name: "ID"},
+ {data : "attempt", name: "Attempt"},
+ {data : "status", name: "Status"},
+ {data : "taskLocality", name: "Locality Level"},
+ {data : "executorId", name: "Executor ID"},
+ {data : "host", name: "Host"},
+ {data : "executorLogs", name: "Logs", render: formatLogsCells},
+ {data : "launchTime", name: "Launch Time", render: formatDate},
+ {
+ data : function (row, type) {
+ if (row.taskMetrics && row.taskMetrics.executorRunTime) {
+ return type === 'display' ? formatDuration(row.taskMetrics.executorRunTime) : row.taskMetrics.executorRunTime;
+ } else {
+ return "";
}
- executorSummaryTableSelector.column(10).visible(dataToShow.showOutputData);
- if (dataToShow.showOutputData) {
- $('#executor-summary-output').attr("data-toggle", "tooltip")
- .attr("data-placement", "top")
- .attr("title", "Bytes and records written to Hadoop.");
- $('#executor-summary-output').tooltip(true);
+ },
+ name: "Duration"
+ },
+ {
+ data : function (row, type) {
+ if (row.taskMetrics && row.taskMetrics.jvmGcTime) {
+ return type === 'display' ? formatDuration(row.taskMetrics.jvmGcTime) : row.taskMetrics.jvmGcTime;
+ } else {
+ return "";
}
- executorSummaryTableSelector.column(11).visible(dataToShow.showShuffleReadData);
- if (dataToShow.showShuffleReadData) {
- $('#executor-summary-shuffle-read').attr("data-toggle", "tooltip")
- .attr("data-placement", "top")
- .attr("title", "Total shuffle bytes and records read (includes both data read locally and data read from remote executors).");
- $('#executor-summary-shuffle-read').tooltip(true);
+ },
+ name: "GC Time"
+ },
+ {
+ data : function (row, type) {
+ if (row.schedulerDelay) {
+ return type === 'display' ? formatDuration(row.schedulerDelay) : row.schedulerDelay;
+ } else {
+ return "";
}
- executorSummaryTableSelector.column(12).visible(dataToShow.showShuffleWriteData);
- if (dataToShow.showShuffleWriteData) {
- $('#executor-summary-shuffle-write').attr("data-toggle", "tooltip")
- .attr("data-placement", "top")
- .attr("title", "Bytes and records written to disk in order to be read by a shuffle in a future stage.");
- $('#executor-summary-shuffle-write').tooltip(true);
+ },
+ name: "Scheduler Delay"
+ },
+ {
+ data : function (row, type) {
+ if (row.taskMetrics && row.taskMetrics.executorDeserializeTime) {
+ return type === 'display' ? formatDuration(row.taskMetrics.executorDeserializeTime) : row.taskMetrics.executorDeserializeTime;
+ } else {
+ return "";
}
- executorSummaryTableSelector.column(13).visible(dataToShow.showBytesSpilledData);
- executorSummaryTableSelector.column(14).visible(dataToShow.showBytesSpilledData);
- });
-
- // prepare data for accumulatorUpdates
- var accumulatorTable = responseBody.accumulatorUpdates.filter(accumUpdate =>
- !(accumUpdate.name).toString().includes("internal."));
-
- var quantiles = "0,0.25,0.5,0.75,1.0";
- $.getJSON(endPoint + "/" + stageAttemptId + "/taskSummary?quantiles=" + quantiles,
- function(taskMetricsResponse, status, jqXHR) {
- var taskMetricKeys = Object.keys(taskMetricsResponse);
- taskMetricKeys.forEach(function (columnKey) {
- switch(columnKey) {
- case "shuffleReadMetrics":
- var row1 = createRowMetadataForColumn(
- columnKey, taskMetricsResponse[columnKey], 3);
- var row2 = createRowMetadataForColumn(
- "shuffleReadBlockedTime", taskMetricsResponse[columnKey], 13);
- var row3 = createRowMetadataForColumn(
- "shuffleRemoteReads", taskMetricsResponse[columnKey], 14);
- if (dataToShow.showShuffleReadData) {
- taskSummaryMetricsTableArray.push(row1);
- taskSummaryMetricsTableArray.push(row2);
- taskSummaryMetricsTableArray.push(row3);
- }
- break;
-
- case "schedulerDelay":
- var row = createRowMetadataForColumn(
- columnKey, taskMetricsResponse[columnKey], 11);
- taskSummaryMetricsTableArray.push(row);
- break;
-
- case "executorDeserializeTime":
- var row = createRowMetadataForColumn(
- columnKey, taskMetricsResponse[columnKey], 12);
- taskSummaryMetricsTableArray.push(row);
- break;
-
- case "resultSerializationTime":
- var row = createRowMetadataForColumn(
- columnKey, taskMetricsResponse[columnKey], 15);
- taskSummaryMetricsTableArray.push(row);
- break;
-
- case "gettingResultTime":
- var row = createRowMetadataForColumn(
- columnKey, taskMetricsResponse[columnKey], 16);
- taskSummaryMetricsTableArray.push(row);
- break;
-
- case "peakExecutionMemory":
- var row = createRowMetadataForColumn(
- columnKey, taskMetricsResponse[columnKey], 17);
- taskSummaryMetricsTableArray.push(row);
- break;
-
- case "inputMetrics":
- var row = createRowMetadataForColumn(
- columnKey, taskMetricsResponse[columnKey], 1);
- if (dataToShow.showInputData) {
- taskSummaryMetricsTableArray.push(row);
- }
- break;
-
- case "outputMetrics":
- var row = createRowMetadataForColumn(
- columnKey, taskMetricsResponse[columnKey], 2);
- if (dataToShow.showOutputData) {
- taskSummaryMetricsTableArray.push(row);
- }
- break;
-
- case "shuffleWriteMetrics":
- var row1 = createRowMetadataForColumn(
- columnKey, taskMetricsResponse[columnKey], 4);
- var row2 = createRowMetadataForColumn(
- "shuffleWriteTime", taskMetricsResponse[columnKey], 21);
- if (dataToShow.showShuffleWriteData) {
- taskSummaryMetricsTableArray.push(row1);
- taskSummaryMetricsTableArray.push(row2);
- }
- break;
-
- case "diskBytesSpilled":
- var row = createRowMetadataForColumn(
- columnKey, taskMetricsResponse[columnKey], 5);
- if (dataToShow.showBytesSpilledData) {
- taskSummaryMetricsTableArray.push(row);
- }
- break;
-
- case "memoryBytesSpilled":
- var row = createRowMetadataForColumn(
- columnKey, taskMetricsResponse[columnKey], 6);
- if (dataToShow.showBytesSpilledData) {
- taskSummaryMetricsTableArray.push(row);
- }
- break;
-
- default:
- if (getColumnNameForTaskMetricSummary(columnKey) != "NA") {
- var row = createRowMetadataForColumn(
- columnKey, taskMetricsResponse[columnKey], 0);
- taskSummaryMetricsTableArray.push(row);
- }
- break;
- }
- });
- var taskSummaryMetricsTableFilteredArray =
- taskSummaryMetricsTableArray.filter(row => row.checkboxId < 11);
- taskSummaryMetricsTableCurrentStateArray = taskSummaryMetricsTableFilteredArray.slice();
- reselectCheckboxesBasedOnTaskTableState();
- });
-
- // building accumulator update table
- var accumulatorConf = {
- "data": accumulatorTable,
- "columns": [
- {data : "id"},
- {data : "name"},
- {data : "value"}
- ],
- "paging": false,
- "searching": false,
- "order": [[0, "asc"]],
- "bAutoWidth": false
- };
- $("#accumulator-table").DataTable(accumulatorConf);
-
- // building tasks table that uses server side functionality
- var totalTasksToShow = responseBody.numCompleteTasks + responseBody.numActiveTasks +
- responseBody.numKilledTasks + responseBody.numFailedTasks;
- var taskTable = "#active-tasks-table";
- var taskConf = {
- "serverSide": true,
- "paging": true,
- "info": true,
- "processing": true,
- "lengthMenu": [[20, 40, 60, 100, -1], [20, 40, 60, 100, "All"]],
- "orderMulti": false,
- "bAutoWidth": false,
- "ajax": {
- "url": endPoint + "/" + stageAttemptId + "/taskTable",
- "data": function (data) {
- var columnIndexToSort = 0;
- var columnNameToSort = "Index";
- if (data.order[0].column && data.order[0].column != "") {
- columnIndexToSort = parseInt(data.order[0].column);
- columnNameToSort = data.columns[columnIndexToSort].name;
- }
- delete data.columns;
- data.numTasks = totalTasksToShow;
- data.columnIndexToSort = columnIndexToSort;
- data.columnNameToSort = columnNameToSort;
- if (data.length === -1) {
- data.length = totalTasksToShow;
- }
- },
- "dataSrc": function (jsons) {
- var jsonStr = JSON.stringify(jsons);
- var tasksToShow = JSON.parse(jsonStr);
- return tasksToShow.aaData;
- },
- "error": function (jqXHR, textStatus, errorThrown) {
- alert("Unable to connect to the server. Looks like the Spark " +
- "application must have ended. Please Switch to the history UI.");
- $("#active-tasks-table_processing").css("display","none");
- }
- },
- "columns": [
- {data: function (row, type) {
- return type !== 'display' ? (isNaN(row.index) ? 0 : row.index ) : row.index;
- },
- name: "Index"
- },
- {data : "taskId", name: "ID"},
- {data : "attempt", name: "Attempt"},
- {data : "status", name: "Status"},
- {data : "taskLocality", name: "Locality Level"},
- {data : "executorId", name: "Executor ID"},
- {data : "host", name: "Host"},
- {data : "executorLogs", name: "Logs", render: formatLogsCells},
- {data : "launchTime", name: "Launch Time", render: formatDate},
- {
- data : function (row, type) {
- if (row.taskMetrics && row.taskMetrics.executorRunTime) {
- return type === 'display' ? formatDuration(row.taskMetrics.executorRunTime) : row.taskMetrics.executorRunTime;
- } else {
- return "";
- }
- },
- name: "Duration"
- },
- {
- data : function (row, type) {
- if (row.taskMetrics && row.taskMetrics.jvmGcTime) {
- return type === 'display' ? formatDuration(row.taskMetrics.jvmGcTime) : row.taskMetrics.jvmGcTime;
- } else {
- return "";
- }
- },
- name: "GC Time"
- },
- {
- data : function (row, type) {
- if (row.schedulerDelay) {
- return type === 'display' ? formatDuration(row.schedulerDelay) : row.schedulerDelay;
- } else {
- return "";
- }
- },
- name: "Scheduler Delay"
- },
- {
- data : function (row, type) {
- if (row.taskMetrics && row.taskMetrics.executorDeserializeTime) {
- return type === 'display' ? formatDuration(row.taskMetrics.executorDeserializeTime) : row.taskMetrics.executorDeserializeTime;
- } else {
- return "";
- }
- },
- name: "Task Deserialization Time"
- },
- {
- data : function (row, type) {
- if (row.taskMetrics && row.taskMetrics.shuffleReadMetrics) {
- return type === 'display' ? formatDuration(row.taskMetrics.shuffleReadMetrics.fetchWaitTime) : row.taskMetrics.shuffleReadMetrics.fetchWaitTime;
- } else {
- return "";
- }
- },
- name: "Shuffle Read Blocked Time"
- },
- {
- data : function (row, type) {
- if (row.taskMetrics && row.taskMetrics.shuffleReadMetrics) {
- return type === 'display' ? formatBytes(row.taskMetrics.shuffleReadMetrics.remoteBytesRead, type) : row.taskMetrics.shuffleReadMetrics.remoteBytesRead;
- } else {
- return "";
- }
- },
- name: "Shuffle Remote Reads"
- },
- {
- data : function (row, type) {
- if (row.taskMetrics && row.taskMetrics.resultSerializationTime) {
- return type === 'display' ? formatDuration(row.taskMetrics.resultSerializationTime) : row.taskMetrics.resultSerializationTime;
- } else {
- return "";
- }
- },
- name: "Result Serialization Time"
- },
- {
- data : function (row, type) {
- if (row.gettingResultTime) {
- return type === 'display' ? formatDuration(row.gettingResultTime) : row.gettingResultTime;
- } else {
- return "";
- }
- },
- name: "Getting Result Time"
- },
- {
- data : function (row, type) {
- if (row.taskMetrics && row.taskMetrics.peakExecutionMemory) {
- return type === 'display' ? formatBytes(row.taskMetrics.peakExecutionMemory, type) : row.taskMetrics.peakExecutionMemory;
- } else {
- return "";
- }
- },
- name: "Peak Execution Memory"
- },
- {
- data : function (row, type) {
- if (accumulatorTable.length > 0 && row.accumulatorUpdates.length > 0) {
- var allAccums = "";
- row.accumulatorUpdates.forEach(function(accumulator) {
- allAccums += accumulator.name + ': ' + accumulator.update + "
";
- });
- return allAccums;
- } else {
- return "";
- }
- },
- name: "Accumulators"
- },
- {
- data : function (row, type) {
- if (row.taskMetrics && row.taskMetrics.inputMetrics && row.taskMetrics.inputMetrics.bytesRead > 0) {
- if (type === 'display') {
- return formatBytes(row.taskMetrics.inputMetrics.bytesRead, type) + " / " + row.taskMetrics.inputMetrics.recordsRead;
- } else {
- return row.taskMetrics.inputMetrics.bytesRead + " / " + row.taskMetrics.inputMetrics.recordsRead;
- }
- } else {
- return "";
- }
- },
- name: "Input Size / Records"
- },
- {
- data : function (row, type) {
- if (row.taskMetrics && row.taskMetrics.outputMetrics && row.taskMetrics.outputMetrics.bytesWritten > 0) {
- if (type === 'display') {
- return formatBytes(row.taskMetrics.outputMetrics.bytesWritten, type) + " / " + row.taskMetrics.outputMetrics.recordsWritten;
- } else {
- return row.taskMetrics.outputMetrics.bytesWritten + " / " + row.taskMetrics.outputMetrics.recordsWritten;
- }
- } else {
- return "";
- }
- },
- name: "Output Size / Records"
- },
- {
- data : function (row, type) {
- if (row.taskMetrics && row.taskMetrics.shuffleWriteMetrics && row.taskMetrics.shuffleWriteMetrics.writeTime > 0) {
- return type === 'display' ? formatDuration(parseInt(row.taskMetrics.shuffleWriteMetrics.writeTime) / 1000000.0) : row.taskMetrics.shuffleWriteMetrics.writeTime;
- } else {
- return "";
- }
- },
- name: "Shuffle Write Time"
- },
- {
- data : function (row, type) {
- if (row.taskMetrics && row.taskMetrics.shuffleWriteMetrics && row.taskMetrics.shuffleWriteMetrics.bytesWritten > 0) {
- if (type === 'display') {
- return formatBytes(row.taskMetrics.shuffleWriteMetrics.bytesWritten, type) + " / " + row.taskMetrics.shuffleWriteMetrics.recordsWritten;
- } else {
- return row.taskMetrics.shuffleWriteMetrics.bytesWritten + " / " + row.taskMetrics.shuffleWriteMetrics.recordsWritten;
- }
- } else {
- return "";
- }
- },
- name: "Shuffle Write Size / Records"
- },
- {
- data : function (row, type) {
- if (row.taskMetrics && row.taskMetrics.shuffleReadMetrics &&
- (row.taskMetrics.shuffleReadMetrics.localBytesRead > 0 || row.taskMetrics.shuffleReadMetrics.remoteBytesRead > 0)) {
- var totalBytesRead = parseInt(row.taskMetrics.shuffleReadMetrics.localBytesRead) + parseInt(row.taskMetrics.shuffleReadMetrics.remoteBytesRead);
- if (type === 'display') {
- return formatBytes(totalBytesRead, type) + " / " + row.taskMetrics.shuffleReadMetrics.recordsRead;
- } else {
- return totalBytesRead + " / " + row.taskMetrics.shuffleReadMetrics.recordsRead;
- }
- } else {
- return "";
- }
- },
- name: "Shuffle Read Size / Records"
- },
- {
- data : function (row, type) {
- if (row.taskMetrics && row.taskMetrics.memoryBytesSpilled && row.taskMetrics.memoryBytesSpilled > 0) {
- return type === 'display' ? formatBytes(row.taskMetrics.memoryBytesSpilled, type) : row.taskMetrics.memoryBytesSpilled;
- } else {
- return "";
- }
- },
- name: "Spill (Memory)"
- },
- {
- data : function (row, type) {
- if (row.taskMetrics && row.taskMetrics.diskBytesSpilled && row.taskMetrics.diskBytesSpilled > 0) {
- return type === 'display' ? formatBytes(row.taskMetrics.diskBytesSpilled, type) : row.taskMetrics.diskBytesSpilled;
- } else {
- return "";
- }
- },
- name: "Spill (Disk)"
- },
- {
- data : function (row, type) {
- var msg = row.errorMessage;
- if (typeof msg === 'undefined') {
- return "";
- } else {
- var indexOfLineSeparator = msg.indexOf("\n");
- var formHead = indexOfLineSeparator > 0 ? msg.substring(0, indexOfLineSeparator) : (msg.length > 100 ? msg.substring(0, 100) : msg);
- var form = "+details";
- var formMsg = "";
- return formHead + form + formMsg;
- }
- },
- name: "Errors"
- }
- ],
- "columnDefs": [
- { "visible": false, "targets": 11 },
- { "visible": false, "targets": 12 },
- { "visible": false, "targets": 13 },
- { "visible": false, "targets": 14 },
- { "visible": false, "targets": 15 },
- { "visible": false, "targets": 16 },
- { "visible": false, "targets": 17 },
- { "visible": false, "targets": 18 },
- { "visible": false, "targets": 21 }
- ],
- "deferRender": true
- };
- taskTableSelector = $(taskTable).DataTable(taskConf);
- $('#active-tasks-table_filter input').unbind();
- var searchEvent;
- $('#active-tasks-table_filter input').bind('keyup', function(e) {
- if (typeof searchEvent !== 'undefined') {
- window.clearTimeout(searchEvent);
+ },
+ name: "Task Deserialization Time"
+ },
+ {
+ data : function (row, type) {
+ if (row.taskMetrics && row.taskMetrics.shuffleReadMetrics) {
+ return type === 'display' ? formatDuration(row.taskMetrics.shuffleReadMetrics.fetchWaitTime) : row.taskMetrics.shuffleReadMetrics.fetchWaitTime;
+ } else {
+ return "";
+ }
+ },
+ name: "Shuffle Read Blocked Time"
+ },
+ {
+ data : function (row, type) {
+ if (row.taskMetrics && row.taskMetrics.shuffleReadMetrics) {
+ return type === 'display' ? formatBytes(row.taskMetrics.shuffleReadMetrics.remoteBytesRead, type) : row.taskMetrics.shuffleReadMetrics.remoteBytesRead;
+ } else {
+ return "";
+ }
+ },
+ name: "Shuffle Remote Reads"
+ },
+ {
+ data : function (row, type) {
+ if (row.taskMetrics && row.taskMetrics.resultSerializationTime) {
+ return type === 'display' ? formatDuration(row.taskMetrics.resultSerializationTime) : row.taskMetrics.resultSerializationTime;
+ } else {
+ return "";
+ }
+ },
+ name: "Result Serialization Time"
+ },
+ {
+ data : function (row, type) {
+ if (row.gettingResultTime) {
+ return type === 'display' ? formatDuration(row.gettingResultTime) : row.gettingResultTime;
+ } else {
+ return "";
+ }
+ },
+ name: "Getting Result Time"
+ },
+ {
+ data : function (row, type) {
+ if (row.taskMetrics && row.taskMetrics.peakExecutionMemory) {
+ return type === 'display' ? formatBytes(row.taskMetrics.peakExecutionMemory, type) : row.taskMetrics.peakExecutionMemory;
+ } else {
+ return "";
+ }
+ },
+ name: "Peak Execution Memory"
+ },
+ {
+ data : function (row, _ignored_type) {
+ if (accumulatorTable.length > 0 && row.accumulatorUpdates.length > 0) {
+ var allAccums = "";
+ row.accumulatorUpdates.forEach(function(accumulator) {
+ allAccums += accumulator.name + ': ' + accumulator.update + "
";
+ });
+ return allAccums;
+ } else {
+ return "";
+ }
+ },
+ name: "Accumulators"
+ },
+ {
+ data : function (row, type) {
+ if (row.taskMetrics && row.taskMetrics.inputMetrics && row.taskMetrics.inputMetrics.bytesRead > 0) {
+ if (type === 'display') {
+ return formatBytes(row.taskMetrics.inputMetrics.bytesRead, type) + " / " + row.taskMetrics.inputMetrics.recordsRead;
+ } else {
+ return row.taskMetrics.inputMetrics.bytesRead + " / " + row.taskMetrics.inputMetrics.recordsRead;
}
- var value = this.value;
- searchEvent = window.setTimeout(function(){
- taskTableSelector.search( value ).draw();}, 500);
- });
- reselectCheckboxesBasedOnTaskTableState();
-
- // hide or show columns dynamically event
- $('input.toggle-vis').on('click', function(e){
- // Get the column
- var para = $(this).attr('data-column');
- if (para == "0") {
- var allColumns = taskTableSelector.columns(optionalColumns);
- var executorAllColumns = executorSummaryTableSelector.columns(executorOptionalColumns);
- if ($(this).is(":checked")) {
- $(".toggle-vis").prop('checked', true);
- allColumns.visible(true);
- executorAllColumns.visible(true);
- createDataTableForTaskSummaryMetricsTable(taskSummaryMetricsTableArray);
- } else {
- $(".toggle-vis").prop('checked', false);
- allColumns.visible(false);
- executorAllColumns.visible(false);
- var taskSummaryMetricsTableFilteredArray =
- taskSummaryMetricsTableArray.filter(row => row.checkboxId < 11);
- createDataTableForTaskSummaryMetricsTable(taskSummaryMetricsTableFilteredArray);
- }
- } else {
- var dataMetricsType = $(this).attr("data-metrics-type");
- if (dataMetricsType === 'task') {
- var column = taskTableSelector.column(para);
- // Toggle the visibility
- column.visible(!column.visible());
- var taskSummaryMetricsTableFilteredArray = [];
- if ($(this).is(":checked")) {
- taskSummaryMetricsTableCurrentStateArray.push(taskSummaryMetricsTableArray.filter(row => (row.checkboxId).toString() == para)[0]);
- taskSummaryMetricsTableFilteredArray = taskSummaryMetricsTableCurrentStateArray.slice();
- } else {
- taskSummaryMetricsTableFilteredArray =
- taskSummaryMetricsTableCurrentStateArray.filter(row => (row.checkboxId).toString() != para);
- }
- createDataTableForTaskSummaryMetricsTable(taskSummaryMetricsTableFilteredArray);
- }
- if (dataMetricsType === "executor") {
- var column = executorSummaryTableSelector.column(para);
- column.visible(!column.visible());
- }
- }
- });
-
- // title number and toggle list
- $("#summaryMetricsTitle").html("Summary Metrics for " + "" + responseBody.numCompleteTasks + " Completed Tasks" + "");
- $("#tasksTitle").html("Tasks (" + totalTasksToShow + ")");
-
- // hide or show the accumulate update table
- if (accumulatorTable.length == 0) {
- $("#accumulator-update-table").hide();
} else {
- taskTableSelector.column(18).visible(true);
- $("#accumulator-update-table").show();
+ return "";
}
- // Showing relevant stage data depending on stage type for task table and executor
- // summary table
- taskTableSelector.column(19).visible(dataToShow.showInputData);
- taskTableSelector.column(20).visible(dataToShow.showOutputData);
- taskTableSelector.column(22).visible(dataToShow.showShuffleWriteData);
- taskTableSelector.column(23).visible(dataToShow.showShuffleReadData);
- taskTableSelector.column(24).visible(dataToShow.showBytesSpilledData);
- taskTableSelector.column(25).visible(dataToShow.showBytesSpilledData);
-
- if (window.localStorage) {
- if (window.localStorage.getItem("arrowtoggle1class") !== null &&
- window.localStorage.getItem("arrowtoggle1class").includes("arrow-open")) {
- $("#arrowtoggle1").toggleClass("arrow-open arrow-closed");
- $("#toggle-metrics").toggleClass("d-none");
- }
- if (window.localStorage.getItem("arrowtoggle2class") !== null &&
- window.localStorage.getItem("arrowtoggle2class").includes("arrow-open")) {
- $("#arrowtoggle2").toggleClass("arrow-open arrow-closed");
- $("#toggle-aggregatedMetrics").toggleClass("d-none");
- }
+ },
+ name: "Input Size / Records"
+ },
+ {
+ data : function (row, type) {
+ if (row.taskMetrics && row.taskMetrics.outputMetrics && row.taskMetrics.outputMetrics.bytesWritten > 0) {
+ if (type === 'display') {
+ return formatBytes(row.taskMetrics.outputMetrics.bytesWritten, type) + " / " + row.taskMetrics.outputMetrics.recordsWritten;
+ } else {
+ return row.taskMetrics.outputMetrics.bytesWritten + " / " + row.taskMetrics.outputMetrics.recordsWritten;
+ }
+ } else {
+ return "";
}
- });
+ },
+ name: "Output Size / Records"
+ },
+ {
+ data : function (row, type) {
+ if (row.taskMetrics && row.taskMetrics.shuffleWriteMetrics && row.taskMetrics.shuffleWriteMetrics.writeTime > 0) {
+ return type === 'display' ? formatDuration(parseInt(row.taskMetrics.shuffleWriteMetrics.writeTime) / 1000000.0) : row.taskMetrics.shuffleWriteMetrics.writeTime;
+ } else {
+ return "";
+ }
+ },
+ name: "Shuffle Write Time"
+ },
+ {
+ data : function (row, type) {
+ if (row.taskMetrics && row.taskMetrics.shuffleWriteMetrics && row.taskMetrics.shuffleWriteMetrics.bytesWritten > 0) {
+ if (type === 'display') {
+ return formatBytes(row.taskMetrics.shuffleWriteMetrics.bytesWritten, type) + " / " + row.taskMetrics.shuffleWriteMetrics.recordsWritten;
+ } else {
+ return row.taskMetrics.shuffleWriteMetrics.bytesWritten + " / " + row.taskMetrics.shuffleWriteMetrics.recordsWritten;
+ }
+ } else {
+ return "";
+ }
+ },
+ name: "Shuffle Write Size / Records"
+ },
+ {
+ data : function (row, type) {
+ if (row.taskMetrics && row.taskMetrics.shuffleReadMetrics &&
+ (row.taskMetrics.shuffleReadMetrics.localBytesRead > 0 || row.taskMetrics.shuffleReadMetrics.remoteBytesRead > 0)) {
+ var totalBytesRead = parseInt(row.taskMetrics.shuffleReadMetrics.localBytesRead) + parseInt(row.taskMetrics.shuffleReadMetrics.remoteBytesRead);
+ if (type === 'display') {
+ return formatBytes(totalBytesRead, type) + " / " + row.taskMetrics.shuffleReadMetrics.recordsRead;
+ } else {
+ return totalBytesRead + " / " + row.taskMetrics.shuffleReadMetrics.recordsRead;
+ }
+ } else {
+ return "";
+ }
+ },
+ name: "Shuffle Read Size / Records"
+ },
+ {
+ data : function (row, type) {
+ if (row.taskMetrics && row.taskMetrics.memoryBytesSpilled && row.taskMetrics.memoryBytesSpilled > 0) {
+ return type === 'display' ? formatBytes(row.taskMetrics.memoryBytesSpilled, type) : row.taskMetrics.memoryBytesSpilled;
+ } else {
+ return "";
+ }
+ },
+ name: "Spill (Memory)"
+ },
+ {
+ data : function (row, type) {
+ if (row.taskMetrics && row.taskMetrics.diskBytesSpilled && row.taskMetrics.diskBytesSpilled > 0) {
+ return type === 'display' ? formatBytes(row.taskMetrics.diskBytesSpilled, type) : row.taskMetrics.diskBytesSpilled;
+ } else {
+ return "";
+ }
+ },
+ name: "Spill (Disk)"
+ },
+ {
+ data : function (row, _ignored_type) {
+ var msg = row.errorMessage;
+ if (typeof msg === 'undefined') {
+ return "";
+ } else {
+ var indexOfLineSeparator = msg.indexOf("\n");
+ var formHead = indexOfLineSeparator > 0 ? msg.substring(0, indexOfLineSeparator) : (msg.length > 100 ? msg.substring(0, 100) : msg);
+ var form = "+details";
+ var formMsg = "";
+ return formHead + form + formMsg;
+ }
+ },
+ name: "Errors"
+ }
+ ],
+ "columnDefs": [
+ { "visible": false, "targets": 11 },
+ { "visible": false, "targets": 12 },
+ { "visible": false, "targets": 13 },
+ { "visible": false, "targets": 14 },
+ { "visible": false, "targets": 15 },
+ { "visible": false, "targets": 16 },
+ { "visible": false, "targets": 17 },
+ { "visible": false, "targets": 18 },
+ { "visible": false, "targets": 21 }
+ ],
+ "deferRender": true
+ };
+ taskTableSelector = $(taskTable).DataTable(taskConf);
+ $('#active-tasks-table_filter input').unbind();
+ var searchEvent;
+ $('#active-tasks-table_filter input').bind('keyup', function(_ignored_e) {
+ if (typeof searchEvent !== 'undefined') {
+ window.clearTimeout(searchEvent);
+ }
+ var value = this.value;
+ searchEvent = window.setTimeout(function(){
+ taskTableSelector.search( value ).draw();}, 500);
});
+ reselectCheckboxesBasedOnTaskTableState();
+
+ // hide or show columns dynamically event
+ $('input.toggle-vis').on('click', function(_ignored_e){
+ var taskSummaryMetricsTableFilteredArray;
+ // Get the column
+ var para = $(this).attr('data-column');
+ if (para == "0") {
+ var allColumns = taskTableSelector.columns(optionalColumns);
+ var executorAllColumns = executorSummaryTableSelector.columns(executorOptionalColumns);
+ if ($(this).is(":checked")) {
+ $(".toggle-vis").prop('checked', true);
+ allColumns.visible(true);
+ executorAllColumns.visible(true);
+ createDataTableForTaskSummaryMetricsTable(taskSummaryMetricsTableArray);
+ } else {
+ $(".toggle-vis").prop('checked', false);
+ allColumns.visible(false);
+ executorAllColumns.visible(false);
+ taskSummaryMetricsTableFilteredArray =
+ taskSummaryMetricsTableArray.filter(row => row.checkboxId < 11);
+ createDataTableForTaskSummaryMetricsTable(taskSummaryMetricsTableFilteredArray);
+ }
+ } else {
+ var dataMetricsType = $(this).attr("data-metrics-type");
+ var column;
+ if (dataMetricsType === 'task') {
+ column = taskTableSelector.column(para);
+ // Toggle the visibility
+ column.visible(!column.visible());
+ taskSummaryMetricsTableFilteredArray = [];
+ if ($(this).is(":checked")) {
+ taskSummaryMetricsTableCurrentStateArray.push(taskSummaryMetricsTableArray.filter(row => (row.checkboxId).toString() == para)[0]);
+ taskSummaryMetricsTableFilteredArray = taskSummaryMetricsTableCurrentStateArray.slice();
+ } else {
+ taskSummaryMetricsTableFilteredArray =
+ taskSummaryMetricsTableCurrentStateArray.filter(row => (row.checkboxId).toString() != para);
+ }
+ createDataTableForTaskSummaryMetricsTable(taskSummaryMetricsTableFilteredArray);
+ }
+ if (dataMetricsType === "executor") {
+ column = executorSummaryTableSelector.column(para);
+ column.visible(!column.visible());
+ }
+ }
+ });
+
+ // title number and toggle list
+ $("#summaryMetricsTitle").html("Summary Metrics for " + "" + responseBody.numCompleteTasks + " Completed Tasks" + "");
+ $("#tasksTitle").html("Tasks (" + totalTasksToShow + ")");
+
+ // hide or show the accumulate update table
+ if (accumulatorTable.length == 0) {
+ $("#accumulator-update-table").hide();
+ } else {
+ taskTableSelector.column(18).visible(true);
+ $("#accumulator-update-table").show();
+ }
+ // Showing relevant stage data depending on stage type for task table and executor
+ // summary table
+ taskTableSelector.column(19).visible(dataToShow.showInputData);
+ taskTableSelector.column(20).visible(dataToShow.showOutputData);
+ taskTableSelector.column(22).visible(dataToShow.showShuffleWriteData);
+ taskTableSelector.column(23).visible(dataToShow.showShuffleReadData);
+ taskTableSelector.column(24).visible(dataToShow.showBytesSpilledData);
+ taskTableSelector.column(25).visible(dataToShow.showBytesSpilledData);
+
+ if (window.localStorage) {
+ if (window.localStorage.getItem("arrowtoggle1class") !== null &&
+ window.localStorage.getItem("arrowtoggle1class").includes("arrow-open")) {
+ $("#arrowtoggle1").toggleClass("arrow-open arrow-closed");
+ $("#toggle-metrics").toggleClass("d-none");
+ }
+ if (window.localStorage.getItem("arrowtoggle2class") !== null &&
+ window.localStorage.getItem("arrowtoggle2class").includes("arrow-open")) {
+ $("#arrowtoggle2").toggleClass("arrow-open arrow-closed");
+ $("#toggle-aggregatedMetrics").toggleClass("d-none");
+ }
+ }
+ });
});
+ });
});
diff --git a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js
index 6d4c8d94b4288..9f366025a8b55 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js
@@ -15,6 +15,7 @@
* limitations under the License.
*/
+/* global $, d3, timeFormat, timeTipStrings */
// timeFormat: StreamingPage.scala will generate a global "timeFormat" dictionary to store the time
// and its formatted string. Because we cannot specify a timezone in JavaScript, to make sure the
@@ -37,87 +38,90 @@ var onClickTimeline = function() {};
// Show a tooltip "text" for "node"
function showBootstrapTooltip(node, text) {
- $(node).tooltip({title: text, trigger: "manual", container: "body"});
- $(node).tooltip("show");
+ $(node).tooltip({title: text, trigger: "manual", container: "body"});
+ $(node).tooltip("show");
}
// Hide the tooltip for "node"
function hideBootstrapTooltip(node) {
- $(node).tooltip("dispose");
+ $(node).tooltip("dispose");
}
+/* eslint-disable no-unused-vars */
// Return the function to scroll to the corresponding
// row on clicking a point of batch in the timeline.
function getOnClickTimelineFunction() {
- // If the user click one point in the graphs, jump to the batch row and highlight it. And
- // recovery the batch row after 3 seconds if necessary.
- // We need to remember the last clicked batch so that we can recovery it.
- var lastClickedBatch = null;
- var lastTimeout = null;
-
- return function(d) {
- var batchSelector = $("#batch-" + d.x);
- // If there is a corresponding batch row, scroll down to it and highlight it.
- if (batchSelector.length > 0) {
- if (lastTimeout != null) {
- window.clearTimeout(lastTimeout);
- }
- if (lastClickedBatch != null) {
- clearBatchRow(lastClickedBatch);
- lastClickedBatch = null;
- }
- lastClickedBatch = d.x;
- highlightBatchRow(lastClickedBatch);
- lastTimeout = window.setTimeout(function () {
- lastTimeout = null;
- if (lastClickedBatch != null) {
- clearBatchRow(lastClickedBatch);
- lastClickedBatch = null;
- }
- }, 3000); // Clean up after 3 seconds
-
- var topOffset = batchSelector.offset().top - 15;
- if (topOffset < 0) {
- topOffset = 0;
- }
- $('html,body').animate({scrollTop: topOffset}, 200);
+ // If the user click one point in the graphs, jump to the batch row and highlight it. And
+ // recovery the batch row after 3 seconds if necessary.
+ // We need to remember the last clicked batch so that we can recovery it.
+ var lastClickedBatch = null;
+ var lastTimeout = null;
+
+ return function(d) {
+ var batchSelector = $("#batch-" + d.x);
+ // If there is a corresponding batch row, scroll down to it and highlight it.
+ if (batchSelector.length > 0) {
+ if (lastTimeout != null) {
+ window.clearTimeout(lastTimeout);
+ }
+ if (lastClickedBatch != null) {
+ clearBatchRow(lastClickedBatch);
+ lastClickedBatch = null;
+ }
+ lastClickedBatch = d.x;
+ highlightBatchRow(lastClickedBatch);
+ lastTimeout = window.setTimeout(function () {
+ lastTimeout = null;
+ if (lastClickedBatch != null) {
+ clearBatchRow(lastClickedBatch);
+ lastClickedBatch = null;
}
+ }, 3000); // Clean up after 3 seconds
+
+ var topOffset = batchSelector.offset().top - 15;
+ if (topOffset < 0) {
+ topOffset = 0;
+ }
+ $('html,body').animate({scrollTop: topOffset}, 200);
}
+ }
}
// Register a timeline graph. All timeline graphs should be register before calling any
// "drawTimeline" so that we can determine the max margin left for all timeline graphs.
function registerTimeline(minY, maxY) {
- var numOfChars = yValueFormat(maxY).length;
- // A least width for "maxY" in the graph
- var pxForMaxY = numOfChars * 8 + 10;
- // Make sure we have enough space to show the ticks in the y axis of timeline
- maxMarginLeftForTimeline = pxForMaxY > maxMarginLeftForTimeline? pxForMaxY : maxMarginLeftForTimeline;
+ var numOfChars = yValueFormat(maxY).length;
+ // A least width for "maxY" in the graph
+ var pxForMaxY = numOfChars * 8 + 10;
+ // Make sure we have enough space to show the ticks in the y axis of timeline
+ maxMarginLeftForTimeline = pxForMaxY > maxMarginLeftForTimeline? pxForMaxY : maxMarginLeftForTimeline;
}
// Register a histogram graph. All histogram graphs should be register before calling any
// "drawHistogram" so that we can determine the max X value for histograms.
function registerHistogram(values, minY, maxY) {
- var data = d3.layout.histogram().range([minY, maxY]).bins(histogramBinCount)(values);
- // d.x is the y values while d.y is the x values
- var maxX = d3.max(data, function(d) { return d.y; });
- maxXForHistogram = maxX > maxXForHistogram ? maxX : maxXForHistogram;
+ var data = d3.layout.histogram().range([minY, maxY]).bins(histogramBinCount)(values);
+ // d.x is the y values while d.y is the x values
+ var maxX = d3.max(data, function(d) { return d.y; });
+ maxXForHistogram = maxX > maxXForHistogram ? maxX : maxXForHistogram;
}
+/* eslint-enable no-unused-vars */
// Draw a line between (x1, y1) and (x2, y2)
function drawLine(svg, xFunc, yFunc, x1, y1, x2, y2) {
- var line = d3.svg.line()
- .x(function(d) { return xFunc(d.x); })
- .y(function(d) { return yFunc(d.y); });
- var data = [{x: x1, y: y1}, {x: x2, y: y2}];
- svg.append("path")
- .datum(data)
- .style("stroke-dasharray", ("6, 6"))
- .style("stroke", "lightblue")
- .attr("class", "line")
- .attr("d", line);
+ var line = d3.svg.line()
+ .x(function(d) { return xFunc(d.x); })
+ .y(function(d) { return yFunc(d.y); });
+ var data = [{x: x1, y: y1}, {x: x2, y: y2}];
+ svg.append("path")
+ .datum(data)
+ .style("stroke-dasharray", ("6, 6"))
+ .style("stroke", "lightblue")
+ .attr("class", "line")
+ .attr("d", line);
}
+/* eslint-disable no-unused-vars */
/**
* @param id the `id` used in the html `div` tag
* @param data the data for the timeline graph
@@ -129,108 +133,105 @@ function drawLine(svg, xFunc, yFunc, x1, y1, x2, y2) {
* @param batchInterval if "batchInterval" is specified, we will draw a line for "batchInterval" in the graph
*/
function drawTimeline(id, data, minX, maxX, minY, maxY, unitY, batchInterval) {
- // Hide the right border of "". We cannot use "css" directly, or "sorttable.js" will override them.
- d3.select(d3.select(id).node().parentNode)
- .style("padding", "8px 0 8px 8px")
- .style("border-right", "0px solid white");
-
- var margin = {top: 20, right: 27, bottom: 30, left: maxMarginLeftForTimeline};
- var width = 500 - margin.left - margin.right;
- var height = 150 - margin.top - margin.bottom;
-
- var x = d3.scale.linear().domain([minX, maxX]).range([0, width]);
- var y = d3.scale.linear().domain([minY, maxY]).range([height, 0]);
-
- var xAxis = d3.svg.axis().scale(x).orient("bottom").tickFormat(function(d) {
- var formattedDate = timeFormat[d];
- var dotIndex = formattedDate.indexOf('.');
- if (dotIndex >= 0) {
- // Remove milliseconds
- return formattedDate.substring(0, dotIndex);
- } else {
- return formattedDate;
- }
- });
- var formatYValue = d3.format(",.2f");
- var yAxis = d3.svg.axis().scale(y).orient("left").ticks(5).tickFormat(formatYValue);
-
- var line = d3.svg.line()
- .x(function(d) { return x(d.x); })
- .y(function(d) { return y(d.y); });
-
- var svg = d3.select(id).append("svg")
- .attr("width", width + margin.left + margin.right)
- .attr("height", height + margin.top + margin.bottom)
- .append("g")
- .attr("transform", "translate(" + margin.left + "," + margin.top + ")");
-
- // Only show the first and last time in the graph
- xAxis.tickValues(x.domain());
-
- svg.append("g")
- .attr("class", "x axis")
- .attr("transform", "translate(0," + height + ")")
- .call(xAxis);
-
- svg.append("g")
- .attr("class", "y axis")
- .call(yAxis)
- .append("text")
- .attr("transform", "translate(0," + unitLabelYOffset + ")")
- .text(unitY);
-
-
- if (batchInterval && batchInterval <= maxY) {
- drawLine(svg, x, y, minX, batchInterval, maxX, batchInterval);
- }
-
- svg.append("path")
- .datum(data)
- .attr("class", "line")
- .attr("d", line);
-
- // If the user click one point in the graphs, jump to the batch row and highlight it. And
- // recovery the batch row after 3 seconds if necessary.
- // We need to remember the last clicked batch so that we can recovery it.
- var lastClickedBatch = null;
- var lastTimeout = null;
-
- function isFailedBatch(batchTime) {
- return $("#batch-" + batchTime).attr("isFailed") == "true";
+ // Hide the right border of " | ". We cannot use "css" directly, or "sorttable.js" will override them.
+ d3.select(d3.select(id).node().parentNode)
+ .style("padding", "8px 0 8px 8px")
+ .style("border-right", "0px solid white");
+ var margin = {top: 20, right: 27, bottom: 30, left: maxMarginLeftForTimeline};
+ var width = 500 - margin.left - margin.right;
+ var height = 150 - margin.top - margin.bottom;
+
+ var x = d3.scale.linear().domain([minX, maxX]).range([0, width]);
+ var y = d3.scale.linear().domain([minY, maxY]).range([height, 0]);
+
+ var xAxis = d3.svg.axis().scale(x).orient("bottom").tickFormat(function(d) {
+ var formattedDate = timeFormat[d];
+ var dotIndex = formattedDate.indexOf('.');
+ if (dotIndex >= 0) {
+ // Remove milliseconds
+ return formattedDate.substring(0, dotIndex);
+ } else {
+ return formattedDate;
}
-
- // Add points to the line. However, we make it invisible at first. But when the user moves mouse
- // over a point, it will be displayed with its detail.
- svg.selectAll(".point")
- .data(data)
- .enter().append("circle")
- .attr("stroke", function(d) { return isFailedBatch(d.x) ? "red" : "white";}) // white and opacity = 0 make it invisible
- .attr("fill", function(d) { return isFailedBatch(d.x) ? "red" : "white";})
- .attr("opacity", function(d) { return isFailedBatch(d.x) ? "1" : "0";})
- .style("cursor", "pointer")
- .attr("cx", function(d) { return x(d.x); })
- .attr("cy", function(d) { return y(d.y); })
- .attr("r", function(d) { return isFailedBatch(d.x) ? "2" : "3";})
- .on('mouseover', function(d) {
- var tip = formatYValue(d.y) + " " + unitY + " at " + timeTipStrings[d.x];
- showBootstrapTooltip(d3.select(this).node(), tip);
- // show the point
- d3.select(this)
- .attr("stroke", function(d) { return isFailedBatch(d.x) ? "red" : "steelblue";})
- .attr("fill", function(d) { return isFailedBatch(d.x) ? "red" : "steelblue";})
- .attr("opacity", "1")
- .attr("r", "3");
- })
- .on('mouseout', function() {
- hideBootstrapTooltip(d3.select(this).node());
- // hide the point
- d3.select(this)
- .attr("stroke", function(d) { return isFailedBatch(d.x) ? "red" : "white";})
- .attr("fill", function(d) { return isFailedBatch(d.x) ? "red" : "white";})
- .attr("opacity", function(d) { return isFailedBatch(d.x) ? "1" : "0";})
- .attr("r", function(d) { return isFailedBatch(d.x) ? "2" : "3";});
- })
- .on("click", onClickTimeline);
+ });
+ var formatYValue = d3.format(",.2f");
+ var yAxis = d3.svg.axis().scale(y).orient("left").ticks(5).tickFormat(formatYValue);
+
+ var line = d3.svg.line()
+ .x(function(d) { return x(d.x); })
+ .y(function(d) { return y(d.y); });
+
+ var svg = d3.select(id).append("svg")
+ .attr("width", width + margin.left + margin.right)
+ .attr("height", height + margin.top + margin.bottom)
+ .append("g")
+ .attr("transform", "translate(" + margin.left + "," + margin.top + ")");
+
+ // Only show the first and last time in the graph
+ xAxis.tickValues(x.domain());
+
+ svg.append("g")
+ .attr("class", "x axis")
+ .attr("transform", "translate(0," + height + ")")
+ .call(xAxis);
+
+ svg.append("g")
+ .attr("class", "y axis")
+ .call(yAxis)
+ .append("text")
+ .attr("transform", "translate(0," + unitLabelYOffset + ")")
+ .text(unitY);
+
+ if (batchInterval && batchInterval <= maxY) {
+ drawLine(svg, x, y, minX, batchInterval, maxX, batchInterval);
+ }
+
+ svg.append("path")
+ .datum(data)
+ .attr("class", "line")
+ .attr("d", line);
+ // If the user click one point in the graphs, jump to the batch row and highlight it. And
+ // recovery the batch row after 3 seconds if necessary.
+ // We need to remember the last clicked batch so that we can recovery it.
+ var lastClickedBatch = null;
+ var lastTimeout = null;
+
+ function isFailedBatch(batchTime) {
+ return $("#batch-" + batchTime).attr("isFailed") == "true";
+ }
+
+ // Add points to the line. However, we make it invisible at first. But when the user moves mouse
+ // over a point, it will be displayed with its detail.
+ svg.selectAll(".point")
+ .data(data)
+ .enter().append("circle")
+ .attr("stroke", function(d) { return isFailedBatch(d.x) ? "red" : "white";}) // white and opacity = 0 make it invisible
+ .attr("fill", function(d) { return isFailedBatch(d.x) ? "red" : "white";})
+ .attr("opacity", function(d) { return isFailedBatch(d.x) ? "1" : "0";})
+ .style("cursor", "pointer")
+ .attr("cx", function(d) { return x(d.x); })
+ .attr("cy", function(d) { return y(d.y); })
+ .attr("r", function(d) { return isFailedBatch(d.x) ? "2" : "3";})
+ .on('mouseover', function(d) {
+ var tip = formatYValue(d.y) + " " + unitY + " at " + timeTipStrings[d.x];
+ showBootstrapTooltip(d3.select(this).node(), tip);
+ // show the point
+ d3.select(this)
+ .attr("stroke", function(d) { return isFailedBatch(d.x) ? "red" : "steelblue";})
+ .attr("fill", function(d) { return isFailedBatch(d.x) ? "red" : "steelblue";})
+ .attr("opacity", "1")
+ .attr("r", "3");
+ })
+ .on('mouseout', function() {
+ hideBootstrapTooltip(d3.select(this).node());
+ // hide the point
+ d3.select(this)
+ .attr("stroke", function(d) { return isFailedBatch(d.x) ? "red" : "white";})
+ .attr("fill", function(d) { return isFailedBatch(d.x) ? "red" : "white";})
+ .attr("opacity", function(d) { return isFailedBatch(d.x) ? "1" : "0";})
+ .attr("r", function(d) { return isFailedBatch(d.x) ? "2" : "3";});
+ })
+ .on("click", onClickTimeline);
}
/**
@@ -242,105 +243,106 @@ function drawTimeline(id, data, minX, maxX, minY, maxY, unitY, batchInterval) {
* @param batchInterval if "batchInterval" is specified, we will draw a line for "batchInterval" in the graph
*/
function drawHistogram(id, values, minY, maxY, unitY, batchInterval) {
- // Hide the left border of " | ". We cannot use "css" directly, or "sorttable.js" will override them.
- d3.select(d3.select(id).node().parentNode)
- .style("padding", "8px 8px 8px 0")
- .style("border-left", "0px solid white");
-
- var margin = {top: 20, right: 30, bottom: 30, left: 10};
- var width = 350 - margin.left - margin.right;
- var height = 150 - margin.top - margin.bottom;
-
- var x = d3.scale.linear().domain([0, maxXForHistogram]).range([0, width - 50]);
- var y = d3.scale.linear().domain([minY, maxY]).range([height, 0]);
-
- var xAxis = d3.svg.axis().scale(x).orient("top").ticks(5);
- var yAxis = d3.svg.axis().scale(y).orient("left").ticks(0).tickFormat(function(d) { return ""; });
-
- var data = d3.layout.histogram().range([minY, maxY]).bins(histogramBinCount)(values);
-
- var svg = d3.select(id).append("svg")
- .attr("width", width + margin.left + margin.right)
- .attr("height", height + margin.top + margin.bottom)
- .append("g")
- .attr("transform", "translate(" + margin.left + "," + margin.top + ")");
-
- if (batchInterval && batchInterval <= maxY) {
- drawLine(svg, x, y, 0, batchInterval, maxXForHistogram, batchInterval);
- }
+ // Hide the left border of " | ". We cannot use "css" directly, or "sorttable.js" will override them.
+ d3.select(d3.select(id).node().parentNode)
+ .style("padding", "8px 8px 8px 0")
+ .style("border-left", "0px solid white");
+
+ var margin = {top: 20, right: 30, bottom: 30, left: 10};
+ var width = 350 - margin.left - margin.right;
+ var height = 150 - margin.top - margin.bottom;
+
+ var x = d3.scale.linear().domain([0, maxXForHistogram]).range([0, width - 50]);
+ var y = d3.scale.linear().domain([minY, maxY]).range([height, 0]);
+
+ var xAxis = d3.svg.axis().scale(x).orient("top").ticks(5);
+ var yAxis = d3.svg.axis().scale(y).orient("left").ticks(0).tickFormat(function(d) { return ""; });
+
+ var data = d3.layout.histogram().range([minY, maxY]).bins(histogramBinCount)(values);
+
+ var svg = d3.select(id).append("svg")
+ .attr("width", width + margin.left + margin.right)
+ .attr("height", height + margin.top + margin.bottom)
+ .append("g")
+ .attr("transform", "translate(" + margin.left + "," + margin.top + ")");
+
+ if (batchInterval && batchInterval <= maxY) {
+ drawLine(svg, x, y, 0, batchInterval, maxXForHistogram, batchInterval);
+ }
+
+ svg.append("g")
+ .attr("class", "x axis")
+ .call(xAxis)
+ .append("text")
+ .attr("transform", "translate(" + (margin.left + width - 45) + ", " + unitLabelYOffset + ")")
+ .text("#batches");
+
+ svg.append("g")
+ .attr("class", "y axis")
+ .call(yAxis);
+
+ svg.selectAll(".bar")
+ .data(data)
+ .enter()
+ .append("g")
+ .attr("transform", function(d) { return "translate(0," + (y(d.x) - height + y(d.dx)) + ")";})
+ .attr("class", "bar").append("rect")
+ .attr("width", function(d) { return x(d.y); })
+ .attr("height", function(d) { return height - y(d.dx); })
+ .on('mouseover', function(d) {
+ var percent = yValueFormat(d.y * 100.0 / values.length) + "%";
+ var tip = d.y + " batches (" + percent + ") between " + yValueFormat(d.x) + " and " + yValueFormat(d.x + d.dx) + " " + unitY;
+ showBootstrapTooltip(d3.select(this).node(), tip);
+ })
+ .on('mouseout', function() {
+ hideBootstrapTooltip(d3.select(this).node());
+ });
- svg.append("g")
- .attr("class", "x axis")
- .call(xAxis)
- .append("text")
- .attr("transform", "translate(" + (margin.left + width - 45) + ", " + unitLabelYOffset + ")")
- .text("#batches");
-
- svg.append("g")
- .attr("class", "y axis")
- .call(yAxis);
-
- svg.selectAll(".bar")
- .data(data)
- .enter()
- .append("g")
- .attr("transform", function(d) { return "translate(0," + (y(d.x) - height + y(d.dx)) + ")";})
- .attr("class", "bar").append("rect")
- .attr("width", function(d) { return x(d.y); })
- .attr("height", function(d) { return height - y(d.dx); })
- .on('mouseover', function(d) {
- var percent = yValueFormat(d.y * 100.0 / values.length) + "%";
- var tip = d.y + " batches (" + percent + ") between " + yValueFormat(d.x) + " and " + yValueFormat(d.x + d.dx) + " " + unitY;
- showBootstrapTooltip(d3.select(this).node(), tip);
- })
- .on('mouseout', function() {
- hideBootstrapTooltip(d3.select(this).node());
- });
-
- if (batchInterval && batchInterval <= maxY) {
- // Add the "stable" text to the graph below the batch interval line.
- var stableXOffset = x(maxXForHistogram) - 20;
- var stableYOffset = y(batchInterval) + 15;
- svg.append("text")
- .style("fill", "lightblue")
- .attr("class", "stable-text")
- .attr("text-anchor", "middle")
- .attr("transform", "translate(" + stableXOffset + "," + stableYOffset + ")")
- .text("stable")
- .on('mouseover', function(d) {
- var tip = "Processing Time <= Batch Interval (" + yValueFormat(batchInterval) +" " + unitY +")";
- showBootstrapTooltip(d3.select(this).node(), tip);
- })
- .on('mouseout', function() {
- hideBootstrapTooltip(d3.select(this).node());
- });
- }
+ if (batchInterval && batchInterval <= maxY) {
+ // Add the "stable" text to the graph below the batch interval line.
+ var stableXOffset = x(maxXForHistogram) - 20;
+ var stableYOffset = y(batchInterval) + 15;
+ svg.append("text")
+ .style("fill", "lightblue")
+ .attr("class", "stable-text")
+ .attr("text-anchor", "middle")
+ .attr("transform", "translate(" + stableXOffset + "," + stableYOffset + ")")
+ .text("stable")
+ .on('mouseover', function(d) {
+ var tip = "Processing Time <= Batch Interval (" + yValueFormat(batchInterval) +" " + unitY +")";
+ showBootstrapTooltip(d3.select(this).node(), tip);
+ })
+ .on('mouseout', function() {
+ hideBootstrapTooltip(d3.select(this).node());
+ });
+ }
}
+/* eslint-enable no-unused-vars */
$(function() {
- var status = window.localStorage && window.localStorage.getItem("show-streams-detail") == "true";
-
- $("span.expand-input-rate").click(function() {
- status = !status;
- $("#inputs-table").toggle('collapsed');
- // Toggle the class of the arrow between open and closed
- $(this).find('.expand-input-rate-arrow').toggleClass('arrow-open').toggleClass('arrow-closed');
- if (window.localStorage) {
- window.localStorage.setItem("show-streams-detail", "" + status);
- }
- });
-
- if (status) {
- $("#inputs-table").toggle('collapsed');
- // Toggle the class of the arrow between open and closed
- $(this).find('.expand-input-rate-arrow').toggleClass('arrow-open').toggleClass('arrow-closed');
+ var status = window.localStorage && window.localStorage.getItem("show-streams-detail") == "true";
+
+ $("span.expand-input-rate").click(function() {
+ status = !status;
+ $("#inputs-table").toggle('collapsed');
+ // Toggle the class of the arrow between open and closed
+ $(this).find('.expand-input-rate-arrow').toggleClass('arrow-open').toggleClass('arrow-closed');
+ if (window.localStorage) {
+ window.localStorage.setItem("show-streams-detail", "" + status);
}
+ });
+
+ if (status) {
+ $("#inputs-table").toggle('collapsed');
+ // Toggle the class of the arrow between open and closed
+ $(this).find('.expand-input-rate-arrow').toggleClass('arrow-open').toggleClass('arrow-closed');
+ }
});
function highlightBatchRow(batch) {
- $("#batch-" + batch).parent().addClass("batch-table-cell-highlight");
+ $("#batch-" + batch).parent().addClass("batch-table-cell-highlight");
}
function clearBatchRow(batch) {
- $("#batch-" + batch).parent().removeClass("batch-table-cell-highlight");
+ $("#batch-" + batch).parent().removeClass("batch-table-cell-highlight");
}
diff --git a/core/src/main/resources/org/apache/spark/ui/static/structured-streaming-page.js b/core/src/main/resources/org/apache/spark/ui/static/structured-streaming-page.js
index c92226b408b6c..9701f5a57e170 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/structured-streaming-page.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/structured-streaming-page.js
@@ -15,157 +15,158 @@
* limitations under the License.
*/
+/* global d3, formattedTimeTipStrings, formattedTimeToValues, hideBootstrapTooltip, maxMarginLeftForTimeline, showBootstrapTooltip, unitLabelYOffset */
// pre-define some colors for legends.
var colorPool = ["#F8C471", "#F39C12", "#B9770E", "#73C6B6", "#16A085", "#117A65", "#B2BABB", "#7F8C8D", "#616A6B"];
+/* eslint-disable no-unused-vars */
function drawAreaStack(id, labels, values, minX, maxX, minY, maxY) {
- d3.select(d3.select(id).node().parentNode)
- .style("padding", "8px 0 8px 8px")
- .style("border-right", "0px solid white");
-
- // Setup svg using Bostock's margin convention
- var margin = {top: 20, right: 40, bottom: 30, left: maxMarginLeftForTimeline};
- var width = 850 - margin.left - margin.right;
- var height = 300 - margin.top - margin.bottom;
-
- var svg = d3.select(id)
- .append("svg")
- .attr("width", width + margin.left + margin.right)
- .attr("height", height + margin.top + margin.bottom)
- .append("g")
- .attr("transform", "translate(" + margin.left + "," + margin.top + ")");
-
- var data = values;
-
- var parse = d3.time.format("%H:%M:%S.%L").parse;
-
- // Transpose the data into layers
- var dataset = d3.layout.stack()(labels.map(function(fruit) {
- return data.map(function(d) {
- return {_x: d.x, x: parse(d.x), y: +d[fruit]};
- });
- }));
-
-
- // Set x, y and colors
- var x = d3.scale.ordinal()
- .domain(dataset[0].map(function(d) { return d.x; }))
- .rangeRoundBands([10, width-10], 0.02);
-
- var y = d3.scale.linear()
- .domain([0, d3.max(dataset, function(d) { return d3.max(d, function(d) { return d.y0 + d.y; }); })])
- .range([height, 0]);
-
- var colors = colorPool.slice(0, labels.length)
-
- // Define and draw axes
- var yAxis = d3.svg.axis()
- .scale(y)
- .orient("left")
- .ticks(7)
- .tickFormat( function(d) { return d } );
-
- var xAxis = d3.svg.axis()
- .scale(x)
- .orient("bottom")
- .tickFormat(d3.time.format("%H:%M:%S.%L"));
-
- // Only show the first and last time in the graph
- var xline = []
- xline.push(x.domain()[0])
- xline.push(x.domain()[x.domain().length - 1])
- xAxis.tickValues(xline);
-
- svg.append("g")
- .attr("class", "y axis")
- .call(yAxis)
- .append("text")
- .attr("transform", "translate(0," + unitLabelYOffset + ")")
- .text("ms");
-
- svg.append("g")
- .attr("class", "x axis")
- .attr("transform", "translate(0," + height + ")")
- .call(xAxis);
-
- // Create groups for each series, rects for each segment
- var groups = svg.selectAll("g.cost")
- .data(dataset)
- .enter().append("g")
- .attr("class", "cost")
- .style("fill", function(d, i) { return colors[i]; });
-
- var rect = groups.selectAll("rect")
- .data(function(d) { return d; })
- .enter()
- .append("rect")
- .attr("x", function(d) { return x(d.x); })
- .attr("y", function(d) { return y(d.y0 + d.y); })
- .attr("height", function(d) { return y(d.y0) - y(d.y0 + d.y); })
- .attr("width", x.rangeBand())
- .on('mouseover', function(d) {
- var tip = '';
- var idx = 0;
- var _values = formattedTimeToValues[d._x];
- _values.forEach(function (k) {
- tip += labels[idx] + ': ' + k + ' ';
- idx += 1;
- });
- tip += " at " + formattedTimeTipStrings[d._x];
- showBootstrapTooltip(d3.select(this).node(), tip);
- })
- .on('mouseout', function() {
- hideBootstrapTooltip(d3.select(this).node());
- })
- .on("mousemove", function(d) {
- var xPosition = d3.mouse(this)[0] - 15;
- var yPosition = d3.mouse(this)[1] - 25;
- tooltip.attr("transform", "translate(" + xPosition + "," + yPosition + ")");
- tooltip.select("text").text(d.y);
- });
-
-
- // Draw legend
- var legend = svg.selectAll(".legend")
- .data(colors)
- .enter().append("g")
- .attr("class", "legend")
- .attr("transform", function(d, i) { return "translate(30," + i * 19 + ")"; });
-
- legend.append("rect")
- .attr("x", width - 20)
- .attr("width", 18)
- .attr("height", 18)
- .style("fill", function(d, i) {return colors.slice().reverse()[i];})
- .on('mouseover', function(d, i) {
- var len = labels.length
- showBootstrapTooltip(d3.select(this).node(), labels[len - 1 - i]);
- })
- .on('mouseout', function() {
- hideBootstrapTooltip(d3.select(this).node());
- })
- .on("mousemove", function(d) {
- var xPosition = d3.mouse(this)[0] - 15;
- var yPosition = d3.mouse(this)[1] - 25;
- tooltip.attr("transform", "translate(" + xPosition + "," + yPosition + ")");
- tooltip.select("text").text(d.y);
- });
-
- // Prep the tooltip bits, initial display is hidden
- var tooltip = svg.append("g")
- .attr("class", "tooltip")
- .style("display", "none");
-
- tooltip.append("rect")
- .attr("width", 30)
- .attr("height", 20)
- .attr("fill", "white")
- .style("opacity", 0.5);
-
- tooltip.append("text")
- .attr("x", 15)
- .attr("dy", "1.2em")
- .style("text-anchor", "middle")
- .attr("font-size", "12px")
- .attr("font-weight", "bold");
+ d3.select(d3.select(id).node().parentNode)
+ .style("padding", "8px 0 8px 8px")
+ .style("border-right", "0px solid white");
+
+ // Setup svg using Bostock's margin convention
+ var margin = {top: 20, right: 40, bottom: 30, left: maxMarginLeftForTimeline};
+ var width = 850 - margin.left - margin.right;
+ var height = 300 - margin.top - margin.bottom;
+
+ var svg = d3.select(id)
+ .append("svg")
+ .attr("width", width + margin.left + margin.right)
+ .attr("height", height + margin.top + margin.bottom)
+ .append("g")
+ .attr("transform", "translate(" + margin.left + "," + margin.top + ")");
+
+ var data = values;
+
+ var parse = d3.time.format("%H:%M:%S.%L").parse;
+
+ // Transpose the data into layers
+ var dataset = d3.layout.stack()(labels.map(function(fruit) {
+ return data.map(function(d) {
+ return {_x: d.x, x: parse(d.x), y: +d[fruit]};
+ });
+ }));
+
+ // Set x, y and colors
+ var x = d3.scale.ordinal()
+ .domain(dataset[0].map(function(d) { return d.x; }))
+ .rangeRoundBands([10, width-10], 0.02);
+
+ var y = d3.scale.linear()
+ .domain([0, d3.max(dataset, function(d) { return d3.max(d, function(d) { return d.y0 + d.y; }); })])
+ .range([height, 0]);
+
+ var colors = colorPool.slice(0, labels.length);
+
+ // Define and draw axes
+ var yAxis = d3.svg.axis()
+ .scale(y)
+ .orient("left")
+ .ticks(7)
+ .tickFormat( function(d) { return d } );
+
+ var xAxis = d3.svg.axis()
+ .scale(x)
+ .orient("bottom")
+ .tickFormat(d3.time.format("%H:%M:%S.%L"));
+
+ // Only show the first and last time in the graph
+ var xline = [];
+ xline.push(x.domain()[0]);
+ xline.push(x.domain()[x.domain().length - 1]);
+ xAxis.tickValues(xline);
+
+ svg.append("g")
+ .attr("class", "y axis")
+ .call(yAxis)
+ .append("text")
+ .attr("transform", "translate(0," + unitLabelYOffset + ")")
+ .text("ms");
+
+ svg.append("g")
+ .attr("class", "x axis")
+ .attr("transform", "translate(0," + height + ")")
+ .call(xAxis);
+
+ // Create groups for each series, rects for each segment
+ var groups = svg.selectAll("g.cost")
+ .data(dataset)
+ .enter().append("g")
+ .attr("class", "cost")
+ .style("fill", function(d, i) { return colors[i]; });
+
+ var rect = groups.selectAll("rect")
+ .data(function(d) { return d; })
+ .enter()
+ .append("rect")
+ .attr("x", function(d) { return x(d.x); })
+ .attr("y", function(d) { return y(d.y0 + d.y); })
+ .attr("height", function(d) { return y(d.y0) - y(d.y0 + d.y); })
+ .attr("width", x.rangeBand())
+ .on('mouseover', function(d) {
+ var tip = '';
+ var idx = 0;
+ var _values = formattedTimeToValues[d._x];
+ _values.forEach(function (k) {
+ tip += labels[idx] + ': ' + k + ' ';
+ idx += 1;
+ });
+ tip += " at " + formattedTimeTipStrings[d._x];
+ showBootstrapTooltip(d3.select(this).node(), tip);
+ })
+ .on('mouseout', function() {
+ hideBootstrapTooltip(d3.select(this).node());
+ })
+ .on("mousemove", function(d) {
+ var xPosition = d3.mouse(this)[0] - 15;
+ var yPosition = d3.mouse(this)[1] - 25;
+ tooltip.attr("transform", "translate(" + xPosition + "," + yPosition + ")");
+ tooltip.select("text").text(d.y);
+ });
+
+ // Draw legend
+ var legend = svg.selectAll(".legend")
+ .data(colors)
+ .enter().append("g")
+ .attr("class", "legend")
+ .attr("transform", function(d, i) { return "translate(30," + i * 19 + ")"; });
+
+ legend.append("rect")
+ .attr("x", width - 20)
+ .attr("width", 18)
+ .attr("height", 18)
+ .style("fill", function(d, i) {return colors.slice().reverse()[i];})
+ .on('mouseover', function(d, i) {
+ var len = labels.length;
+ showBootstrapTooltip(d3.select(this).node(), labels[len - 1 - i]);
+ })
+ .on('mouseout', function() {
+ hideBootstrapTooltip(d3.select(this).node());
+ })
+ .on("mousemove", function(d) {
+ var xPosition = d3.mouse(this)[0] - 15;
+ var yPosition = d3.mouse(this)[1] - 25;
+ tooltip.attr("transform", "translate(" + xPosition + "," + yPosition + ")");
+ tooltip.select("text").text(d.y);
+ });
+
+ // Prep the tooltip bits, initial display is hidden
+ var tooltip = svg.append("g")
+ .attr("class", "tooltip")
+ .style("display", "none");
+
+ tooltip.append("rect")
+ .attr("width", 30)
+ .attr("height", 20)
+ .attr("fill", "white")
+ .style("opacity", 0.5);
+
+ tooltip.append("text")
+ .attr("x", 15)
+ .attr("dy", "1.2em")
+ .style("text-anchor", "middle")
+ .attr("font-size", "12px")
+ .attr("font-weight", "bold");
}
+/* eslint-enable no-unused-vars */
\ No newline at end of file
diff --git a/core/src/main/resources/org/apache/spark/ui/static/table.js b/core/src/main/resources/org/apache/spark/ui/static/table.js
index 32b7a6522d050..0203748cf7dbc 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/table.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/table.js
@@ -15,6 +15,8 @@
* limitations under the License.
*/
+/* global $ */
+/* eslint-disable no-unused-vars */
/* Adds background colors to stripe table rows in the summary table (on the stage page). This is
* necessary (instead of using css or the table striping provided by bootstrap) because the summary
* table has hidden rows.
@@ -22,84 +24,88 @@
* An ID selector (rather than a class selector) is used to ensure this runs quickly even on pages
* with thousands of task rows (ID selectors are much faster than class selectors). */
function stripeSummaryTable() {
- $("#task-summary-table").find("tr:not(:hidden)").each(function (index) {
- if (index % 2 == 1) {
- $(this).css("background-color", "#f9f9f9");
- } else {
- $(this).css("background-color", "#ffffff");
- }
- });
+ $("#task-summary-table").find("tr:not(:hidden)").each(function (index) {
+ if (index % 2 == 1) {
+ $(this).css("background-color", "#f9f9f9");
+ } else {
+ $(this).css("background-color", "#ffffff");
+ }
+ });
}
+/* eslint-enable no-unused-vars */
function toggleThreadStackTrace(threadId, forceAdd) {
- var stackTrace = $("#" + threadId + "_stacktrace")
- if (stackTrace.length == 0) {
- var stackTraceText = $('#' + threadId + "_td_stacktrace").html()
- var threadCell = $("#thread_" + threadId + "_tr")
- threadCell.after(" | " +
- stackTraceText + " |
")
- } else {
- if (!forceAdd) {
- stackTrace.remove()
- }
+ var stackTrace = $("#" + threadId + "_stacktrace");
+ if (stackTrace.length == 0) {
+ var stackTraceText = $('#' + threadId + "_td_stacktrace").html();
+ var threadCell = $("#thread_" + threadId + "_tr");
+ threadCell.after("" +
+ stackTraceText + " |
")
+ } else {
+ if (!forceAdd) {
+ stackTrace.remove()
}
+ }
}
+/* eslint-disable no-unused-vars */
function expandAllThreadStackTrace(toggleButton) {
- $('.accordion-heading').each(function() {
- //get thread ID
- if (!$(this).hasClass("d-none")) {
- var trId = $(this).attr('id').match(/thread_([0-9]+)_tr/m)[1]
- toggleThreadStackTrace(trId, true)
- }
- })
- if (toggleButton) {
- $('.expandbutton').toggleClass('d-none')
+ $('.accordion-heading').each(function() {
+ //get thread ID
+ if (!$(this).hasClass("d-none")) {
+ var trId = $(this).attr('id').match(/thread_([0-9]+)_tr/m)[1];
+ toggleThreadStackTrace(trId, true)
}
+ });
+ if (toggleButton) {
+ $('.expandbutton').toggleClass('d-none')
+ }
}
+/* eslint-enable no-unused-vars */
function collapseAllThreadStackTrace(toggleButton) {
- $('.accordion-body').each(function() {
- $(this).remove()
- })
- if (toggleButton) {
- $('.expandbutton').toggleClass('d-none');
- }
+ $('.accordion-body').each(function() {
+ $(this).remove()
+ });
+ if (toggleButton) {
+ $('.expandbutton').toggleClass('d-none');
+ }
}
-
+/* eslint-disable no-unused-vars */
// inOrOut - true: over, false: out
function onMouseOverAndOut(threadId) {
- $("#" + threadId + "_td_id").toggleClass("threaddump-td-mouseover");
- $("#" + threadId + "_td_name").toggleClass("threaddump-td-mouseover");
- $("#" + threadId + "_td_state").toggleClass("threaddump-td-mouseover");
- $("#" + threadId + "_td_locking").toggleClass("threaddump-td-mouseover");
+ $("#" + threadId + "_td_id").toggleClass("threaddump-td-mouseover");
+ $("#" + threadId + "_td_name").toggleClass("threaddump-td-mouseover");
+ $("#" + threadId + "_td_state").toggleClass("threaddump-td-mouseover");
+ $("#" + threadId + "_td_locking").toggleClass("threaddump-td-mouseover");
}
function onSearchStringChange() {
- var searchString = $('#search').val().toLowerCase();
- //remove the stacktrace
- collapseAllThreadStackTrace(false)
- if (searchString.length == 0) {
- $('tr').each(function() {
- $(this).removeClass('d-none')
- })
- } else {
- $('tr').each(function(){
- if($(this).attr('id') && $(this).attr('id').match(/thread_[0-9]+_tr/) ) {
- var children = $(this).children()
- var found = false
- for (var i = 0; i < children.length; i++) {
- if (children.eq(i).text().toLowerCase().indexOf(searchString) >= 0) {
- found = true
- }
- }
- if (found) {
- $(this).removeClass('d-none')
- } else {
- $(this).addClass('d-none')
- }
- }
- });
- }
+ var searchString = $('#search').val().toLowerCase();
+ //remove the stacktrace
+ collapseAllThreadStackTrace(false);
+ if (searchString.length == 0) {
+ $('tr').each(function() {
+ $(this).removeClass('d-none')
+ })
+ } else {
+ $('tr').each(function(){
+ if($(this).attr('id') && $(this).attr('id').match(/thread_[0-9]+_tr/) ) {
+ var children = $(this).children();
+ var found = false;
+ for (var i = 0; i < children.length; i++) {
+ if (children.eq(i).text().toLowerCase().indexOf(searchString) >= 0) {
+ found = true;
+ }
+ }
+ if (found) {
+ $(this).removeClass('d-none')
+ } else {
+ $(this).addClass('d-none')
+ }
+ }
+ });
+ }
}
+/* eslint-enable no-unused-vars */
diff --git a/core/src/main/resources/org/apache/spark/ui/static/timeline-view.js b/core/src/main/resources/org/apache/spark/ui/static/timeline-view.js
index 220b76a0f1b27..274848d5e62b5 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/timeline-view.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/timeline-view.js
@@ -15,6 +15,8 @@
* limitations under the License.
*/
+/* global $, vis, uiRoot, appBasePath */
+/* eslint-disable no-unused-vars */
function drawApplicationTimeline(groupArray, eventObjArray, startTime, offset) {
var groups = new vis.DataSet(groupArray);
var items = new vis.DataSet(eventObjArray);
@@ -249,7 +251,7 @@ function drawTaskAssignmentTimeline(groupArray, eventObjArray, minLaunchTime, ma
var visibilityState = status ? "" : "none";
$("#task-assignment-timeline").css("display", visibilityState);
- // Switch the class of the arrow from open to closed.
+ // Switch the class of the arrow from open to closed.
$(this).find(".expand-task-assignment-timeline-arrow").toggleClass("arrow-open");
$(this).find(".expand-task-assignment-timeline-arrow").toggleClass("arrow-closed");
@@ -280,6 +282,7 @@ function setupExecutorEventAction() {
);
});
}
+/* eslint-enable no-unused-vars */
function setupZoomable(id, timeline) {
$(id + ' > input[type="checkbox"]').click(function() {
diff --git a/core/src/main/resources/org/apache/spark/ui/static/utils.js b/core/src/main/resources/org/apache/spark/ui/static/utils.js
index 0026850654447..a2d4f5568cb26 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/utils.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/utils.js
@@ -15,42 +15,46 @@
* limitations under the License.
*/
+/* global $, uiRoot */
+/* eslint-disable no-unused-vars */
// this function works exactly the same as UIUtils.formatDuration
function formatDuration(milliseconds) {
- if (milliseconds < 100) {
- return parseInt(milliseconds).toFixed(1) + " ms";
- }
- var seconds = milliseconds * 1.0 / 1000;
- if (seconds < 1) {
- return seconds.toFixed(1) + " s";
- }
- if (seconds < 60) {
- return seconds.toFixed(0) + " s";
- }
- var minutes = seconds / 60;
- if (minutes < 10) {
- return minutes.toFixed(1) + " min";
- } else if (minutes < 60) {
- return minutes.toFixed(0) + " min";
- }
- var hours = minutes / 60;
- return hours.toFixed(1) + " h";
+ if (milliseconds < 100) {
+ return parseInt(milliseconds).toFixed(1) + " ms";
+ }
+ var seconds = milliseconds * 1.0 / 1000;
+ if (seconds < 1) {
+ return seconds.toFixed(1) + " s";
+ }
+ if (seconds < 60) {
+ return seconds.toFixed(0) + " s";
+ }
+ var minutes = seconds / 60;
+ if (minutes < 10) {
+ return minutes.toFixed(1) + " min";
+ } else if (minutes < 60) {
+ return minutes.toFixed(0) + " min";
+ }
+ var hours = minutes / 60;
+ return hours.toFixed(1) + " h";
}
function formatBytes(bytes, type) {
- if (type !== 'display') return bytes;
- if (bytes <= 0) return '0.0 B';
- var k = 1024;
- var dm = 1;
- var sizes = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'];
- var i = Math.floor(Math.log(bytes) / Math.log(k));
- return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i];
+ if (type !== 'display') return bytes;
+ if (bytes <= 0) return '0.0 B';
+ var k = 1024;
+ var dm = 1;
+ var sizes = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'];
+ var i = Math.floor(Math.log(bytes) / Math.log(k));
+ return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i];
}
+/* eslint-enable no-unused-vars */
function padZeroes(num) {
return ("0" + num).slice(-2);
}
+/* eslint-disable no-unused-vars */
function formatTimeMillis(timeMillis) {
if (timeMillis <= 0) {
return "-";
@@ -59,16 +63,18 @@ function formatTimeMillis(timeMillis) {
return formatDateString(dt);
}
}
+/* eslint-enable no-unused-vars */
function formatDateString(dt) {
- return dt.getFullYear() + "-" +
- padZeroes(dt.getMonth() + 1) + "-" +
- padZeroes(dt.getDate()) + " " +
- padZeroes(dt.getHours()) + ":" +
- padZeroes(dt.getMinutes()) + ":" +
- padZeroes(dt.getSeconds());
+ return dt.getFullYear() + "-" +
+ padZeroes(dt.getMonth() + 1) + "-" +
+ padZeroes(dt.getDate()) + " " +
+ padZeroes(dt.getHours()) + ":" +
+ padZeroes(dt.getMinutes()) + ":" +
+ padZeroes(dt.getSeconds());
}
+/* eslint-disable no-unused-vars */
function getTimeZone() {
try {
return Intl.DateTimeFormat().resolvedOptions().timeZone;
@@ -92,14 +98,15 @@ function formatLogsCells(execLogs, type) {
function getStandAloneAppId(cb) {
var words = document.baseURI.split('/');
var ind = words.indexOf("proxy");
+ var appId;
if (ind > 0) {
- var appId = words[ind + 1];
+ appId = words[ind + 1];
cb(appId);
return;
}
ind = words.indexOf("history");
if (ind > 0) {
- var appId = words[ind + 1];
+ appId = words[ind + 1];
cb(appId);
return;
}
@@ -119,7 +126,7 @@ function getStandAloneAppId(cb) {
// It will convert the string into integer for correct ordering
function ConvertDurationString(data) {
data = data.toString();
- var units = data.replace(/[\d\.]/g, '' )
+ var units = data.replace(/[\d.]/g, '' )
.replace(' ', '')
.toLowerCase();
var multiplier = 1;
@@ -143,13 +150,14 @@ function ConvertDurationString(data) {
function createTemplateURI(appId, templateName) {
var words = document.baseURI.split('/');
var ind = words.indexOf("proxy");
+ var baseURI;
if (ind > 0) {
- var baseURI = words.slice(0, ind + 1).join('/') + '/' + appId + '/static/' + templateName + '-template.html';
+ baseURI = words.slice(0, ind + 1).join('/') + '/' + appId + '/static/' + templateName + '-template.html';
return baseURI;
}
ind = words.indexOf("history");
if(ind > 0) {
- var baseURI = words.slice(0, ind).join('/') + '/static/' + templateName + '-template.html';
+ baseURI = words.slice(0, ind).join('/') + '/static/' + templateName + '-template.html';
return baseURI;
}
return uiRoot + "/static/" + templateName + "-template.html";
@@ -159,7 +167,7 @@ function setDataTableDefaults() {
$.extend($.fn.dataTable.defaults, {
stateSave: true,
stateSaveParams: function(_, data) {
- data.search.search = "";
+ data.search.search = "";
},
lengthMenu: [[20, 40, 60, 100, -1], [20, 40, 60, 100, "All"]],
pageLength: 20
@@ -169,51 +177,54 @@ function setDataTableDefaults() {
function formatDate(date) {
if (date <= 0) return "-";
else {
- var dt = new Date(date.replace("GMT", "Z"));
- return formatDateString(dt);
+ var dt = new Date(date.replace("GMT", "Z"));
+ return formatDateString(dt);
}
}
function createRESTEndPointForExecutorsPage(appId) {
- var words = document.baseURI.split('/');
- var ind = words.indexOf("proxy");
- if (ind > 0) {
- var appId = words[ind + 1];
- var newBaseURI = words.slice(0, ind + 2).join('/');
- return newBaseURI + "/api/v1/applications/" + appId + "/allexecutors";
- }
- ind = words.indexOf("history");
- if (ind > 0) {
- var appId = words[ind + 1];
- var attemptId = words[ind + 2];
- var newBaseURI = words.slice(0, ind).join('/');
- if (isNaN(attemptId)) {
- return newBaseURI + "/api/v1/applications/" + appId + "/allexecutors";
- } else {
- return newBaseURI + "/api/v1/applications/" + appId + "/" + attemptId + "/allexecutors";
- }
+ var words = document.baseURI.split('/');
+ var ind = words.indexOf("proxy");
+ var newBaseURI;
+ if (ind > 0) {
+ appId = words[ind + 1];
+ newBaseURI = words.slice(0, ind + 2).join('/');
+ return newBaseURI + "/api/v1/applications/" + appId + "/allexecutors";
+ }
+ ind = words.indexOf("history");
+ if (ind > 0) {
+ appId = words[ind + 1];
+ var attemptId = words[ind + 2];
+ newBaseURI = words.slice(0, ind).join('/');
+ if (isNaN(attemptId)) {
+ return newBaseURI + "/api/v1/applications/" + appId + "/allexecutors";
+ } else {
+ return newBaseURI + "/api/v1/applications/" + appId + "/" + attemptId + "/allexecutors";
}
- return uiRoot + "/api/v1/applications/" + appId + "/allexecutors";
+ }
+ return uiRoot + "/api/v1/applications/" + appId + "/allexecutors";
}
function createRESTEndPointForMiscellaneousProcess(appId) {
- var words = document.baseURI.split('/');
- var ind = words.indexOf("proxy");
- if (ind > 0) {
- var appId = words[ind + 1];
- var newBaseURI = words.slice(0, ind + 2).join('/');
- return newBaseURI + "/api/v1/applications/" + appId + "/allmiscellaneousprocess";
- }
- ind = words.indexOf("history");
- if (ind > 0) {
- var appId = words[ind + 1];
- var attemptId = words[ind + 2];
- var newBaseURI = words.slice(0, ind).join('/');
- if (isNaN(attemptId)) {
- return newBaseURI + "/api/v1/applications/" + appId + "/allmiscellaneousprocess";
- } else {
- return newBaseURI + "/api/v1/applications/" + appId + "/" + attemptId + "/allmiscellaneousprocess";
- }
+ var words = document.baseURI.split('/');
+ var ind = words.indexOf("proxy");
+ var newBaseURI;
+ if (ind > 0) {
+ appId = words[ind + 1];
+ newBaseURI = words.slice(0, ind + 2).join('/');
+ return newBaseURI + "/api/v1/applications/" + appId + "/allmiscellaneousprocess";
+ }
+ ind = words.indexOf("history");
+ if (ind > 0) {
+ appId = words[ind + 1];
+ var attemptId = words[ind + 2];
+ newBaseURI = words.slice(0, ind).join('/');
+ if (isNaN(attemptId)) {
+ return newBaseURI + "/api/v1/applications/" + appId + "/allmiscellaneousprocess";
+ } else {
+ return newBaseURI + "/api/v1/applications/" + appId + "/" + attemptId + "/allmiscellaneousprocess";
}
- return uiRoot + "/api/v1/applications/" + appId + "/allmiscellaneousprocess";
+ }
+ return uiRoot + "/api/v1/applications/" + appId + "/allmiscellaneousprocess";
}
+/* eslint-enable no-unused-vars */
\ No newline at end of file
diff --git a/core/src/main/resources/org/apache/spark/ui/static/webui.js b/core/src/main/resources/org/apache/spark/ui/static/webui.js
index bb3725650c667..c149f2d84337e 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/webui.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/webui.js
@@ -15,16 +15,19 @@
* limitations under the License.
*/
+/* global $ */
+/* eslint-disable no-unused-vars */
var uiRoot = "";
var appBasePath = "";
function setUIRoot(val) {
- uiRoot = val;
+ uiRoot = val;
}
function setAppBasePath(path) {
- appBasePath = path;
+ appBasePath = path;
}
+/* eslint-enable no-unused-vars */
function collapseTablePageLoad(name, table){
if (window.localStorage.getItem(name) == "true") {
@@ -35,20 +38,20 @@ function collapseTablePageLoad(name, table){
}
function collapseTable(thisName, table){
- var status = window.localStorage.getItem(thisName) == "true";
- status = !status;
+ var status = window.localStorage.getItem(thisName) == "true";
+ status = !status;
- var thisClass = '.' + thisName;
+ var thisClass = '.' + thisName;
- // Expand the list of additional metrics.
- var tableDiv = $(thisClass).parent().find('.' + table);
- $(tableDiv).toggleClass('collapsed');
+ // Expand the list of additional metrics.
+ var tableDiv = $(thisClass).parent().find('.' + table);
+ $(tableDiv).toggleClass('collapsed');
- // Switch the class of the arrow from open to closed.
- $(thisClass).find('.collapse-table-arrow').toggleClass('arrow-open');
- $(thisClass).find('.collapse-table-arrow').toggleClass('arrow-closed');
+ // Switch the class of the arrow from open to closed.
+ $(thisClass).find('.collapse-table-arrow').toggleClass('arrow-open');
+ $(thisClass).find('.collapse-table-arrow').toggleClass('arrow-closed');
- window.localStorage.setItem(thisName, "" + status);
+ window.localStorage.setItem(thisName, "" + status);
}
// Add a call to collapseTablePageLoad() on each collapsible table
@@ -101,8 +104,8 @@ $(function() {
});
$(function() {
- // Show/hide full job description on click event.
- $(".description-input").click(function() {
- $(this).toggleClass("description-input-full");
- });
+ // Show/hide full job description on click event.
+ $(".description-input").click(function() {
+ $(this).toggleClass("description-input-full");
+ });
});
diff --git a/core/src/main/scala/org/apache/spark/executor/Executor.scala b/core/src/main/scala/org/apache/spark/executor/Executor.scala
index 8fc1c80cb7f69..acb83f33b2e3c 100644
--- a/core/src/main/scala/org/apache/spark/executor/Executor.scala
+++ b/core/src/main/scala/org/apache/spark/executor/Executor.scala
@@ -55,7 +55,7 @@ import org.apache.spark.util.io.ChunkedByteBuffer
/**
* Spark executor, backed by a threadpool to run tasks.
*
- * This can be used with Mesos, YARN, and the standalone scheduler.
+ * This can be used with Mesos, YARN, kubernetes and the standalone scheduler.
* An internal RPC interface is used for communication with the driver,
* except in the case of Mesos fine-grained mode.
*/
diff --git a/dev/.rat-excludes b/dev/.rat-excludes
index 167cf224f92c2..6c809f43418cd 100644
--- a/dev/.rat-excludes
+++ b/dev/.rat-excludes
@@ -134,3 +134,4 @@ flights_tiny.txt.1
over1k
over10k
exported_table/*
+node_modules
\ No newline at end of file
diff --git a/dev/eslint.json b/dev/eslint.json
new file mode 100644
index 0000000000000..ee1fd3dcc6e71
--- /dev/null
+++ b/dev/eslint.json
@@ -0,0 +1,24 @@
+{
+ "env": {
+ "browser": true,
+ "es6": true
+ },
+ "extends": "eslint:recommended",
+ "rules": {
+ "indent": [
+ "error",
+ 2,
+ {
+ "SwitchCase": 1,
+ "MemberExpression": "off"
+ }
+ ],
+ "no-unused-vars": ["error", {"argsIgnorePattern": "^_ignored_.*"}]
+ },
+ "ignorePatterns": [
+ "*.min.js",
+ "sorttable.js",
+ "jquery.mustache.js",
+ "dataTables.rowsGroup.js"
+ ]
+}
diff --git a/dev/lint-js b/dev/lint-js
new file mode 100755
index 0000000000000..ce06e282192a0
--- /dev/null
+++ b/dev/lint-js
@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -o pipefail
+
+SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"
+SPARK_ROOT_DIR="$(dirname $SCRIPT_DIR)"
+LINT_JS_REPORT_FILE_NAME="$SPARK_ROOT_DIR/dev/lint-js-report.log"
+LINT_TARGET_FILES=(
+ "$SPARK_ROOT_DIR/core/src/main/resources/org/apache/spark/ui/static/"
+ "$SPARK_ROOT_DIR/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/"
+ "$SPARK_ROOT_DIR/docs/js"
+)
+
+if ! type "npm" > /dev/null; then
+ echo "ERROR: You should install npm"
+ exit 1
+fi
+
+if ! type "npx" > /dev/null; then
+ echo "ERROR: You should install npx"
+ exit 1
+fi
+
+cd $SCRIPT_DIR
+
+if ! npm ls eslint > /dev/null; then
+ npm ci eslint
+fi
+
+npx eslint -c "$SPARK_ROOT_DIR/dev/eslint.json" $LINT_TARGET_FILES | tee "$LINT_JS_REPORT_FILE_NAME"
+lint_status=$?
+
+if [ "$lint_status" = "0" ] ; then
+ echo "lint-js checks passed."
+else
+ echo "lint-js checks failed."
+fi
+
+exit "$lint_status"
diff --git a/dev/package-lock.json b/dev/package-lock.json
new file mode 100644
index 0000000000000..a57f45bcf7184
--- /dev/null
+++ b/dev/package-lock.json
@@ -0,0 +1,979 @@
+{
+ "requires": true,
+ "lockfileVersion": 1,
+ "dependencies": {
+ "@babel/code-frame": {
+ "version": "7.12.11",
+ "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.12.11.tgz",
+ "integrity": "sha512-Zt1yodBx1UcyiePMSkWnU4hPqhwq7hGi2nFL1LeA3EUl+q2LQx16MISgJ0+z7dnmgvP9QtIleuETGOiOH1RcIw==",
+ "dev": true,
+ "requires": {
+ "@babel/highlight": "^7.10.4"
+ }
+ },
+ "@babel/helper-validator-identifier": {
+ "version": "7.14.0",
+ "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.14.0.tgz",
+ "integrity": "sha512-V3ts7zMSu5lfiwWDVWzRDGIN+lnCEUdaXgtVHJgLb1rGaA6jMrtB9EmE7L18foXJIE8Un/A/h6NJfGQp/e1J4A==",
+ "dev": true
+ },
+ "@babel/highlight": {
+ "version": "7.14.0",
+ "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.14.0.tgz",
+ "integrity": "sha512-YSCOwxvTYEIMSGaBQb5kDDsCopDdiUGsqpatp3fOlI4+2HQSkTmEVWnVuySdAC5EWCqSWWTv0ib63RjR7dTBdg==",
+ "dev": true,
+ "requires": {
+ "@babel/helper-validator-identifier": "^7.14.0",
+ "chalk": "^2.0.0",
+ "js-tokens": "^4.0.0"
+ },
+ "dependencies": {
+ "chalk": {
+ "version": "2.4.2",
+ "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
+ "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==",
+ "dev": true,
+ "requires": {
+ "ansi-styles": "^3.2.1",
+ "escape-string-regexp": "^1.0.5",
+ "supports-color": "^5.3.0"
+ }
+ }
+ }
+ },
+ "@eslint/eslintrc": {
+ "version": "0.4.0",
+ "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.4.0.tgz",
+ "integrity": "sha512-2ZPCc+uNbjV5ERJr+aKSPRwZgKd2z11x0EgLvb1PURmUrn9QNRXFqje0Ldq454PfAVyaJYyrDvvIKSFP4NnBog==",
+ "dev": true,
+ "requires": {
+ "ajv": "^6.12.4",
+ "debug": "^4.1.1",
+ "espree": "^7.3.0",
+ "globals": "^12.1.0",
+ "ignore": "^4.0.6",
+ "import-fresh": "^3.2.1",
+ "js-yaml": "^3.13.1",
+ "minimatch": "^3.0.4",
+ "strip-json-comments": "^3.1.1"
+ },
+ "dependencies": {
+ "globals": {
+ "version": "12.4.0",
+ "resolved": "https://registry.npmjs.org/globals/-/globals-12.4.0.tgz",
+ "integrity": "sha512-BWICuzzDvDoH54NHKCseDanAhE3CeDorgDL5MT6LMXXj2WCnd9UC2szdk4AWLfjdgNBCXLUanXYcpBBKOSWGwg==",
+ "dev": true,
+ "requires": {
+ "type-fest": "^0.8.1"
+ }
+ }
+ }
+ },
+ "acorn": {
+ "version": "7.4.1",
+ "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz",
+ "integrity": "sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==",
+ "dev": true
+ },
+ "acorn-jsx": {
+ "version": "5.3.1",
+ "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.1.tgz",
+ "integrity": "sha512-K0Ptm/47OKfQRpNQ2J/oIN/3QYiK6FwW+eJbILhsdxh2WTLdl+30o8aGdTbm5JbffpFFAg/g+zi1E+jvJha5ng==",
+ "dev": true
+ },
+ "ajv": {
+ "version": "6.12.6",
+ "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
+ "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
+ "dev": true,
+ "requires": {
+ "fast-deep-equal": "^3.1.1",
+ "fast-json-stable-stringify": "^2.0.0",
+ "json-schema-traverse": "^0.4.1",
+ "uri-js": "^4.2.2"
+ }
+ },
+ "ansi-colors": {
+ "version": "4.1.1",
+ "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.1.tgz",
+ "integrity": "sha512-JoX0apGbHaUJBNl6yF+p6JAFYZ666/hhCGKN5t9QFjbJQKUU/g8MNbFDbvfrgKXvI1QpZplPOnwIo99lX/AAmA==",
+ "dev": true
+ },
+ "ansi-regex": {
+ "version": "5.0.0",
+ "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz",
+ "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==",
+ "dev": true
+ },
+ "ansi-styles": {
+ "version": "3.2.1",
+ "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
+ "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
+ "dev": true,
+ "requires": {
+ "color-convert": "^1.9.0"
+ }
+ },
+ "argparse": {
+ "version": "1.0.10",
+ "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz",
+ "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==",
+ "dev": true,
+ "requires": {
+ "sprintf-js": "~1.0.2"
+ }
+ },
+ "astral-regex": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-2.0.0.tgz",
+ "integrity": "sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==",
+ "dev": true
+ },
+ "balanced-match": {
+ "version": "1.0.2",
+ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
+ "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==",
+ "dev": true
+ },
+ "brace-expansion": {
+ "version": "1.1.11",
+ "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
+ "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
+ "dev": true,
+ "requires": {
+ "balanced-match": "^1.0.0",
+ "concat-map": "0.0.1"
+ }
+ },
+ "callsites": {
+ "version": "3.1.0",
+ "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
+ "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==",
+ "dev": true
+ },
+ "chalk": {
+ "version": "4.1.1",
+ "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.1.tgz",
+ "integrity": "sha512-diHzdDKxcU+bAsUboHLPEDQiw0qEe0qd7SYUn3HgcFlWgbDcfLGswOHYeGrHKzG9z6UYf01d9VFMfZxPM1xZSg==",
+ "dev": true,
+ "requires": {
+ "ansi-styles": "^4.1.0",
+ "supports-color": "^7.1.0"
+ },
+ "dependencies": {
+ "ansi-styles": {
+ "version": "4.3.0",
+ "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+ "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
+ "dev": true,
+ "requires": {
+ "color-convert": "^2.0.1"
+ }
+ },
+ "color-convert": {
+ "version": "2.0.1",
+ "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+ "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+ "dev": true,
+ "requires": {
+ "color-name": "~1.1.4"
+ }
+ },
+ "color-name": {
+ "version": "1.1.4",
+ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+ "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+ "dev": true
+ },
+ "has-flag": {
+ "version": "4.0.0",
+ "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+ "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+ "dev": true
+ },
+ "supports-color": {
+ "version": "7.2.0",
+ "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+ "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+ "dev": true,
+ "requires": {
+ "has-flag": "^4.0.0"
+ }
+ }
+ }
+ },
+ "color-convert": {
+ "version": "1.9.3",
+ "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
+ "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==",
+ "dev": true,
+ "requires": {
+ "color-name": "1.1.3"
+ }
+ },
+ "color-name": {
+ "version": "1.1.3",
+ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
+ "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=",
+ "dev": true
+ },
+ "concat-map": {
+ "version": "0.0.1",
+ "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
+ "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=",
+ "dev": true
+ },
+ "cross-spawn": {
+ "version": "7.0.3",
+ "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
+ "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
+ "dev": true,
+ "requires": {
+ "path-key": "^3.1.0",
+ "shebang-command": "^2.0.0",
+ "which": "^2.0.1"
+ }
+ },
+ "debug": {
+ "version": "4.3.1",
+ "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz",
+ "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==",
+ "dev": true,
+ "requires": {
+ "ms": "2.1.2"
+ }
+ },
+ "deep-is": {
+ "version": "0.1.3",
+ "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz",
+ "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=",
+ "dev": true
+ },
+ "doctrine": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz",
+ "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==",
+ "dev": true,
+ "requires": {
+ "esutils": "^2.0.2"
+ }
+ },
+ "emoji-regex": {
+ "version": "8.0.0",
+ "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
+ "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==",
+ "dev": true
+ },
+ "enquirer": {
+ "version": "2.3.6",
+ "resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.3.6.tgz",
+ "integrity": "sha512-yjNnPr315/FjS4zIsUxYguYUPP2e1NK4d7E7ZOLiyYCcbFBiTMyID+2wvm2w6+pZ/odMA7cRkjhsPbltwBOrLg==",
+ "dev": true,
+ "requires": {
+ "ansi-colors": "^4.1.1"
+ }
+ },
+ "escape-string-regexp": {
+ "version": "1.0.5",
+ "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz",
+ "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=",
+ "dev": true
+ },
+ "eslint": {
+ "version": "7.25.0",
+ "resolved": "https://registry.npmjs.org/eslint/-/eslint-7.25.0.tgz",
+ "integrity": "sha512-TVpSovpvCNpLURIScDRB6g5CYu/ZFq9GfX2hLNIV4dSBKxIWojeDODvYl3t0k0VtMxYeR8OXPCFE5+oHMlGfhw==",
+ "dev": true,
+ "requires": {
+ "@babel/code-frame": "7.12.11",
+ "@eslint/eslintrc": "^0.4.0",
+ "ajv": "^6.10.0",
+ "chalk": "^4.0.0",
+ "cross-spawn": "^7.0.2",
+ "debug": "^4.0.1",
+ "doctrine": "^3.0.0",
+ "enquirer": "^2.3.5",
+ "eslint-scope": "^5.1.1",
+ "eslint-utils": "^2.1.0",
+ "eslint-visitor-keys": "^2.0.0",
+ "espree": "^7.3.1",
+ "esquery": "^1.4.0",
+ "esutils": "^2.0.2",
+ "file-entry-cache": "^6.0.1",
+ "functional-red-black-tree": "^1.0.1",
+ "glob-parent": "^5.0.0",
+ "globals": "^13.6.0",
+ "ignore": "^4.0.6",
+ "import-fresh": "^3.0.0",
+ "imurmurhash": "^0.1.4",
+ "is-glob": "^4.0.0",
+ "js-yaml": "^3.13.1",
+ "json-stable-stringify-without-jsonify": "^1.0.1",
+ "levn": "^0.4.1",
+ "lodash": "^4.17.21",
+ "minimatch": "^3.0.4",
+ "natural-compare": "^1.4.0",
+ "optionator": "^0.9.1",
+ "progress": "^2.0.0",
+ "regexpp": "^3.1.0",
+ "semver": "^7.2.1",
+ "strip-ansi": "^6.0.0",
+ "strip-json-comments": "^3.1.0",
+ "table": "^6.0.4",
+ "text-table": "^0.2.0",
+ "v8-compile-cache": "^2.0.3"
+ }
+ },
+ "eslint-scope": {
+ "version": "5.1.1",
+ "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz",
+ "integrity": "sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==",
+ "dev": true,
+ "requires": {
+ "esrecurse": "^4.3.0",
+ "estraverse": "^4.1.1"
+ }
+ },
+ "eslint-utils": {
+ "version": "2.1.0",
+ "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-2.1.0.tgz",
+ "integrity": "sha512-w94dQYoauyvlDc43XnGB8lU3Zt713vNChgt4EWwhXAP2XkBvndfxF0AgIqKOOasjPIPzj9JqgwkwbCYD0/V3Zg==",
+ "dev": true,
+ "requires": {
+ "eslint-visitor-keys": "^1.1.0"
+ },
+ "dependencies": {
+ "eslint-visitor-keys": {
+ "version": "1.3.0",
+ "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz",
+ "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==",
+ "dev": true
+ }
+ }
+ },
+ "eslint-visitor-keys": {
+ "version": "2.1.0",
+ "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.1.0.tgz",
+ "integrity": "sha512-0rSmRBzXgDzIsD6mGdJgevzgezI534Cer5L/vyMX0kHzT/jiB43jRhd9YUlMGYLQy2zprNmoT8qasCGtY+QaKw==",
+ "dev": true
+ },
+ "espree": {
+ "version": "7.3.1",
+ "resolved": "https://registry.npmjs.org/espree/-/espree-7.3.1.tgz",
+ "integrity": "sha512-v3JCNCE64umkFpmkFGqzVKsOT0tN1Zr+ueqLZfpV1Ob8e+CEgPWa+OxCoGH3tnhimMKIaBm4m/vaRpJ/krRz2g==",
+ "dev": true,
+ "requires": {
+ "acorn": "^7.4.0",
+ "acorn-jsx": "^5.3.1",
+ "eslint-visitor-keys": "^1.3.0"
+ },
+ "dependencies": {
+ "eslint-visitor-keys": {
+ "version": "1.3.0",
+ "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz",
+ "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==",
+ "dev": true
+ }
+ }
+ },
+ "esprima": {
+ "version": "4.0.1",
+ "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz",
+ "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==",
+ "dev": true
+ },
+ "esquery": {
+ "version": "1.4.0",
+ "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.4.0.tgz",
+ "integrity": "sha512-cCDispWt5vHHtwMY2YrAQ4ibFkAL8RbH5YGBnZBc90MolvvfkkQcJro/aZiAQUlQ3qgrYS6D6v8Gc5G5CQsc9w==",
+ "dev": true,
+ "requires": {
+ "estraverse": "^5.1.0"
+ },
+ "dependencies": {
+ "estraverse": {
+ "version": "5.2.0",
+ "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.2.0.tgz",
+ "integrity": "sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ==",
+ "dev": true
+ }
+ }
+ },
+ "esrecurse": {
+ "version": "4.3.0",
+ "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz",
+ "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==",
+ "dev": true,
+ "requires": {
+ "estraverse": "^5.2.0"
+ },
+ "dependencies": {
+ "estraverse": {
+ "version": "5.2.0",
+ "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.2.0.tgz",
+ "integrity": "sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ==",
+ "dev": true
+ }
+ }
+ },
+ "estraverse": {
+ "version": "4.3.0",
+ "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz",
+ "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==",
+ "dev": true
+ },
+ "esutils": {
+ "version": "2.0.3",
+ "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz",
+ "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==",
+ "dev": true
+ },
+ "fast-deep-equal": {
+ "version": "3.1.3",
+ "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
+ "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==",
+ "dev": true
+ },
+ "fast-json-stable-stringify": {
+ "version": "2.1.0",
+ "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz",
+ "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==",
+ "dev": true
+ },
+ "fast-levenshtein": {
+ "version": "2.0.6",
+ "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz",
+ "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=",
+ "dev": true
+ },
+ "file-entry-cache": {
+ "version": "6.0.1",
+ "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz",
+ "integrity": "sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==",
+ "dev": true,
+ "requires": {
+ "flat-cache": "^3.0.4"
+ }
+ },
+ "flat-cache": {
+ "version": "3.0.4",
+ "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.0.4.tgz",
+ "integrity": "sha512-dm9s5Pw7Jc0GvMYbshN6zchCA9RgQlzzEZX3vylR9IqFfS8XciblUXOKfW6SiuJ0e13eDYZoZV5wdrev7P3Nwg==",
+ "dev": true,
+ "requires": {
+ "flatted": "^3.1.0",
+ "rimraf": "^3.0.2"
+ }
+ },
+ "flatted": {
+ "version": "3.1.1",
+ "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.1.1.tgz",
+ "integrity": "sha512-zAoAQiudy+r5SvnSw3KJy5os/oRJYHzrzja/tBDqrZtNhUw8bt6y8OBzMWcjWr+8liV8Eb6yOhw8WZ7VFZ5ZzA==",
+ "dev": true
+ },
+ "fs.realpath": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
+ "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=",
+ "dev": true
+ },
+ "functional-red-black-tree": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz",
+ "integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=",
+ "dev": true
+ },
+ "glob": {
+ "version": "7.1.6",
+ "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz",
+ "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==",
+ "dev": true,
+ "requires": {
+ "fs.realpath": "^1.0.0",
+ "inflight": "^1.0.4",
+ "inherits": "2",
+ "minimatch": "^3.0.4",
+ "once": "^1.3.0",
+ "path-is-absolute": "^1.0.0"
+ }
+ },
+ "glob-parent": {
+ "version": "5.1.2",
+ "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz",
+ "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==",
+ "dev": true,
+ "requires": {
+ "is-glob": "^4.0.1"
+ }
+ },
+ "globals": {
+ "version": "13.8.0",
+ "resolved": "https://registry.npmjs.org/globals/-/globals-13.8.0.tgz",
+ "integrity": "sha512-rHtdA6+PDBIjeEvA91rpqzEvk/k3/i7EeNQiryiWuJH0Hw9cpyJMAt2jtbAwUaRdhD+573X4vWw6IcjKPasi9Q==",
+ "dev": true,
+ "requires": {
+ "type-fest": "^0.20.2"
+ },
+ "dependencies": {
+ "type-fest": {
+ "version": "0.20.2",
+ "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz",
+ "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==",
+ "dev": true
+ }
+ }
+ },
+ "has-flag": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
+ "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=",
+ "dev": true
+ },
+ "ignore": {
+ "version": "4.0.6",
+ "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz",
+ "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==",
+ "dev": true
+ },
+ "import-fresh": {
+ "version": "3.3.0",
+ "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz",
+ "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==",
+ "dev": true,
+ "requires": {
+ "parent-module": "^1.0.0",
+ "resolve-from": "^4.0.0"
+ }
+ },
+ "imurmurhash": {
+ "version": "0.1.4",
+ "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz",
+ "integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o=",
+ "dev": true
+ },
+ "inflight": {
+ "version": "1.0.6",
+ "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
+ "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
+ "dev": true,
+ "requires": {
+ "once": "^1.3.0",
+ "wrappy": "1"
+ }
+ },
+ "inherits": {
+ "version": "2.0.4",
+ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
+ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
+ "dev": true
+ },
+ "is-extglob": {
+ "version": "2.1.1",
+ "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz",
+ "integrity": "sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=",
+ "dev": true
+ },
+ "is-fullwidth-code-point": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
+ "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
+ "dev": true
+ },
+ "is-glob": {
+ "version": "4.0.1",
+ "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.1.tgz",
+ "integrity": "sha512-5G0tKtBTFImOqDnLB2hG6Bp2qcKEFduo4tZu9MT/H6NQv/ghhy30o55ufafxJ/LdH79LLs2Kfrn85TLKyA7BUg==",
+ "dev": true,
+ "requires": {
+ "is-extglob": "^2.1.1"
+ }
+ },
+ "isexe": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
+ "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=",
+ "dev": true
+ },
+ "js-tokens": {
+ "version": "4.0.0",
+ "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
+ "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==",
+ "dev": true
+ },
+ "js-yaml": {
+ "version": "3.14.1",
+ "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz",
+ "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==",
+ "dev": true,
+ "requires": {
+ "argparse": "^1.0.7",
+ "esprima": "^4.0.0"
+ }
+ },
+ "json-schema-traverse": {
+ "version": "0.4.1",
+ "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
+ "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==",
+ "dev": true
+ },
+ "json-stable-stringify-without-jsonify": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz",
+ "integrity": "sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE=",
+ "dev": true
+ },
+ "levn": {
+ "version": "0.4.1",
+ "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz",
+ "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==",
+ "dev": true,
+ "requires": {
+ "prelude-ls": "^1.2.1",
+ "type-check": "~0.4.0"
+ }
+ },
+ "lodash": {
+ "version": "4.17.21",
+ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
+ "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==",
+ "dev": true
+ },
+ "lodash.clonedeep": {
+ "version": "4.5.0",
+ "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz",
+ "integrity": "sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8=",
+ "dev": true
+ },
+ "lodash.flatten": {
+ "version": "4.4.0",
+ "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz",
+ "integrity": "sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8=",
+ "dev": true
+ },
+ "lodash.truncate": {
+ "version": "4.4.2",
+ "resolved": "https://registry.npmjs.org/lodash.truncate/-/lodash.truncate-4.4.2.tgz",
+ "integrity": "sha1-WjUNoLERO4N+z//VgSy+WNbq4ZM=",
+ "dev": true
+ },
+ "lru-cache": {
+ "version": "6.0.0",
+ "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
+ "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
+ "dev": true,
+ "requires": {
+ "yallist": "^4.0.0"
+ }
+ },
+ "minimatch": {
+ "version": "3.0.4",
+ "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
+ "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
+ "dev": true,
+ "requires": {
+ "brace-expansion": "^1.1.7"
+ }
+ },
+ "ms": {
+ "version": "2.1.2",
+ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
+ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
+ "dev": true
+ },
+ "natural-compare": {
+ "version": "1.4.0",
+ "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz",
+ "integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=",
+ "dev": true
+ },
+ "once": {
+ "version": "1.4.0",
+ "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
+ "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
+ "dev": true,
+ "requires": {
+ "wrappy": "1"
+ }
+ },
+ "optionator": {
+ "version": "0.9.1",
+ "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.1.tgz",
+ "integrity": "sha512-74RlY5FCnhq4jRxVUPKDaRwrVNXMqsGsiW6AJw4XK8hmtm10wC0ypZBLw5IIp85NZMr91+qd1RvvENwg7jjRFw==",
+ "dev": true,
+ "requires": {
+ "deep-is": "^0.1.3",
+ "fast-levenshtein": "^2.0.6",
+ "levn": "^0.4.1",
+ "prelude-ls": "^1.2.1",
+ "type-check": "^0.4.0",
+ "word-wrap": "^1.2.3"
+ }
+ },
+ "parent-module": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz",
+ "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==",
+ "dev": true,
+ "requires": {
+ "callsites": "^3.0.0"
+ }
+ },
+ "path-is-absolute": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
+ "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=",
+ "dev": true
+ },
+ "path-key": {
+ "version": "3.1.1",
+ "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",
+ "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==",
+ "dev": true
+ },
+ "prelude-ls": {
+ "version": "1.2.1",
+ "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz",
+ "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==",
+ "dev": true
+ },
+ "progress": {
+ "version": "2.0.3",
+ "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz",
+ "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==",
+ "dev": true
+ },
+ "punycode": {
+ "version": "2.1.1",
+ "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz",
+ "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==",
+ "dev": true
+ },
+ "regexpp": {
+ "version": "3.1.0",
+ "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-3.1.0.tgz",
+ "integrity": "sha512-ZOIzd8yVsQQA7j8GCSlPGXwg5PfmA1mrq0JP4nGhh54LaKN3xdai/vHUDu74pKwV8OxseMS65u2NImosQcSD0Q==",
+ "dev": true
+ },
+ "require-from-string": {
+ "version": "2.0.2",
+ "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz",
+ "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==",
+ "dev": true
+ },
+ "resolve-from": {
+ "version": "4.0.0",
+ "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz",
+ "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==",
+ "dev": true
+ },
+ "rimraf": {
+ "version": "3.0.2",
+ "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
+ "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==",
+ "dev": true,
+ "requires": {
+ "glob": "^7.1.3"
+ }
+ },
+ "semver": {
+ "version": "7.3.5",
+ "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz",
+ "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==",
+ "dev": true,
+ "requires": {
+ "lru-cache": "^6.0.0"
+ }
+ },
+ "shebang-command": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
+ "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==",
+ "dev": true,
+ "requires": {
+ "shebang-regex": "^3.0.0"
+ }
+ },
+ "shebang-regex": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz",
+ "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==",
+ "dev": true
+ },
+ "slice-ansi": {
+ "version": "4.0.0",
+ "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-4.0.0.tgz",
+ "integrity": "sha512-qMCMfhY040cVHT43K9BFygqYbUPFZKHOg7K73mtTWJRb8pyP3fzf4Ixd5SzdEJQ6MRUg/WBnOLxghZtKKurENQ==",
+ "dev": true,
+ "requires": {
+ "ansi-styles": "^4.0.0",
+ "astral-regex": "^2.0.0",
+ "is-fullwidth-code-point": "^3.0.0"
+ },
+ "dependencies": {
+ "ansi-styles": {
+ "version": "4.3.0",
+ "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+ "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
+ "dev": true,
+ "requires": {
+ "color-convert": "^2.0.1"
+ }
+ },
+ "color-convert": {
+ "version": "2.0.1",
+ "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+ "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+ "dev": true,
+ "requires": {
+ "color-name": "~1.1.4"
+ }
+ },
+ "color-name": {
+ "version": "1.1.4",
+ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+ "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+ "dev": true
+ }
+ }
+ },
+ "sprintf-js": {
+ "version": "1.0.3",
+ "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",
+ "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=",
+ "dev": true
+ },
+ "string-width": {
+ "version": "4.2.2",
+ "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz",
+ "integrity": "sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA==",
+ "dev": true,
+ "requires": {
+ "emoji-regex": "^8.0.0",
+ "is-fullwidth-code-point": "^3.0.0",
+ "strip-ansi": "^6.0.0"
+ }
+ },
+ "strip-ansi": {
+ "version": "6.0.0",
+ "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz",
+ "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==",
+ "dev": true,
+ "requires": {
+ "ansi-regex": "^5.0.0"
+ }
+ },
+ "strip-json-comments": {
+ "version": "3.1.1",
+ "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz",
+ "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==",
+ "dev": true
+ },
+ "supports-color": {
+ "version": "5.5.0",
+ "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz",
+ "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==",
+ "dev": true,
+ "requires": {
+ "has-flag": "^3.0.0"
+ }
+ },
+ "table": {
+ "version": "6.6.0",
+ "resolved": "https://registry.npmjs.org/table/-/table-6.6.0.tgz",
+ "integrity": "sha512-iZMtp5tUvcnAdtHpZTWLPF0M7AgiQsURR2DwmxnJwSy8I3+cY+ozzVvYha3BOLG2TB+L0CqjIz+91htuj6yCXg==",
+ "dev": true,
+ "requires": {
+ "ajv": "^8.0.1",
+ "lodash.clonedeep": "^4.5.0",
+ "lodash.flatten": "^4.4.0",
+ "lodash.truncate": "^4.4.2",
+ "slice-ansi": "^4.0.0",
+ "string-width": "^4.2.0",
+ "strip-ansi": "^6.0.0"
+ },
+ "dependencies": {
+ "ajv": {
+ "version": "8.2.0",
+ "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.2.0.tgz",
+ "integrity": "sha512-WSNGFuyWd//XO8n/m/EaOlNLtO0yL8EXT/74LqT4khdhpZjP7lkj/kT5uwRmGitKEVp/Oj7ZUHeGfPtgHhQ5CA==",
+ "dev": true,
+ "requires": {
+ "fast-deep-equal": "^3.1.1",
+ "json-schema-traverse": "^1.0.0",
+ "require-from-string": "^2.0.2",
+ "uri-js": "^4.2.2"
+ }
+ },
+ "json-schema-traverse": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz",
+ "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==",
+ "dev": true
+ }
+ }
+ },
+ "text-table": {
+ "version": "0.2.0",
+ "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz",
+ "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=",
+ "dev": true
+ },
+ "type-check": {
+ "version": "0.4.0",
+ "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz",
+ "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==",
+ "dev": true,
+ "requires": {
+ "prelude-ls": "^1.2.1"
+ }
+ },
+ "type-fest": {
+ "version": "0.8.1",
+ "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.8.1.tgz",
+ "integrity": "sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==",
+ "dev": true
+ },
+ "uri-js": {
+ "version": "4.4.1",
+ "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz",
+ "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==",
+ "dev": true,
+ "requires": {
+ "punycode": "^2.1.0"
+ }
+ },
+ "v8-compile-cache": {
+ "version": "2.3.0",
+ "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz",
+ "integrity": "sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA==",
+ "dev": true
+ },
+ "which": {
+ "version": "2.0.2",
+ "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
+ "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==",
+ "dev": true,
+ "requires": {
+ "isexe": "^2.0.0"
+ }
+ },
+ "word-wrap": {
+ "version": "1.2.3",
+ "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz",
+ "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==",
+ "dev": true
+ },
+ "wrappy": {
+ "version": "1.0.2",
+ "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
+ "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=",
+ "dev": true
+ },
+ "yallist": {
+ "version": "4.0.0",
+ "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+ "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
+ "dev": true
+ }
+ }
+}
diff --git a/dev/package.json b/dev/package.json
new file mode 100644
index 0000000000000..0391a3983f78f
--- /dev/null
+++ b/dev/package.json
@@ -0,0 +1,5 @@
+{
+ "devDependencies": {
+ "eslint": "^7.25.0"
+ }
+}
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/functions/ScalarFunction.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/functions/ScalarFunction.java
index ef755aae3fb07..858ab923490fc 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/functions/ScalarFunction.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/functions/ScalarFunction.java
@@ -29,33 +29,62 @@
*
* The JVM type of result values produced by this function must be the type used by Spark's
* InternalRow API for the {@link DataType SQL data type} returned by {@link #resultType()}.
+ * The mapping between {@link DataType} and the corresponding JVM type is defined below.
*
* IMPORTANT: the default implementation of {@link #produceResult} throws
- * {@link UnsupportedOperationException}. Users can choose to override this method, or implement
- * a "magic method" with name {@link #MAGIC_METHOD_NAME} which takes individual parameters
- * instead of a {@link InternalRow}. The magic method will be loaded by Spark through Java
- * reflection and will also provide better performance in general, due to optimizations such as
- * codegen, removal of Java boxing, etc.
- *
+ * {@link UnsupportedOperationException}. Users must choose to either override this method, or
+ * implement a magic method with name {@link #MAGIC_METHOD_NAME}, which takes individual parameters
+ * instead of a {@link InternalRow}. The magic method approach is generally recommended because it
+ * provides better performance over the default {@link #produceResult}, due to optimizations such
+ * as whole-stage codegen, elimination of Java boxing, etc.
+ *
+ * In addition, for stateless Java functions, users can optionally define the
+ * {@link #MAGIC_METHOD_NAME} as a static method, which further avoids certain runtime costs such
+ * as Java dynamic dispatch.
+ *
* For example, a scalar UDF for adding two integers can be defined as follow with the magic
* method approach:
*
*
* public class IntegerAdd implements{@code ScalarFunction} {
+ * public DataType[] inputTypes() {
+ * return new DataType[] { DataTypes.IntegerType, DataTypes.IntegerType };
+ * }
* public int invoke(int left, int right) {
* return left + right;
* }
* }
*
- * In this case, since {@link #MAGIC_METHOD_NAME} is defined, Spark will use it over
- * {@link #produceResult} to evalaute the inputs. In general Spark looks up the magic method by
- * first converting the actual input SQL data types to their corresponding Java types following
- * the mapping defined below, and then checking if there is a matching method from all the
- * declared methods in the UDF class, using method name (i.e., {@link #MAGIC_METHOD_NAME}) and
- * the Java types. If no magic method is found, Spark will falls back to use {@link #produceResult}.
+ * In the above, since {@link #MAGIC_METHOD_NAME} is defined, and also that it has
+ * matching parameter types and return type, Spark will use it to evaluate inputs.
+ *
+ * As another example, in the following:
+ *
+ * public class IntegerAdd implements{@code ScalarFunction} {
+ * public DataType[] inputTypes() {
+ * return new DataType[] { DataTypes.IntegerType, DataTypes.IntegerType };
+ * }
+ * public static int invoke(int left, int right) {
+ * return left + right;
+ * }
+ * public Integer produceResult(InternalRow input) {
+ * return input.getInt(0) + input.getInt(1);
+ * }
+ * }
+ *
+ *
+ * the class defines both the magic method and the {@link #produceResult}, and Spark will use
+ * {@link #MAGIC_METHOD_NAME} over the {@link #produceResult(InternalRow)} as it takes higher
+ * precedence. Also note that the magic method is annotated as a static method in this case.
+ *
+ * Resolution on magic method is done during query analysis, where Spark looks up the magic
+ * method by first converting the actual input SQL data types to their corresponding Java types
+ * following the mapping defined below, and then checking if there is a matching method from all the
+ * declared methods in the UDF class, using method name and the Java types.
*
- * The following are the mapping from {@link DataType SQL data type} to Java type through
- * the magic method approach:
+ * The following are the mapping from {@link DataType SQL data type} to Java type which is used
+ * by Spark to infer parameter types for the magic methods as well as return value type for
+ * {@link #produceResult}:
*
* - {@link org.apache.spark.sql.types.BooleanType}: {@code boolean}
* - {@link org.apache.spark.sql.types.ByteType}: {@code byte}
@@ -80,7 +109,8 @@
* {@link org.apache.spark.sql.catalyst.util.MapData}
*
*
- * @param the JVM type of result values
+ * @param the JVM type of result values, MUST be consistent with the {@link DataType}
+ * returned via {@link #resultType()}, according to the mapping above.
*/
public interface ScalarFunction extends BoundFunction {
String MAGIC_METHOD_NAME = "invoke";
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index b3310635cdcb6..757778b66c345 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -17,7 +17,7 @@
package org.apache.spark.sql.catalyst.analysis
-import java.lang.reflect.Method
+import java.lang.reflect.{Method, Modifier}
import java.util
import java.util.Locale
import java.util.concurrent.atomic.AtomicBoolean
@@ -2181,6 +2181,9 @@ class Analyzer(override val catalogManager: CatalogManager)
// match the input type through `BoundFunction.inputTypes`.
val argClasses = inputType.fields.map(_.dataType)
findMethod(scalarFunc, MAGIC_METHOD_NAME, argClasses) match {
+ case Some(m) if Modifier.isStatic(m.getModifiers) =>
+ StaticInvoke(scalarFunc.getClass, scalarFunc.resultType(),
+ MAGIC_METHOD_NAME, arguments, returnNullable = scalarFunc.isResultNullable)
case Some(_) =>
val caller = Literal.create(scalarFunc, ObjectType(scalarFunc.getClass))
Invoke(caller, MAGIC_METHOD_NAME, scalarFunc.resultType(),
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
index 469c89572aed8..5d79774a303d2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
@@ -33,7 +33,7 @@ import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.expressions.codegen._
import org.apache.spark.sql.catalyst.expressions.codegen.Block._
import org.apache.spark.sql.catalyst.trees.TernaryLike
-import org.apache.spark.sql.catalyst.trees.TreePattern.{NULL_CHECK, TreePattern}
+import org.apache.spark.sql.catalyst.trees.TreePattern._
import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, ArrayData, GenericArrayData, MapData}
import org.apache.spark.sql.errors.QueryExecutionErrors
import org.apache.spark.sql.types._
@@ -145,6 +145,34 @@ trait InvokeLike extends Expression with NonSQLExpression {
}
}
}
+
+ final def findMethod(cls: Class[_], functionName: String, argClasses: Seq[Class[_]]): Method = {
+ // Looking with function name + argument classes first.
+ try {
+ cls.getMethod(functionName, argClasses: _*)
+ } catch {
+ case _: NoSuchMethodException =>
+ // For some cases, e.g. arg class is Object, `getMethod` cannot find the method.
+ // We look at function name + argument length
+ val m = cls.getMethods.filter { m =>
+ m.getName == functionName && m.getParameterCount == arguments.length
+ }
+ if (m.isEmpty) {
+ sys.error(s"Couldn't find $functionName on $cls")
+ } else if (m.length > 1) {
+ // More than one matched method signature. Exclude synthetic one, e.g. generic one.
+ val realMethods = m.filter(!_.isSynthetic)
+ if (realMethods.length > 1) {
+ // Ambiguous case, we don't know which method to choose, just fail it.
+ sys.error(s"Found ${realMethods.length} $functionName on $cls")
+ } else {
+ realMethods.head
+ }
+ } else {
+ m.head
+ }
+ }
+ }
}
/**
@@ -236,7 +264,7 @@ case class StaticInvoke(
override def children: Seq[Expression] = arguments
lazy val argClasses = ScalaReflection.expressionJavaClasses(arguments)
- @transient lazy val method = cls.getDeclaredMethod(functionName, argClasses : _*)
+ @transient lazy val method = findMethod(cls, functionName, argClasses)
override def eval(input: InternalRow): Any = {
invoke(null, method, arguments, input, dataType)
@@ -326,31 +354,7 @@ case class Invoke(
@transient lazy val method = targetObject.dataType match {
case ObjectType(cls) =>
- // Looking with function name + argument classes first.
- try {
- Some(cls.getMethod(encodedFunctionName, argClasses: _*))
- } catch {
- case _: NoSuchMethodException =>
- // For some cases, e.g. arg class is Object, `getMethod` cannot find the method.
- // We look at function name + argument length
- val m = cls.getMethods.filter { m =>
- m.getName == encodedFunctionName && m.getParameterCount == arguments.length
- }
- if (m.isEmpty) {
- sys.error(s"Couldn't find $encodedFunctionName on $cls")
- } else if (m.length > 1) {
- // More than one matched method signature. Exclude synthetic one, e.g. generic one.
- val realMethods = m.filter(!_.isSynthetic)
- if (realMethods.length > 1) {
- // Ambiguous case, we don't know which method to choose, just fail it.
- sys.error(s"Found ${realMethods.length} $encodedFunctionName on $cls")
- } else {
- Some(realMethods.head)
- }
- } else {
- Some(m.head)
- }
- }
+ Some(findMethod(cls, encodedFunctionName, argClasses))
case _ => None
}
@@ -669,6 +673,8 @@ case class LambdaVariable(
private val accessor: (InternalRow, Int) => Any = InternalRow.getAccessor(dataType, nullable)
+ final override val nodePatterns: Seq[TreePattern] = Seq(LAMBDA_VARIABLE)
+
// Interpreted execution of `LambdaVariable` always get the 0-index element from input row.
override def eval(input: InternalRow): Any = {
assert(input.numFields == 1,
@@ -781,6 +787,8 @@ case class MapObjects private(
override def second: Expression = lambdaFunction
override def third: Expression = inputData
+ final override val nodePatterns: Seq[TreePattern] = Seq(MAP_OBJECTS)
+
// The data with UserDefinedType are actually stored with the data type of its sqlType.
// When we want to apply MapObjects on it, we have to use it.
lazy private val inputDataType = inputData.dataType match {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/objects.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/objects.scala
index 97712a05b7c21..52544ff3e241d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/objects.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/objects.scala
@@ -24,6 +24,7 @@ import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.expressions.objects._
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.catalyst.rules._
+import org.apache.spark.sql.catalyst.trees.TreePattern._
import org.apache.spark.sql.types.{ArrayType, DataType, MapType, StructType, UserDefinedType}
/*
@@ -35,7 +36,8 @@ import org.apache.spark.sql.types.{ArrayType, DataType, MapType, StructType, Use
* representation of data item. For example back to back map operations.
*/
object EliminateSerialization extends Rule[LogicalPlan] {
- def apply(plan: LogicalPlan): LogicalPlan = plan transform {
+ def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
+ _.containsAnyPattern(DESERIALIZE_TO_OBJECT, APPEND_COLUMNS, TYPED_FILTER), ruleId) {
case d @ DeserializeToObject(_, _, s: SerializeFromObject)
if d.outputObjAttr.dataType == s.inputObjAttr.dataType =>
// Adds an extra Project here, to preserve the output expr id of `DeserializeToObject`.
@@ -72,7 +74,8 @@ object EliminateSerialization extends Rule[LogicalPlan] {
* merging the filter functions into one conjunctive function.
*/
object CombineTypedFilters extends Rule[LogicalPlan] {
- def apply(plan: LogicalPlan): LogicalPlan = plan transform {
+ def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
+ _.containsPattern(TYPED_FILTER), ruleId) {
case t1 @ TypedFilter(_, _, _, _, t2 @ TypedFilter(_, _, _, _, child))
if t1.deserializer.dataType == t2.deserializer.dataType =>
TypedFilter(
@@ -108,7 +111,8 @@ object CombineTypedFilters extends Rule[LogicalPlan] {
* 2. no custom collection class specified representation of data item.
*/
object EliminateMapObjects extends Rule[LogicalPlan] {
- def apply(plan: LogicalPlan): LogicalPlan = plan transformAllExpressions {
+ def apply(plan: LogicalPlan): LogicalPlan = plan.transformAllExpressionsWithPruning(
+ _.containsAllPatterns(MAP_OBJECTS, LAMBDA_VARIABLE), ruleId) {
case MapObjects(_, LambdaVariable(_, _, false, _), inputData, None) => inputData
}
}
@@ -207,7 +211,8 @@ object ObjectSerializerPruning extends Rule[LogicalPlan] {
}
}
- def apply(plan: LogicalPlan): LogicalPlan = plan transform {
+ def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
+ _.containsAllPatterns(SERIALIZE_FROM_OBJECT, PROJECT), ruleId) {
case p @ Project(_, s: SerializeFromObject) =>
// Prunes individual serializer if it is not used at all by above projection.
val usedRefs = p.references
@@ -252,7 +257,7 @@ object ReassignLambdaVariableID extends Rule[LogicalPlan] {
var hasNegativeIds = false
var hasPositiveIds = false
- plan.transformAllExpressions {
+ plan.transformAllExpressionsWithPruning(_.containsPattern(LAMBDA_VARIABLE), ruleId) {
case lr: LambdaVariable if lr.id == 0 =>
throw new IllegalStateException("LambdaVariable should never has 0 as its ID.")
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
index f5e92fb2deb0a..1bd16661fddee 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
@@ -70,6 +70,8 @@ case class Project(projectList: Seq[NamedExpression], child: LogicalPlan)
override def output: Seq[Attribute] = projectList.map(_.toAttribute)
override def maxRows: Option[Long] = child.maxRows
+ final override val nodePatterns: Seq[TreePattern] = Seq(PROJECT)
+
override lazy val resolved: Boolean = {
val hasSpecialExpressions = projectList.exists ( _.collect {
case agg: AggregateExpression => agg
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/object.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/object.scala
index 6d61a86ab5ef7..1f7eb67bf1726 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/object.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/object.scala
@@ -24,6 +24,7 @@ import org.apache.spark.sql.catalyst.analysis.UnresolvedDeserializer
import org.apache.spark.sql.catalyst.encoders._
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.expressions.objects.Invoke
+import org.apache.spark.sql.catalyst.trees.TreePattern._
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.streaming.{GroupStateTimeout, OutputMode}
import org.apache.spark.sql.types._
@@ -80,6 +81,7 @@ case class DeserializeToObject(
deserializer: Expression,
outputObjAttr: Attribute,
child: LogicalPlan) extends UnaryNode with ObjectProducer {
+ final override val nodePatterns: Seq[TreePattern] = Seq(DESERIALIZE_TO_OBJECT)
override protected def withNewChildInternal(newChild: LogicalPlan): DeserializeToObject =
copy(child = newChild)
}
@@ -94,6 +96,8 @@ case class SerializeFromObject(
override def output: Seq[Attribute] = serializer.map(_.toAttribute)
+ final override val nodePatterns: Seq[TreePattern] = Seq(SERIALIZE_FROM_OBJECT)
+
override protected def withNewChildInternal(newChild: LogicalPlan): SerializeFromObject =
copy(child = newChild)
}
@@ -256,6 +260,8 @@ case class TypedFilter(
override def output: Seq[Attribute] = child.output
+ final override val nodePatterns: Seq[TreePattern] = Seq(TYPED_FILTER)
+
def withObjectProducerChild(obj: LogicalPlan): Filter = {
assert(obj.output.length == 1)
Filter(typedCondition(obj.output.head), obj)
@@ -354,6 +360,8 @@ case class AppendColumns(
override def output: Seq[Attribute] = child.output ++ newColumns
+ final override val nodePatterns: Seq[TreePattern] = Seq(APPEND_COLUMNS)
+
def newColumns: Seq[Attribute] = serializer.map(_.toAttribute)
override protected def withNewChildInternal(newChild: LogicalPlan): AppendColumns =
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleIdCollection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleIdCollection.scala
index 3a1fa6a8e523d..62f09d02ea146 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleIdCollection.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleIdCollection.scala
@@ -89,13 +89,17 @@ object RuleIdCollection {
// Catalyst Optimizer rules
"org.apache.spark.sql.catalyst.optimizer.BooleanSimplification" ::
"org.apache.spark.sql.catalyst.optimizer.CombineConcats" ::
+ "org.apache.spark.sql.catalyst.optimizer.CombineTypedFilters" ::
"org.apache.spark.sql.catalyst.optimizer.ConstantFolding" ::
"org.apache.spark.sql.catalyst.optimizer.ConstantPropagation" ::
"org.apache.spark.sql.catalyst.optimizer.CostBasedJoinReorder" ::
+ "org.apache.spark.sql.catalyst.optimizer.EliminateMapObjects" ::
"org.apache.spark.sql.catalyst.optimizer.EliminateOuterJoin" ::
+ "org.apache.spark.sql.catalyst.optimizer.EliminateSerialization" ::
"org.apache.spark.sql.catalyst.optimizer.LikeSimplification" ::
"org.apache.spark.sql.catalyst.optimizer.LimitPushDownThroughWindow" ::
"org.apache.spark.sql.catalyst.optimizer.NullPropagation" ::
+ "org.apache.spark.sql.catalyst.optimizer.ObjectSerializerPruning" ::
"org.apache.spark.sql.catalyst.optimizer.OptimizeCsvJsonExprs" ::
"org.apache.spark.sql.catalyst.optimizer.OptimizeIn" ::
"org.apache.spark.sql.catalyst.optimizer.Optimizer$OptimizeSubqueries" ::
@@ -105,6 +109,7 @@ object RuleIdCollection {
"org.apache.spark.sql.catalyst.optimizer.PushExtraPredicateThroughJoin" ::
"org.apache.spark.sql.catalyst.optimizer.PushFoldableIntoBranches" ::
"org.apache.spark.sql.catalyst.optimizer.PushLeftSemiLeftAntiThroughJoin" ::
+ "org.apache.spark.sql.catalyst.optimizer.ReassignLambdaVariableID" ::
"org.apache.spark.sql.catalyst.optimizer.RemoveDispensableExpressions" ::
"org.apache.spark.sql.catalyst.optimizer.ReorderAssociativeOperator" ::
"org.apache.spark.sql.catalyst.optimizer.ReorderJoin" ::
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreePatterns.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreePatterns.scala
index b44847cfef0e2..fb384cd946d3c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreePatterns.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreePatterns.scala
@@ -25,6 +25,7 @@ object TreePattern extends Enumeration {
// Expression patterns (alphabetically ordered)
val AND_OR: Value = Value(0)
val ATTRIBUTE_REFERENCE: Value = Value
+ val APPEND_COLUMNS: Value = Value
val BINARY_ARITHMETIC: Value = Value
val BINARY_COMPARISON: Value = Value
val CASE_WHEN: Value = Value
@@ -32,6 +33,7 @@ object TreePattern extends Enumeration {
val CONCAT: Value = Value
val COUNT: Value = Value
val CREATE_NAMED_STRUCT: Value = Value
+ val DESERIALIZE_TO_OBJECT: Value = Value
val DYNAMIC_PRUNING_SUBQUERY: Value = Value
val EXISTS_SUBQUERY = Value
val EXPRESSION_WITH_RANDOM_SEED: Value = Value
@@ -41,12 +43,15 @@ object TreePattern extends Enumeration {
val IN_SUBQUERY: Value = Value
val INSET: Value = Value
val JSON_TO_STRUCT: Value = Value
+ val LAMBDA_VARIABLE: Value = Value
val LIKE_FAMLIY: Value = Value
val LIST_SUBQUERY: Value = Value
val LITERAL: Value = Value
+ val MAP_OBJECTS: Value = Value
val NOT: Value = Value
val NULL_CHECK: Value = Value
val NULL_LITERAL: Value = Value
+ val SERIALIZE_FROM_OBJECT: Value = Value
val OUTER_REFERENCE: Value = Value
val PLAN_EXPRESSION: Value = Value
val SCALAR_SUBQUERY: Value = Value
@@ -66,5 +71,7 @@ object TreePattern extends Enumeration {
val LOCAL_RELATION: Value = Value
val NATURAL_LIKE_JOIN: Value = Value
val OUTER_JOIN: Value = Value
+ val PROJECT: Value = Value
+ val TYPED_FILTER: Value = Value
val WINDOW: Value = Value
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Implicits.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Implicits.scala
index cc41d8ca9007f..39642fd541706 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Implicits.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Implicits.scala
@@ -17,12 +17,12 @@
package org.apache.spark.sql.connector.catalog
-import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier}
import org.apache.spark.sql.catalyst.catalog.BucketSpec
import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
import org.apache.spark.sql.catalyst.util.quoteIfNeeded
import org.apache.spark.sql.connector.expressions.{BucketTransform, IdentityTransform, LogicalExpressions, Transform}
+import org.apache.spark.sql.errors.QueryCompilationErrors
/**
* Conversion helpers for working with v2 [[CatalogPlugin]].
@@ -39,8 +39,7 @@ private[sql] object CatalogV2Implicits {
implicit class BucketSpecHelper(spec: BucketSpec) {
def asTransform: BucketTransform = {
if (spec.sortColumnNames.nonEmpty) {
- throw new AnalysisException(
- s"Cannot convert bucketing with sort columns to a transform: $spec")
+ throw QueryCompilationErrors.cannotConvertBucketWithSortColumnsToTransformError(spec)
}
val references = spec.bucketColumnNames.map(col => reference(Seq(col)))
@@ -53,14 +52,13 @@ private[sql] object CatalogV2Implicits {
val (idTransforms, nonIdTransforms) = transforms.partition(_.isInstanceOf[IdentityTransform])
if (nonIdTransforms.nonEmpty) {
- throw new AnalysisException("Transforms cannot be converted to partition columns: " +
- nonIdTransforms.map(_.describe).mkString(", "))
+ throw QueryCompilationErrors.cannotConvertTransformsToPartitionColumnsError(nonIdTransforms)
}
idTransforms.map(_.asInstanceOf[IdentityTransform]).map(_.reference).map { ref =>
val parts = ref.fieldNames
if (parts.size > 1) {
- throw new AnalysisException(s"Cannot partition by nested column: $ref")
+ throw QueryCompilationErrors.cannotPartitionByNestedColumnError(ref)
} else {
parts(0)
}
@@ -73,15 +71,14 @@ private[sql] object CatalogV2Implicits {
case tableCatalog: TableCatalog =>
tableCatalog
case _ =>
- throw new AnalysisException(s"Cannot use catalog ${plugin.name}: not a TableCatalog")
+ throw QueryCompilationErrors.cannotUseCatalogError(plugin, "not a TableCatalog")
}
def asNamespaceCatalog: SupportsNamespaces = plugin match {
case namespaceCatalog: SupportsNamespaces =>
namespaceCatalog
case _ =>
- throw new AnalysisException(
- s"Cannot use catalog ${plugin.name}: does not support namespaces")
+ throw QueryCompilationErrors.cannotUseCatalogError(plugin, "does not support namespaces")
}
def isFunctionCatalog: Boolean = plugin match {
@@ -93,8 +90,7 @@ private[sql] object CatalogV2Implicits {
case functionCatalog: FunctionCatalog =>
functionCatalog
case _ =>
- throw new AnalysisException(
- s"Cannot use catalog '${plugin.name}': not a FunctionCatalog")
+ throw QueryCompilationErrors.cannotUseCatalogError(plugin, "not a FunctionCatalog")
}
}
@@ -128,22 +124,22 @@ private[sql] object CatalogV2Implicits {
case ns if ns.isEmpty => TableIdentifier(ident.name)
case Array(dbName) => TableIdentifier(ident.name, Some(dbName))
case _ =>
- throw new AnalysisException(
- s"$quoted is not a valid TableIdentifier as it has more than 2 name parts.")
+ throw QueryCompilationErrors.identifierHavingMoreThanTwoNamePartsError(
+ quoted, "TableIdentifier")
}
def asFunctionIdentifier: FunctionIdentifier = ident.namespace() match {
case ns if ns.isEmpty => FunctionIdentifier(ident.name())
case Array(dbName) => FunctionIdentifier(ident.name(), Some(dbName))
case _ =>
- throw new AnalysisException(
- s"$quoted is not a valid FunctionIdentifier as it has more than 2 name parts.")
+ throw QueryCompilationErrors.identifierHavingMoreThanTwoNamePartsError(
+ quoted, "FunctionIdentifier")
}
}
implicit class MultipartIdentifierHelper(parts: Seq[String]) {
if (parts.isEmpty) {
- throw new AnalysisException("multi-part identifier cannot be empty.")
+ throw QueryCompilationErrors.emptyMultipartIdentifierError()
}
def asIdentifier: Identifier = Identifier.of(parts.init.toArray, parts.last)
@@ -152,16 +148,16 @@ private[sql] object CatalogV2Implicits {
case Seq(tblName) => TableIdentifier(tblName)
case Seq(dbName, tblName) => TableIdentifier(tblName, Some(dbName))
case _ =>
- throw new AnalysisException(
- s"$quoted is not a valid TableIdentifier as it has more than 2 name parts.")
+ throw QueryCompilationErrors.identifierHavingMoreThanTwoNamePartsError(
+ quoted, "TableIdentifier")
}
def asFunctionIdentifier: FunctionIdentifier = parts match {
case Seq(funcName) => FunctionIdentifier(funcName)
case Seq(dbName, funcName) => FunctionIdentifier(funcName, Some(dbName))
case _ =>
- throw new AnalysisException(
- s"$quoted is not a valid FunctionIdentifier as it has more than 2 name parts.")
+ throw QueryCompilationErrors.identifierHavingMoreThanTwoNamePartsError(
+ quoted, "FunctionIdentifier")
}
def quoted: String = parts.map(quoteIfNeeded).mkString(".")
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Util.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Util.scala
index 02db2293ec64a..a779e50a1f214 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Util.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Util.scala
@@ -22,10 +22,10 @@ import java.util.Collections
import scala.collection.JavaConverters._
-import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.catalyst.analysis.{NamedRelation, NoSuchDatabaseException, NoSuchNamespaceException, NoSuchTableException, UnresolvedV2Relation}
import org.apache.spark.sql.catalyst.plans.logical.{AlterTable, CreateTableAsSelectStatement, CreateTableStatement, ReplaceTableAsSelectStatement, ReplaceTableStatement, SerdeInfo}
import org.apache.spark.sql.connector.catalog.TableChange._
+import org.apache.spark.sql.errors.QueryCompilationErrors
import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation
import org.apache.spark.sql.types.{ArrayType, DataType, MapType, NullType, StructField, StructType}
import org.apache.spark.sql.util.CaseInsensitiveStringMap
@@ -386,8 +386,7 @@ private[sql] object CatalogV2Util {
case _ => dt.isInstanceOf[NullType]
}
if (containsNullType(dt)) {
- throw new AnalysisException(
- s"Cannot create tables with ${NullType.simpleString} type.")
+ throw QueryCompilationErrors.cannotCreateTablesWithNullTypeError()
}
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/LookupCatalog.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/LookupCatalog.scala
index dcd352267a178..06358590a1e46 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/LookupCatalog.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/LookupCatalog.scala
@@ -18,8 +18,8 @@
package org.apache.spark.sql.connector.catalog
import org.apache.spark.internal.Logging
-import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier}
+import org.apache.spark.sql.errors.QueryCompilationErrors
import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
/**
@@ -192,11 +192,11 @@ private[sql] trait LookupCatalog extends Logging {
ident.namespace match {
case Array(db) => FunctionIdentifier(ident.name, Some(db))
case _ =>
- throw new AnalysisException(s"Unsupported function name '$ident'")
+ throw QueryCompilationErrors.unsupportedFunctionNameError(ident.toString)
}
}
- case _ => throw new AnalysisException("function is only supported in v1 catalog")
+ case _ => throw QueryCompilationErrors.functionUnsupportedInV2CatalogError()
}
}
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala
index b73df7ac844d1..ec15ae295d0b8 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala
@@ -22,17 +22,18 @@ import org.apache.hadoop.fs.Path
import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.catalyst.{FunctionIdentifier, QualifiedTableName, TableIdentifier}
import org.apache.spark.sql.catalyst.analysis.{CannotReplaceMissingTableException, NamespaceAlreadyExistsException, NoSuchNamespaceException, NoSuchTableException, ResolvedNamespace, ResolvedTable, ResolvedView, TableAlreadyExistsException}
-import org.apache.spark.sql.catalyst.catalog.{CatalogTable, InvalidUDFClassException}
+import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogTable, InvalidUDFClassException}
import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeReference, CreateMap, Expression, GroupingID, NamedExpression, SpecifiedWindowFrame, WindowFrame, WindowFunction, WindowSpecDefinition}
import org.apache.spark.sql.catalyst.plans.logical.{InsertIntoStatement, LogicalPlan, SerdeInfo}
import org.apache.spark.sql.catalyst.trees.TreeNode
import org.apache.spark.sql.catalyst.util.{toPrettySQL, FailFastMode, ParseMode, PermissiveMode}
-import org.apache.spark.sql.connector.catalog.{Identifier, NamespaceChange, Table, TableCapability, TableChange, V1Table}
+import org.apache.spark.sql.connector.catalog.{CatalogPlugin, Identifier, NamespaceChange, Table, TableCapability, TableChange, V1Table}
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+import org.apache.spark.sql.connector.expressions.{NamedReference, Transform}
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.sources.Filter
import org.apache.spark.sql.streaming.OutputMode
-import org.apache.spark.sql.types.{AbstractDataType, DataType, StructField, StructType}
+import org.apache.spark.sql.types.{AbstractDataType, DataType, NullType, StructField, StructType}
/**
* Object for grouping error messages from exceptions thrown during query compilation.
@@ -1356,6 +1357,41 @@ private[spark] object QueryCompilationErrors {
new AnalysisException("Cannot use interval type in the table schema.")
}
+ def cannotConvertBucketWithSortColumnsToTransformError(spec: BucketSpec): Throwable = {
+ new AnalysisException(
+ s"Cannot convert bucketing with sort columns to a transform: $spec")
+ }
+
+ def cannotConvertTransformsToPartitionColumnsError(nonIdTransforms: Seq[Transform]): Throwable = {
+ new AnalysisException("Transforms cannot be converted to partition columns: " +
+ nonIdTransforms.map(_.describe).mkString(", "))
+ }
+
+ def cannotPartitionByNestedColumnError(reference: NamedReference): Throwable = {
+ new AnalysisException(s"Cannot partition by nested column: $reference")
+ }
+
+ def cannotUseCatalogError(plugin: CatalogPlugin, msg: String): Throwable = {
+ new AnalysisException(s"Cannot use catalog ${plugin.name}: $msg")
+ }
+
+ def identifierHavingMoreThanTwoNamePartsError(
+ quoted: String, identifier: String): Throwable = {
+ new AnalysisException(s"$quoted is not a valid $identifier as it has more than 2 name parts.")
+ }
+
+ def emptyMultipartIdentifierError(): Throwable = {
+ new AnalysisException("multi-part identifier cannot be empty.")
+ }
+
+ def cannotCreateTablesWithNullTypeError(): Throwable = {
+ new AnalysisException(s"Cannot create tables with ${NullType.simpleString} type.")
+ }
+
+ def functionUnsupportedInV2CatalogError(): Throwable = {
+ new AnalysisException("function is only supported in v1 catalog")
+ }
+
def operateHiveDataSourceDirectlyError(operation: String): Throwable = {
new AnalysisException("Hive data source can only be used with tables, you can not " +
s"$operation files of Hive data source directly.")
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ObjectExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ObjectExpressionsSuite.scala
index ea38f190a0067..5e4c5bcd0ecc9 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ObjectExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ObjectExpressionsSuite.scala
@@ -638,8 +638,22 @@ class ObjectExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
val clsType = ObjectType(classOf[ConcreteClass])
val obj = new ConcreteClass
+ val input = (1, 2)
checkObjectExprEvaluation(
- Invoke(Literal(obj, clsType), "testFunc", IntegerType, Seq(Literal(1))), 0)
+ Invoke(Literal(obj, clsType), "testFunc", IntegerType,
+ Seq(Literal(input, ObjectType(input.getClass)))), 2)
+ }
+
+ test("SPARK-35288: static invoke should find method without exact param type match") {
+ val input = (1, 2)
+
+ checkObjectExprEvaluation(
+ StaticInvoke(TestStaticInvoke.getClass, IntegerType, "func",
+ Seq(Literal(input, ObjectType(input.getClass)))), 3)
+
+ checkObjectExprEvaluation(
+ StaticInvoke(TestStaticInvoke.getClass, IntegerType, "func",
+ Seq(Literal(1, IntegerType))), -1)
}
test("SPARK-35281: StaticInvoke shouldn't box primitive when result is nullable") {
@@ -659,12 +673,24 @@ class TestBean extends Serializable {
assert(i != null, "this setter should not be called with null.")
}
+object TestStaticInvoke {
+ def func(param: Any): Int = param match {
+ case pair: Tuple2[_, _] =>
+ pair.asInstanceOf[Tuple2[Int, Int]]._1 + pair.asInstanceOf[Tuple2[Int, Int]]._2
+ case _ => -1
+ }
+}
+
abstract class BaseClass[T] {
- def testFunc(param: T): T
+ def testFunc(param: T): Int
}
-class ConcreteClass extends BaseClass[Int] with Serializable {
- override def testFunc(param: Int): Int = param - 1
+class ConcreteClass extends BaseClass[Product] with Serializable {
+ override def testFunc(param: Product): Int = param match {
+ case _: Tuple2[_, _] => 2
+ case _: Tuple3[_, _, _] => 3
+ case _ => 4
+ }
}
case object TestFun {
diff --git a/sql/core/benchmarks/V2FunctionBenchmark-jdk11-results.txt b/sql/core/benchmarks/V2FunctionBenchmark-jdk11-results.txt
new file mode 100644
index 0000000000000..564742f653e57
--- /dev/null
+++ b/sql/core/benchmarks/V2FunctionBenchmark-jdk11-results.txt
@@ -0,0 +1,44 @@
+OpenJDK 64-Bit Server VM 11.0.11+9-LTS on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
+scalar function (long + long) -> long, result_nullable = true codegen = true: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------------------------------------------
+native_long_add 17789 18405 580 28.1 35.6 1.0X
+java_long_add_default 85058 87073 NaN 5.9 170.1 0.2X
+java_long_add_magic 20262 20641 352 24.7 40.5 0.9X
+java_long_add_static_magic 19458 19524 105 25.7 38.9 0.9X
+scala_long_add_default 85892 86496 560 5.8 171.8 0.2X
+scala_long_add_magic 20164 20330 212 24.8 40.3 0.9X
+
+OpenJDK 64-Bit Server VM 11.0.11+9-LTS on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
+scalar function (long + long) -> long, result_nullable = false codegen = true: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+-------------------------------------------------------------------------------------------------------------------------------------------------------------
+native_long_add 18290 18467 157 27.3 36.6 1.0X
+java_long_add_default 82415 82687 270 6.1 164.8 0.2X
+java_long_add_magic 19941 20032 85 25.1 39.9 0.9X
+java_long_add_static_magic 17861 17940 92 28.0 35.7 1.0X
+scala_long_add_default 83800 85639 NaN 6.0 167.6 0.2X
+scala_long_add_magic 20103 20123 18 24.9 40.2 0.9X
+
+OpenJDK 64-Bit Server VM 11.0.11+9-LTS on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
+scalar function (long + long) -> long, result_nullable = true codegen = false: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+-------------------------------------------------------------------------------------------------------------------------------------------------------------
+native_long_add 46039 46199 162 10.9 92.1 1.0X
+java_long_add_default 113199 113773 720 4.4 226.4 0.4X
+java_long_add_magic 158252 159419 1075 3.2 316.5 0.3X
+java_long_add_static_magic 157162 157676 516 3.2 314.3 0.3X
+scala_long_add_default 112363 113264 1503 4.4 224.7 0.4X
+scala_long_add_magic 158122 159010 835 3.2 316.2 0.3X
+
+OpenJDK 64-Bit Server VM 11.0.11+9-LTS on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz
+scalar function (long + long) -> long, result_nullable = false codegen = false: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+--------------------------------------------------------------------------------------------------------------------------------------------------------------
+native_long_add 42685 42743 54 11.7 85.4 1.0X
+java_long_add_default 92041 92236 202 5.4 184.1 0.5X
+java_long_add_magic 148299 148722 397 3.4 296.6 0.3X
+java_long_add_static_magic 140599 141064 442 3.6 281.2 0.3X
+scala_long_add_default 91896 92980 959 5.4 183.8 0.5X
+scala_long_add_magic 148031 148802 759 3.4 296.1 0.3X
+
diff --git a/sql/core/benchmarks/V2FunctionBenchmark-results.txt b/sql/core/benchmarks/V2FunctionBenchmark-results.txt
new file mode 100644
index 0000000000000..2035aa3633b80
--- /dev/null
+++ b/sql/core/benchmarks/V2FunctionBenchmark-results.txt
@@ -0,0 +1,44 @@
+OpenJDK 64-Bit Server VM 1.8.0_292-b10 on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+scalar function (long + long) -> long, result_nullable = true codegen = true: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------------------------------------------
+native_long_add 10559 11585 903 47.4 21.1 1.0X
+java_long_add_default 78979 80089 987 6.3 158.0 0.1X
+java_long_add_magic 14061 14326 305 35.6 28.1 0.8X
+java_long_add_static_magic 11971 12150 242 41.8 23.9 0.9X
+scala_long_add_default 77254 78565 1254 6.5 154.5 0.1X
+scala_long_add_magic 13174 13232 51 38.0 26.3 0.8X
+
+OpenJDK 64-Bit Server VM 1.8.0_292-b10 on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+scalar function (long + long) -> long, result_nullable = false codegen = true: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+-------------------------------------------------------------------------------------------------------------------------------------------------------------
+native_long_add 10489 10665 162 47.7 21.0 1.0X
+java_long_add_default 66636 68422 NaN 7.5 133.3 0.2X
+java_long_add_magic 13504 14213 883 37.0 27.0 0.8X
+java_long_add_static_magic 11726 11984 240 42.6 23.5 0.9X
+scala_long_add_default 75906 76130 196 6.6 151.8 0.1X
+scala_long_add_magic 14480 14770 261 34.5 29.0 0.7X
+
+OpenJDK 64-Bit Server VM 1.8.0_292-b10 on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+scalar function (long + long) -> long, result_nullable = true codegen = false: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+-------------------------------------------------------------------------------------------------------------------------------------------------------------
+native_long_add 39178 39548 323 12.8 78.4 1.0X
+java_long_add_default 84756 85509 1092 5.9 169.5 0.5X
+java_long_add_magic 199140 200801 1823 2.5 398.3 0.2X
+java_long_add_static_magic 203500 207050 NaN 2.5 407.0 0.2X
+scala_long_add_default 101180 101421 387 4.9 202.4 0.4X
+scala_long_add_magic 193277 197006 1138 2.6 386.6 0.2X
+
+OpenJDK 64-Bit Server VM 1.8.0_292-b10 on Linux 5.4.0-1046-azure
+Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz
+scalar function (long + long) -> long, result_nullable = false codegen = false: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+--------------------------------------------------------------------------------------------------------------------------------------------------------------
+native_long_add 37064 37333 235 13.5 74.1 1.0X
+java_long_add_default 104439 107802 NaN 4.8 208.9 0.4X
+java_long_add_magic 212496 214321 NaN 2.4 425.0 0.2X
+java_long_add_static_magic 239551 240619 1652 2.1 479.1 0.2X
+scala_long_add_default 122413 123171 788 4.1 244.8 0.3X
+scala_long_add_magic 215912 222715 NaN 2.3 431.8 0.2X
+
diff --git a/sql/core/src/test/java/test/org/apache/spark/sql/connector/catalog/functions/JavaLongAdd.java b/sql/core/src/test/java/test/org/apache/spark/sql/connector/catalog/functions/JavaLongAdd.java
new file mode 100644
index 0000000000000..e2e7136d6f44c
--- /dev/null
+++ b/sql/core/src/test/java/test/org/apache/spark/sql/connector/catalog/functions/JavaLongAdd.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package test.org.apache.spark.sql.connector.catalog.functions;
+
+import org.apache.spark.sql.catalyst.InternalRow;
+import org.apache.spark.sql.connector.catalog.functions.BoundFunction;
+import org.apache.spark.sql.connector.catalog.functions.ScalarFunction;
+import org.apache.spark.sql.connector.catalog.functions.UnboundFunction;
+import org.apache.spark.sql.types.DataType;
+import org.apache.spark.sql.types.DataTypes;
+import org.apache.spark.sql.types.LongType;
+import org.apache.spark.sql.types.StructField;
+import org.apache.spark.sql.types.StructType;
+
+public class JavaLongAdd implements UnboundFunction {
+ private final ScalarFunction impl;
+
+ public JavaLongAdd(ScalarFunction impl) {
+ this.impl = impl;
+ }
+
+ @Override
+ public String name() {
+ return "long_add";
+ }
+
+ @Override
+ public BoundFunction bind(StructType inputType) {
+ if (inputType.fields().length != 2) {
+ throw new UnsupportedOperationException("Expect two arguments");
+ }
+ StructField[] fields = inputType.fields();
+ if (!(fields[0].dataType() instanceof LongType)) {
+ throw new UnsupportedOperationException("Expect first argument to be LongType");
+ }
+ if (!(fields[1].dataType() instanceof LongType)) {
+ throw new UnsupportedOperationException("Expect second argument to be LongType");
+ }
+ return impl;
+ }
+
+ @Override
+ public String description() {
+ return "long_add";
+ }
+
+ private abstract static class JavaLongAddBase implements ScalarFunction {
+ private final boolean isResultNullable;
+
+ JavaLongAddBase(boolean isResultNullable) {
+ this.isResultNullable = isResultNullable;
+ }
+
+ @Override
+ public DataType[] inputTypes() {
+ return new DataType[] { DataTypes.LongType, DataTypes.LongType };
+ }
+
+ @Override
+ public DataType resultType() {
+ return DataTypes.LongType;
+ }
+
+ @Override
+ public boolean isResultNullable() {
+ return isResultNullable;
+ }
+ }
+
+ public static class JavaLongAddDefault extends JavaLongAddBase {
+ public JavaLongAddDefault(boolean isResultNullable) {
+ super(isResultNullable);
+ }
+
+ @Override
+ public String name() {
+ return "long_add_default";
+ }
+
+ @Override
+ public Long produceResult(InternalRow input) {
+ return input.getLong(0) + input.getLong(1);
+ }
+ }
+
+ public static class JavaLongAddMagic extends JavaLongAddBase {
+ public JavaLongAddMagic(boolean isResultNullable) {
+ super(isResultNullable);
+ }
+
+ @Override
+ public String name() {
+ return "long_add_magic";
+ }
+
+ public long invoke(long left, long right) {
+ return left + right;
+ }
+ }
+
+ public static class JavaLongAddStaticMagic extends JavaLongAddBase {
+ public JavaLongAddStaticMagic(boolean isResultNullable) {
+ super(isResultNullable);
+ }
+
+ @Override
+ public String name() {
+ return "long_add_static_magic";
+ }
+
+ public static long invoke(long left, long right) {
+ return left + right;
+ }
+ }
+}
diff --git a/sql/core/src/test/java/test/org/apache/spark/sql/connector/catalog/functions/JavaStrLen.java b/sql/core/src/test/java/test/org/apache/spark/sql/connector/catalog/functions/JavaStrLen.java
index 8b2d883a3703f..7cd010b9365bb 100644
--- a/sql/core/src/test/java/test/org/apache/spark/sql/connector/catalog/functions/JavaStrLen.java
+++ b/sql/core/src/test/java/test/org/apache/spark/sql/connector/catalog/functions/JavaStrLen.java
@@ -58,7 +58,7 @@ public String description() {
" strlen(string) -> int";
}
- public static class JavaStrLenDefault implements ScalarFunction {
+ private abstract static class JavaStrLenBase implements ScalarFunction {
@Override
public DataType[] inputTypes() {
return new DataType[] { DataTypes.StringType };
@@ -73,7 +73,9 @@ public DataType resultType() {
public String name() {
return "strlen";
}
+ }
+ public static class JavaStrLenDefault extends JavaStrLenBase {
@Override
public Integer produceResult(InternalRow input) {
String str = input.getString(0);
@@ -81,42 +83,42 @@ public Integer produceResult(InternalRow input) {
}
}
- public static class JavaStrLenMagic implements ScalarFunction {
- @Override
- public DataType[] inputTypes() {
- return new DataType[] { DataTypes.StringType };
+ public static class JavaStrLenMagic extends JavaStrLenBase {
+ public int invoke(UTF8String str) {
+ return str.toString().length();
}
+ }
- @Override
- public DataType resultType() {
- return DataTypes.IntegerType;
+ public static class JavaStrLenStaticMagic extends JavaStrLenBase {
+ public static int invoke(UTF8String str) {
+ return str.toString().length();
}
+ }
+ public static class JavaStrLenBoth extends JavaStrLenBase {
@Override
- public String name() {
- return "strlen";
+ public Integer produceResult(InternalRow input) {
+ String str = input.getString(0);
+ return str.length();
}
-
public int invoke(UTF8String str) {
- return str.toString().length();
+ return str.toString().length() + 100;
}
}
- public static class JavaStrLenNoImpl implements ScalarFunction {
- @Override
- public DataType[] inputTypes() {
- return new DataType[] { DataTypes.StringType };
+ // even though the static magic method is present, it has incorrect parameter type and so Spark
+ // should fallback to the non-static magic method
+ public static class JavaStrLenBadStaticMagic extends JavaStrLenBase {
+ public static int invoke(String str) {
+ return str.length();
}
- @Override
- public DataType resultType() {
- return DataTypes.IntegerType;
+ public int invoke(UTF8String str) {
+ return str.toString().length() + 100;
}
+ }
- @Override
- public String name() {
- return "strlen";
- }
+ public static class JavaStrLenNoImpl extends JavaStrLenBase {
}
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2FunctionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2FunctionSuite.scala
index fe856ffecb84a..b269da39daf38 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2FunctionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2FunctionSuite.scala
@@ -183,6 +183,28 @@ class DataSourceV2FunctionSuite extends DatasourceV2SQLBase {
checkAnswer(sql("SELECT testcat.ns.strlen('abc')"), Row(3) :: Nil)
}
+ test("scalar function: static magic method in Java") {
+ catalog("testcat").asInstanceOf[SupportsNamespaces].createNamespace(Array("ns"), emptyProps)
+ addFunction(Identifier.of(Array("ns"), "strlen"),
+ new JavaStrLen(new JavaStrLenStaticMagic))
+ checkAnswer(sql("SELECT testcat.ns.strlen('abc')"), Row(3) :: Nil)
+ }
+
+ test("scalar function: magic method should take higher precedence in Java") {
+ catalog("testcat").asInstanceOf[SupportsNamespaces].createNamespace(Array("ns"), emptyProps)
+ addFunction(Identifier.of(Array("ns"), "strlen"),
+ new JavaStrLen(new JavaStrLenBoth))
+ // to differentiate, the static method returns string length + 100
+ checkAnswer(sql("SELECT testcat.ns.strlen('abc')"), Row(103) :: Nil)
+ }
+
+ test("scalar function: bad static magic method should fallback to non-static") {
+ catalog("testcat").asInstanceOf[SupportsNamespaces].createNamespace(Array("ns"), emptyProps)
+ addFunction(Identifier.of(Array("ns"), "strlen"),
+ new JavaStrLen(new JavaStrLenBadStaticMagic))
+ checkAnswer(sql("SELECT testcat.ns.strlen('abc')"), Row(103) :: Nil)
+ }
+
test("scalar function: no implementation found in Java") {
catalog("testcat").asInstanceOf[SupportsNamespaces].createNamespace(Array("ns"), emptyProps)
addFunction(Identifier.of(Array("ns"), "strlen"),
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/functions/V2FunctionBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/functions/V2FunctionBenchmark.scala
new file mode 100644
index 0000000000000..9328a9a8e93e3
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/functions/V2FunctionBenchmark.scala
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.connector.functions
+
+import test.org.apache.spark.sql.connector.catalog.functions.JavaLongAdd
+import test.org.apache.spark.sql.connector.catalog.functions.JavaLongAdd.{JavaLongAddDefault, JavaLongAddMagic, JavaLongAddStaticMagic}
+
+import org.apache.spark.benchmark.Benchmark
+import org.apache.spark.sql.Column
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.expressions.{BinaryArithmetic, Expression}
+import org.apache.spark.sql.catalyst.expressions.CodegenObjectFactoryMode._
+import org.apache.spark.sql.catalyst.util.TypeUtils
+import org.apache.spark.sql.connector.catalog.{Identifier, InMemoryCatalog}
+import org.apache.spark.sql.connector.catalog.functions.{BoundFunction, ScalarFunction, UnboundFunction}
+import org.apache.spark.sql.execution.benchmark.SqlBasedBenchmark
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.types.{AbstractDataType, DataType, LongType, NumericType, StructType}
+
+/**
+ * Benchmark to measure DataSourceV2 UDF performance
+ * {{{
+ * To run this benchmark:
+ * 1. without sbt:
+ * bin/spark-submit --class
+ * --jars ,
+ * 2. build/sbt "sql/test:runMain "
+ * 3. generate result: SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain "
+ * Results will be written to "benchmarks/V2FunctionBenchmark-results.txt".
+ * }}}
+ * '''NOTE''': to update the result of this benchmark, please use Github benchmark action:
+ * https://spark.apache.org/developer-tools.html#github-workflow-benchmarks
+ */
+object V2FunctionBenchmark extends SqlBasedBenchmark {
+ val catalogName: String = "benchmark_catalog"
+
+ override def runBenchmarkSuite(mainArgs: Array[String]): Unit = {
+ val N = 500L * 1000 * 1000
+ Seq(true, false).foreach { codegenEnabled =>
+ Seq(true, false).foreach { resultNullable =>
+ scalarFunctionBenchmark(N, codegenEnabled = codegenEnabled,
+ resultNullable = resultNullable)
+ }
+ }
+ }
+
+ private def scalarFunctionBenchmark(
+ N: Long,
+ codegenEnabled: Boolean,
+ resultNullable: Boolean): Unit = {
+ withSQLConf(s"spark.sql.catalog.$catalogName" -> classOf[InMemoryCatalog].getName) {
+ createFunction("java_long_add_default",
+ new JavaLongAdd(new JavaLongAddDefault(resultNullable)))
+ createFunction("java_long_add_magic", new JavaLongAdd(new JavaLongAddMagic(resultNullable)))
+ createFunction("java_long_add_static_magic",
+ new JavaLongAdd(new JavaLongAddStaticMagic(resultNullable)))
+ createFunction("scala_long_add_default",
+ LongAddUnbound(new LongAddWithProduceResult(resultNullable)))
+ createFunction("scala_long_add_magic", LongAddUnbound(new LongAddWithMagic(resultNullable)))
+
+ val codeGenFactoryMode = if (codegenEnabled) FALLBACK else NO_CODEGEN
+ withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> codegenEnabled.toString,
+ SQLConf.CODEGEN_FACTORY_MODE.key -> codeGenFactoryMode.toString) {
+ val name = s"scalar function (long + long) -> long, result_nullable = $resultNullable " +
+ s"codegen = $codegenEnabled"
+ val benchmark = new Benchmark(name, N, output = output)
+ benchmark.addCase(s"native_long_add", numIters = 3) { _ =>
+ spark.range(N).select(Column(NativeAdd($"id".expr, $"id".expr, resultNullable))).noop()
+ }
+ Seq("java_long_add_default", "java_long_add_magic", "java_long_add_static_magic",
+ "scala_long_add_default", "scala_long_add_magic").foreach { functionName =>
+ benchmark.addCase(s"$functionName", numIters = 3) { _ =>
+ spark.range(N).selectExpr(s"$catalogName.$functionName(id, id)").noop()
+ }
+ }
+ benchmark.run()
+ }
+ }
+ }
+
+ private def createFunction(name: String, fn: UnboundFunction): Unit = {
+ val catalog = spark.sessionState.catalogManager.catalog(catalogName)
+ val ident = Identifier.of(Array.empty, name)
+ catalog.asInstanceOf[InMemoryCatalog].createFunction(ident, fn)
+ }
+
+ case class NativeAdd(
+ left: Expression,
+ right: Expression,
+ override val nullable: Boolean) extends BinaryArithmetic {
+ override protected val failOnError: Boolean = true
+ override def inputType: AbstractDataType = NumericType
+ override def symbol: String = "+"
+ override def exactMathMethod: Option[String] = Some("addExact")
+
+ private lazy val numeric = TypeUtils.getNumeric(dataType, failOnError)
+ protected override def nullSafeEval(input1: Any, input2: Any): Any =
+ numeric.plus(input1, input2)
+
+ override protected def withNewChildrenInternal(
+ newLeft: Expression,
+ newRight: Expression): NativeAdd = copy(left = newLeft, right = newRight)
+ }
+
+ case class LongAddUnbound(impl: ScalarFunction[Long]) extends UnboundFunction {
+ override def bind(inputType: StructType): BoundFunction = impl
+ override def description(): String = name()
+ override def name(): String = "long_add_unbound"
+ }
+
+ abstract class LongAddBase(resultNullable: Boolean) extends ScalarFunction[Long] {
+ override def inputTypes(): Array[DataType] = Array(LongType, LongType)
+ override def resultType(): DataType = LongType
+ override def isResultNullable: Boolean = resultNullable
+ }
+
+ class LongAddWithProduceResult(resultNullable: Boolean) extends LongAddBase(resultNullable) {
+ override def produceResult(input: InternalRow): Long = {
+ input.getLong(0) + input.getLong(1)
+ }
+ override def name(): String = "long_add_default"
+ }
+
+ class LongAddWithMagic(resultNullable: Boolean) extends LongAddBase(resultNullable) {
+ def invoke(left: Long, right: Long): Long = {
+ left + right
+ }
+ override def name(): String = "long_add_magic"
+ }
+}
+
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
index e9728b8c0461d..9bb3f45b96c11 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
@@ -273,7 +273,7 @@ private[hive] class HiveClientImpl(
if (clientLoader.cachedHive != null) {
clientLoader.cachedHive.asInstanceOf[Hive]
} else {
- val c = Hive.get(conf)
+ val c = shim.getHive(conf)
clientLoader.cachedHive = c
c
}
@@ -303,7 +303,7 @@ private[hive] class HiveClientImpl(
// with the side-effect of Hive.get(conf) to avoid using out-of-date HiveConf.
// See discussion in https://github.com/apache/spark/pull/16826/files#r104606859
// for more details.
- Hive.get(conf)
+ shim.getHive(conf)
// setCurrentSessionState will use the classLoader associated
// with the HiveConf in `state` to override the context class loader of the current
// thread.
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala
index 2f7fe96013dea..b0a877d732be7 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala
@@ -177,6 +177,8 @@ private[client] sealed abstract class Shim {
def getMSC(hive: Hive): IMetaStoreClient
+ def getHive(hiveConf: HiveConf): Hive
+
protected def findMethod(klass: Class[_], name: String, args: Class[_]*): Method = {
klass.getMethod(name, args: _*)
}
@@ -199,6 +201,8 @@ private[client] class Shim_v0_12 extends Shim with Logging {
getMSCMethod.invoke(hive).asInstanceOf[IMetaStoreClient]
}
+ override def getHive(hiveConf: HiveConf): Hive = Hive.get(hiveConf)
+
private lazy val startMethod =
findStaticMethod(
classOf[SessionState],
@@ -1316,6 +1320,13 @@ private[client] class Shim_v2_1 extends Shim_v2_0 {
override def alterPartitions(hive: Hive, tableName: String, newParts: JList[Partition]): Unit = {
alterPartitionsMethod.invoke(hive, tableName, newParts, environmentContextInAlterTable)
}
+
+ // HIVE-10319 introduced a new HMS thrift API `get_all_functions` which is used by
+ // `Hive.get` since version 2.1.0, when it loads all Hive permanent functions during
+ // initialization. This breaks compatibility with HMS server of lower versions.
+ // To mitigate here we use `Hive.getWithFastCheck` instead which skips loading the permanent
+ // functions and therefore avoids calling `get_all_functions`.
+ override def getHive(hiveConf: HiveConf): Hive = Hive.getWithFastCheck(hiveConf, false)
}
private[client] class Shim_v2_2 extends Shim_v2_1