/*!
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
/* global document, window, $, d3, STATE_COLOR, isoDateToTimeEl, autoRefreshInterval,
localStorage */
import { throttle } from "lodash";
import { getMetaValue } from "./utils";
import tiTooltip from "./task_instances";
import { approxTimeFromNow, formatDateTime } from "./datetime_utils";
import { openDatasetModal, getDatasetTooltipInfo } from "./datasetUtils";
const DAGS_INDEX = getMetaValue("dags_index");
const ENTER_KEY_CODE = 13;
const pausedUrl = getMetaValue("paused_url");
const statusFilter = getMetaValue("status_filter");
const autocompleteUrl = getMetaValue("autocomplete_url");
const graphUrl = getMetaValue("graph_url");
const dagRunUrl = getMetaValue("dag_run_url");
const taskInstanceUrl = getMetaValue("task_instance_url");
const blockedUrl = getMetaValue("blocked_url");
const csrfToken = getMetaValue("csrf_token");
const lastDagRunsUrl = getMetaValue("last_dag_runs_url");
const dagStatsUrl = getMetaValue("dag_stats_url");
const taskStatsUrl = getMetaValue("task_stats_url");
const gridUrl = getMetaValue("grid_url");
const datasetsUrl = getMetaValue("datasets_url");
const nextRunDatasetsSummaryUrl = getMetaValue("next_run_datasets_summary_url");
const nextDatasets = {};
let nextDatasetsError;
const DAG_RUN = "dag-run";
const TASK_INSTANCE = "task-instance";
// auto refresh interval in milliseconds
// (x2 the interval in tree/graph view since this page can take longer to refresh )
const refreshIntervalMs = 2000;
$("#tags_filter").select2({
placeholder: "Filter DAGs by tag",
allowClear: true,
});
$("#tags_filter").on("change", (e) => {
e.preventDefault();
const query = new URLSearchParams(window.location.search);
const tags = $(e.target).select2("val");
if (tags.length) {
if (query.has("tags")) query.delete("tags");
tags.forEach((value) => {
query.append("tags", value);
});
} else {
query.delete("tags");
query.set("reset_tags", "reset");
}
if (query.has("page")) query.delete("page");
window.location = `${DAGS_INDEX}?${query.toString()}`;
});
$("#tags_form").on("reset", (e) => {
e.preventDefault();
const query = new URLSearchParams(window.location.search);
query.delete("tags");
if (query.has("page")) query.delete("page");
query.set("reset_tags", "reset");
window.location = `${DAGS_INDEX}?${query.toString()}`;
});
$("#dag_query").on("keypress", (e) => {
// check for key press on ENTER (key code 13) to trigger the search
if (e.which === ENTER_KEY_CODE) {
const query = new URLSearchParams(window.location.search);
query.set("search", e.target.value.trim());
query.delete("page");
window.location = `${DAGS_INDEX}?${query.toString()}`;
e.preventDefault();
}
});
$.each($("[id^=toggle]"), function toggleId() {
const $input = $(this);
const dagId = $input.data("dag-id");
$input.on("change", () => {
const isPaused = $input.is(":checked");
const url = `${pausedUrl}?is_paused=${isPaused}&dag_id=${encodeURIComponent(
dagId
)}`;
$input.removeClass("switch-input--error");
// Remove focus on element so the tooltip will go away
$input.trigger("blur");
$.post(url).fail(() => {
setTimeout(() => {
$input.prop("checked", !isPaused);
$input.addClass("switch-input--error");
}, 500);
});
});
});
$(".typeahead").typeahead({
source(query, callback) {
return $.ajax(autocompleteUrl, {
data: {
query: encodeURIComponent(query),
status: statusFilter,
},
success: callback,
});
},
displayText(value) {
return value.dag_display_name || value.name;
},
autoSelect: false,
afterSelect(value) {
const query = new URLSearchParams(window.location.search);
query.set("search", value.name);
if (value.type === "owner") {
window.location = `${DAGS_INDEX}?${query}`;
}
if (value.type === "dag") {
window.location = `${gridUrl.replace("__DAG_ID__", value.name)}?${query}`;
}
},
});
$("#search_form").on("reset", () => {
const query = new URLSearchParams(window.location.search);
query.delete("search");
query.delete("page");
window.location = `${DAGS_INDEX}?${query}`;
});
$("#main_content").show(250);
const diameter = 25;
const circleMargin = 4;
const strokeWidth = 2;
const strokeWidthHover = 6;
function blockedHandler(error, json) {
$.each(json, function handleBlock() {
const a = document.querySelector(`[data-dag-id="${this.dag_id}"]`);
a.title = `${this.active_dag_run}/${this.max_active_runs} active dag runs`;
if (this.active_dag_run >= this.max_active_runs) {
a.style.color = "#e43921";
}
});
}
function lastDagRunsHandler(error, json) {
$(".js-loading-last-run").remove();
Object.keys(json).forEach((safeDagId) => {
const dagId = json[safeDagId].dag_id;
const executionDate = json[safeDagId].execution_date;
const g = d3.select(`#last-run-${safeDagId}`);
// Show last run as a link to the graph view
g.selectAll("a")
.attr(
"href",
`${graphUrl}?dag_id=${encodeURIComponent(
dagId
)}&execution_date=${encodeURIComponent(executionDate)}`
)
.html("")
.insert(isoDateToTimeEl.bind(null, executionDate, { title: false }));
// Only show the tooltip when we have a last run and add the json to a custom data- attribute
g.selectAll("span")
.style("display", null)
.attr("data-lastrun", JSON.stringify(json[safeDagId]));
});
}
// Load data-lastrun attribute data to populate the tooltip on hover
d3.selectAll(".js-last-run-tooltip").on(
"mouseover",
function mouseoverLastRun() {
const lastRunData = JSON.parse(d3.select(this).attr("data-lastrun"));
d3.select(this).attr("data-original-title", tiTooltip(lastRunData));
}
);
function formatCount(count) {
if (count >= 1000000) return `${Math.floor(count / 1000000)}M`;
if (count >= 1000) return `${Math.floor(count / 1000)}k`;
return count;
}
function drawDagStats(selector, dagId, states) {
const g = d3
.select(`svg#${selector}-${dagId.replace(/\./g, "__dot__")}`)
.attr("height", diameter + strokeWidthHover * 2)
.attr("width", states.length * (diameter + circleMargin) + circleMargin)
.selectAll("g")
.data(states)
.enter()
.append("g")
.attr("transform", (d, i) => {
const x = i * (diameter + circleMargin) + (diameter / 2 + circleMargin);
const y = diameter / 2 + strokeWidthHover;
return `translate(${x},${y})`;
});
g.append("svg:a")
.attr("href", (d) => {
const params = new URLSearchParams();
params.append("_flt_3_dag_id", dagId);
/* eslint no-unused-expressions: ["error", { "allowTernary": true }] */
d.state
? params.append("_flt_3_state", d.state)
: params.append("_flt_8_state", "");
switch (selector) {
case DAG_RUN:
return `${dagRunUrl}?${params.toString()}`;
case TASK_INSTANCE:
return `${taskInstanceUrl}?${params.toString()}`;
default:
return "";
}
})
.append("circle")
.attr(
"id",
(d) => `${selector}-${dagId.replace(/\./g, "_")}-${d.state || "none"}`
)
.attr("class", "has-svg-tooltip")
.attr("stroke-width", (d) => {
if (d.count > 0) return strokeWidth;
return 1;
})
.attr("stroke", (d) => {
if (d.count > 0) return STATE_COLOR[d.state];
return "gainsboro";
})
.attr("fill", "#fff")
.attr("r", diameter / 2)
.attr("title", (d) => `${d.state || "none"}: ${d.count}`)
.on("mouseover", (d) => {
if (d.count > 0) {
d3.select(d3.event.currentTarget)
.transition()
.duration(400)
.attr("fill", "#e2e2e2")
.style("stroke-width", strokeWidthHover);
}
})
.on("mouseout", (d) => {
if (d.count > 0) {
d3.select(d3.event.currentTarget)
.transition()
.duration(400)
.attr("fill", "#fff")
.style("stroke-width", strokeWidth);
}
})
.style("opacity", 0)
.transition()
.duration(300)
.delay((d, i) => i * 50)
.style("opacity", 1);
d3.select(`.js-loading-${selector}-stats`).remove();
g.append("text")
.attr("fill", "#51504f")
.attr("text-anchor", "middle")
.attr("vertical-align", "middle")
.attr("font-size", 9)
.attr("y", 3)
.style("pointer-events", "none")
.text((d) => (d.count > 0 ? formatCount(d.count) : ""));
}
function dagStatsHandler(selector, json) {
Object.keys(json).forEach((dagId) => {
const states = json[dagId];
drawDagStats(selector, dagId, states);
});
}
function nextRunDatasetsSummaryHandler(_, json) {
[...document.getElementsByClassName("next-dataset-triggered")].forEach(
(el) => {
const dagId = $(el).attr("data-dag-id");
const previousSummary = $(el).attr("data-summary");
const nextDatasetsInfo = json[dagId];
// Only update dags that depend on multiple datasets
if (nextDatasetsInfo && !nextDatasetsInfo.uri) {
const newSummary = `${nextDatasetsInfo.ready} of ${nextDatasetsInfo.total} datasets updated`;
// Only update the element if the summary has changed
if (previousSummary !== newSummary) {
$(el).attr("data-summary", newSummary);
$(el).text(newSummary);
}
}
}
);
}
function getDagIds({ activeDagsOnly = false } = {}) {
let dagIds = $("[id^=toggle]");
if (activeDagsOnly) {
dagIds = dagIds.filter(":checked");
}
dagIds = dagIds
// eslint-disable-next-line func-names
.map(function () {
return $(this).data("dag-id");
})
.get();
return dagIds;
}
function getDagStats() {
const dagIds = getDagIds();
const params = new URLSearchParams();
dagIds.forEach((dagId) => {
params.append("dag_ids", dagId);
});
if (params.has("dag_ids")) {
d3.json(blockedUrl)
.header("X-CSRFToken", csrfToken)
.post(params, blockedHandler);
d3.json(lastDagRunsUrl)
.header("X-CSRFToken", csrfToken)
.post(params, lastDagRunsHandler);
d3.json(dagStatsUrl)
.header("X-CSRFToken", csrfToken)
.post(params, (error, json) => dagStatsHandler(DAG_RUN, json));
d3.json(taskStatsUrl)
.header("X-CSRFToken", csrfToken)
.post(params, (error, json) => dagStatsHandler(TASK_INSTANCE, json));
} else {
// no dags, hide the loading dots
$(`.js-loading-${DAG_RUN}-stats`).remove();
$(`.js-loading-${TASK_INSTANCE}-stats`).remove();
}
}
function showSvgTooltip(text, circ) {
const tip = $("#svg-tooltip");
tip.children(".tooltip-inner").text(text);
const centeringOffset = tip.width() / 2;
tip.css({
display: "block",
left: `${circ.left + 12.5 - centeringOffset}px`, // 12.5 == half of circle width
top: `${circ.top - 25}px`, // 25 == position above circle
});
}
function hideSvgTooltip() {
$("#svg-tooltip").css("display", "none");
}
function refreshDagStats(selector, dagId, states) {
d3.select(`svg#${selector}-${dagId.replace(/\./g, "__dot__")}`)
.selectAll("circle")
.data(states)
.attr("stroke-width", (d) => {
if (d.count > 0) return strokeWidth;
return 1;
})
.attr("stroke", (d) => {
if (d.count > 0) return STATE_COLOR[d.state];
return "gainsboro";
});
d3.select(`svg#${selector}-${dagId.replace(/\./g, "__dot__")}`)
.selectAll("text")
.data(states)
.text((d) => {
if (d.count > 0) {
return d.count;
}
return "";
});
}
let refreshInterval;
function checkActiveRuns(json) {
// filter latest dag runs and check if there are still running dags
const activeRuns = Object.keys(json).filter((dagId) => {
const dagRuns = json[dagId]
.filter(({ state }) => state === "running" || state === "queued")
.filter((r) => r.count > 0);
return dagRuns.length > 0;
});
if (activeRuns.length === 0) {
// in case there are no active runs increase the interval for auto refresh
$("#auto_refresh").prop("checked", false);
clearInterval(refreshInterval);
}
}
function refreshDagStatsHandler(selector, json) {
if (selector === DAG_RUN) checkActiveRuns(json);
Object.keys(json).forEach((dagId) => {
const states = json[dagId];
refreshDagStats(selector, dagId, states);
});
}
function handleRefresh({ activeDagsOnly = false } = {}) {
const dagIds = getDagIds({ activeDagsOnly });
const params = new URLSearchParams();
dagIds.forEach((dagId) => {
params.append("dag_ids", dagId);
});
$("#loading-dots").css("display", "inline-block");
if (params.has("dag_ids")) {
d3.json(lastDagRunsUrl)
.header("X-CSRFToken", csrfToken)
.post(params, lastDagRunsHandler);
d3.json(dagStatsUrl)
.header("X-CSRFToken", csrfToken)
.post(params, (error, json) => refreshDagStatsHandler(DAG_RUN, json));
d3.json(taskStatsUrl)
.header("X-CSRFToken", csrfToken)
.post(params, (error, json) =>
refreshDagStatsHandler(TASK_INSTANCE, json)
);
d3.json(nextRunDatasetsSummaryUrl)
.header("X-CSRFToken", csrfToken)
.post(params, nextRunDatasetsSummaryHandler);
}
setTimeout(() => {
$("#loading-dots").css("display", "none");
}, refreshIntervalMs);
}
function startOrStopRefresh() {
if ($("#auto_refresh").is(":checked")) {
refreshInterval = setInterval(() => {
handleRefresh({ activeDagsOnly: true });
}, autoRefreshInterval * refreshIntervalMs);
} else {
clearInterval(refreshInterval);
}
}
function initAutoRefresh() {
const isDisabled = localStorage.getItem("dagsDisableAutoRefresh");
$("#auto_refresh").prop("checked", !isDisabled);
startOrStopRefresh();
d3.select("#refresh_button").on("click", () => handleRefresh());
}
// pause autorefresh when the page is not active
const handleVisibilityChange = () => {
if (document.hidden) {
clearInterval(refreshInterval);
} else {
initAutoRefresh();
}
};
document.addEventListener("visibilitychange", handleVisibilityChange);
$(window).on("load", () => {
initAutoRefresh();
$("body").on("mouseover", ".has-svg-tooltip", (e) => {
const elem = e.target;
const text = elem.getAttribute("title");
const circ = elem.getBoundingClientRect();
showSvgTooltip(text, circ);
});
$("body").on("mouseout", ".has-svg-tooltip", () => {
hideSvgTooltip();
});
getDagStats();
});
$(".js-next-run-tooltip").each((i, run) => {
$(run).on("mouseover", () => {
$(run).attr("data-original-title", () => {
const nextRunData = $(run).attr("data-nextrun");
const [createAfter, intervalStart, intervalEnd] = nextRunData.split(",");
let newTitle = "";
newTitle += `Run After: ${formatDateTime(
createAfter
)}
`;
newTitle += `Next Run: ${approxTimeFromNow(createAfter)}
`;
newTitle += "Data Interval
";
newTitle += `Start: ${formatDateTime(intervalStart)}
`;
newTitle += `End: ${formatDateTime(intervalEnd)}`;
return newTitle;
});
});
});
$("#auto_refresh").change(() => {
if ($("#auto_refresh").is(":checked")) {
// Run an initial refresh before starting interval if manually turned on
handleRefresh({ activeDagsOnly: true });
localStorage.removeItem("dagsDisableAutoRefresh");
} else {
localStorage.setItem("dagsDisableAutoRefresh", "true");
$("#loading-dots").css("display", "none");
}
startOrStopRefresh();
});
$(".next-dataset-triggered").on("click", (e) => {
const dagId = $(e.target).data("dag-id");
const summary = $(e.target).data("summary");
const singleDatasetUri = $(e.target).data("uri");
// If there are multiple datasets, open a modal, otherwise link directly to the dataset
if (!singleDatasetUri) {
if (dagId)
openDatasetModal(dagId, summary, nextDatasets[dagId], nextDatasetsError);
} else {
window.location.href = `${datasetsUrl}?uri=${encodeURIComponent(
singleDatasetUri
)}`;
}
});
const getTooltipInfo = throttle(
(dagId, run, setNextDatasets) =>
getDatasetTooltipInfo(dagId, run, setNextDatasets),
1000
);
$(".js-dataset-triggered").each((i, cell) => {
$(cell).on("mouseover", () => {
const run = $(cell).children();
const dagId = $(run).data("dag-id");
const singleDatasetUri = $(run).data("uri");
const setNextDatasets = (datasets, error) => {
nextDatasets[dagId] = datasets;
nextDatasetsError = error;
};
// Only update the tooltip info if there are multiple datasets
if (!singleDatasetUri) {
getTooltipInfo(dagId, run, setNextDatasets);
}
});
});