Spark History Server metrics mapping

Spark Application

spark_duration

job

/{appId}/1/jobs/{jobId}

.duration

spark_completed_tasks

job

/{appId}/1/jobs/{jobId}

.numCompletedTasks

spark_active_tasks

job

/{appId}/1/jobs/{jobId}

.numActiveTasks

spark_skipped_tasks

job

/{appId}/1/jobs/{jobId}

.numSkippedTasks

spark_failed_tasks

job

/{appId}/1/jobs/{jobId}

.numFailedTasks

spark_killed_tasks

job

/{appId}/1/jobs/{jobId}

.numKilledTasks

spark_completed_stages

job

/{appId}/1/jobs/{jobId}

.numCompletedStages

spark_failed_stages

job

/{appId}/1/jobs/{jobId}

.numFailedStages

spark_skipped_stages

job

/{appId}/1/jobs/{jobId}

.numSkippedStages

spark_active_stages

job

/{appId}/1/jobs/{jobId}

.numActiveStages

spark_duration

stage

/{appId}/1/stages/{stageId}

.getDuration

spark_task_stage_executor_run_time

stage

/{appId}/1/stages/{stageId}

.getExecutorRunTime

spark_task_stage_executor_cpu_time

stage

/{appId}/1/stages/{stageId}

.getExecutorCpuTime

spark_active_tasks

stage

/{appId}/1/stages/{stageId}

.getNumActiveTasks

spark_completed_tasks

stage

/{appId}/1/stages/{stageId}

.getNumCompleteTasks

spark_failed_tasks

stage

/{appId}/1/stages/{stageId}

.getNumFailedTasks

spark_killed_tasks

stage

/{appId}/1/stages/{stageId}

.getNumKilledTasks

spark_task_stage_input_bytes_read

stage

/{appId}/1/stages/{stageId}

.getInputBytes

spark_task_stage_input_records_read

stage

/{appId}/1/stages/{stageId}

.getInputRecords

spark_task_stage_output_bytes_written

stage

/{appId}/1/stages/{stageId}

.getOutputBytes

spark_task_stage_output_records_written

stage

/{appId}/1/stages/{stageId}

.getOutputRecords

spark_stage_shuffle_read_bytes

stage

/{appId}/1/stages/{stageId}

.getShuffleReadBytes

spark_task_stage_shuffle_read_records

stage

/{appId}/1/stages/{stageId}

.getShuffleReadRecords

spark_task_stage_shuffle_write_bytes

stage

/{appId}/1/stages/{stageId}

.getShuffleWriteBytes

spark_task_stage_shuffle_write_records

stage

/{appId}/1/stages/{stageId}

.getShuffleWriteRecords

spark_task_stage_memory_bytes_spilled

stage

/{appId}/1/stages/{stageId}

.getMemoryBytesSpilled

spark_task_stage_disk_bytes_spilled

stage

/{appId}/1/stages/{stageId}

.getDiskBytesSpilled

spark_duration

task

/{appId}/1/stages/{stageId}

.tasks[].duration

spark_task_executor_deserialize_time

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.executorDeserializeTime

spark_task_executor_deserialize_cpu_time

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.executorDeserializeCpuTime

spark_task_stage_executor_run_time

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.executorRunTime

spark_task_stage_executor_cpu_time

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.executorCpuTime

spark_task_result_size

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.resultSize

spark_task_jvm_gc_duration

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.jvmGcTime

spark_task_result_serialization_time

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.resultSerializationTime

spark_task_stage_memory_bytes_spilled

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.memoryBytesSpilled

spark_task_stage_disk_bytes_spilled

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.diskBytesSpilled

spark_task_peak_execution_memory

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.peakExecutionMemory

spark_task_stage_input_bytes_read

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.inputMetrics.bytesRead

spark_task_stage_input_records_read

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.inputMetrics.recordsRead

spark_task_stage_output_bytes_written

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.outputMetrics.bytesWritten

spark_task_stage_output_records_written

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.outputMetrics.recordsWritten

spark_task_shuffle_read_remote_blocks_fetched

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.shuffleReadMetrics.remoteBlocksFetched

spark_task_shuffle_read_local_blocks_fetched

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.shuffleReadMetrics.localBlocksFetched

spark_task_shuffle_read_fetch_wait_time

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.shuffleReadMetrics.fetchWaitTime

spark_task_shuffle_read_remote_bytes

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.shuffleReadMetrics.remoteBytesRead

spark_task_shuffle_read_remote_bytes_to_disk

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.shuffleReadMetrics.remoteBytesReadToDisk

spark_task_shuffle_read_local_bytes

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.shuffleReadMetrics.localBytesRead

spark_task_stage_shuffle_read_records

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.shuffleReadMetrics.recordsRead

spark_task_stage_shuffle_write_bytes

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.shuffleWriteMetrics.bytesWritten

spark_task_shuffle_write_time

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.shuffleWriteMetrics.writeTime

spark_task_stage_shuffle_write_records

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.shuffleWriteMetrics.recordsWritten

spark_executor_rdd_blocks

executor

/{appId}/1/allexecutors

select(.id!='driver) | .rddBlocks

spark_executor_mem_used

executor

/{appId}/1/allexecutors

select(.id!='driver) | .memoryUsed

spark_executor_disk_used

executor

/{appId}/1/allexecutors

select(.id!='driver) | .diskUsed

spark_executor_cores

executor

/{appId}/1/allexecutors

select(.id!='driver) | .totalCores

spark_active_tasks

executor

/{appId}/1/allexecutors

select(.id!='driver) | .activeTasks

spark_failed_tasks

executor

/{appId}/1/allexecutors

select(.id!='driver) | .failedTasks

spark_completed_tasks

executor

/{appId}/1/allexecutors

select(.id!='driver) | .completedTasks

spark_executor_total_tasks

executor

/{appId}/1/allexecutors

select(.id!='driver) | .totalTasks

spark_executor_total_duration

executor

/{appId}/1/allexecutors

select(.id!='driver) | .totalDuration

spark_executor_total_jvm_gc_duration

executor

/{appId}/1/allexecutors

select(.id!='driver) | .totalGCTime

spark_executor_total_input_bytes

executor

/{appId}/1/allexecutors

select(.id!='driver) | .totalInputBytes

spark_executor_total_shuffle_read

executor

/{appId}/1/allexecutors

select(.id!='driver) | .totalShuffleRead

spark_executor_total_shuffle_write

executor

/{appId}/1/allexecutors

select(.id!='driver) | .totalShuffleWrite

spark_executor_max_mem_used

executor

/{appId}/1/allexecutors

select(.id!='driver) | .maxMemory

spark_executor_used_on_heap_storage_memory

executor

/{appId}/1/allexecutors

select(.id!='driver) | .memoryMetrics.usedOnHeapStorageMemory

spark_executor_used_off_heap_storage_memory

executor

/{appId}/1/allexecutors

select(.id!='driver) | .memoryMetrics.usedOffHeapStorageMemory

spark_executor_total_on_heap_storage_memory

executor

/{appId}/1/allexecutors

select(.id!='driver) | .memoryMetrics.totalOnHeapStorageMemory

spark_executor_total_off_heap_storage_memory

executor

/{appId}/1/allexecutors

select(.id!='driver) | .memoryMetrics.totalOffHeapStorageMemory

spark_driver_rdd_blocks

driver

/{appId}/1/allexecutors

select(.id=='driver') | .rddBlocks

spark_driver_mem_used

driver

/{appId}/1/allexecutors

select(.id=='driver') | .memoryUsed

spark_driver_disk_used

driver

/{appId}/1/allexecutors

select(.id=='driver') | .diskUsed

spark_driver_cores

driver

/{appId}/1/allexecutors

select(.id=='driver') | .totalCores

spark_driver_total_duration

driver

/{appId}/1/allexecutors

select(.id=='driver') | .totalDuration

spark_driver_total_jvm_gc_duration

driver

/{appId}/1/allexecutors

select(.id=='driver') | .totalGCTime

spark_driver_total_input_bytes

driver

/{appId}/1/allexecutors

select(.id=='driver') | .totalInputBytes

spark_driver_total_shuffle_read

driver

/{appId}/1/allexecutors

select(.id=='driver') | .totalShuffleRead

spark_driver_total_shuffle_write

driver

/{appId}/1/allexecutors

select(.id=='driver') | .totalShuffleWrite

spark_driver_max_mem_used

driver

/{appId}/1/allexecutors

select(.id=='driver') | .maxMemory

spark_driver_used_on_heap_storage_memory

driver

/{appId}/1/allexecutors

select(.id=='driver') | .memoryMetrics.usedOnHeapStorageMemory

spark_driver_used_off_heap_storage_memory

driver

/{appId}/1/allexecutors

select(.id=='driver') | .memoryMetrics.usedOffHeapStorageMemory

spark_driver_total_on_heap_storage_memory

driver

/{appId}/1/allexecutors

select(.id=='driver') | .memoryMetrics.totalOnHeapStorageMemory

spark_driver_total_off_heap_storage_memory

driver

/{appId}/1/allexecutors

select(.id=='driver') | .memoryMetrics.totalOffHeapStorageMemory