Spark History Server metrics mapping

This page describes the mapping between metrics provided by Spark History Server to Akamas metrics for each supported component type

Component Type
Notes

Spark Application

Spark Application

Component metric
Granularity
Document Path
JSON query

spark_duration

job

/{appId}/1/jobs/{jobId}

.duration

spark_completed_tasks

job

/{appId}/1/jobs/{jobId}

.numCompletedTasks

spark_active_tasks

job

/{appId}/1/jobs/{jobId}

.numActiveTasks

spark_skipped_tasks

job

/{appId}/1/jobs/{jobId}

.numSkippedTasks

spark_failed_tasks

job

/{appId}/1/jobs/{jobId}

.numFailedTasks

spark_killed_tasks

job

/{appId}/1/jobs/{jobId}

.numKilledTasks

spark_completed_stages

job

/{appId}/1/jobs/{jobId}

.numCompletedStages

spark_failed_stages

job

/{appId}/1/jobs/{jobId}

.numFailedStages

spark_skipped_stages

job

/{appId}/1/jobs/{jobId}

.numSkippedStages

spark_active_stages

job

/{appId}/1/jobs/{jobId}

.numActiveStages

spark_duration

stage

/{appId}/1/stages/{stageId}

.getDuration

spark_task_stage_executor_run_time

stage

/{appId}/1/stages/{stageId}

.getExecutorRunTime

spark_task_stage_executor_cpu_time

stage

/{appId}/1/stages/{stageId}

.getExecutorCpuTime

spark_active_tasks

stage

/{appId}/1/stages/{stageId}

.getNumActiveTasks

spark_completed_tasks

stage

/{appId}/1/stages/{stageId}

.getNumCompleteTasks

spark_failed_tasks

stage

/{appId}/1/stages/{stageId}

.getNumFailedTasks

spark_killed_tasks

stage

/{appId}/1/stages/{stageId}

.getNumKilledTasks

spark_task_stage_input_bytes_read

stage

/{appId}/1/stages/{stageId}

.getInputBytes

spark_task_stage_input_records_read

stage

/{appId}/1/stages/{stageId}

.getInputRecords

spark_task_stage_output_bytes_written

stage

/{appId}/1/stages/{stageId}

.getOutputBytes

spark_task_stage_output_records_written

stage

/{appId}/1/stages/{stageId}

.getOutputRecords

spark_stage_shuffle_read_bytes

stage

/{appId}/1/stages/{stageId}

.getShuffleReadBytes

spark_task_stage_shuffle_read_records

stage

/{appId}/1/stages/{stageId}

.getShuffleReadRecords

spark_task_stage_shuffle_write_bytes

stage

/{appId}/1/stages/{stageId}

.getShuffleWriteBytes

spark_task_stage_shuffle_write_records

stage

/{appId}/1/stages/{stageId}

.getShuffleWriteRecords

spark_task_stage_memory_bytes_spilled

stage

/{appId}/1/stages/{stageId}

.getMemoryBytesSpilled

spark_task_stage_disk_bytes_spilled

stage

/{appId}/1/stages/{stageId}

.getDiskBytesSpilled

spark_duration

task

/{appId}/1/stages/{stageId}

.tasks[].duration

spark_task_executor_deserialize_time

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.executorDeserializeTime

spark_task_executor_deserialize_cpu_time

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.executorDeserializeCpuTime

spark_task_stage_executor_run_time

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.executorRunTime

spark_task_stage_executor_cpu_time

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.executorCpuTime

spark_task_result_size

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.resultSize

spark_task_jvm_gc_duration

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.jvmGcTime

spark_task_result_serialization_time

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.resultSerializationTime

spark_task_stage_memory_bytes_spilled

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.memoryBytesSpilled

spark_task_stage_disk_bytes_spilled

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.diskBytesSpilled

spark_task_peak_execution_memory

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.peakExecutionMemory

spark_task_stage_input_bytes_read

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.inputMetrics.bytesRead

spark_task_stage_input_records_read

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.inputMetrics.recordsRead

spark_task_stage_output_bytes_written

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.outputMetrics.bytesWritten

spark_task_stage_output_records_written

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.outputMetrics.recordsWritten

spark_task_shuffle_read_remote_blocks_fetched

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.shuffleReadMetrics.remoteBlocksFetched

spark_task_shuffle_read_local_blocks_fetched

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.shuffleReadMetrics.localBlocksFetched

spark_task_shuffle_read_fetch_wait_time

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.shuffleReadMetrics.fetchWaitTime

spark_task_shuffle_read_remote_bytes

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.shuffleReadMetrics.remoteBytesRead

spark_task_shuffle_read_remote_bytes_to_disk

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.shuffleReadMetrics.remoteBytesReadToDisk

spark_task_shuffle_read_local_bytes

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.shuffleReadMetrics.localBytesRead

spark_task_stage_shuffle_read_records

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.shuffleReadMetrics.recordsRead

spark_task_stage_shuffle_write_bytes

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.shuffleWriteMetrics.bytesWritten

spark_task_shuffle_write_time

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.shuffleWriteMetrics.writeTime

spark_task_stage_shuffle_write_records

task

/{appId}/1/stages/{stageId}

.tasks[].taskMetrics.shuffleWriteMetrics.recordsWritten

spark_executor_rdd_blocks

executor

/{appId}/1/allexecutors

select(.id!='driver) | .rddBlocks

spark_executor_mem_used

executor

/{appId}/1/allexecutors

select(.id!='driver) | .memoryUsed

spark_executor_disk_used

executor

/{appId}/1/allexecutors

select(.id!='driver) | .diskUsed

spark_executor_cores

executor

/{appId}/1/allexecutors

select(.id!='driver) | .totalCores

spark_active_tasks

executor

/{appId}/1/allexecutors

select(.id!='driver) | .activeTasks

spark_failed_tasks

executor

/{appId}/1/allexecutors

select(.id!='driver) | .failedTasks

spark_completed_tasks

executor

/{appId}/1/allexecutors

select(.id!='driver) | .completedTasks

spark_executor_total_tasks

executor

/{appId}/1/allexecutors

select(.id!='driver) | .totalTasks

spark_executor_total_duration

executor

/{appId}/1/allexecutors

select(.id!='driver) | .totalDuration

spark_executor_total_jvm_gc_duration

executor

/{appId}/1/allexecutors

select(.id!='driver) | .totalGCTime

spark_executor_total_input_bytes

executor

/{appId}/1/allexecutors

select(.id!='driver) | .totalInputBytes

spark_executor_total_shuffle_read

executor

/{appId}/1/allexecutors

select(.id!='driver) | .totalShuffleRead

spark_executor_total_shuffle_write

executor

/{appId}/1/allexecutors

select(.id!='driver) | .totalShuffleWrite

spark_executor_max_mem_used

executor

/{appId}/1/allexecutors

select(.id!='driver) | .maxMemory

spark_executor_used_on_heap_storage_memory

executor

/{appId}/1/allexecutors

select(.id!='driver) | .memoryMetrics.usedOnHeapStorageMemory

spark_executor_used_off_heap_storage_memory

executor

/{appId}/1/allexecutors

select(.id!='driver) | .memoryMetrics.usedOffHeapStorageMemory

spark_executor_total_on_heap_storage_memory

executor

/{appId}/1/allexecutors

select(.id!='driver) | .memoryMetrics.totalOnHeapStorageMemory

spark_executor_total_off_heap_storage_memory

executor

/{appId}/1/allexecutors

select(.id!='driver) | .memoryMetrics.totalOffHeapStorageMemory

spark_driver_rdd_blocks

driver

/{appId}/1/allexecutors

select(.id=='driver') | .rddBlocks

spark_driver_mem_used

driver

/{appId}/1/allexecutors

select(.id=='driver') | .memoryUsed

spark_driver_disk_used

driver

/{appId}/1/allexecutors

select(.id=='driver') | .diskUsed

spark_driver_cores

driver

/{appId}/1/allexecutors

select(.id=='driver') | .totalCores

spark_driver_total_duration

driver

/{appId}/1/allexecutors

select(.id=='driver') | .totalDuration

spark_driver_total_jvm_gc_duration

driver

/{appId}/1/allexecutors

select(.id=='driver') | .totalGCTime

spark_driver_total_input_bytes

driver

/{appId}/1/allexecutors

select(.id=='driver') | .totalInputBytes

spark_driver_total_shuffle_read

driver

/{appId}/1/allexecutors

select(.id=='driver') | .totalShuffleRead

spark_driver_total_shuffle_write

driver

/{appId}/1/allexecutors

select(.id=='driver') | .totalShuffleWrite

spark_driver_max_mem_used

driver

/{appId}/1/allexecutors

select(.id=='driver') | .maxMemory

spark_driver_used_on_heap_storage_memory

driver

/{appId}/1/allexecutors

select(.id=='driver') | .memoryMetrics.usedOnHeapStorageMemory

spark_driver_used_off_heap_storage_memory

driver

/{appId}/1/allexecutors

select(.id=='driver') | .memoryMetrics.usedOffHeapStorageMemory

spark_driver_total_on_heap_storage_memory

driver

/{appId}/1/allexecutors

select(.id=='driver') | .memoryMetrics.totalOnHeapStorageMemory

spark_driver_total_off_heap_storage_memory

driver

/{appId}/1/allexecutors

select(.id=='driver') | .memoryMetrics.totalOffHeapStorageMemory

Last updated