Spark History Server metrics mapping

This page describes the mapping between metrics provided by Spark History Server to Akamas metrics for each supported component type

Component Type	Notes
Spark Application

Spark Application

Component metric	Granularity	Document Path	JSON query
spark_duration	job	/{appId}/1/jobs/{jobId}	.duration
spark_completed_tasks	job	/{appId}/1/jobs/{jobId}	.numCompletedTasks
spark_active_tasks	job	/{appId}/1/jobs/{jobId}	.numActiveTasks
spark_skipped_tasks	job	/{appId}/1/jobs/{jobId}	.numSkippedTasks
spark_failed_tasks	job	/{appId}/1/jobs/{jobId}	.numFailedTasks
spark_killed_tasks	job	/{appId}/1/jobs/{jobId}	.numKilledTasks
spark_completed_stages	job	/{appId}/1/jobs/{jobId}	.numCompletedStages
spark_failed_stages	job	/{appId}/1/jobs/{jobId}	.numFailedStages
spark_skipped_stages	job	/{appId}/1/jobs/{jobId}	.numSkippedStages
spark_active_stages	job	/{appId}/1/jobs/{jobId}	.numActiveStages
spark_duration	stage	/{appId}/1/stages/{stageId}	.getDuration
spark_task_stage_executor_run_time	stage	/{appId}/1/stages/{stageId}	.getExecutorRunTime
spark_task_stage_executor_cpu_time	stage	/{appId}/1/stages/{stageId}	.getExecutorCpuTime
spark_active_tasks	stage	/{appId}/1/stages/{stageId}	.getNumActiveTasks
spark_completed_tasks	stage	/{appId}/1/stages/{stageId}	.getNumCompleteTasks
spark_failed_tasks	stage	/{appId}/1/stages/{stageId}	.getNumFailedTasks
spark_killed_tasks	stage	/{appId}/1/stages/{stageId}	.getNumKilledTasks
spark_task_stage_input_bytes_read	stage	/{appId}/1/stages/{stageId}	.getInputBytes
spark_task_stage_input_records_read	stage	/{appId}/1/stages/{stageId}	.getInputRecords
spark_task_stage_output_bytes_written	stage	/{appId}/1/stages/{stageId}	.getOutputBytes
spark_task_stage_output_records_written	stage	/{appId}/1/stages/{stageId}	.getOutputRecords
spark_stage_shuffle_read_bytes	stage	/{appId}/1/stages/{stageId}	.getShuffleReadBytes
spark_task_stage_shuffle_read_records	stage	/{appId}/1/stages/{stageId}	.getShuffleReadRecords
spark_task_stage_shuffle_write_bytes	stage	/{appId}/1/stages/{stageId}	.getShuffleWriteBytes
spark_task_stage_shuffle_write_records	stage	/{appId}/1/stages/{stageId}	.getShuffleWriteRecords
spark_task_stage_memory_bytes_spilled	stage	/{appId}/1/stages/{stageId}	.getMemoryBytesSpilled
spark_task_stage_disk_bytes_spilled	stage	/{appId}/1/stages/{stageId}	.getDiskBytesSpilled
spark_duration	task	/{appId}/1/stages/{stageId}	.tasks[].duration
spark_task_executor_deserialize_time	task	/{appId}/1/stages/{stageId}	.tasks[].taskMetrics.executorDeserializeTime
spark_task_executor_deserialize_cpu_time	task	/{appId}/1/stages/{stageId}	.tasks[].taskMetrics.executorDeserializeCpuTime
spark_task_stage_executor_run_time	task	/{appId}/1/stages/{stageId}	.tasks[].taskMetrics.executorRunTime
spark_task_stage_executor_cpu_time	task	/{appId}/1/stages/{stageId}	.tasks[].taskMetrics.executorCpuTime
spark_task_result_size	task	/{appId}/1/stages/{stageId}	.tasks[].taskMetrics.resultSize
spark_task_jvm_gc_duration	task	/{appId}/1/stages/{stageId}	.tasks[].taskMetrics.jvmGcTime
spark_task_result_serialization_time	task	/{appId}/1/stages/{stageId}	.tasks[].taskMetrics.resultSerializationTime
spark_task_stage_memory_bytes_spilled	task	/{appId}/1/stages/{stageId}	.tasks[].taskMetrics.memoryBytesSpilled
spark_task_stage_disk_bytes_spilled	task	/{appId}/1/stages/{stageId}	.tasks[].taskMetrics.diskBytesSpilled
spark_task_peak_execution_memory	task	/{appId}/1/stages/{stageId}	.tasks[].taskMetrics.peakExecutionMemory
spark_task_stage_input_bytes_read	task	/{appId}/1/stages/{stageId}	.tasks[].taskMetrics.inputMetrics.bytesRead
spark_task_stage_input_records_read	task	/{appId}/1/stages/{stageId}	.tasks[].taskMetrics.inputMetrics.recordsRead
spark_task_stage_output_bytes_written	task	/{appId}/1/stages/{stageId}	.tasks[].taskMetrics.outputMetrics.bytesWritten
spark_task_stage_output_records_written	task	/{appId}/1/stages/{stageId}	.tasks[].taskMetrics.outputMetrics.recordsWritten
spark_task_shuffle_read_remote_blocks_fetched	task	/{appId}/1/stages/{stageId}	.tasks[].taskMetrics.shuffleReadMetrics.remoteBlocksFetched
spark_task_shuffle_read_local_blocks_fetched	task	/{appId}/1/stages/{stageId}	.tasks[].taskMetrics.shuffleReadMetrics.localBlocksFetched
spark_task_shuffle_read_fetch_wait_time	task	/{appId}/1/stages/{stageId}	.tasks[].taskMetrics.shuffleReadMetrics.fetchWaitTime
spark_task_shuffle_read_remote_bytes	task	/{appId}/1/stages/{stageId}	.tasks[].taskMetrics.shuffleReadMetrics.remoteBytesRead
spark_task_shuffle_read_remote_bytes_to_disk	task	/{appId}/1/stages/{stageId}	.tasks[].taskMetrics.shuffleReadMetrics.remoteBytesReadToDisk
spark_task_shuffle_read_local_bytes	task	/{appId}/1/stages/{stageId}	.tasks[].taskMetrics.shuffleReadMetrics.localBytesRead
spark_task_stage_shuffle_read_records	task	/{appId}/1/stages/{stageId}	.tasks[].taskMetrics.shuffleReadMetrics.recordsRead
spark_task_stage_shuffle_write_bytes	task	/{appId}/1/stages/{stageId}	.tasks[].taskMetrics.shuffleWriteMetrics.bytesWritten
spark_task_shuffle_write_time	task	/{appId}/1/stages/{stageId}	.tasks[].taskMetrics.shuffleWriteMetrics.writeTime
spark_task_stage_shuffle_write_records	task	/{appId}/1/stages/{stageId}	.tasks[].taskMetrics.shuffleWriteMetrics.recordsWritten
spark_executor_rdd_blocks	executor	/{appId}/1/allexecutors	select(.id!='driver) | .rddBlocks
spark_executor_mem_used	executor	/{appId}/1/allexecutors	select(.id!='driver) | .memoryUsed
spark_executor_disk_used	executor	/{appId}/1/allexecutors	select(.id!='driver) | .diskUsed
spark_executor_cores	executor	/{appId}/1/allexecutors	select(.id!='driver) | .totalCores
spark_active_tasks	executor	/{appId}/1/allexecutors	select(.id!='driver) | .activeTasks
spark_failed_tasks	executor	/{appId}/1/allexecutors	select(.id!='driver) | .failedTasks
spark_completed_tasks	executor	/{appId}/1/allexecutors	select(.id!='driver) | .completedTasks
spark_executor_total_tasks	executor	/{appId}/1/allexecutors	select(.id!='driver) | .totalTasks
spark_executor_total_duration	executor	/{appId}/1/allexecutors	select(.id!='driver) | .totalDuration
spark_executor_total_jvm_gc_duration	executor	/{appId}/1/allexecutors	select(.id!='driver) | .totalGCTime
spark_executor_total_input_bytes	executor	/{appId}/1/allexecutors	select(.id!='driver) | .totalInputBytes
spark_executor_total_shuffle_read	executor	/{appId}/1/allexecutors	select(.id!='driver) | .totalShuffleRead
spark_executor_total_shuffle_write	executor	/{appId}/1/allexecutors	select(.id!='driver) | .totalShuffleWrite
spark_executor_max_mem_used	executor	/{appId}/1/allexecutors	select(.id!='driver) | .maxMemory
spark_executor_used_on_heap_storage_memory	executor	/{appId}/1/allexecutors	select(.id!='driver) | .memoryMetrics.usedOnHeapStorageMemory
spark_executor_used_off_heap_storage_memory	executor	/{appId}/1/allexecutors	select(.id!='driver) | .memoryMetrics.usedOffHeapStorageMemory
spark_executor_total_on_heap_storage_memory	executor	/{appId}/1/allexecutors	select(.id!='driver) | .memoryMetrics.totalOnHeapStorageMemory
spark_executor_total_off_heap_storage_memory	executor	/{appId}/1/allexecutors	select(.id!='driver) | .memoryMetrics.totalOffHeapStorageMemory
spark_driver_rdd_blocks	driver	/{appId}/1/allexecutors	select(.id=='driver') | .rddBlocks
spark_driver_mem_used	driver	/{appId}/1/allexecutors	select(.id=='driver') | .memoryUsed
spark_driver_disk_used	driver	/{appId}/1/allexecutors	select(.id=='driver') | .diskUsed
spark_driver_cores	driver	/{appId}/1/allexecutors	select(.id=='driver') | .totalCores
spark_driver_total_duration	driver	/{appId}/1/allexecutors	select(.id=='driver') | .totalDuration
spark_driver_total_jvm_gc_duration	driver	/{appId}/1/allexecutors	select(.id=='driver') | .totalGCTime
spark_driver_total_input_bytes	driver	/{appId}/1/allexecutors	select(.id=='driver') | .totalInputBytes
spark_driver_total_shuffle_read	driver	/{appId}/1/allexecutors	select(.id=='driver') | .totalShuffleRead
spark_driver_total_shuffle_write	driver	/{appId}/1/allexecutors	select(.id=='driver') | .totalShuffleWrite
spark_driver_max_mem_used	driver	/{appId}/1/allexecutors	select(.id=='driver') | .maxMemory
spark_driver_used_on_heap_storage_memory	driver	/{appId}/1/allexecutors	select(.id=='driver') | .memoryMetrics.usedOnHeapStorageMemory
spark_driver_used_off_heap_storage_memory	driver	/{appId}/1/allexecutors	select(.id=='driver') | .memoryMetrics.usedOffHeapStorageMemory
spark_driver_total_on_heap_storage_memory	driver	/{appId}/1/allexecutors	select(.id=='driver') | .memoryMetrics.totalOnHeapStorageMemory
spark_driver_total_off_heap_storage_memory	driver	/{appId}/1/allexecutors	select(.id=='driver') | .memoryMetrics.totalOffHeapStorageMemory