Akamas Docs
3.3.0
Ask or search…
K
Links
Comment on page

Spark History Server metrics mapping

This page describes the mapping between metrics provided by Spark History Server to Akamas metrics for each supported component type
Component Type
Notes

Spark Application

Component metric
Granularity
Document Path
JSON query
spark_duration
job
/{appId}/1/jobs/{jobId}
.duration
spark_completed_tasks
job
/{appId}/1/jobs/{jobId}
.numCompletedTasks
spark_active_tasks
job
/{appId}/1/jobs/{jobId}
.numActiveTasks
spark_skipped_tasks
job
/{appId}/1/jobs/{jobId}
.numSkippedTasks
spark_failed_tasks
job
/{appId}/1/jobs/{jobId}
.numFailedTasks
spark_killed_tasks
job
/{appId}/1/jobs/{jobId}
.numKilledTasks
spark_completed_stages
job
/{appId}/1/jobs/{jobId}
.numCompletedStages
spark_failed_stages
job
/{appId}/1/jobs/{jobId}
.numFailedStages
spark_skipped_stages
job
/{appId}/1/jobs/{jobId}
.numSkippedStages
spark_active_stages
job
/{appId}/1/jobs/{jobId}
.numActiveStages
spark_duration
stage
/{appId}/1/stages/{stageId}
.getDuration
spark_task_stage_executor_run_time
stage
/{appId}/1/stages/{stageId}
.getExecutorRunTime
spark_task_stage_executor_cpu_time
stage
/{appId}/1/stages/{stageId}
.getExecutorCpuTime
spark_active_tasks
stage
/{appId}/1/stages/{stageId}
.getNumActiveTasks
spark_completed_tasks
stage
/{appId}/1/stages/{stageId}
.getNumCompleteTasks
spark_failed_tasks
stage
/{appId}/1/stages/{stageId}
.getNumFailedTasks
spark_killed_tasks
stage
/{appId}/1/stages/{stageId}
.getNumKilledTasks
spark_task_stage_input_bytes_read
stage
/{appId}/1/stages/{stageId}
.getInputBytes
spark_task_stage_input_records_read
stage
/{appId}/1/stages/{stageId}
.getInputRecords
spark_task_stage_output_bytes_written
stage
/{appId}/1/stages/{stageId}
.getOutputBytes
spark_task_stage_output_records_written
stage
/{appId}/1/stages/{stageId}
.getOutputRecords
spark_stage_shuffle_read_bytes
stage
/{appId}/1/stages/{stageId}
.getShuffleReadBytes
spark_task_stage_shuffle_read_records
stage
/{appId}/1/stages/{stageId}
.getShuffleReadRecords
spark_task_stage_shuffle_write_bytes
stage
/{appId}/1/stages/{stageId}
.getShuffleWriteBytes
spark_task_stage_shuffle_write_records
stage
/{appId}/1/stages/{stageId}
.getShuffleWriteRecords
spark_task_stage_memory_bytes_spilled
stage
/{appId}/1/stages/{stageId}
.getMemoryBytesSpilled
spark_task_stage_disk_bytes_spilled
stage
/{appId}/1/stages/{stageId}
.getDiskBytesSpilled
spark_duration
task
/{appId}/1/stages/{stageId}
.tasks[].duration
spark_task_executor_deserialize_time
task
/{appId}/1/stages/{stageId}
.tasks[].taskMetrics.executorDeserializeTime
spark_task_executor_deserialize_cpu_time
task
/{appId}/1/stages/{stageId}
.tasks[].taskMetrics.executorDeserializeCpuTime
spark_task_stage_executor_run_time
task
/{appId}/1/stages/{stageId}
.tasks[].taskMetrics.executorRunTime
spark_task_stage_executor_cpu_time
task
/{appId}/1/stages/{stageId}
.tasks[].taskMetrics.executorCpuTime
spark_task_result_size
task
/{appId}/1/stages/{stageId}
.tasks[].taskMetrics.resultSize
spark_task_jvm_gc_duration
task
/{appId}/1/stages/{stageId}
.tasks[].taskMetrics.jvmGcTime
spark_task_result_serialization_time
task
/{appId}/1/stages/{stageId}
.tasks[].taskMetrics.resultSerializationTime
spark_task_stage_memory_bytes_spilled
task
/{appId}/1/stages/{stageId}
.tasks[].taskMetrics.memoryBytesSpilled
spark_task_stage_disk_bytes_spilled
task
/{appId}/1/stages/{stageId}
.tasks[].taskMetrics.diskBytesSpilled
spark_task_peak_execution_memory
task
/{appId}/1/stages/{stageId}
.tasks[].taskMetrics.peakExecutionMemory
spark_task_stage_input_bytes_read
task
/{appId}/1/stages/{stageId}
.tasks[].taskMetrics.inputMetrics.bytesRead
spark_task_stage_input_records_read
task
/{appId}/1/stages/{stageId}
.tasks[].taskMetrics.inputMetrics.recordsRead
spark_task_stage_output_bytes_written
task
/{appId}/1/stages/{stageId}
.tasks[].taskMetrics.outputMetrics.bytesWritten
spark_task_stage_output_records_written
task
/{appId}/1/stages/{stageId}
.tasks[].taskMetrics.outputMetrics.recordsWritten
spark_task_shuffle_read_remote_blocks_fetched
task
/{appId}/1/stages/{stageId}
.tasks[].taskMetrics.shuffleReadMetrics.remoteBlocksFetched
spark_task_shuffle_read_local_blocks_fetched
task
/{appId}/1/stages/{stageId}
.tasks[].taskMetrics.shuffleReadMetrics.localBlocksFetched
spark_task_shuffle_read_fetch_wait_time
task
/{appId}/1/stages/{stageId}
.tasks[].taskMetrics.shuffleReadMetrics.fetchWaitTime
spark_task_shuffle_read_remote_bytes
task
/{appId}/1/stages/{stageId}
.tasks[].taskMetrics.shuffleReadMetrics.remoteBytesRead
spark_task_shuffle_read_remote_bytes_to_disk
task
/{appId}/1/stages/{stageId}
.tasks[].taskMetrics.shuffleReadMetrics.remoteBytesReadToDisk
spark_task_shuffle_read_local_bytes
task
/{appId}/1/stages/{stageId}
.tasks[].taskMetrics.shuffleReadMetrics.localBytesRead
spark_task_stage_shuffle_read_records
task
/{appId}/1/stages/{stageId}
.tasks[].taskMetrics.shuffleReadMetrics.recordsRead
spark_task_stage_shuffle_write_bytes
task
/{appId}/1/stages/{stageId}
.tasks[].taskMetrics.shuffleWriteMetrics.bytesWritten
spark_task_shuffle_write_time
task
/{appId}/1/stages/{stageId}
.tasks[].taskMetrics.shuffleWriteMetrics.writeTime
spark_task_stage_shuffle_write_records
task
/{appId}/1/stages/{stageId}
.tasks[].taskMetrics.shuffleWriteMetrics.recordsWritten
spark_executor_rdd_blocks
executor
/{appId}/1/allexecutors
select(.id!='driver) | .rddBlocks
spark_executor_mem_used
executor
/{appId}/1/allexecutors
select(.id!='driver) | .memoryUsed
spark_executor_disk_used
executor
/{appId}/1/allexecutors
select(.id!='driver) | .diskUsed
spark_executor_cores
executor
/{appId}/1/allexecutors
select(.id!='driver) | .totalCores
spark_active_tasks
executor
/{appId}/1/allexecutors
select(.id!='driver) | .activeTasks
spark_failed_tasks
executor
/{appId}/1/allexecutors
select(.id!='driver) | .failedTasks
spark_completed_tasks
executor
/{appId}/1/allexecutors
select(.id!='driver) | .completedTasks
spark_executor_total_tasks
executor
/{appId}/1/allexecutors
select(.id!='driver) | .totalTasks
spark_executor_total_duration
executor
/{appId}/1/allexecutors
select(.id!='driver) | .totalDuration
spark_executor_total_jvm_gc_duration
executor
/{appId}/1/allexecutors
select(.id!='driver) | .totalGCTime
spark_executor_total_input_bytes
executor
/{appId}/1/allexecutors
select(.id!='driver) | .totalInputBytes
spark_executor_total_shuffle_read
executor
/{appId}/1/allexecutors
select(.id!='driver) | .totalShuffleRead
spark_executor_total_shuffle_write
executor
/{appId}/1/allexecutors
select(.id!='driver) | .totalShuffleWrite
spark_executor_max_mem_used
executor
/{appId}/1/allexecutors
select(.id!='driver) | .maxMemory
spark_executor_used_on_heap_storage_memory
executor
/{appId}/1/allexecutors
select(.id!='driver) | .memoryMetrics.usedOnHeapStorageMemory
spark_executor_used_off_heap_storage_memory
executor
/{appId}/1/allexecutors
select(.id!='driver) | .memoryMetrics.usedOffHeapStorageMemory
spark_executor_total_on_heap_storage_memory
executor
/{appId}/1/allexecutors
select(.id!='driver) | .memoryMetrics.totalOnHeapStorageMemory
spark_executor_total_off_heap_storage_memory
executor
/{appId}/1/allexecutors
select(.id!='driver) | .memoryMetrics.totalOffHeapStorageMemory
spark_driver_rdd_blocks
driver
/{appId}/1/allexecutors
select(.id=='driver') | .rddBlocks
spark_driver_mem_used
driver
/{appId}/1/allexecutors
select(.id=='driver') | .memoryUsed
spark_driver_disk_used
driver
/{appId}/1/allexecutors
select(.id=='driver') | .diskUsed
spark_driver_cores
driver
/{appId}/1/allexecutors
select(.id=='driver') | .totalCores
spark_driver_total_duration
driver
/{appId}/1/allexecutors
select(.id=='driver') | .totalDuration
spark_driver_total_jvm_gc_duration
driver
/{appId}/1/allexecutors
select(.id=='driver') | .totalGCTime
spark_driver_total_input_bytes
driver
/{appId}/1/allexecutors
select(.id=='driver') | .totalInputBytes
spark_driver_total_shuffle_read
driver
/{appId}/1/allexecutors
select(.id=='driver') | .totalShuffleRead
spark_driver_total_shuffle_write
driver
/{appId}/1/allexecutors
select(.id=='driver') | .totalShuffleWrite
spark_driver_max_mem_used
driver
/{appId}/1/allexecutors
select(.id=='driver') | .maxMemory
spark_driver_used_on_heap_storage_memory
driver
/{appId}/1/allexecutors
select(.id=='driver') | .memoryMetrics.usedOnHeapStorageMemory
spark_driver_used_off_heap_storage_memory
driver
/{appId}/1/allexecutors
select(.id=='driver') | .memoryMetrics.usedOffHeapStorageMemory
spark_driver_total_on_heap_storage_memory
driver
/{appId}/1/allexecutors
select(.id=='driver') | .memoryMetrics.totalOnHeapStorageMemory
spark_driver_total_off_heap_storage_memory
driver
/{appId}/1/allexecutors
select(.id=='driver') | .memoryMetrics.totalOffHeapStorageMemory