# Spark History Server metrics mapping

This page describes the mapping between metrics provided by Spark History Server to Akamas metrics for each supported component type

| Component Type                                          | Notes |
| ------------------------------------------------------- | ----- |
| [Spark Application](#spark-application-component-types) |       |

## Spark Application <a href="#spark-application-component-types" id="spark-application-component-types"></a>

<table><thead><tr><th width="244">Component metric</th><th>Granularity</th><th width="262">Document Path</th><th width="346">JSON query</th></tr></thead><tbody><tr><td>spark_duration</td><td>job</td><td>/{appId}/1/jobs/{jobId}</td><td><code>.duration</code></td></tr><tr><td>spark_completed_tasks</td><td>job</td><td>/{appId}/1/jobs/{jobId}</td><td><code>.numCompletedTasks</code></td></tr><tr><td>spark_active_tasks</td><td>job</td><td>/{appId}/1/jobs/{jobId}</td><td><code>.numActiveTasks</code></td></tr><tr><td>spark_skipped_tasks</td><td>job</td><td>/{appId}/1/jobs/{jobId}</td><td><code>.numSkippedTasks</code></td></tr><tr><td>spark_failed_tasks</td><td>job</td><td>/{appId}/1/jobs/{jobId}</td><td><code>.numFailedTasks</code></td></tr><tr><td>spark_killed_tasks</td><td>job</td><td>/{appId}/1/jobs/{jobId}</td><td><code>.numKilledTasks</code></td></tr><tr><td>spark_completed_stages</td><td>job</td><td>/{appId}/1/jobs/{jobId}</td><td><code>.numCompletedStages</code></td></tr><tr><td>spark_failed_stages</td><td>job</td><td>/{appId}/1/jobs/{jobId}</td><td><code>.numFailedStages</code></td></tr><tr><td>spark_skipped_stages</td><td>job</td><td>/{appId}/1/jobs/{jobId}</td><td><code>.numSkippedStages</code></td></tr><tr><td>spark_active_stages</td><td>job</td><td>/{appId}/1/jobs/{jobId}</td><td><code>.numActiveStages</code></td></tr><tr><td>spark_duration</td><td>stage</td><td>/{appId}/1/stages/{stageId}</td><td><code>.getDuration</code></td></tr><tr><td>spark_task_stage_executor_run_time</td><td>stage</td><td>/{appId}/1/stages/{stageId}</td><td><code>.getExecutorRunTime</code></td></tr><tr><td>spark_task_stage_executor_cpu_time</td><td>stage</td><td>/{appId}/1/stages/{stageId}</td><td><code>.getExecutorCpuTime</code></td></tr><tr><td>spark_active_tasks</td><td>stage</td><td>/{appId}/1/stages/{stageId}</td><td><code>.getNumActiveTasks</code></td></tr><tr><td>spark_completed_tasks</td><td>stage</td><td>/{appId}/1/stages/{stageId}</td><td><code>.getNumCompleteTasks</code></td></tr><tr><td>spark_failed_tasks</td><td>stage</td><td>/{appId}/1/stages/{stageId}</td><td><code>.getNumFailedTasks</code></td></tr><tr><td>spark_killed_tasks</td><td>stage</td><td>/{appId}/1/stages/{stageId}</td><td><code>.getNumKilledTasks</code></td></tr><tr><td>spark_task_stage_input_bytes_read</td><td>stage</td><td>/{appId}/1/stages/{stageId}</td><td><code>.getInputBytes</code></td></tr><tr><td>spark_task_stage_input_records_read</td><td>stage</td><td>/{appId}/1/stages/{stageId}</td><td><code>.getInputRecords</code></td></tr><tr><td>spark_task_stage_output_bytes_written</td><td>stage</td><td>/{appId}/1/stages/{stageId}</td><td><code>.getOutputBytes</code></td></tr><tr><td>spark_task_stage_output_records_written</td><td>stage</td><td>/{appId}/1/stages/{stageId}</td><td><code>.getOutputRecords</code></td></tr><tr><td>spark_stage_shuffle_read_bytes</td><td>stage</td><td>/{appId}/1/stages/{stageId}</td><td><code>.getShuffleReadBytes</code></td></tr><tr><td>spark_task_stage_shuffle_read_records</td><td>stage</td><td>/{appId}/1/stages/{stageId}</td><td><code>.getShuffleReadRecords</code></td></tr><tr><td>spark_task_stage_shuffle_write_bytes</td><td>stage</td><td>/{appId}/1/stages/{stageId}</td><td><code>.getShuffleWriteBytes</code></td></tr><tr><td>spark_task_stage_shuffle_write_records</td><td>stage</td><td>/{appId}/1/stages/{stageId}</td><td><code>.getShuffleWriteRecords</code></td></tr><tr><td>spark_task_stage_memory_bytes_spilled</td><td>stage</td><td>/{appId}/1/stages/{stageId}</td><td><code>.getMemoryBytesSpilled</code></td></tr><tr><td>spark_task_stage_disk_bytes_spilled</td><td>stage</td><td>/{appId}/1/stages/{stageId}</td><td><code>.getDiskBytesSpilled</code></td></tr><tr><td>spark_duration</td><td>task</td><td>/{appId}/1/stages/{stageId}</td><td><code>.tasks[].duration</code></td></tr><tr><td>spark_task_executor_deserialize_time</td><td>task</td><td>/{appId}/1/stages/{stageId}</td><td><code>.tasks[].taskMetrics.executorDeserializeTime</code></td></tr><tr><td>spark_task_executor_deserialize_cpu_time</td><td>task</td><td>/{appId}/1/stages/{stageId}</td><td><code>.tasks[].taskMetrics.executorDeserializeCpuTime</code></td></tr><tr><td>spark_task_stage_executor_run_time</td><td>task</td><td>/{appId}/1/stages/{stageId}</td><td><code>.tasks[].taskMetrics.executorRunTime</code></td></tr><tr><td>spark_task_stage_executor_cpu_time</td><td>task</td><td>/{appId}/1/stages/{stageId}</td><td><code>.tasks[].taskMetrics.executorCpuTime</code></td></tr><tr><td>spark_task_result_size</td><td>task</td><td>/{appId}/1/stages/{stageId}</td><td><code>.tasks[].taskMetrics.resultSize</code></td></tr><tr><td>spark_task_jvm_gc_duration</td><td>task</td><td>/{appId}/1/stages/{stageId}</td><td><code>.tasks[].taskMetrics.jvmGcTime</code></td></tr><tr><td>spark_task_result_serialization_time</td><td>task</td><td>/{appId}/1/stages/{stageId}</td><td><code>.tasks[].taskMetrics.resultSerializationTime</code></td></tr><tr><td>spark_task_stage_memory_bytes_spilled</td><td>task</td><td>/{appId}/1/stages/{stageId}</td><td><code>.tasks[].taskMetrics.memoryBytesSpilled</code></td></tr><tr><td>spark_task_stage_disk_bytes_spilled</td><td>task</td><td>/{appId}/1/stages/{stageId}</td><td><code>.tasks[].taskMetrics.diskBytesSpilled</code></td></tr><tr><td>spark_task_peak_execution_memory</td><td>task</td><td>/{appId}/1/stages/{stageId}</td><td><code>.tasks[].taskMetrics.peakExecutionMemory</code></td></tr><tr><td>spark_task_stage_input_bytes_read</td><td>task</td><td>/{appId}/1/stages/{stageId}</td><td><code>.tasks[].taskMetrics.inputMetrics.bytesRead</code></td></tr><tr><td>spark_task_stage_input_records_read</td><td>task</td><td>/{appId}/1/stages/{stageId}</td><td><code>.tasks[].taskMetrics.inputMetrics.recordsRead</code></td></tr><tr><td>spark_task_stage_output_bytes_written</td><td>task</td><td>/{appId}/1/stages/{stageId}</td><td><code>.tasks[].taskMetrics.outputMetrics.bytesWritten</code></td></tr><tr><td>spark_task_stage_output_records_written</td><td>task</td><td>/{appId}/1/stages/{stageId}</td><td><code>.tasks[].taskMetrics.outputMetrics.recordsWritten</code></td></tr><tr><td>spark_task_shuffle_read_remote_blocks_fetched</td><td>task</td><td>/{appId}/1/stages/{stageId}</td><td><code>.tasks[].taskMetrics.shuffleReadMetrics.remoteBlocksFetched</code></td></tr><tr><td>spark_task_shuffle_read_local_blocks_fetched</td><td>task</td><td>/{appId}/1/stages/{stageId}</td><td><code>.tasks[].taskMetrics.shuffleReadMetrics.localBlocksFetched</code></td></tr><tr><td>spark_task_shuffle_read_fetch_wait_time</td><td>task</td><td>/{appId}/1/stages/{stageId}</td><td><code>.tasks[].taskMetrics.shuffleReadMetrics.fetchWaitTime</code></td></tr><tr><td>spark_task_shuffle_read_remote_bytes</td><td>task</td><td>/{appId}/1/stages/{stageId}</td><td><code>.tasks[].taskMetrics.shuffleReadMetrics.remoteBytesRead</code></td></tr><tr><td>spark_task_shuffle_read_remote_bytes_to_disk</td><td>task</td><td>/{appId}/1/stages/{stageId}</td><td><code>.tasks[].taskMetrics.shuffleReadMetrics.remoteBytesReadToDisk</code></td></tr><tr><td>spark_task_shuffle_read_local_bytes</td><td>task</td><td>/{appId}/1/stages/{stageId}</td><td><code>.tasks[].taskMetrics.shuffleReadMetrics.localBytesRead</code></td></tr><tr><td>spark_task_stage_shuffle_read_records</td><td>task</td><td>/{appId}/1/stages/{stageId}</td><td><code>.tasks[].taskMetrics.shuffleReadMetrics.recordsRead</code></td></tr><tr><td>spark_task_stage_shuffle_write_bytes</td><td>task</td><td>/{appId}/1/stages/{stageId}</td><td><code>.tasks[].taskMetrics.shuffleWriteMetrics.bytesWritten</code></td></tr><tr><td>spark_task_shuffle_write_time</td><td>task</td><td>/{appId}/1/stages/{stageId}</td><td><code>.tasks[].taskMetrics.shuffleWriteMetrics.writeTime</code></td></tr><tr><td>spark_task_stage_shuffle_write_records</td><td>task</td><td>/{appId}/1/stages/{stageId}</td><td><code>.tasks[].taskMetrics.shuffleWriteMetrics.recordsWritten</code></td></tr><tr><td>spark_executor_rdd_blocks</td><td>executor</td><td>/{appId}/1/allexecutors</td><td><code>select(.id!='driver) | .rddBlocks</code></td></tr><tr><td>spark_executor_mem_used</td><td>executor</td><td>/{appId}/1/allexecutors</td><td><code>select(.id!='driver) | .memoryUsed</code></td></tr><tr><td>spark_executor_disk_used</td><td>executor</td><td>/{appId}/1/allexecutors</td><td><code>select(.id!='driver) | .diskUsed</code></td></tr><tr><td>spark_executor_cores</td><td>executor</td><td>/{appId}/1/allexecutors</td><td><code>select(.id!='driver) | .totalCores</code></td></tr><tr><td>spark_active_tasks</td><td>executor</td><td>/{appId}/1/allexecutors</td><td><code>select(.id!='driver) | .activeTasks</code></td></tr><tr><td>spark_failed_tasks</td><td>executor</td><td>/{appId}/1/allexecutors</td><td><code>select(.id!='driver) | .failedTasks</code></td></tr><tr><td>spark_completed_tasks</td><td>executor</td><td>/{appId}/1/allexecutors</td><td><code>select(.id!='driver) | .completedTasks</code></td></tr><tr><td>spark_executor_total_tasks</td><td>executor</td><td>/{appId}/1/allexecutors</td><td><code>select(.id!='driver) | .totalTasks</code></td></tr><tr><td>spark_executor_total_duration</td><td>executor</td><td>/{appId}/1/allexecutors</td><td><code>select(.id!='driver) | .totalDuration</code></td></tr><tr><td>spark_executor_total_jvm_gc_duration</td><td>executor</td><td>/{appId}/1/allexecutors</td><td><code>select(.id!='driver) | .totalGCTime</code></td></tr><tr><td>spark_executor_total_input_bytes</td><td>executor</td><td>/{appId}/1/allexecutors</td><td><code>select(.id!='driver) | .totalInputBytes</code></td></tr><tr><td>spark_executor_total_shuffle_read</td><td>executor</td><td>/{appId}/1/allexecutors</td><td><code>select(.id!='driver) | .totalShuffleRead</code></td></tr><tr><td>spark_executor_total_shuffle_write</td><td>executor</td><td>/{appId}/1/allexecutors</td><td><code>select(.id!='driver) | .totalShuffleWrite</code></td></tr><tr><td>spark_executor_max_mem_used</td><td>executor</td><td>/{appId}/1/allexecutors</td><td><code>select(.id!='driver) | .maxMemory</code></td></tr><tr><td>spark_executor_used_on_heap_storage_memory</td><td>executor</td><td>/{appId}/1/allexecutors</td><td><code>select(.id!='driver) | .memoryMetrics.usedOnHeapStorageMemory</code></td></tr><tr><td>spark_executor_used_off_heap_storage_memory</td><td>executor</td><td>/{appId}/1/allexecutors</td><td><code>select(.id!='driver) | .memoryMetrics.usedOffHeapStorageMemory</code></td></tr><tr><td>spark_executor_total_on_heap_storage_memory</td><td>executor</td><td>/{appId}/1/allexecutors</td><td><code>select(.id!='driver) | .memoryMetrics.totalOnHeapStorageMemory</code></td></tr><tr><td>spark_executor_total_off_heap_storage_memory</td><td>executor</td><td>/{appId}/1/allexecutors</td><td><code>select(.id!='driver) | .memoryMetrics.totalOffHeapStorageMemory</code></td></tr><tr><td>spark_driver_rdd_blocks</td><td>driver</td><td>/{appId}/1/allexecutors</td><td><code>select(.id=='driver') | .rddBlocks</code></td></tr><tr><td>spark_driver_mem_used</td><td>driver</td><td>/{appId}/1/allexecutors</td><td><code>select(.id=='driver') | .memoryUsed</code></td></tr><tr><td>spark_driver_disk_used</td><td>driver</td><td>/{appId}/1/allexecutors</td><td><code>select(.id=='driver') | .diskUsed</code></td></tr><tr><td>spark_driver_cores</td><td>driver</td><td>/{appId}/1/allexecutors</td><td><code>select(.id=='driver') | .totalCores</code></td></tr><tr><td>spark_driver_total_duration</td><td>driver</td><td>/{appId}/1/allexecutors</td><td><code>select(.id=='driver') | .totalDuration</code></td></tr><tr><td>spark_driver_total_jvm_gc_duration</td><td>driver</td><td>/{appId}/1/allexecutors</td><td><code>select(.id=='driver') | .totalGCTime</code></td></tr><tr><td>spark_driver_total_input_bytes</td><td>driver</td><td>/{appId}/1/allexecutors</td><td><code>select(.id=='driver') | .totalInputBytes</code></td></tr><tr><td>spark_driver_total_shuffle_read</td><td>driver</td><td>/{appId}/1/allexecutors</td><td><code>select(.id=='driver') | .totalShuffleRead</code></td></tr><tr><td>spark_driver_total_shuffle_write</td><td>driver</td><td>/{appId}/1/allexecutors</td><td><code>select(.id=='driver') | .totalShuffleWrite</code></td></tr><tr><td>spark_driver_max_mem_used</td><td>driver</td><td>/{appId}/1/allexecutors</td><td><code>select(.id=='driver') | .maxMemory</code></td></tr><tr><td>spark_driver_used_on_heap_storage_memory</td><td>driver</td><td>/{appId}/1/allexecutors</td><td><code>select(.id=='driver') | .memoryMetrics.usedOnHeapStorageMemory</code></td></tr><tr><td>spark_driver_used_off_heap_storage_memory</td><td>driver</td><td>/{appId}/1/allexecutors</td><td><code>select(.id=='driver') | .memoryMetrics.usedOffHeapStorageMemory</code></td></tr><tr><td>spark_driver_total_on_heap_storage_memory</td><td>driver</td><td>/{appId}/1/allexecutors</td><td><code>select(.id=='driver') | .memoryMetrics.totalOnHeapStorageMemory</code></td></tr><tr><td>spark_driver_total_off_heap_storage_memory</td><td>driver</td><td>/{appId}/1/allexecutors</td><td><code>select(.id=='driver') | .memoryMetrics.totalOffHeapStorageMemory</code></td></tr></tbody></table>