Comment on page
Spark History Server metrics mapping
This page describes the mapping between metrics provided by Spark History Server to Akamas metrics for each supported component type
Component Type | Notes |
|---|---|
|
Component metric | Granularity | Document Path | JSON query |
|---|---|---|---|
spark_duration | job | /{appId}/1/jobs/{jobId} | .duration |
spark_completed_tasks | job | /{appId}/1/jobs/{jobId} | .numCompletedTasks |
spark_active_tasks | job | /{appId}/1/jobs/{jobId} | .numActiveTasks |
spark_skipped_tasks | job | /{appId}/1/jobs/{jobId} | .numSkippedTasks |
spark_failed_tasks | job | /{appId}/1/jobs/{jobId} | .numFailedTasks |
spark_killed_tasks | job | /{appId}/1/jobs/{jobId} | .numKilledTasks |
spark_completed_stages | job | /{appId}/1/jobs/{jobId} | .numCompletedStages |
spark_failed_stages | job | /{appId}/1/jobs/{jobId} | .numFailedStages |
spark_skipped_stages | job | /{appId}/1/jobs/{jobId} | .numSkippedStages |
spark_active_stages | job | /{appId}/1/jobs/{jobId} | .numActiveStages |
spark_duration | stage | /{appId}/1/stages/{stageId} | .getDuration |
spark_task_stage_executor_run_time | stage | /{appId}/1/stages/{stageId} | .getExecutorRunTime |
spark_task_stage_executor_cpu_time | stage | /{appId}/1/stages/{stageId} | .getExecutorCpuTime |
spark_active_tasks | stage | /{appId}/1/stages/{stageId} | .getNumActiveTasks |
spark_completed_tasks | stage | /{appId}/1/stages/{stageId} | .getNumCompleteTasks |
spark_failed_tasks | stage | /{appId}/1/stages/{stageId} | .getNumFailedTasks |
spark_killed_tasks | stage | /{appId}/1/stages/{stageId} | .getNumKilledTasks |
spark_task_stage_input_bytes_read | stage | /{appId}/1/stages/{stageId} | .getInputBytes |
spark_task_stage_input_records_read | stage | /{appId}/1/stages/{stageId} | .getInputRecords |
spark_task_stage_output_bytes_written | stage | /{appId}/1/stages/{stageId} | .getOutputBytes |
spark_task_stage_output_records_written | stage | /{appId}/1/stages/{stageId} | .getOutputRecords |
spark_stage_shuffle_read_bytes | stage | /{appId}/1/stages/{stageId} | .getShuffleReadBytes |
spark_task_stage_shuffle_read_records | stage | /{appId}/1/stages/{stageId} | .getShuffleReadRecords |
spark_task_stage_shuffle_write_bytes | stage | /{appId}/1/stages/{stageId} | .getShuffleWriteBytes |
spark_task_stage_shuffle_write_records | stage | /{appId}/1/stages/{stageId} | .getShuffleWriteRecords |
spark_task_stage_memory_bytes_spilled | stage | /{appId}/1/stages/{stageId} | .getMemoryBytesSpilled |
spark_task_stage_disk_bytes_spilled | stage | /{appId}/1/stages/{stageId} | .getDiskBytesSpilled |
spark_duration | task | /{appId}/1/stages/{stageId} | .tasks[].duration |
spark_task_executor_deserialize_time | task | /{appId}/1/stages/{stageId} | .tasks[].taskMetrics.executorDeserializeTime |
spark_task_executor_deserialize_cpu_time | task | /{appId}/1/stages/{stageId} | .tasks[].taskMetrics.executorDeserializeCpuTime |
spark_task_stage_executor_run_time | task | /{appId}/1/stages/{stageId} | .tasks[].taskMetrics.executorRunTime |
spark_task_stage_executor_cpu_time | task | /{appId}/1/stages/{stageId} | .tasks[].taskMetrics.executorCpuTime |
spark_task_result_size | task | /{appId}/1/stages/{stageId} | .tasks[].taskMetrics.resultSize |
spark_task_jvm_gc_duration | task | /{appId}/1/stages/{stageId} | .tasks[].taskMetrics.jvmGcTime |
spark_task_result_serialization_time | task | /{appId}/1/stages/{stageId} | .tasks[].taskMetrics.resultSerializationTime |
spark_task_stage_memory_bytes_spilled | task | /{appId}/1/stages/{stageId} | .tasks[].taskMetrics.memoryBytesSpilled |
spark_task_stage_disk_bytes_spilled | task | /{appId}/1/stages/{stageId} | .tasks[].taskMetrics.diskBytesSpilled |
spark_task_peak_execution_memory | task | /{appId}/1/stages/{stageId} | .tasks[].taskMetrics.peakExecutionMemory |
spark_task_stage_input_bytes_read | task | /{appId}/1/stages/{stageId} | .tasks[].taskMetrics.inputMetrics.bytesRead |
spark_task_stage_input_records_read | task | /{appId}/1/stages/{stageId} | .tasks[].taskMetrics.inputMetrics.recordsRead |
spark_task_stage_output_bytes_written | task | /{appId}/1/stages/{stageId} | .tasks[].taskMetrics.outputMetrics.bytesWritten |
spark_task_stage_output_records_written | task | /{appId}/1/stages/{stageId} | .tasks[].taskMetrics.outputMetrics.recordsWritten |
spark_task_shuffle_read_remote_blocks_fetched | task | /{appId}/1/stages/{stageId} | .tasks[].taskMetrics.shuffleReadMetrics.remoteBlocksFetched |
spark_task_shuffle_read_local_blocks_fetched | task | /{appId}/1/stages/{stageId} | .tasks[].taskMetrics.shuffleReadMetrics.localBlocksFetched |
spark_task_shuffle_read_fetch_wait_time | task | /{appId}/1/stages/{stageId} | .tasks[].taskMetrics.shuffleReadMetrics.fetchWaitTime |
spark_task_shuffle_read_remote_bytes | task | /{appId}/1/stages/{stageId} | .tasks[].taskMetrics.shuffleReadMetrics.remoteBytesRead |
spark_task_shuffle_read_remote_bytes_to_disk | task | /{appId}/1/stages/{stageId} | .tasks[].taskMetrics.shuffleReadMetrics.remoteBytesReadToDisk |
spark_task_shuffle_read_local_bytes | task | /{appId}/1/stages/{stageId} | .tasks[].taskMetrics.shuffleReadMetrics.localBytesRead |
spark_task_stage_shuffle_read_records | task | /{appId}/1/stages/{stageId} | .tasks[].taskMetrics.shuffleReadMetrics.recordsRead |
spark_task_stage_shuffle_write_bytes | task | /{appId}/1/stages/{stageId} | .tasks[].taskMetrics.shuffleWriteMetrics.bytesWritten |
spark_task_shuffle_write_time | task | /{appId}/1/stages/{stageId} | .tasks[].taskMetrics.shuffleWriteMetrics.writeTime |
spark_task_stage_shuffle_write_records | task | /{appId}/1/stages/{stageId} | .tasks[].taskMetrics.shuffleWriteMetrics.recordsWritten |
spark_executor_rdd_blocks | executor | /{appId}/1/allexecutors | select(.id!='driver) | .rddBlocks |
spark_executor_mem_used | executor | /{appId}/1/allexecutors | select(.id!='driver) | .memoryUsed |
spark_executor_disk_used | executor | /{appId}/1/allexecutors | select(.id!='driver) | .diskUsed |
spark_executor_cores | executor | /{appId}/1/allexecutors | select(.id!='driver) | .totalCores |
spark_active_tasks | executor | /{appId}/1/allexecutors | select(.id!='driver) | .activeTasks |
spark_failed_tasks | executor | /{appId}/1/allexecutors | select(.id!='driver) | .failedTasks |
spark_completed_tasks | executor | /{appId}/1/allexecutors | select(.id!='driver) | .completedTasks |
spark_executor_total_tasks | executor | /{appId}/1/allexecutors | select(.id!='driver) | .totalTasks |
spark_executor_total_duration | executor | /{appId}/1/allexecutors | select(.id!='driver) | .totalDuration |
spark_executor_total_jvm_gc_duration | executor | /{appId}/1/allexecutors | select(.id!='driver) | .totalGCTime |
spark_executor_total_input_bytes | executor | /{appId}/1/allexecutors | select(.id!='driver) | .totalInputBytes |
spark_executor_total_shuffle_read | executor | /{appId}/1/allexecutors | select(.id!='driver) | .totalShuffleRead |
spark_executor_total_shuffle_write | executor | /{appId}/1/allexecutors | select(.id!='driver) | .totalShuffleWrite |
spark_executor_max_mem_used | executor | /{appId}/1/allexecutors | select(.id!='driver) | .maxMemory |
spark_executor_used_on_heap_storage_memory | executor | /{appId}/1/allexecutors | select(.id!='driver) | .memoryMetrics.usedOnHeapStorageMemory |
spark_executor_used_off_heap_storage_memory | executor | /{appId}/1/allexecutors | select(.id!='driver) | .memoryMetrics.usedOffHeapStorageMemory |
spark_executor_total_on_heap_storage_memory | executor | /{appId}/1/allexecutors | select(.id!='driver) | .memoryMetrics.totalOnHeapStorageMemory |
spark_executor_total_off_heap_storage_memory | executor | /{appId}/1/allexecutors | select(.id!='driver) | .memoryMetrics.totalOffHeapStorageMemory |
spark_driver_rdd_blocks | driver | /{appId}/1/allexecutors | select(.id=='driver') | .rddBlocks |
spark_driver_mem_used | driver | /{appId}/1/allexecutors | select(.id=='driver') | .memoryUsed |
spark_driver_disk_used | driver | /{appId}/1/allexecutors | select(.id=='driver') | .diskUsed |
spark_driver_cores | driver | /{appId}/1/allexecutors | select(.id=='driver') | .totalCores |
spark_driver_total_duration | driver | /{appId}/1/allexecutors | select(.id=='driver') | .totalDuration |
spark_driver_total_jvm_gc_duration | driver | /{appId}/1/allexecutors | select(.id=='driver') | .totalGCTime |
spark_driver_total_input_bytes | driver | /{appId}/1/allexecutors | select(.id=='driver') | .totalInputBytes |
spark_driver_total_shuffle_read | driver | /{appId}/1/allexecutors | select(.id=='driver') | .totalShuffleRead |
spark_driver_total_shuffle_write | driver | /{appId}/1/allexecutors | select(.id=='driver') | .totalShuffleWrite |
spark_driver_max_mem_used | driver | /{appId}/1/allexecutors | select(.id=='driver') | .maxMemory |
spark_driver_used_on_heap_storage_memory | driver | /{appId}/1/allexecutors | select(.id=='driver') | .memoryMetrics.usedOnHeapStorageMemory |
spark_driver_used_off_heap_storage_memory | driver | /{appId}/1/allexecutors | select(.id=='driver') | .memoryMetrics.usedOffHeapStorageMemory |
spark_driver_total_on_heap_storage_memory | driver | /{appId}/1/allexecutors | select(.id=='driver') | .memoryMetrics.totalOnHeapStorageMemory |
spark_driver_total_off_heap_storage_memory | driver | /{appId}/1/allexecutors | select(.id=='driver') | .memoryMetrics.totalOffHeapStorageMemory |
Last modified 4mo ago