example_spark_event_log_message.json

{
  "Event": "SparkListenerTaskEnd",
  "Stage ID": 0,
  "Stage Attempt ID": 0,
  "Task Type": "ShuffleMapTask",
  "Task End Reason": { "Reason": "Success" },
  "Task Info": {
    "Task ID": 3,
    "Index": 3,
    "Attempt": 0,
    "Launch Time": 1589888002688,
    "Executor ID": "1",
    "Host": "172.16.7.4",
    "Locality": "PROCESS_LOCAL",
    "Speculative": false,
    "Getting Result Time": 0,
    "Finish Time": 1589888003614,
    "Failed": false,
    "Killed": false,
    "Accumulables": [
      {
        "ID": 23,
        "Name": "internal.metrics.input.recordsRead",
        "Update": 54004,
        "Value": 229579,
        "Internal": true,
        "Count Failed Values": true
      },
      {
        "ID": 22,
        "Name": "internal.metrics.input.bytesRead",
        "Update": 888082,
        "Value": 3777387,
        "Internal": true,
        "Count Failed Values": true
      },
      {
        "ID": 21,
        "Name": "internal.metrics.shuffle.write.writeTime",
        "Update": 18853810,
        "Value": 621332704,
        "Internal": true,
        "Count Failed Values": true
      },
      {
        "ID": 20,
        "Name": "internal.metrics.shuffle.write.recordsWritten",
        "Update": 700,
        "Value": 4100,
        "Internal": true,
        "Count Failed Values": true
      },
      {
        "ID": 19,
        "Name": "internal.metrics.shuffle.write.bytesWritten",
        "Update": 246849,
        "Value": 1343032,
        "Internal": true,
        "Count Failed Values": true
      },
      {
        "ID": 9,
        "Name": "internal.metrics.diskBytesSpilled",
        "Update": 0,
        "Value": 0,
        "Internal": true,
        "Count Failed Values": true
      },
      {
        "ID": 8,
        "Name": "internal.metrics.memoryBytesSpilled",
        "Update": 0,
        "Value": 0,
        "Internal": true,
        "Count Failed Values": true
      },
      {
        "ID": 5,
        "Name": "internal.metrics.resultSize",
        "Update": 1855,
        "Value": 11388,
        "Internal": true,
        "Count Failed Values": true
      },
      {
        "ID": 4,
        "Name": "internal.metrics.executorCpuTime",
        "Update": 211029780,
        "Value": 4303158247,
        "Internal": true,
        "Count Failed Values": true
      },
      {
        "ID": 3,
        "Name": "internal.metrics.executorRunTime",
        "Update": 902,
        "Value": 13775,
        "Internal": true,
        "Count Failed Values": true
      },
      {
        "ID": 2,
        "Name": "internal.metrics.executorDeserializeCpuTime",
        "Update": 4541231,
        "Value": 2076672103,
        "Internal": true,
        "Count Failed Values": true
      },
      {
        "ID": 1,
        "Name": "internal.metrics.executorDeserializeTime",
        "Update": 12,
        "Value": 3972,
        "Internal": true,
        "Count Failed Values": true
      }
    ]
  },
  "Task Executor Metrics": {
    "JVMHeapMemory": 0,
    "JVMOffHeapMemory": 0,
    "OnHeapExecutionMemory": 0,
    "OffHeapExecutionMemory": 0,
    "OnHeapStorageMemory": 0,
    "OffHeapStorageMemory": 0,
    "OnHeapUnifiedMemory": 0,
    "OffHeapUnifiedMemory": 0,
    "DirectPoolMemory": 0,
    "MappedPoolMemory": 0,
    "ProcessTreeJVMVMemory": 0,
    "ProcessTreeJVMRSSMemory": 0,
    "ProcessTreePythonVMemory": 0,
    "ProcessTreePythonRSSMemory": 0,
    "ProcessTreeOtherVMemory": 0,
    "ProcessTreeOtherRSSMemory": 0,
    "MinorGCCount": 0,
    "MinorGCTime": 0,
    "MajorGCCount": 0,
    "MajorGCTime": 0
  },
  "Task Metrics": {
    "Executor Deserialize Time": 12,
    "Executor Deserialize CPU Time": 4541231,
    "Executor Run Time": 902,
    "Executor CPU Time": 211029780,
    "Peak Execution Memory": 0,
    "Result Size": 1855,
    "JVM GC Time": 0,
    "Result Serialization Time": 0,
    "Memory Bytes Spilled": 0,
    "Disk Bytes Spilled": 0,
    "Shuffle Read Metrics": {
      "Remote Blocks Fetched": 0,
      "Local Blocks Fetched": 0,
      "Fetch Wait Time": 0,
      "Remote Bytes Read": 0,
      "Remote Bytes Read To Disk": 0,
      "Local Bytes Read": 0,
      "Total Records Read": 0
    },
    "Shuffle Write Metrics": {
      "Shuffle Bytes Written": 246849,
      "Shuffle Write Time": 18853810,
      "Shuffle Records Written": 700
    },
    "Input Metrics": { "Bytes Read": 888082, "Records Read": 54004 },
    "Output Metrics": { "Bytes Written": 0, "Records Written": 0 },
    "Updated Blocks": []
  }
}