Skip to content

Commit

Permalink
Added log analysis notebooks
Browse files Browse the repository at this point in the history
  • Loading branch information
@hehlinge42
hehlinge42 committed Mar 17, 2021
2 parents 8d44366 + c9b94ef commit 3e23662
Show file tree
Hide file tree
Showing 23 changed files with 3,701,889 additions and 1,953,299 deletions.
Binary file modified .DS_Store
View file
Binary file not shown.
558 changes: 558 additions & 0 deletions cleaned_logs/scala test 1/log_accumulables.csv
View file

Large diffs are not rendered by default.

11 changes: 11 additions & 0 deletions cleaned_logs/scala test 1/log_jobs.csv
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
Job ID;Submission Time;Stage IDs;Completion Time;Job Result;Duration;spark.scheduler.pool;spark.databricks.env;spark.job.interruptOnCancel;spark.databricks.api.url;spark.rdd.scope;spark.rdd.scope.noOverride;orgId;spark.job.description;spark.databricks.replId;spark.jobGroup.id;spark.databricks.notebook.id;spark.databricks.isolationID;spark.databricks.inherited.credentials.keys.spark.databricks.api.token;spark.databricks.notebook.path;userId;user;spark.databricks.token;Filename
5;2021-03-16 09:00:43.938;[30, 27, 31, 28, 29];2021-03-16 09:04:04.834;{'Result': 'JobSucceeded'};200.896;4023510570700290162;databricks-scala-repl;true;https://community.cloud.databricks.com;"{""id"":""138"",""name"":""collect""}";true;191748078717668;"val filename = ""graph_45M.txt""
val filestore_na...";ReplId-37d66-18bec-9eb87-2;4023510570700290162_7789929736263496251_dfd4b4b665dd4272ba77ee0f6c0de65f;4199483808298115;ca8b4387-f26c-4e1d-8889-c0fb32538e9d;*********(redacted);/Users/[email protected]/ScalaRDD;8470066847461284;[email protected];[REDACTED];
6;2021-03-16 09:05:12.649;[33, 37, 34, 38, 35, 32, 36];2021-03-16 09:11:22.337;{'Result': 'JobSucceeded'};369.688;4023510570700290162;databricks-scala-repl;true;https://community.cloud.databricks.com;"{""id"":""160"",""name"":""collect""}";true;191748078717668;"val filename = ""graph_45M.txt""
val filestore_na...";ReplId-37d66-18bec-9eb87-2;4023510570700290162_7789929736263496251_dfd4b4b665dd4272ba77ee0f6c0de65f;4199483808298115;ca8b4387-f26c-4e1d-8889-c0fb32538e9d;*********(redacted);/Users/[email protected]/ScalaRDD;8470066847461284;[email protected];[REDACTED];
7;2021-03-16 09:12:57.766;[45, 46, 42, 39, 43, 40, 47, 44, 41];2021-03-16 09:33:25.887;{'Result': 'JobSucceeded'};1228.121;4023510570700290162;databricks-scala-repl;true;https://community.cloud.databricks.com;"{""id"":""182"",""name"":""collect""}";true;191748078717668;"val filename = ""graph_45M.txt""
val filestore_na...";ReplId-37d66-18bec-9eb87-2;4023510570700290162_7789929736263496251_dfd4b4b665dd4272ba77ee0f6c0de65f;4199483808298115;ca8b4387-f26c-4e1d-8889-c0fb32538e9d;*********(redacted);/Users/[email protected]/ScalaRDD;8470066847461284;[email protected];[REDACTED];
8;2021-03-16 09:33:33.465;[51, 48, 52, 49, 56, 53, 50, 57, 54, 58, 55];2021-03-16 09:37:25.275;{'Result': 'JobSucceeded'};231.81;4023510570700290162;databricks-scala-repl;true;https://community.cloud.databricks.com;"{""id"":""204"",""name"":""collect""}";true;191748078717668;"val filename = ""graph_45M.txt""
val filestore_na...";ReplId-37d66-18bec-9eb87-2;4023510570700290162_7789929736263496251_dfd4b4b665dd4272ba77ee0f6c0de65f;4199483808298115;ca8b4387-f26c-4e1d-8889-c0fb32538e9d;*********(redacted);/Users/[email protected]/ScalaRDD;8470066847461284;[email protected];[REDACTED];
9;2021-03-16 09:37:29.127;[66, 63, 60, 70, 67, 64, 71, 68, 69, 61, 65, 62, 59];2021-03-16 09:37:49.610;{'Result': 'JobSucceeded'};20.483;4023510570700290162;databricks-scala-repl;true;https://community.cloud.databricks.com;"{""id"":""226"",""name"":""collect""}";true;191748078717668;"val filename = ""graph_45M.txt""
val filestore_na...";ReplId-37d66-18bec-9eb87-2;4023510570700290162_7789929736263496251_dfd4b4b665dd4272ba77ee0f6c0de65f;4199483808298115;ca8b4387-f26c-4e1d-8889-c0fb32538e9d;*********(redacted);/Users/[email protected]/ScalaRDD;8470066847461284;[email protected];[REDACTED];
51 changes: 51 additions & 0 deletions cleaned_logs/scala test 1/log_rdds.csv
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
RDD ID;Name;Scope;Callsite;Parent IDs;Storage Level;Barrier;Number of Partitions;Number of Cached Partitions;Memory Size;Disk Size;Stage ID
46;ShuffledRDD;"{""id"":""109"",""name"":""groupByKey""}";groupByKey at command-4199483808298120:37;[45];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;29
47;MapPartitionsRDD;"{""id"":""112"",""name"":""map""}";map at command-4199483808298120:38;[46];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;29
48;MapPartitionsRDD;"{""id"":""126"",""name"":""flatMap""}";flatMap at command-4199483808298120:30;[47];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;29
49;MapPartitionsRDD;"{""id"":""127"",""name"":""map""}";map at command-4199483808298120:31;[48];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;29
50;ShuffledRDD;"{""id"":""128"",""name"":""groupByKey""}";groupByKey at command-4199483808298120:32;[49];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;30
51;MapPartitionsRDD;"{""id"":""131"",""name"":""mapValues""}";mapValues at command-4199483808298120:32;[50];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;30
52;MapPartitionsRDD;"{""id"":""132"",""name"":""flatMap""}";flatMap at command-4199483808298120:33;[51];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;30
53;MapPartitionsRDD;"{""id"":""133"",""name"":""map""}";map at command-4199483808298120:36;[52];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;30
54;ShuffledRDD;"{""id"":""134"",""name"":""groupByKey""}";groupByKey at command-4199483808298120:37;[53];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;36
54;ShuffledRDD;"{""id"":""134"",""name"":""groupByKey""}";groupByKey at command-4199483808298120:37;[53];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;31
55;MapPartitionsRDD;"{""id"":""137"",""name"":""map""}";map at command-4199483808298120:38;[54];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;31
55;MapPartitionsRDD;"{""id"":""137"",""name"":""map""}";map at command-4199483808298120:38;[54];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;36
56;MapPartitionsRDD;"{""id"":""148"",""name"":""flatMap""}";flatMap at command-4199483808298120:30;[55];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;36
57;MapPartitionsRDD;"{""id"":""149"",""name"":""map""}";map at command-4199483808298120:31;[56];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;36
58;ShuffledRDD;"{""id"":""150"",""name"":""groupByKey""}";groupByKey at command-4199483808298120:32;[57];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;37
59;MapPartitionsRDD;"{""id"":""153"",""name"":""mapValues""}";mapValues at command-4199483808298120:32;[58];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;37
60;MapPartitionsRDD;"{""id"":""154"",""name"":""flatMap""}";flatMap at command-4199483808298120:33;[59];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;37
61;MapPartitionsRDD;"{""id"":""155"",""name"":""map""}";map at command-4199483808298120:36;[60];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;37
62;ShuffledRDD;"{""id"":""156"",""name"":""groupByKey""}";groupByKey at command-4199483808298120:37;[61];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;38
62;ShuffledRDD;"{""id"":""156"",""name"":""groupByKey""}";groupByKey at command-4199483808298120:37;[61];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;45
63;MapPartitionsRDD;"{""id"":""159"",""name"":""map""}";map at command-4199483808298120:38;[62];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;38
63;MapPartitionsRDD;"{""id"":""159"",""name"":""map""}";map at command-4199483808298120:38;[62];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;45
64;MapPartitionsRDD;"{""id"":""170"",""name"":""flatMap""}";flatMap at command-4199483808298120:30;[63];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;45
65;MapPartitionsRDD;"{""id"":""171"",""name"":""map""}";map at command-4199483808298120:31;[64];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;45
66;ShuffledRDD;"{""id"":""172"",""name"":""groupByKey""}";groupByKey at command-4199483808298120:32;[65];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;46
67;MapPartitionsRDD;"{""id"":""175"",""name"":""mapValues""}";mapValues at command-4199483808298120:32;[66];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;46
68;MapPartitionsRDD;"{""id"":""176"",""name"":""flatMap""}";flatMap at command-4199483808298120:33;[67];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;46
69;MapPartitionsRDD;"{""id"":""177"",""name"":""map""}";map at command-4199483808298120:36;[68];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;46
70;ShuffledRDD;"{""id"":""178"",""name"":""groupByKey""}";groupByKey at command-4199483808298120:37;[69];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;47
70;ShuffledRDD;"{""id"":""178"",""name"":""groupByKey""}";groupByKey at command-4199483808298120:37;[69];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;56
71;MapPartitionsRDD;"{""id"":""181"",""name"":""map""}";map at command-4199483808298120:38;[70];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;47
71;MapPartitionsRDD;"{""id"":""181"",""name"":""map""}";map at command-4199483808298120:38;[70];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;56
72;MapPartitionsRDD;"{""id"":""192"",""name"":""flatMap""}";flatMap at command-4199483808298120:30;[71];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;56
73;MapPartitionsRDD;"{""id"":""193"",""name"":""map""}";map at command-4199483808298120:31;[72];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;56
74;ShuffledRDD;"{""id"":""194"",""name"":""groupByKey""}";groupByKey at command-4199483808298120:32;[73];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;57
75;MapPartitionsRDD;"{""id"":""197"",""name"":""mapValues""}";mapValues at command-4199483808298120:32;[74];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;57
76;MapPartitionsRDD;"{""id"":""198"",""name"":""flatMap""}";flatMap at command-4199483808298120:33;[75];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;57
77;MapPartitionsRDD;"{""id"":""199"",""name"":""map""}";map at command-4199483808298120:36;[76];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;57
78;ShuffledRDD;"{""id"":""200"",""name"":""groupByKey""}";groupByKey at command-4199483808298120:37;[77];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;58
78;ShuffledRDD;"{""id"":""200"",""name"":""groupByKey""}";groupByKey at command-4199483808298120:37;[77];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;69
79;MapPartitionsRDD;"{""id"":""203"",""name"":""map""}";map at command-4199483808298120:38;[78];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;58
79;MapPartitionsRDD;"{""id"":""203"",""name"":""map""}";map at command-4199483808298120:38;[78];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;69
80;MapPartitionsRDD;"{""id"":""214"",""name"":""flatMap""}";flatMap at command-4199483808298120:30;[79];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;69
81;MapPartitionsRDD;"{""id"":""215"",""name"":""map""}";map at command-4199483808298120:31;[80];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;69
82;ShuffledRDD;"{""id"":""216"",""name"":""groupByKey""}";groupByKey at command-4199483808298120:32;[81];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;70
83;MapPartitionsRDD;"{""id"":""219"",""name"":""mapValues""}";mapValues at command-4199483808298120:32;[82];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;70
84;MapPartitionsRDD;"{""id"":""220"",""name"":""flatMap""}";flatMap at command-4199483808298120:33;[83];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;70
85;MapPartitionsRDD;"{""id"":""221"",""name"":""map""}";map at command-4199483808298120:36;[84];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;70
86;ShuffledRDD;"{""id"":""222"",""name"":""groupByKey""}";groupByKey at command-4199483808298120:37;[85];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;71
87;MapPartitionsRDD;"{""id"":""225"",""name"":""map""}";map at command-4199483808298120:38;[86];{'Use Disk': False, 'Use Memory': False, 'Deserialized': False, 'Replication': 1};False;2;0;0;0;71
Loading

0 comments on commit 3e23662

Please sign in to comment.