Initial commit.

.gitignore

@@ -0,0 +1,4 @@
+bin
+obj
+*.suo
+packages

LICENSE.txt

@@ -0,0 +1,19 @@
+Copyright (c) 2017 J.D. Purcell
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,10 @@
+# RechatTool
+Command line tool to download Twith chat replays. Saves the full JSON data and optionally processes it to produce a simple text file.
+
+Sample usage:
+```
+RechatTool -D 111111111
+```
+Downloads the chat replay for video id 111111111 and saves the .json and processed .txt output in the current directory.
+
+Run without any arguments to see full list of modes.

RechatTool.sln

@@ -0,0 +1,22 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 15
+VisualStudioVersion = 15.0.26430.14
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RechatTool", "RechatTool\RechatTool.csproj", "{D1E4F04E-E71C-469A-A476-476124D2F7EC}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{D1E4F04E-E71C-469A-A476-476124D2F7EC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{D1E4F04E-E71C-469A-A476-476124D2F7EC}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{D1E4F04E-E71C-469A-A476-476124D2F7EC}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{D1E4F04E-E71C-469A-A476-476124D2F7EC}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal

RechatTool/App.config

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<configuration>
+    <startup> 
+        <supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5" />
+    </startup>
+</configuration>

RechatTool/Program.cs

@@ -0,0 +1,203 @@
+// --------------------------------------------------------------------------------
+// Copyright (c) J.D. Purcell
+//
+// Licensed under the MIT License (see LICENSE.txt)
+// --------------------------------------------------------------------------------
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Net;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace RechatTool {
+	internal class Program {
+		private static int Main(string[] args) {
+			int iArg = 0;
+			string GetArg(bool optional = false) =>
+				iArg < args.Length ? args[iArg++] : optional ? (string)null : throw new InvalidArgumentException();
+
+			try {
+				string mode = GetArg();
+				if (mode == "-d" || mode == "-D") {
+					if (!Int64.TryParse(GetArg(), out long videoId)) {
+						throw new InvalidArgumentException();
+					}
+					string path = GetArg(true) ?? $"{videoId}.json";;
+					void UpdateProgress(int downloaded, int total) {
+						Console.Write($"\rDownloaded {downloaded} of {total}");
+					}
+					DownloadFile(videoId, path, false, UpdateProgress);
+					if (mode == "-D") {
+						ProcessFile(path);
+					}
+					Console.WriteLine();
+					Console.WriteLine("Done!");
+				}
+				else if (mode == "-p") {
+					string path = GetArg();
+					if (path.Contains('*') || path.Contains('?')) {
+						string[] paths = Directory.GetFiles(Path.GetDirectoryName(path), Path.GetFileName(path));
+						foreach (string p in paths) {
+							ProcessFile(p);
+						}
+					}
+					else {
+						ProcessFile(path);
+						Console.WriteLine("Done!");
+					}
+				}
+				else {
+					throw new InvalidArgumentException();
+				}
+
+				return 0;
+			}
+			catch (InvalidArgumentException) {
+				Console.WriteLine("Modes:");
+				Console.WriteLine("   -d videoid [path]");
+				Console.WriteLine("      Downloads chat replay for the specified videoid. If path is not");
+				Console.WriteLine("      specified, the output is saved to the current directory.");
+				Console.WriteLine("   -D videoid [path]");
+				Console.WriteLine("      Downloads and processes chat replay (combines -d and -p).");
+				Console.WriteLine("   -p path");
+				Console.WriteLine("      Processes a JSON chat replay file and outputs a human-readable text file.");
+				Console.WriteLine("      Output is written to same folder as the input file with the extension");
+				Console.WriteLine("      changed to .txt.");
+				return 1;
+			}
+			catch (Exception ex) {
+				Console.WriteLine("\rError: " + ex.Message);
+				return 1;
+			}
+		}
+
+		private static void DownloadFile(long videoId, string path, bool overwrite, Action<int, int> progressCallback) {
+			const int timestampStep = 30;
+			const int threadCount = 6;
+			if (File.Exists(path) && !overwrite) {
+				throw new Exception("Output file already exists.");
+			}
+			string MakeUrl(long timestamp) => $"https://rechat.twitch.tv/rechat-messages?video_id=v{videoId}&start={timestamp}";
+			string videoInfo = (string)JObject.Parse(DownloadUrlAsString(MakeUrl(0), true))["errors"][0]["detail"];
+			string[] videoInfoSplit = videoInfo.Split(' ');
+			if (!videoInfo.StartsWith("0 is not between ", StringComparison.Ordinal) || videoInfoSplit.Length != 7) {
+				throw new Exception("Unrecognized response: " + videoInfo);
+			}
+			long firstTimestamp = Int64.Parse(videoInfoSplit[4]);
+			long lastTimestamp = Int64.Parse(videoInfoSplit[6]);
+			int segmentCount = ((int)(lastTimestamp - firstTimestamp) / timestampStep) + 1;
+			object syncObj = new object();
+			JArray[] segments = new JArray[segmentCount];
+			int downloadedSegmentCount = 0;
+			void DownloadSegment(int iSegment) {
+				long segmentTimestamp = firstTimestamp + (iSegment * timestampStep);
+				JArray segment = (JArray)JObject.Parse(DownloadUrlAsString(MakeUrl(segmentTimestamp)))["data"];
+				lock (syncObj) {
+					segments[iSegment] = segment;
+					downloadedSegmentCount++;
+					progressCallback?.Invoke(downloadedSegmentCount, segmentCount);
+				}
+			}
+			progressCallback?.Invoke(0, segmentCount);
+			Parallel.ForEach(
+				Enumerable.Range(0, segmentCount),
+				new ParallelOptions { MaxDegreeOfParallelism = threadCount },
+				DownloadSegment);
+			JArray combined = new JArray(segments.SelectMany(s => s).ToArray());
+			File.WriteAllText(path, combined.ToString(Formatting.None), new UTF8Encoding(true));
+		}
+
+		private static string DownloadUrlAsString(string url, bool allowErrors = false) {
+			HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
+			HttpWebResponse response;
+			try {
+				response = (HttpWebResponse)request.GetResponse();
+			}
+			catch (WebException ex) when (allowErrors) {
+				response = (HttpWebResponse)ex.Response;
+			}
+			using (response)
+			using (StreamReader responseStream = new StreamReader(response.GetResponseStream())) {
+				return responseStream.ReadToEnd();
+			}
+		}
+
+		private static void ProcessFile(string pathIn, string pathOut = null, bool overwrite = false) {
+			if (pathOut == null) {
+				bool isAlreadyTxt = Path.GetExtension(pathIn).Equals(".txt", StringComparison.OrdinalIgnoreCase);
+				pathOut = Path.Combine(
+					Path.GetDirectoryName(pathIn),
+					Path.GetFileNameWithoutExtension(pathIn) + (isAlreadyTxt ? "-p" : "") + ".txt");
+			}
+			if (File.Exists(pathOut) && !overwrite) {
+				throw new Exception("Output file already exists.");
+			}
+			JArray items = JArray.Parse(File.ReadAllText(pathIn));
+			List<Message> messages = items
+				.OfType<JObject>()
+				.Where(n => (string)n["type"] == "rechat-message")
+				.Select(n => new Message(n.ToObject<RawMessage>()))
+				.ToList();
+			IEnumerable<string> lines = messages
+				.Select(m => $"[{m.VideoOffset:hh\\:mm\\:ss\\.fff}] {m.UserDisplayName}: {m.MessageText ?? "<message deleted>"}");
+			File.WriteAllLines(pathOut, lines, new UTF8Encoding(true));
+		}
+
+		private class Message {
+			private static readonly DateTime BaseTime = new DateTime(1970, 1, 1);
+
+			private RawMessage Main { get; }
+			private RawMessageAttributes Attributes => Main.Attributes;
+			private RawMessageTags Tags => Main.Attributes.Tags;
+
+			public Message(RawMessage main) {
+				Main = main;
+			}
+
+			public DateTime Timestamp => BaseTime.AddMilliseconds(Attributes.Timestamp);
+
+			public TimeSpan VideoOffset => TimeSpan.FromMilliseconds(Attributes.VideoOffset);
+
+			public string MessageText => Attributes.Deleted ? null : Attributes.Message;
+
+			public string UserDisplayName => Tags.DisplayName;
+
+			public bool UserIsModerator => Tags.Mod;
+
+			public bool UserIsSubscriber => Tags.Subscriber;
+		}
+
+		private class RawMessage {
+			[JsonProperty("attributes")]
+			public RawMessageAttributes Attributes { get; set; }
+		}
+
+		private class RawMessageAttributes {
+			[JsonProperty("timestamp")]
+			public long Timestamp { get; set; }
+			[JsonProperty("video-offset")]
+			public int VideoOffset { get; set; }
+			[JsonProperty("deleted")]
+			public bool Deleted { get; set; }
+			[JsonProperty("message")]
+			public string Message { get; set; }
+			[JsonProperty("tags")]
+			public RawMessageTags Tags { get; set; }
+		}
+
+		private class RawMessageTags {
+			[JsonProperty("display-name")]
+			public string DisplayName { get; set; }
+			[JsonProperty("mod")]
+			public bool Mod { get; set; }
+			[JsonProperty("subscriber")]
+			public bool Subscriber { get; set; }
+		}
+
+		private class InvalidArgumentException : Exception { }
+	}
+}

RechatTool/Properties/AssemblyInfo.cs

@@ -0,0 +1,35 @@
+using System.Reflection;
+using System.Runtime.InteropServices;
+
+// General Information about an assembly is controlled through the following
+// set of attributes. Change these attribute values to modify the information
+// associated with an assembly.
+[assembly: AssemblyTitle("RechatTool")]
+[assembly: AssemblyDescription("")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("J.D. Purcell")]
+[assembly: AssemblyProduct("RechatTool")]
+[assembly: AssemblyCopyright("© J.D. Purcell")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+// Setting ComVisible to false makes the types in this assembly not visible
+// to COM components.  If you need to access a type in this assembly from
+// COM, set the ComVisible attribute to true on that type.
+[assembly: ComVisible(false)]
+
+// The following GUID is for the ID of the typelib if this project is exposed to COM
+[assembly: Guid("d1e4f04e-e71c-469a-a476-476124d2f7ec")]
+
+// Version information for an assembly consists of the following four values:
+//
+//      Major Version
+//      Minor Version
+//      Build Number
+//      Revision
+//
+// You can specify all the values or you can default the Build and Revision Numbers
+// by using the '*' as shown below:
+// [assembly: AssemblyVersion("1.0.*")]
+[assembly: AssemblyVersion("1.0.0.0")]
+[assembly: AssemblyFileVersion("1.0.0.0")]

RechatTool/RechatTool.csproj

@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <ProjectGuid>{D1E4F04E-E71C-469A-A476-476124D2F7EC}</ProjectGuid>
+    <OutputType>Exe</OutputType>
+    <RootNamespace>RechatTool</RootNamespace>
+    <AssemblyName>RechatTool</AssemblyName>
+    <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
+    <FileAlignment>512</FileAlignment>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+    <PlatformTarget>AnyCPU</PlatformTarget>
+    <DebugSymbols>true</DebugSymbols>
+    <DebugType>full</DebugType>
+    <Optimize>false</Optimize>
+    <OutputPath>bin\Debug\</OutputPath>
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+    <PlatformTarget>AnyCPU</PlatformTarget>
+    <DebugType>pdbonly</DebugType>
+    <Optimize>true</Optimize>
+    <OutputPath>bin\Release\</OutputPath>
+    <DefineConstants>TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="Newtonsoft.Json, Version=10.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
+      <HintPath>..\packages\Newtonsoft.Json.10.0.3\lib\net45\Newtonsoft.Json.dll</HintPath>
+    </Reference>
+    <Reference Include="System" />
+    <Reference Include="System.Core" />
+    <Reference Include="System.Xml.Linq" />
+    <Reference Include="System.Data.DataSetExtensions" />
+    <Reference Include="Microsoft.CSharp" />
+    <Reference Include="System.Data" />
+    <Reference Include="System.Net.Http" />
+    <Reference Include="System.Xml" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="Program.cs" />
+    <Compile Include="Properties\AssemblyInfo.cs" />
+  </ItemGroup>
+  <ItemGroup>
+    <None Include="App.config" />
+    <None Include="packages.config" />
+  </ItemGroup>
+  <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+</Project>

RechatTool/packages.config

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="utf-8"?>
+<packages>
+  <package id="Newtonsoft.Json" version="10.0.3" targetFramework="net45" />
+</packages>