Initial commit, added a working Twitch bot with OpenAI TTS integration and VLC support for playing audio files.

Signed-off-by: Nathan <nat@natfan.io>

.dockerignore

@@ -0,0 +1,25 @@
+**/.dockerignore
+**/.env
+**/.git
+**/.gitignore
+**/.project
+**/.settings
+**/.toolstarget
+**/.vs
+**/.vscode
+**/.idea
+**/*.*proj.user
+**/*.dbmdl
+**/*.jfm
+**/azds.yaml
+**/bin
+**/charts
+**/docker-compose*
+**/Dockerfile*
+**/node_modules
+**/npm-debug.log
+**/obj
+**/secrets.dev.yaml
+**/values.dev.yaml
+LICENSE
+README.md

.gitignore

@@ -0,0 +1,5 @@
+bin/
+obj/
+/packages/
+riderModule.iml
+/_ReSharper.Caches/

.idea/.idea.TwitchUtils/.idea/.gitignore

@@ -0,0 +1,13 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Rider ignored files
+/projectSettingsUpdater.xml
+/modules.xml
+/contentModel.xml
+/.idea.TwitchUtils.iml
+# Editor-based HTTP Client requests
+/httpRequests/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml

.idea/.idea.TwitchUtils/.idea/encodings.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="Encoding" addBOMForNewFiles="with BOM under Windows, with no BOM otherwise" />
+</project>

TTSMe/Bot.cs

@@ -0,0 +1,81 @@
+using TwitchLib.Api;
+using TwitchLib.Client;
+using TwitchLib.Client.Events;
+using TwitchLib.Client.Models;
+using TwitchLib.Communication.Clients;
+
+namespace TTSMe;
+
+public class Bot {
+	public const bool VERBOSE = false;
+	private TwitchClient client { get; }
+
+	private TwitchAPI apiClient { get; }
+
+	private string _channel { get; }
+
+	private static Voices voices => Voices.Instance();
+	private static bool SEND_WHISPERS => false;
+
+	public Bot(string clientId, ConnectionCredentials credentials, bool verbose = false) {
+		_channel = credentials.TwitchUsername;
+		client = new TwitchClient(new WebSocketClient {
+			Options = { MessagesAllowedInPeriod = 750, ThrottlingPeriod = TimeSpan.FromSeconds(90) }
+		});
+		apiClient = new TwitchAPI { Settings = { ClientId = clientId, AccessToken = credentials.TwitchOAuth } };
+		client.Initialize(credentials, credentials.TwitchUsername);
+
+		if (VERBOSE) client.OnLog += Log;
+		client.OnMessageReceived += OnTtsMeMessageReceived;
+
+		AppDomain.CurrentDomain.ProcessExit += OnProcessExit!;
+		
+		Connect();
+	}
+
+
+	private void SendMessage(string message, string? username = null) {
+		client.SendMessage(_channel, message);
+		if (!SEND_WHISPERS || username is null) return; // TODO: Change this when I figure out how whispers work...
+		client.SendWhisper(username, message);
+		apiClient.Helix.Whispers.SendWhisperAsync(_channel, username, message, true);
+	}
+
+	private void Connect() {
+		while (client is ({IsInitialized: false})) {
+			Console.Error.WriteLine("client is not yet initialized.");
+			Thread.Sleep(250);
+		}
+		Console.WriteLine($"Connecting to channel '{_channel}'...");
+		client.Connect();
+		try {
+			Console.WriteLine($"Connected to channel '{_channel}'.");
+		} catch (Exception e) {
+			Console.WriteLine($"Failed to connect to channel '{_channel}': {e.Message}");
+			Disconnect();
+		}
+	}
+
+	private void Disconnect() {
+		client.Disconnect();
+		Console.WriteLine($"Disconnected from channel '{_channel}'.");
+	}
+	private void OnProcessExit(object? _, EventArgs e) => Disconnect();
+	private static void Log(object? _, OnLogArgs e) => Console.WriteLine($"{e.DateTime:yyyy-MM-ddTHH:mm:ssZ} - {e.Data}");
+
+	private void OnTtsMeMessageReceived(object? _, OnMessageReceivedArgs e) {
+		ChatMessage chatMessage = e.ChatMessage;
+		string username = chatMessage.Username;
+		string message = chatMessage.GetMessageText();
+		if (!chatMessage.HasStoredVoice()) voices.SetRandomVoice(username);
+
+		string? ttsMeCommandResponse = TtsMeCommand.ProcessVoiceChange(chatMessage);
+		if (ttsMeCommandResponse is not null) SendMessage(ttsMeCommandResponse);
+		
+		if (message.StartsWith("!ttsme")) return; // Don't process the command message
+		
+		Console.WriteLine($"{username} [{voices.GetVoice(username)!}]: !ttsme {message}");
+		
+		Voices.ProcessVoiceMessage(chatMessage).Wait();
+	}
+}

TTSMe/Dockerfile

@@ -0,0 +1,21 @@
+FROM mcr.microsoft.com/dotnet/runtime:9.0 AS base
+USER $APP_UID
+WORKDIR /app
+
+FROM mcr.microsoft.com/dotnet/sdk:9.0 AS build
+ARG BUILD_CONFIGURATION=Release
+WORKDIR /src
+COPY ["TTSMe/TTSMe.csproj", "TTSMe/"]
+RUN dotnet restore "TTSMe/TTSMe.csproj"
+COPY . .
+WORKDIR "/src/TTSMe"
+RUN dotnet build "TTSMe.csproj" -c $BUILD_CONFIGURATION -o /app/build
+
+FROM build AS publish
+ARG BUILD_CONFIGURATION=Release
+RUN dotnet publish "TTSMe.csproj" -c $BUILD_CONFIGURATION -o /app/publish /p:UseAppHost=false
+
+FROM base AS final
+WORKDIR /app
+COPY --from=publish /app/publish .
+ENTRYPOINT ["dotnet", "TTSMe.dll"]

TTSMe/ExtensionMethods.cs

@@ -0,0 +1,16 @@
+using EmojiCSharp;
+using TwitchLib.Client.Models;
+
+namespace TTSMe;
+
+public static class ExtensionMethods {
+	internal static string? GetStoredVoice(this ChatMessage chatMessage) => Voices.Instance().GetVoice(chatMessage.Username);
+	internal static bool HasStoredVoice(this ChatMessage chatMessage) => Voices.Instance().HasVoice(chatMessage.Username);
+	// parse out all emojis into their full names
+	internal static string GetMessageText(this ChatMessage chatMessage) =>
+		EmojiParser.ParseToAliases(chatMessage.Message)
+			.Replace(":", " colon ")
+			.Replace("_", " underscore ");
+
+	internal static TextToSpeechRequest ToTTSRequest(this ChatMessage chatMessage) => new(chatMessage.GetMessageText(), chatMessage.GetStoredVoice()!);
+}

TTSMe/Program.cs

@@ -0,0 +1,13 @@
+EmojiCSharp.EmojiManager.Init();
+LibVLCSharp.Shared.Core.Initialize();
+
+_ = new TTSMe.Bot(
+	clientId: Environment.GetEnvironmentVariable("CLIENT_ID")!,
+	credentials: new(
+		twitchUsername: Environment.GetEnvironmentVariable("USER_CHANNEL")!, 
+		twitchOAuth: Environment.GetEnvironmentVariable("USER_TOKEN")!
+	),
+	verbose: TTSMe.TtsMeCommand.VERBOSE
+);
+Console.WriteLine("Press any key to exit...");
+Console.ReadKey();

TTSMe/TTSClient.cs

@@ -0,0 +1,57 @@
+using System.Net.Http.Headers;
+using System.Net.Http.Json;
+using System.Security.Cryptography;
+using System.Text.Json.Serialization;
+using LibVLCSharp.Shared;
+using TwitchLib.Client.Models;
+
+namespace TTSMe;
+
+public static class TTSClient {
+	private static readonly HttpClient httpClient = new() {
+		DefaultRequestHeaders = {
+			Authorization = new AuthenticationHeaderValue("Bearer", Environment.GetEnvironmentVariable("OPENAI_API_KEY")),
+		},
+		BaseAddress = new Uri("https://api.openai.com")
+	};
+	private const string basePath = "C:/data/twitch/tts";
+
+	public static async Task<string> GenerateSpeech(ChatMessage chatMessage) {
+		HttpResponseMessage response = await httpClient.PostAsJsonAsync("v1/audio/speech", chatMessage.ToTTSRequest());
+		byte[] bytes = await response.Content.ReadAsByteArrayAsync();
+		string outputPath = Path.Combine(basePath, $"{Guid.NewGuid()}.mp3");
+		await File.WriteAllBytesAsync(outputPath, bytes);
+		return outputPath;
+	}
+	
+	public static async Task PlaySpeech(string path) {
+		try {
+			await Task.Run(() => {
+				TaskCompletionSource<bool> taskCompletionSource = new();
+				using LibVLC libVLC = new();
+				if (TtsMeCommand.VERBOSE) libVLC.Log += (_, e) => Console.WriteLine($"LibVLC [{e.Level}] {e.Module}:{e.Message}");
+				using Media media = new(libVLC, path);
+				using MediaPlayer player = new(media);
+				
+				player.EndReached += OnEndReached;
+				player.Play();
+				player.EndReached += (_, _) => taskCompletionSource.TrySetResult(true);
+				taskCompletionSource.Task.Wait();
+				player.Stop();
+			});
+		} catch (Exception ex) {
+			Console.WriteLine($"An error occurred during playback: {ex.Message}");
+		}
+	}
+
+	private static void OnEndReached(object? sender, EventArgs _) {
+		if (sender is MediaPlayer mediaPlayer) mediaPlayer.Stop();
+	}
+}
+internal record TextToSpeechRequest(
+	[property: JsonPropertyName("input")] string Input,
+	[property: JsonPropertyName("voice")] string Voice,
+	[property: JsonPropertyName("model")] string Model = "tts-1",
+	[property: JsonPropertyName("response_format")] string ResponseFormat = "mp3",
+	[property: JsonPropertyName("speed")] double Speed = 1.0
+);

TTSMe/TTSMe.csproj

@@ -0,0 +1,26 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <OutputType>Exe</OutputType>
+        <TargetFramework>net9.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+        <DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <Content Include="..\.dockerignore">
+        <Link>.dockerignore</Link>
+      </Content>
+    </ItemGroup>
+
+    <ItemGroup>
+      <PackageReference Include="EmojiCSharp" Version="1.0.1" />
+      <PackageReference Include="LibVLCSharp" Version="3.8.5" />
+      <PackageReference Include="OpenAI" Version="2.0.0-beta.7" />
+      <PackageReference Include="TwitchLib" Version="3.5.3" />
+      <PackageReference Include="TwitchLib.Api" Version="3.10.0-preview-e47ba7f" />
+      <PackageReference Include="VideoLAN.LibVLC.Windows" Version="3.0.20" />
+    </ItemGroup>
+
+</Project>

TTSMe/TtsMeCommand.cs

@@ -0,0 +1,42 @@
+using TwitchLib.Client.Models;
+
+namespace TTSMe;
+
+public abstract class TtsMeCommand {
+	private const string UsageMessage = "Usage: !ttsme <help|get|set [voice]|randomize|optout>.";
+	public const bool VERBOSE = Bot.VERBOSE || false;
+	private static Voices voices => Voices.Instance();
+
+	private abstract class Messages {
+		public static string VoiceSet(string username) => $"ttsme voice for '{username}' is '{Voices.Instance().GetVoice(username) ?? "none"}'";
+		public static string InvalidVoice(string voice) => $"Invalid voice '{voice}'. Valid voices are: {string.Join(", ", Voices.Instance().ValidVoices)}.";
+	}
+	
+	
+	public static string? ProcessVoiceChange(ChatMessage chatMessage) {
+		string username = chatMessage.Username;
+		string[] args = chatMessage.Message.Split(' ');
+		if (args[0] != "!ttsme") return null;
+		if (args.Length is 1 or > 3) return UsageMessage;
+		string action = args[1];
+		string voice = args[^1];
+		
+		switch (action) {
+			case "help": return UsageMessage;
+			
+			case "get" when args.Length == 2: return Messages.VoiceSet(username);
+			case "get" when args.Length == 3 && username == chatMessage.Channel: return Messages.VoiceSet(voice);
+			
+			case "randomise" or "randomize": voices.SetRandomVoice(username); return $"ttsme voice for '{username}' has been {action}d and is now '{voices.GetVoice(username)}'.";
+			
+			case "set" when args.Length != 3: return Messages.InvalidVoice("NULL");
+			case "set" when !voices.IsValidVoice(voice): return Messages.InvalidVoice(voice);
+			case "set" when voices.IsVoice(username, voice): return $"ttsme voice for '{username}' is already set to '{voice}'.";
+			case "set": voices.SetVoice(username, voice); return $"Changing ttsme for {username} to '{voice}'";
+			
+			case "optout": voices.RemoveVoice(username); return $"Opting {username} out of ttsme, their voice will no longer be heard by the masses (and me!)";
+			
+			default: return UsageMessage;
+		}
+	}
+}

TTSMe/Voices.cs

@@ -0,0 +1,27 @@
+using TwitchLib.Client.Models;
+
+namespace TTSMe;
+
+public class Voices {
+	private static Voices? instance;
+	public static Voices Instance() => instance ??= new Voices();
+
+	private Dictionary<string, string> UserVoices { get; } = new();
+	internal readonly List<string> ValidVoices = [ "alloy", "echo", "fable", "onyx", "nova", "shimmer" ];
+
+	public bool HasVoice(string username) => UserVoices.ContainsKey(username);
+	public bool IsValidVoice(string voice) => ValidVoices.Contains(voice);
+	public bool IsVoice(string username, string voice) => HasVoice(username) && UserVoices.GetValueOrDefault(username) == voice;
+	public string? GetVoice(string username) => UserVoices.GetValueOrDefault(username);
+	public void SetVoice(string username, string voice) => UserVoices[username] = voice;
+	public void SetRandomVoice(string username) => UserVoices[username] = ValidVoices[new Random().Next(ValidVoices.Count)];
+	public void RemoveVoice(string username) => UserVoices.Remove(username);
+    
+	
+	public static async Task ProcessVoiceMessage(ChatMessage chatMessage) {
+		string? voice = chatMessage.GetStoredVoice();
+		if (voice is null) return;
+		string filePath = await TTSClient.GenerateSpeech(chatMessage);
+		await TTSClient.PlaySpeech(filePath);
+	}
+}

TwitchUtils.sln

@@ -0,0 +1,21 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TTSMe", "TTSMe\TTSMe.csproj", "{02DF3D65-BBE1-4888-8A6A-B4051CFAFB8B}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{8B637A1E-14A4-4F70-8440-68F6544B99B5}"
+	ProjectSection(SolutionItems) = preProject
+		docker-compose.yml = docker-compose.yml
+	EndProjectSection
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{02DF3D65-BBE1-4888-8A6A-B4051CFAFB8B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{02DF3D65-BBE1-4888-8A6A-B4051CFAFB8B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{02DF3D65-BBE1-4888-8A6A-B4051CFAFB8B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{02DF3D65-BBE1-4888-8A6A-B4051CFAFB8B}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+EndGlobal

docker-compose.yml

@@ -0,0 +1,6 @@
+services:
+  ttsme:
+    image: ttsme
+    build:
+      context: .
+      dockerfile: TTSMe/Dockerfile