diff --git a/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/conversation-object.ts b/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/conversation-object.ts index 6cbab9d2d..52212a4aa 100644 --- a/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/conversation-object.ts +++ b/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/conversation-object.ts @@ -8,6 +8,7 @@ import { PlayableAudioStream, GetHashFn, MessageCache, + MessageReactionEventData, } from '../types' import { SceneObject } from '../classes/scene-object'; import { Player } from 'react-agents-client/util/player.mjs'; diff --git a/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/discord-manager.ts b/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/discord-manager.ts index 52e5d37eb..db64ae619 100644 --- a/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/discord-manager.ts +++ b/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/discord-manager.ts @@ -97,6 +97,42 @@ const bindOutgoing = ({ channelId, userId, }); + } else if (method === 'messageReaction') { + const { + reaction, + messageId, + userId, + } = args as { + reaction: string, + messageId: string, + userId: string, + }; + + + // TODO: current agent mentionId needs to be set + const getDiscordIdForUserId = (userId: string) => { + const agents = conversation.getAgents(); + const currentAgent = conversation.agent; + const agent = agents.find( + agent => agent.playerId === userId + ) || (currentAgent.id === userId ? currentAgent : undefined); + + const discordId = agent?.playerSpec?.mentionId; + return discordId; + }; + + const discordId = getDiscordIdForUserId(userId); + console.log('discord manager message reaction', { + reaction, + messageId, + userId, + channelId, + discordId, + }); + discordBotClient.input.reactToMessage(reaction, messageId, { + channelId, + userId: discordId, + }); } else { // ignore } @@ -370,6 +406,7 @@ export class DiscordBot extends EventTarget { text, channelId, // if there is no channelId, it's a DM // XXX discord channel/dm distinction can be made more explicit with a type: string field... + messageId, } = e.data; // look up conversation @@ -390,6 +427,7 @@ export class DiscordBot extends EventTarget { method: 'say', args: { text: formattedMessage, + messageId, }, }; const id = getIdFromUserId(userId); @@ -411,11 +449,71 @@ export class DiscordBot extends EventTarget { }); }; + // message reactions + const _bindIncomingMessageReactions = () => { + const handleReaction = (e: MessageEvent, eventType: string) => { + const { + userId, + messageId, + emoji, + channelId, + userDisplayName, + } = e.data; + + console.log(eventType, { + userId, + userDisplayName, + messageId, + emoji, + channelId, + }); + + // look up conversation + const conversation = this.dmConversations.has(userId) + ? this.dmConversations.get(userId) ?? null + : this.channelConversations.get(channelId) ?? null; + + if (!conversation) return; + + const rawMessageReaction = { + userId, + name: userDisplayName, + method: 'messageReaction', + args: { + reaction: emoji, + messageId, + userId, + context: { + action: eventType === 'messagereactionadd' ? 'Reaction added' : 'Reaction removed', + }, + }, + }; + + const newMessageReaction = formatConversationMessage(rawMessageReaction, { + agent: { + id: getIdFromUserId(userId), + name: userDisplayName, + }, + }); + + conversation.addLocalMessage(newMessageReaction); + }; + + discordBotClient.output.addEventListener('messagereactionadd', + (e) => handleReaction(e, 'messagereactionadd') + ); + + discordBotClient.output.addEventListener('messagereactionremove', + (e) => handleReaction(e, 'messagereactionremove') + ); + }; + (async () => { _bindChannels(); _bindGuildMemberAdd(); _bindGuildMemberRemove(); _bindIncoming(); + _bindIncomingMessageReactions(); await _connect(); })().catch(err => { console.warn('discord bot error', err); diff --git a/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/generative-agent-object.ts b/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/generative-agent-object.ts index a3841cc45..159f7c09f 100644 --- a/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/generative-agent-object.ts +++ b/packages/usdk/packages/upstreet-agent/packages/react-agents/classes/generative-agent-object.ts @@ -11,6 +11,7 @@ import type { ActionStep, Evaluator, DebugOptions, + EvaluateOpts, } from '../types'; import { ConversationObject, @@ -110,14 +111,28 @@ export class GenerativeAgentObject { }); // }); } - async evaluate(evaluator: Evaluator) { - return await this.conversation.typing(async () => { + async evaluate(evaluator: Evaluator, opts?: EvaluateOpts) { + const { + sendTyping = true, + } = opts ?? {}; + + const evaluateAndExecuteStep = async () => { const step = await evaluator.evaluate({ - generativeAgent: this, + generativeAgent: this as GenerativeAgentObject, }); await executeAgentActionStep(this, step); return step; - }); + }; + + if (sendTyping) { + let result: ActionStep; + await this.conversation.typing(async () => { + result = await evaluateAndExecuteStep(); + }); + return result; + } else { + return await evaluateAndExecuteStep(); + } } /* async generate(hint: string, schema?: ZodTypeAny) { // console.log('agent renderer generate 1'); diff --git a/packages/usdk/packages/upstreet-agent/packages/react-agents/components/core/chat.tsx b/packages/usdk/packages/upstreet-agent/packages/react-agents/components/core/chat.tsx index 0b00c4520..a3d396c87 100644 --- a/packages/usdk/packages/upstreet-agent/packages/react-agents/components/core/chat.tsx +++ b/packages/usdk/packages/upstreet-agent/packages/react-agents/components/core/chat.tsx @@ -2,6 +2,7 @@ import React from 'react'; import dedent from 'dedent'; import { z } from 'zod'; import { Action } from './action'; +import { PendingActionEvent } from '../../types/react-agents'; export const ChatActions = () => { return ( @@ -27,6 +28,40 @@ export const ChatActions = () => { // await e.commit(); // }} /> + { + await e.commit(); + }} + /> ); }; \ No newline at end of file diff --git a/packages/usdk/packages/upstreet-agent/packages/react-agents/lib/discord/discord-client.js b/packages/usdk/packages/upstreet-agent/packages/react-agents/lib/discord/discord-client.js index 4089113fc..592df8783 100644 --- a/packages/usdk/packages/upstreet-agent/packages/react-agents/lib/discord/discord-client.js +++ b/packages/usdk/packages/upstreet-agent/packages/react-agents/lib/discord/discord-client.js @@ -193,6 +193,22 @@ export class DiscordInput { this.queueManager.completeStream(streamId); } + reactToMessage(reaction, messageId, { + channelId, + userId, + } = {}) { + const m = { + method: 'reactToMessage', + args: { + channelId, + reaction, + messageId, + userId, + }, + }; + this.ws.send(JSON.stringify(m)); + } + destroy() { // Clean up any remaining buffers for (const streamId of this.bufferManager.buffers.keys()) { @@ -340,6 +356,20 @@ export class DiscordOutput extends EventTarget { } } + handleMessageReactionAdd(args) { + console.log('handleMessageReactionAdd', args); + this.dispatchEvent(new MessageEvent('messagereactionadd', { + data: args, + })); + } + + handleMessageReactionRemove(args) { + console.log('handleMessageReactionRemove', args); + this.dispatchEvent(new MessageEvent('messagereactionremove', { + data: args, + })); + } + destroy() { for (const stream of this.streams.values()) { stream.destroy(); @@ -536,6 +566,14 @@ export class DiscordBotClient extends EventTarget { this.input.handleVoiceIdle(args); break; } + case 'messagereactionadd': { + this.output.handleMessageReactionAdd(args); + break; + } + case 'messagereactionremove': { + this.output.handleMessageReactionRemove(args); + break; + } default: { console.warn('unhandled json method', method); break; diff --git a/packages/usdk/packages/upstreet-agent/packages/react-agents/loops/chat-loop.tsx b/packages/usdk/packages/upstreet-agent/packages/react-agents/loops/chat-loop.tsx index c2621f38c..fc2e8567f 100644 --- a/packages/usdk/packages/upstreet-agent/packages/react-agents/loops/chat-loop.tsx +++ b/packages/usdk/packages/upstreet-agent/packages/react-agents/loops/chat-loop.tsx @@ -1,7 +1,7 @@ import React, { useState } from 'react'; import { useAgent } from '../hooks'; import { Perception } from '../components/core/perception'; -import { LoopProps } from '../types'; +import { EvaluateOpts, LoopProps } from '../types'; import { BasicEvaluator } from '../evaluators/basic-evaluator'; export const ChatLoop = (props: LoopProps) => { @@ -39,6 +39,27 @@ export const ChatLoop = (props: LoopProps) => { }} priority={-1} /> + { + const { targetAgent } = e.data; + + + console.log('messageReaction: ', e.data); + (async () => { + const abortController = new AbortController(); + const { signal } = abortController; + + const opts: EvaluateOpts = { + signal, + sendTyping: false, + generativeAgent: targetAgent, + }; + await targetAgent.evaluate(evaluator, opts); + })(); + }} + priority={-1} + /> { diff --git a/packages/usdk/packages/upstreet-agent/packages/react-agents/types/react-agents.d.ts b/packages/usdk/packages/upstreet-agent/packages/react-agents/types/react-agents.d.ts index a1851d0a1..ac80375bf 100644 --- a/packages/usdk/packages/upstreet-agent/packages/react-agents/types/react-agents.d.ts +++ b/packages/usdk/packages/upstreet-agent/packages/react-agents/types/react-agents.d.ts @@ -194,6 +194,7 @@ export type EvaluatorOpts = { export type EvaluateOpts = { generativeAgent: GenerativeAgentObject, signal?: AbortSignal, + sendTyping?: boolean, }; export type Evaluator = { evaluate: (opts: EvaluateOpts) => Promise;