Parameter completion emoji in Schema Config (#1504)

curtisman · web-flow · commit f05a5f972456 · 2025-08-23T03:36:02.000Z
Allow agent to specify the emoji for parameters in completions using the SchemaConfig file. Closes #1487
diff --git a/ts/packages/actionSchema/src/index.ts b/ts/packages/actionSchema/src/index.ts
@@ -12,6 +12,7 @@ export {
     ParsedActionSchema,
     ActionSchemaObject,
     ActionSchemaUnion,
+    SchemaObjectFields,
 } from "./type.js";
 
 export { parseActionSchemaSource } from "./parser.js";
@@ -49,7 +50,12 @@ export {
 export { validateType } from "./validate.js";
 
 // Schema Config
-export { SchemaConfig, ParamSpec, ActionParamSpecs } from "./schemaConfig.js";
+export {
+    SchemaConfig,
+    ParamSpec,
+    ActionParamSpecs,
+    CompletionEmojis,
+} from "./schemaConfig.js";
 
 // Legacy (to be deprecated)
 export { NodeType, SchemaParser, ISymbol, SymbolNode } from "./schemaParser.js";
diff --git a/ts/packages/actionSchema/src/parser.ts b/ts/packages/actionSchema/src/parser.ts
@@ -193,6 +193,17 @@ function checkActionSchema(
         }
         actionDefinition.paramSpecs = paramSpecs;
     }
+
+    const paramCompletionEmojis =
+        schemaConfig?.paramCompletionEmojis?.[actionNameString];
+    if (paramCompletionEmojis) {
+        actionDefinition.paramCompletionEmojis = paramCompletionEmojis;
+    }
+    const entityCompletionEmojis =
+        schemaConfig?.entityCompletionEmojis?.[actionNameString];
+    if (entityCompletionEmojis) {
+        actionDefinition.entityCompletionEmojis = entityCompletionEmojis;
+    }
     return [actionNameString, actionDefinition];
 }
 
diff --git a/ts/packages/actionSchema/src/schemaConfig.ts b/ts/packages/actionSchema/src/schemaConfig.ts
@@ -12,12 +12,16 @@ export type ParamSpec =
     | "literal";
 
 export type ActionParamSpecs = Record<string, ParamSpec> | false;
+export type CompletionEmojis = Record<string, string>; // Key is parameter/entity name, value is emoji string
 export type SchemaConfig = {
     // Key is the action name.
     // If the value is false, then explanation/caching is disabled.
     // Otherwise, the value is an object where the key is the parameter name, and the value the one of the ParamSpec above.
     paramSpec?: Record<string, ActionParamSpecs>;
 
+    paramCompletionEmojis?: Record<string, CompletionEmojis>; // Key is action name
+    entityCompletionEmojis?: Record<string, CompletionEmojis>; // Key is entity type name
+
     // separate the cache by action name
     actionNamespace?: boolean; // default to false
 };
diff --git a/ts/packages/actionSchema/src/type.ts b/ts/packages/actionSchema/src/type.ts
@@ -1,7 +1,7 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-import { ActionParamSpecs } from "./schemaConfig.js";
+import { ActionParamSpecs, CompletionEmojis } from "./schemaConfig.js";
 
 export interface SchemaBase {
     type:
@@ -130,7 +130,11 @@ export interface ActionSchemaObject extends SchemaTypeObject {
 export type ActionSchemaTypeDefinition = (
     | SchemaTypeInterfaceDefinition<ActionSchemaObject>
     | SchemaTypeAliasDefinition<ActionSchemaObject>
-) & { paramSpecs?: ActionParamSpecs };
+) & {
+    paramSpecs?: ActionParamSpecs;
+    paramCompletionEmojis?: CompletionEmojis;
+    entityCompletionEmojis?: CompletionEmojis;
+};
 
 export type ActionSchemaEntryTypeDefinition =
     | ActionSchemaTypeDefinition
diff --git a/ts/packages/agentSdk/src/command.ts b/ts/packages/agentSdk/src/command.ts
@@ -53,7 +53,7 @@ export type CompletionGroup = {
     name: string; // The group name for the completion
     completions: string[]; // The list of completions in the group
     needQuotes?: boolean; // If true, the completion should be quoted if it has spaces.
-    emojiChar?: string; // Optional icon for the completion category
+    emojiChar?: string | undefined; // Optional icon for the completion category
 };
 
 export interface AppAgentCommandInterface {
diff --git a/ts/packages/agents/player/src/agent/playerSchema.json b/ts/packages/agents/player/src/agent/playerSchema.json
@@ -40,5 +40,23 @@
     "getFavorites": {
       "count": "number"
     }
+  },
+  "paramCompletionEmojis": {
+    "playTrack": {
+      "trackName": "🎶",
+      "artists.*": "🎼"
+    },
+    "playAlbum": {
+      "albumName": "💿",
+      "artists.*": "🎼"
+    },
+    "playAlbumTrack": {
+      "albumName": "💿",
+      "trackName": "🎶",
+      "artists.*": "🎼"
+    },
+    "playArtist": {
+      "artist": "🎼"
+    }
   }
 }
diff --git a/ts/packages/cli/src/commands/schema.ts b/ts/packages/cli/src/commands/schema.ts
@@ -6,7 +6,7 @@ import { composeTranslatorSchemas } from "common-utils";
 import {
     getAssistantSelectionSchemas,
     getFullSchemaText,
-    getActionSchema,
+    tryGetActionSchema,
     getAllActionConfigProvider,
 } from "agent-dispatcher/internal";
 import { getInstanceDir } from "agent-dispatcher/helpers/data";
@@ -65,7 +65,7 @@ export default class Schema extends Command {
         const { args, flags } = await this.parse(Schema);
         if (!flags.assistant) {
             if (args.actionName) {
-                const actionSchema = getActionSchema(
+                const actionSchema = tryGetActionSchema(
                     {
                         schemaName: args.schemaName,
                         actionName: args.actionName,
diff --git a/ts/packages/dispatcher/src/context/system/handlers/actionCommandHandler.ts b/ts/packages/dispatcher/src/context/system/handlers/actionCommandHandler.ts
@@ -15,9 +15,10 @@ import { CommandHandlerContext } from "../../commandHandlerContext.js";
 import { getParameterNames, validateAction } from "action-schema";
 import { executeActions } from "../../../execute/actionHandlers.js";
 import { FullAction, toExecutableActions } from "agent-cache";
-import { getActionSchema } from "../../../internal.js";
 import { DeepPartialUndefined, getObjectProperty } from "common-utils";
 import { getActionParamCompletion } from "../../../translation/requestCompletion.js";
+import { getActionSchema } from "../../../translation/actionSchemaUtils.js";
+import { tryGetActionSchema } from "../../../translation/actionSchemaFileCache.js";
 
 export class ActionCommandHandler implements CommandHandler {
     public readonly description = "Execute an action";
@@ -50,14 +51,7 @@ export class ActionCommandHandler implements CommandHandler {
             throw new Error(`Invalid schema name ${schemaName}`);
         }
 
-        const actionSchema =
-            actionSchemaFile.parsedActionSchema.actionSchemas.get(actionName);
-        if (actionSchema === undefined) {
-            throw new Error(
-                `Invalid action name ${actionName} for schema ${schemaName}`,
-            );
-        }
-
+        const actionSchema = getActionSchema(actionSchemaFile, actionName);
         const action: AppAction = {
             schemaName: schemaName,
             actionName,
@@ -118,7 +112,7 @@ export class ActionCommandHandler implements CommandHandler {
                     actionName: params.args?.actionName,
                     parameters: params.flags?.parameters,
                 };
-                const actionInfo = getActionSchema(
+                const actionInfo = tryGetActionSchema(
                     action,
                     systemContext.agents,
                 );
@@ -156,10 +150,10 @@ export class ActionCommandHandler implements CommandHandler {
                     name.substring("--parameters.".length),
                 );
 
-                if (parameterCompletion && parameterCompletion.length > 0) {
+                if (parameterCompletion) {
                     completions.push({
                         name,
-                        completions: parameterCompletion,
+                        ...parameterCompletion,
                     });
                 }
 
diff --git a/ts/packages/dispatcher/src/execute/actionHandlers.ts b/ts/packages/dispatcher/src/execute/actionHandlers.ts
@@ -42,7 +42,6 @@ import {
     isSwitchEnabled,
     translatePendingRequestAction,
 } from "../translation/translateRequest.js";
-import { getActionSchema } from "../internal.js";
 import { validateAction } from "action-schema";
 import {
     PendingAction,
@@ -55,6 +54,7 @@ import {
     addResultToMemory,
 } from "../context/memory.js";
 import { setActivityContext } from "./activityContext.js";
+import { tryGetActionSchema } from "../translation/actionSchemaFileCache.js";
 
 const debugActions = registerDebug("typeagent:dispatcher:actions");
 
@@ -458,7 +458,7 @@ function getAdditionalExecutableActions(
             }
         }
 
-        const actionInfo = getActionSchema(fullAction, context.agents);
+        const actionInfo = tryGetActionSchema(fullAction, context.agents);
         if (actionInfo === undefined) {
             throw new Error(
                 `Action not found ${fullAction.schemaName}.${fullAction.actionName}`,
diff --git a/ts/packages/dispatcher/src/execute/pendingActions.ts b/ts/packages/dispatcher/src/execute/pendingActions.ts
@@ -39,6 +39,7 @@ import { conversation as kp } from "knowledge-processor";
 import { getObjectProperty } from "common-utils";
 import { ActionSchemaFile } from "../translation/actionConfigProvider.js";
 import { ActionSchemaEntityTypeDefinition } from "../../../actionSchema/dist/type.js";
+import { getActionParametersType } from "../translation/actionSchemaUtils.js";
 
 const debugEntities = registerDebug("typeagent:dispatcher:actions:entities");
 
@@ -257,8 +258,8 @@ export function getEntityPropertyTypeName(
         return undefined;
     }
     const actionParametersType = getActionParametersType(
-        actionName,
         actionSchemaFile,
+        actionName,
     );
     const parameterType = getPropertyType(actionParametersType, paramName);
     if (parameterType === undefined) {
@@ -769,28 +770,6 @@ async function getParameterEntities(
     }
 }
 
-export function getActionParametersType(
-    actionName: string,
-    actionSchemaFile: ActionSchemaFile,
-): ActionParamObject {
-    const schema =
-        actionSchemaFile.parsedActionSchema.actionSchemas.get(actionName);
-
-    if (schema === undefined) {
-        throw new Error(
-            `Action schema not found for ${actionSchemaFile.schemaName}.${actionName}`,
-        );
-    }
-
-    const actionParametersType = schema.type.fields.parameters?.type;
-    if (actionParametersType?.type !== "object") {
-        throw new Error(
-            `Action schema parameter type mismatch: ${actionSchemaFile.schemaName}.${actionName}`,
-        );
-    }
-    return actionParametersType;
-}
-
 export async function resolveEntities(
     agents: AppAgentManager,
     action: TypeAgentAction<FullAction>,
@@ -804,8 +783,8 @@ export async function resolveEntities(
     const config = agents.getActionConfig(action.schemaName);
     const actionSchemaFile = agents.getActionSchemaFileForConfig(config);
     const actionParametersType = getActionParametersType(
-        action.actionName,
         actionSchemaFile,
+        action.actionName,
     );
 
     let resolvedEntities: Entity[] | undefined;
diff --git a/ts/packages/dispatcher/src/internal.ts b/ts/packages/dispatcher/src/internal.ts
@@ -19,9 +19,8 @@ export {
 } from "./utils/test/explanationTestData.js";
 
 export { getAssistantSelectionSchemas } from "./translation/unknownSwitcher.js";
-export { getActionSchema } from "./translation/actionSchemaFileCache.js";
 export { getFullSchemaText } from "./translation/agentTranslators.js";
-
+export { tryGetActionSchema } from "./translation/actionSchemaFileCache.js";
 export { getAppAgentName } from "./translation/agentTranslators.js";
 export { createSchemaInfoProvider } from "./translation/actionSchemaFileCache.js";
 export { getAllActionConfigProvider } from "./context/inlineAgentProvider.js";
diff --git a/ts/packages/dispatcher/src/translation/actionSchemaFileCache.ts b/ts/packages/dispatcher/src/translation/actionSchemaFileCache.ts
@@ -27,6 +27,7 @@ import {
     getActivitySchemaTypeName,
 } from "./agentTranslators.js";
 import { getEntityPropertyTypeName } from "../execute/pendingActions.js";
+import { getActionSchema } from "./actionSchemaUtils.js";
 
 const debug = registerDebug("typeagent:dispatcher:schema:cache");
 const debugError = registerDebug("typeagent:dispatcher:schema:cache:error");
@@ -288,7 +289,7 @@ export class ActionSchemaFileCache {
     }
 }
 
-export function getActionSchema(
+export function tryGetActionSchema(
     action: DeepPartialUndefined<AppAction>,
     provider: ActionConfigProvider,
 ): ActionSchemaTypeDefinition | undefined {
@@ -305,6 +306,26 @@ export function getActionSchema(
     return actionSchemaFile.parsedActionSchema.actionSchemas.get(actionName);
 }
 
+export function getParamPatternValue<T>(
+    patternMap: Record<string, T> | undefined,
+    paramName: string,
+): T | undefined {
+    if (patternMap === undefined) {
+        return undefined;
+    }
+    for (const [key, value] of Object.entries(patternMap)) {
+        if (key.includes("*")) {
+            const regex = simpleStarRegex(key);
+            if (regex.test(paramName)) {
+                return value;
+            }
+        } else if (key === paramName) {
+            return value;
+        }
+    }
+    return undefined;
+}
+
 export function createSchemaInfoProvider(
     provider: ActionConfigProvider,
 ): SchemaInfoProvider {
@@ -314,19 +335,6 @@ export function createSchemaInfoProvider(
         );
     };
 
-    const getActionSchema = (
-        actionSchemaFile: ActionSchemaFile,
-        actionName: string,
-    ) => {
-        const actionSchema =
-            actionSchemaFile.parsedActionSchema.actionSchemas.get(actionName);
-        if (actionSchema === undefined) {
-            throw new Error(
-                `Invalid action name ${actionName} for schema ${actionSchemaFile.schemaName}`,
-            );
-        }
-        return actionSchema;
-    };
     const result: SchemaInfoProvider = {
         getActionSchemaFileHash: (schemaName) =>
             getActionSchemaFile(schemaName).sourceHash,
@@ -345,16 +353,7 @@ export function createSchemaInfoProvider(
             }
             // Check the paramSpec if there are explicit entries.
             if (paramSpecs !== undefined) {
-                for (const [key, value] of Object.entries(paramSpecs)) {
-                    if (key.includes("*")) {
-                        const regex = simpleStarRegex(key);
-                        if (regex.test(paramName)) {
-                            return value;
-                        }
-                    } else if (key === paramName) {
-                        return value;
-                    }
-                }
+                return getParamPatternValue(paramSpecs, paramName);
             }
 
             // Check if it is an entity-type.
diff --git a/ts/packages/dispatcher/src/translation/actionSchemaUtils.ts b/ts/packages/dispatcher/src/translation/actionSchemaUtils.ts
@@ -0,0 +1,42 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import { ActionParamObject, ActionSchemaTypeDefinition } from "action-schema";
+import { ActionSchemaFile } from "./actionConfigProvider.js";
+
+export function getActionSchema(
+    actionSchemaFile: ActionSchemaFile,
+    actionName: string,
+) {
+    const schema =
+        actionSchemaFile.parsedActionSchema.actionSchemas.get(actionName);
+
+    if (schema === undefined) {
+        throw new Error(
+            `Action schema not found for ${actionSchemaFile.schemaName}.${actionName}`,
+        );
+    }
+    return schema;
+}
+
+export function getActionSchemaParameterType(
+    actionSchemaFile: ActionSchemaFile,
+    actionName: string,
+    schema: ActionSchemaTypeDefinition,
+) {
+    const actionParametersType = schema.type.fields.parameters?.type;
+    if (actionParametersType?.type !== "object") {
+        throw new Error(
+            `Action schema parameter type mismatch: ${actionSchemaFile.schemaName}.${actionName}`,
+        );
+    }
+    return actionParametersType;
+}
+
+export function getActionParametersType(
+    actionSchemaFile: ActionSchemaFile,
+    actionName: string,
+): ActionParamObject {
+    const schema = getActionSchema(actionSchemaFile, actionName);
+    return getActionSchemaParameterType(actionSchemaFile, actionName, schema);
+}
diff --git a/ts/packages/dispatcher/src/translation/actionTemplate.ts b/ts/packages/dispatcher/src/translation/actionTemplate.ts
diff --git a/ts/packages/dispatcher/src/translation/confirmTranslation.ts b/ts/packages/dispatcher/src/translation/confirmTranslation.ts
diff --git a/ts/packages/dispatcher/src/translation/requestCompletion.ts b/ts/packages/dispatcher/src/translation/requestCompletion.ts
diff --git a/ts/packages/shell/src/renderer/src/partial.ts b/ts/packages/shell/src/renderer/src/partial.ts

Original file line number	Diff line number	Diff line change
`@@ -193,6 +193,17 @@ function checkActionSchema(`
`193`	`193`	`}`
`194`	`194`	`actionDefinition.paramSpecs = paramSpecs;`
`195`	`195`	`}`
	`196`	`+`
	`197`	`+ const paramCompletionEmojis =`
	`198`	`+ schemaConfig?.paramCompletionEmojis?.[actionNameString];`
	`199`	`+ if (paramCompletionEmojis) {`
	`200`	`+ actionDefinition.paramCompletionEmojis = paramCompletionEmojis;`
	`201`	`+ }`
	`202`	`+ const entityCompletionEmojis =`
	`203`	`+ schemaConfig?.entityCompletionEmojis?.[actionNameString];`
	`204`	`+ if (entityCompletionEmojis) {`
	`205`	`+ actionDefinition.entityCompletionEmojis = entityCompletionEmojis;`
	`206`	`+ }`
`196`	`207`	`return [actionNameString, actionDefinition];`
`197`	`208`	`}`
`198`	`209`