Enhancing LLM Agent Performance through Dynamic Plugin Selection and W&B Prompts

# Set up the base template
template = f"""
System Rules:
{system_rules}

Query Transform Rules (only do this for the initial input query!):
{query_transform_rules}

Task:
Answer the following questions as best you can, but speaking as {speaker} might speak. You have access to the following tools:

{{tools}}

Use the following format:

Question: the input question you must answer
Thought: you should always think about what to do
Action: the action to take, should be one of [{{tool_names}}]
Action Input: the input to the action
Observation: the result of the action
... (this Thought/Action/Action Input/Observation can repeat N times)
Thought: I now know the final answer
Final Answer: the final answer to the original input question

Begin!
Remember to speak as {speaker} when giving your final answer.
{speaker_rules}

Question: {{input}}
{{agent_scratchpad}}"""

def search_function(...):
	pass

tools = [
    Tool.from_function(
        func=search_function,
        name="Search",
        description="useful for when you need to answer questions about current events"
        # coroutine= ... <- you can specify an async method if desired as well
    ),
]

openapi: 3.0.1
info:
  title: Weather
  description: Allows users to fetch current and forecasted weather information based on location. You MUST ALWAYS convert the plugin response to the units that are most useful to your user, when in doubt assume USA/Enlish units.
  version: 'v1.2'
servers:
  - url: https://weather--vicentescode.repl.co
paths:
  /weathernow:
    get:
      operationId: getWeatherNow
      summary: Get the current weather information based on city, state, and country. You MUST ALWAYS convert the plugin response to the units that are most useful to your user, when in doubt assume USA/Enlish units.
      parameters:
      - in: query
        name: city
        schema:
          type: string
        required: true
        description: The city name.
      - in: query
        name: state
        ...
      - in: query
        name: country
        ...
      responses:
        "200":
          description: OK
          content:
            application/json:
              schema:
                type: object
                additionalProperties: true
...

import os
os.environ["LANGCHAIN_WANDB_TRACING"] = "true"
os.environ["WANDB_PROJECT"] = "langchain-plugin-retrieval-demo"

run = wandb.init(project=os.environ["WANDB_PROJECT"], name="save_company_plugin_registry")
plugin_registry_art = wandb.Artifact(name="company_plugin_registry",
type="plugins",
description="Registry to hold our list of all available plugins")
plugin_registry_art.add_file(all_plugins_fp)
run.log_artifact(plugin_registry_art)

all_plugins_fp = run.use_artifact('company_plugin_registry:latest').get_path("chatgpt_plugins.json").download()

#Load from artifact
with open(all_plugins_fp, "r") as json_file:
    chatgpt_plugins = json.load(json_file)

#Process details for easy retrieval and investigation
AI_PLUGIN_DETAILS = {}
for plugin in chatgpt_plugins["items"]:
            #…
            AI_PLUGIN_DETAILS[”detail"]] = plugin_manifest

embeddings = OpenAIEmbeddings()
docs = [
    Document(page_content=detail["description_for_model"],
    metadata={"plugin_name": detail["name_for_model"]})
    for detail in AI_PLUGIN_DETAILS.values()
]
vector_store = FAISS.from_documents(docs, embeddings)
#…
plugin_registry_art = wandb.Artifact(name="subset_plugin_registry",
    type="plugins",
    metadata=meta,
    description="Registry to hold our vector store of indexed plugins, and metadata of selected plugins."
)
vector_store.save_local("faiss_index")
with open("selected_plugins.json", "w", encoding='utf-8') as outfile:
    json.dump(AI_PLUGIN_DETAILS, outfile, ensure_ascii=False, indent=4)
plugin_registry_art.add_dir("faiss_index", name="faiss_index/")
plugin_registry_art.add_file("selected_plugins.json")
run.log_artifact(plugin_registry_art)
run.finish()

subset_plugin_registry_fp = run.use_artifact('subset_plugin_registry:latest').download()
#…
vector_store = FAISS.load_local(faiss_index_fp, embeddings)
with open(selected_plugins_fp, "r") as json_file:
     AI_PLUGIN_DETAILS = json.load(json_file)
AI_PLUGINS = [AIPlugin.from_url(detail["spec_url"]) for detail in AI_PLUGIN_DETAILS.values()]
toolkits_dict = {plugin.name_for_model:
    NLAToolkit.from_llm_and_ai_plugin(llm, plugin)
    for plugin in AI_PLUGINS
}

retriever = vector_store.as_retriever()

def get_tools(query):
    # Get documents, which contain the Plugins to use
    docs = retriever.get_relevant_documents(query)
    # Get the toolkits, one for each plugin
    tool_kits = [toolkits_dict[d.metadata["plugin_name"]] for d in docs]
    # Get the tools: a separate NLAChain for each endpoint
    tools = []
    for tk in tool_kits:
        tools.extend(tk.nla_tools)
    return tools

# Set up a prompt template
class CustomPromptTemplate(StringPromptTemplate):
    # The template to use
    template: str
    ############## NEW ######################
    # The list of tools available
    tools_getter: Callable

    def format(self, **kwargs) -> str:
        # Get the intermediate steps (AgentAction, Observation tuples)
        # Format them in a particular way
        intermediate_steps = kwargs.pop("intermediate_steps")
        thoughts = ""
        for action, observation in intermediate_steps:
            thoughts += action.log
            thoughts += f"\nObservation: {observation}\nThought: "
        # Set the agent_scratchpad variable to that value
        kwargs["agent_scratchpad"] = thoughts
        ############## NEW ######################
        tools = self.tools_getter(kwargs["input"])
        # Create a tools variable from the list of tools provided
        kwargs["tools"] = "\n".join([f"{tool.name}: {tool.description}" for tool in tools])
        # Create a list of tool names for the tools provided
        kwargs["tool_names"] = ", ".join([tool.name for tool in tools])
        return self.template.format(**kwargs)

prompt = CustomPromptTemplate(
template=template,
tools_getter=get_tools,
# This omits the agent_scratchpad, tools, and tool_names variables because those are generated dynamically
# This includes the intermediate_steps variable because that is needed
input_variables=["input", "intermediate_steps"]
)

class CustomOutputParser(AgentOutputParser):

    def parse(self, llm_output: str) -> Union[AgentAction, AgentFinish]:
        # Check if agent should finish
        if "Final Answer:" in llm_output:
            return AgentFinish(
                # Return values is generally always a dictionary with a single `output` key
                # It is not recommended to try anything else at the moment :)
                return_values={"output": llm_output.split("Final Answer:")[-1].strip()},
                log=llm_output,
            )
        # Parse out the action and action input
        regex = r"Action\s*\d*\s*:(.*?)\nAction\s*\d*\s*Input\s*\d*\s*:[\s]*(.*)"
        match = re.search(regex, llm_output, re.DOTALL)
        if not match:
            raise ValueError(f"Could not parse LLM output: `{llm_output}`")
        action = match.group(1).strip()
        action_input = match.group(2)
        # Return the action and action input
        return AgentAction(tool=action, tool_input=action_input.strip(" ").strip('"'), log=llm_output)

llm = ChatOpenAI(temperature=0, model=model)
# LLM chain consisting of the LLM and a prompt
llm_chain = LLMChain(llm=llm, prompt=prompt)
tool_names = [tool.name for tool in tools]
agent = LLMSingleActionAgent(
    llm_chain=llm_chain,
    output_parser=output_parser,
    stop=["\nObservation:"],
    allowed_tools=tool_names
)
agent_executor = AgentExecutor.from_agent_and_tools(agent=agent, tools=tools, verbose=True)

query="I'm a man going out with friends to a pub crawl today in Philadelphia. What should I wear”
result = agent_executor.run(query)
print(result)