Mixture-of-Agents
Paper that proposes this idea (https://arxiv.org/pdf/2406.04692) Model is avialabe on HUgging face (https://huggingface.co/papers/2406.04692)
Implementation by Together AI Implemntation by Groq Implementation by Swarm
#!pip install together
import os
os.environ["TOGETHER_API_KEY"] = "dd26ca3ddf031d1c399077914b9824f9e7f9dc5c256d872c14d8a23383731ed2"
# Advanced Mixture-of-Agents example – 3 layers
import asyncio
import os
import together
from together import AsyncTogether, Together
client = Together()
async_client = AsyncTogether()
user_prompt = "What are 3 fun things to do in SF?"
reference_models = [
"Qwen/Qwen2-72B-Instruct",
"Qwen/Qwen1.5-72B-Chat",
"mistralai/Mixtral-8x22B-Instruct-v0.1",
"databricks/dbrx-instruct",
]
aggregator_model = "mistralai/Mixtral-8x22B-Instruct-v0.1"
aggreagator_system_prompt = """You have been provided with a set of responses from various open-source models to the latest user query. Your task is to synthesize these responses into a single, high-quality response. It is crucial to critically evaluate the information provided in these responses, recognizing that some of it may be biased or incorrect. Your response should not simply replicate the given answers but should offer a refined, accurate, and comprehensive reply to the instruction. Ensure your response is well-structured, coherent, and adheres to the highest standards of accuracy and reliability.
Responses from models:"""
layers = 3
def getFinalSystemPrompt(system_prompt, results):
"""Construct a system prompt for layers 2+ that includes the previous responses to synthesize."""
return (
system_prompt
+ "\n"
+ "\n".join([f"{i+1}. {str(element)}" for i, element in enumerate(results)])
)
async def run_llm(model, prev_response=None):
"""Run a single LLM call with a model while accounting for previous responses + rate limits."""
for sleep_time in [1, 2, 4]:
try:
messages = (
[
{
"role": "system",
"content": getFinalSystemPrompt(
aggreagator_system_prompt, prev_response
),
},
{"role": "user", "content": user_prompt},
]
if prev_response
else [{"role": "user", "content": user_prompt}]
)
response = await async_client.chat.completions.create(
model=model,
messages=messages,
temperature=0.7,
max_tokens=512,
)
print("Model: ", model)
break
except together.error.RateLimitError as e:
print(e)
await asyncio.sleep(sleep_time)
return response.choices[0].message.content
async def main():
"""Run the main loop of the MOA process."""
results = await asyncio.gather(*[run_llm(model) for model in reference_models])
for _ in range(1, layers - 1):
results = await asyncio.gather(
*[run_llm(model, prev_response=results) for model in reference_models]
)
finalStream = client.chat.completions.create(
model=aggregator_model,
messages=[
{
"role": "system",
"content": getFinalSystemPrompt(aggreagator_system_prompt, results),
},
{"role": "user", "content": user_prompt},
],
stream=True,
)
for chunk in finalStream:
print(chunk.choices[0].delta.content or "", end="", flush=True)
asyncio.run(main())
---------------------------------------------------------------------------
ModuleNotFoundError Traceback (most recent call last)
Cell In[3], line 4
1 # Advanced Mixture-of-Agents example – 3 layers
2 import asyncio
3 import os
----> 4 import together
5 from together import AsyncTogether, Together
6
7 client = Together()
ModuleNotFoundError: No module named 'together'
import asyncio
import nest_asyncio
import together
from together import AsyncTogether, Together
nest_asyncio.apply() # Allows nested event loops
client = Together()
async_client = AsyncTogether()
user_prompt = "What are 3 fun things to do in SF?"
# Update models based on Together's available models
reference_models = [
"Qwen/Qwen2-72B-Instruct", # Ensure this model is available
"mistralai/Mixtral-8x22B-Instruct-v0.1",
"databricks/dbrx-instruct",
]
aggregator_model = "mistralai/Mixtral-8x22B-Instruct-v0.1"
aggregator_system_prompt = """You have been provided with a set of responses from various open-source models..."""
layers = 3
def get_final_system_prompt(system_prompt, results):
"""Construct a system prompt for layers 2+ that includes the previous responses to synthesize."""
return (
system_prompt
+ "\n"
+ "\n".join([f"{i+1}. {str(element)}" for i, element in enumerate(results)])
)
async def run_llm(model, prev_response=None):
"""Run a single LLM call with a model while accounting for previous responses + rate limits."""
for sleep_time in [1, 2, 4, 8]: # Increasing wait times for retry
try:
messages = (
[
{
"role": "system",
"content": get_final_system_prompt(
aggregator_system_prompt, prev_response
),
},
{"role": "user", "content": user_prompt},
]
if prev_response
else [{"role": "user", "content": user_prompt}]
)
response = await async_client.chat.completions.create(
model=model,
messages=messages,
temperature=0.7,
max_tokens=512,
)
print("Model:", model)
return response.choices[0].message.content
except together.error.RateLimitError as e:
print(f"Rate limit reached: {e}. Retrying in {sleep_time} seconds.")
await asyncio.sleep(sleep_time) # Wait before retrying
except together.error.InvalidRequestError as e:
print(f"Model {model} is not available. Skipping.")
return None # Skip unavailable model
async def main():
"""Run the main loop of the MOA process."""
results = await asyncio.gather(*[run_llm(model) for model in reference_models])
for _ in range(1, layers - 1):
await asyncio.sleep(1) # Pause to avoid rate limits
results = await asyncio.gather(
*[run_llm(model, prev_response=results) for model in reference_models]
)
finalStream = client.chat.completions.create(
model=aggregator_model,
messages=[
{
"role": "system",
"content": get_final_system_prompt(aggregator_system_prompt, results),
},
{"role": "user", "content": user_prompt},
],
stream=True,
)
for chunk in finalStream:
print(chunk.choices[0].delta.content or "", end="", flush=True)
await main()
Rate limit reached: Error code: 429 - {"message": "Request was rejected due to request rate limiting. Your rate limits are 60 RPM (1 QPS) and 60000 TPM (1000 TPS). See details: https://docs.together.ai/docs/rate-limits", "type_": "credit_limit"}. Retrying in 1 seconds.
Rate limit reached: Error code: 429 - {"message": "Request was rejected due to request rate limiting. Your rate limits are 60 RPM (1 QPS) and 60000 TPM (1000 TPS). See details: https://docs.together.ai/docs/rate-limits", "type_": "credit_limit"}. Retrying in 2 seconds.
Model: mistralai/Mixtral-8x22B-Instruct-v0.1
Model: Qwen/Qwen2-72B-Instruct
Model: databricks/dbrx-instruct
Rate limit reached: Error code: 429 - {"message": "Request was rejected due to request rate limiting. Your rate limits are 60 RPM (1 QPS) and 60000 TPM (1000 TPS). See details: https://docs.together.ai/docs/rate-limits", "type_": "credit_limit"}. Retrying in 1 seconds.
Rate limit reached: Error code: 429 - {"message": "Request was rejected due to request rate limiting. Your rate limits are 60 RPM (1 QPS) and 60000 TPM (1000 TPS). See details: https://docs.together.ai/docs/rate-limits", "type_": "credit_limit"}. Retrying in 2 seconds.
Model: mistralai/Mixtral-8x22B-Instruct-v0.1
Model: databricks/dbrx-instruct
Model: Qwen/Qwen2-72B-Instruct
1. Cross the Golden Gate Bridge: This iconic landmark offers breathtaking views of the San Francisco Bay and the city skyline. You can walk or bike across the bridge and visit the Golden Gate Bridge Welcome Center to learn about its history and engineering.
2. Discover Alcatraz Island: Take a ferry to this historic island and explore the infamous federal prison that once housed notorious criminals. The self-guided audio tour provides a fascinating look into the island's history, and you can enjoy panoramic views of the city and bay.
3. Experience the Exploratorium: This interactive science museum offers a fun and educational experience for all ages with over 600 hands-on exhibits covering topics such as physics, biology, and human perception.