llama stack build

代码示例

llama stack build --list-templates

代码示例

llama stack build --template together --image-type venv

代码示例

你可以在以下路径找到新构建的模板：/.venv/lib/python3.12/site-packages/llama_stack/templates/together/run.yaml
可通过以下命令运行新的 Llama Stack 发行版：llama stack run /.venv/lib/python3.12/site-packages/llama_stack/templates/together/run.yaml --image-type venv

代码示例

cat /.venv/lib/python3.12/site-packages/llama_stack/templates/together/run.yaml

version: '2' image_name: together apis:

• agents

• datasetio

• eval

• inference

• safety

• scoring

• telemetry

• tool_runtime

• vector_io providers: inference:

  • provider_id: together provider_type: 'remote::together' config: url: 'https://api.together.xyz/v1' api_key: '${env.TOGETHER_API_KEY:}'
  • provider_id: sentence-transformers provider_type: 'inline::sentence-transformers' config: {} vector_io:
  • provider_id: faiss provider_type: 'inline::faiss' config: kvstore: type: sqlite namespace: null db_path: >- ${env.SQLITE_STORE_DIR:~/.llama/distributions/together}/faiss_store.db safety:
  • provider_id: llama-guard provider_type: 'inline::llama-guard' config: excluded_categories: [] agents:
  • provider_id: meta-reference provider_type: 'inline::meta-reference' config: persistence_store: type: sqlite namespace: null db_path: >- ${env.SQLITE_STORE_DIR:~/.llama/distributions/together}/agents_store.db telemetry:
  • provider_id: meta-reference provider_type: 'inline::meta-reference' config: service_name: '${env.OTEL_SERVICE_NAME:}' sinks: '${env.TELEMETRY_SINKS:console,sqlite}' sqlite_db_path: '${env.SQLITE_STORE_DIR:~/.llama/distributions/together}/trace_store.db' eval:
  • provider_id: meta-reference provider_type: 'inline::meta-reference' config: kvstore: type: sqlite namespace: null db_path: >- ${env.SQLITE_STORE_DIR:~/.llama/distributions/together}/meta_reference_eval.db' datasetio:
  • provider_id: huggingface provider_type: 'remote::huggingface' config: kvstore: type: sqlite namespace: null db_path: >- ${env.SQLITE_STORE_DIR:~/.llama/distributions/together}/huggingface_datasetio.db
  • provider_id: localfs provider_type: 'inline::localfs' config: kvstore: type: sqlite namespace: null db_path: >- ${env.SQLITE_STORE_DIR:~/.llama/distributions/together}/localfs_datasetio.db scoring:
  • provider_id: basic provider_type: 'inline::basic' config: {}
  • provider_id: llm-as-judge provider_type: 'inline::llm-as-judge' config: {}
  • provider_id: braintrust provider_type: 'inline::braintrust' config: openai_api_key: '${env.OPENAI_API_KEY:}' tool_runtime:
  • provider_id: brave-search provider_type: 'remote::brave-search' config: api_key: '${env.BRAVE_SEARCH_API_KEY:}' max_results: 3
  • provider_id: tavily-search provider_type: 'remote::tavily-search' config: api_key: '${env.TAVILY_SEARCH_API_KEY:}' max_results: 3
  • provider_id: rag-runtime provider_type: 'inline::rag-runtime' config: {}
  • provider_id: model-context-protocol provider_type: 'remote::model-context-protocol' config: {}
  • provider_id: wolfram-alpha provider_type: 'remote::wolfram-alpha' config: api_key: '${env.WOLFRAM_ALPHA_API_KEY:}' metadata_store: type: sqlite db_path: '${env.SQLITE_STORE_DIR:~/.llama/distributions/together}/registry.db' models:

• metadata: {} model_id: meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo provider_id: together provider_model_id: meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo model_type: llm

• metadata: {} model_id: meta-llama/Llama-3.1-8B-Instruct provider_id: together provider_model_id: meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo model_type: llm

• metadata: {} model_id: meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo provider_id: together provider_model_id: meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo model_type: llm

• metadata: {} model_id: meta-llama/Llama-3.1-70B-Instruct provider_id: together provider_model_id: meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo model_type: llm

• metadata: {} model_id: meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo provider_id: together provider_model_id: meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo model_type: llm

• metadata: {} model_id: meta-llama/Llama-3.1-405B-Instruct-FP8 provider_id: together provider_model_id: meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo model_type: llm

• metadata: {} model_id: meta-llama/Llama-3.2-3B-Instruct-Turbo provider_id: together provider_model_id: meta-llama/Llama-3.2-3B-Instruct-Turbo model_type: llm

• metadata: {} model_id: meta-llama/Llama-3.2-3B-Instruct provider_id: together provider_model_id: meta-llama/Llama-3.2-3B-Instruct-Turbo model_type: llm

• metadata: {} model_id: meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo provider_id: together provider_model_id: meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo model_type: llm

• metadata: {} model_id: meta-llama/Llama-3.2-11B-Vision-Instruct provider_id: together provider_model_id: meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo model_type: llm

• metadata: {} model_id: meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo provider_id: together provider_model_id: meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo model_type: llm

• metadata: {} model_id: meta-llama/Llama-3.2-90B-Vision-Instruct provider_id: together provider_model_id: meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo model_type: llm

• metadata: {} model_id: meta-llama/Llama-3.3-70B-Instruct-Turbo provider_id: together provider_model_id: meta-llama/Llama-3.3-70B-Instruct-Turbo model_type: llm

• metadata: {} model_id: meta-llama/Llama-3.3-70B-Instruct provider_id: together provider_model_id: meta-llama/Llama-3.3-70B-Instruct-Turbo model_type: llm

• metadata: {} model_id: meta-llama/Meta-Llama-Guard-3-8B provider_id: together provider_model_id: meta-llama/Meta-Llama-Guard-3-8B model_type: llm

• metadata: {} model_id: meta-llama/Llama-Guard-3-8B provider_id: together provider_model_id: meta-llama/Meta-Llama-Guard-3-8B model_type: llm

• metadata: {} model_id: meta-llama/Llama-Guard-3-11B-Vision-Turbo provider_id: together provider_model_id: meta-llama/Llama-Guard-3-11B-Vision-Turbo model_type: llm

• metadata: {} model_id: meta-llama/Llama-Guard-3-11B-Vision provider_id: together provider_model_id: meta-llama/Llama-Guard-3-11B-Vision-Turbo model_type: llm

• metadata: embedding_dimension: 768 context_length: 8192 model_id: togethercomputer/m2-bert-80M-8k-retrieval provider_id: together provider_model_id: togethercomputer/m2-bert-80M-8k-retrieval model_type: embedding

• metadata: embedding_dimension: 768 context_length: 32768 model_id: togethercomputer/m2-bert-80M-32k-retrieval provider_id: together provider_model_id: togethercomputer/m2-bert-80M-32k-retrieval model_type: embedding

• metadata: {} model_id: meta-llama/Llama-4-Scout-17B-16E-Instruct provider_id: together provider_model_id: meta-llama/Llama-4-Scout-17B-16E-Instruct model_type: llm

• metadata: {} model_id: meta-llama/Llama-4-Scout-17B-16E-Instruct provider_id: together provider_model_id: meta-llama/Llama-4-Scout-17B-16E-Instruct model_type: llm

• metadata: {} model_id: together/meta-llama/Llama-4-Scout-17B-16E-Instruct provider_id: together provider_model_id: meta-llama/Llama-4-Scout-17B-16E-Instruct model_type: llm

• metadata: {} model_id: meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 provider_id: together provider_model_id: meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 model_type: llm

• metadata: {} model_id: meta-llama/Llama-4-Maverick-17B-128E-Instruct provider_id: together provider_model_id: meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 model_type: llm

• metadata: {} model_id: together/meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 provider_id: together provider_model_id: meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 model_type: llm

• metadata: embedding_dimension: 384 model_id: all-MiniLM-L6-v2 provider_id: sentence-transformers model_type: embedding shields:

• shield_id: meta-llama/Llama-Guard-3-8B vector_dbs: [] datasets: [] scoring_fns: [] benchmarks: [] tool_groups:

• toolgroup_id: 'builtin::websearch' provider_id: tavily-search

• toolgroup_id: 'builtin::rag' provider_id: rag-runtime

• toolgroup_id: 'builtin::wolfram_alpha' provider_id: wolfram-alpha server: port: 8321

• provider_id: ollama provider_type: remote::ollama config: url: ${env.OLLAMA_URL:http://localhost:11434}

providers: inference:

• provider_id: ollama provider_type: remote::ollama config: url: ${env.OLLAMA_URL:http://localhost:11434}
• provider_id: together provider_type: remote::together config: url: https://api.together.xyz/v1 api_key: ${env.TOGETHER_API_KEY:}
• provider_id: sentence-transformers provider_type: inline::sentence-transformers config: {}

models:

... 已有的 Together AI 模型

• metadata: {} model_id: '${env.INFERENCE_MODEL}' provider_id: ollama model_type: llm
• metadata: embedding_dimension: 384 model_id: all-MiniLM-L6-v2 provider_id: ollama provider_model_id: 'all-minilm:latest' model_type: embedding

version: '2' image_name: together apis:

• agents
• datasetio
• eval
• inference
• safety
• scoring
• telemetry
• tool_runtime
• vector_io providers: inference:
  • provider_id: ollama provider_type: 'remote::ollama' config: url: '${env.OLLAMA_URL:http://localhost:11434}'
  • provider_id: together provider_type: 'remote::together' config: url: 'https://api.together.xyz/v1' api_key: '${env.TOGETHER_API_KEY:}'
  • provider_id: sentence-transformers provider_type: 'inline::sentence-transformers' config: {} vector_io:
  • provider_id: faiss provider_type: 'inline::faiss' config: kvstore: type: sqlite namespace: null db_path: >- ${env.SQLITE_STORE_DIR:~/.llama/distributions/together}/faiss_store.db safety:
  • provider_id: llama-guard provider_type: 'inline::llama-guard' config: excluded_categories: [] agents:
  • provider_id: meta-reference provider_type: 'inline::meta-reference' config: persistence_store: type: sqlite namespace: null db_path: >- ${env.SQLITE_STORE_DIR:~/.llama/distributions/together}/agents_store.db telemetry:
  • provider_id: meta-reference provider_type: 'inline::meta-reference' config: service_name: '${env.OTEL_SERVICE_NAME:}' sinks: '${env.TELEMETRY_SINKS:console,sqlite}' sqlite_db_path: '${env.SQLITE_STORE_DIR:~/.llama/distributions/together}/trace_store.db' eval:
  • provider_id: meta-reference provider_type: 'inline::meta-reference' config: kvstore: type: sqlite namespace: null db_path: >- ${env.SQLITE_STORE_DIR:~/.llama/distributions/together}/meta_reference_eval.db datasetio:
  • provider_id: huggingface provider_type: 'remote::huggingface' config: kvstore: type: sqlite namespace: null db_path: >- ${env.SQLITE_STORE_DIR:~/.llama/distributions/together}/huggingface_datasetio.db
  • provider_id: localfs provider_type: 'inline::localfs' config: kvstore: type: sqlite namespace: null db_path: >- ${env.SQLITE_STORE_DIR:~/.llama/distributions/together}/localfs_datasetio.db scoring:
  • provider_id: basic provider_type: 'inline::basic' config: {}
  • provider_id: llm-as-judge provider_type: 'inline::llm-as-judge' config: {}
  • provider_id: braintrust provider_type: 'inline::braintrust' config: openai_api_key: '${env.OPENAI_API_KEY:}' tool_runtime:
  • provider_id: brave-search provider_type: 'remote::brave-search' config: api_key: '${env.BRAVE_SEARCH_API_KEY:}' max_results: 3
  • provider_id: tavily-search provider_type: 'remote::tavily-search' config: api_key: '${env.TAVILY_SEARCH_API_KEY:}' max_results: 3
  • provider_id: rag-runtime provider_type: 'inline::rag-runtime' config: {}
  • provider_id: model-context-protocol provider_type: 'remote::model-context-protocol' config: {}
  • provider_id: wolfram-alpha provider_type: 'remote::wolfram-alpha' config: api_key: '${env.WOLFRAM_ALPHA_API_KEY:}' metadata_store: type: sqlite db_path: '${env.SQLITE_STORE_DIR:~/.llama/distributions/together}/registry.db' models:
• metadata: {} model_id: meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo provider_id: together provider_model_id: meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo model_type: llm
• metadata: {} model_id: meta-llama/Llama-3.1-8B-Instruct provider_id: together provider_model_id: meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo model_type: llm
• metadata: {} model_id: meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo provider_id: together provider_model_id: meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo model_type: llm
• metadata: {} model_id: meta-llama/Llama-3.1-70B-Instruct provider_id: together provider_model_id: meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo model_type: llm
• metadata: {} model_id: meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo provider_id: together provider_model_id: meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo model_type: llm
• metadata: {} model_id: meta-llama/Llama-3.1-405B-Instruct-FP8 provider_id: together provider_model_id: meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo model_type: llm
• metadata: {} model_id: meta-llama/Llama-3.2-3B-Instruct-Turbo provider_id: together provider_model_id: meta-llama/Llama-3.2-3B-Instruct-Turbo model_type: llm
• metadata: {} model_id: meta-llama/Llama-3.2-3B-Instruct provider_id: together provider_model_id: meta-llama/Llama-3.2-3B-Instruct-Turbo model_type: llm
• metadata: {} model_id: meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo provider_id: together provider_model_id: meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo model_type: llm
• metadata: {} model_id: meta-llama/Llama-3.2-11B-Vision-Instruct provider_id: together provider_model_id: meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo model_type: llm
• metadata: {} model_id: meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo provider_id: together provider_model_id: meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo model_type: llm
• metadata: {} model_id: meta-llama/Llama-3.2-90B-Vision-Instruct provider_id: together provider_model_id: meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo model_type: llm
• metadata: {} model_id: meta-llama/Llama-3.3-70B-Instruct-Turbo provider_id: together provider_model_id: meta-llama/Llama-3.3-70B-Instruct-Turbo model_type: llm
• metadata: {} model_id: meta-llama/Llama-3.3-70B-Instruct provider_id: together provider_model_id: meta-llama/Llama-3.3-70B-Instruct-Turbo model_type: llm
• metadata: {} model_id: meta-llama/Meta-Llama-Guard-3-8B provider_id: together provider_model_id: meta-llama/Meta-Llama-Guard-3-8B model_type: llm
• metadata: {} model_id: meta-llama/Llama-Guard-3-8B provider_id: together provider_model_id: meta-llama/Meta-Llama-Guard-3-8B model_type: llm
• metadata: {} model_id: meta-llama/Llama-Guard-3-11B-Vision-Turbo provider_id: together provider_model_id: meta-llama/Llama-Guard-3-11B-Vision-Turbo model_type: llm
• metadata: {} model_id: meta-llama/Llama-Guard-3-11B-Vision provider_id: together provider_model_id: meta-llama/Llama-Guard-3-11B-Vision-Turbo model_type: llm
• metadata: embedding_dimension: 768 context_length: 8192 model_id: togethercomputer/m2-bert-80M-8k-retrieval provider_id: together provider_model_id: togethercomputer/m2-bert-80M-8k-retrieval model_type: embedding
• metadata: embedding_dimension: 768 context_length: 32768 model_id: togethercomputer/m2-bert-80M-32k-retrieval provider_id: together provider_model_id: togethercomputer/m2-bert-80M-32k-retrieval model_type: embedding
• metadata: {} model_id: meta-llama/Llama-4-Scout-17B-16E-Instruct provider_id: together provider_model_id: meta-llama/Llama-4-Scout-17B-16E-Instruct model_type: llm
• metadata: {} model_id: meta-llama/Llama-4-Scout-17B-16E-Instruct provider_id: together provider_model_id: meta-llama/Llama-4-Scout-17B-16E-Instruct model_type: llm
• metadata: {} model_id: together/meta-llama/Llama-4-Scout-17B-16E-Instruct provider_id: together provider_model_id: meta-llama/Llama-4-Scout-17B-16E-Instruct model_type: llm
• metadata: {} model_id: meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 provider_id: together provider_model_id: meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 model_type: llm
• metadata: {} model_id: meta-llama/Llama-4-Maverick-17B-128E-Instruct provider_id: together provider_model_id: meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 model_type: llm
• metadata: {} model_id: together/meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 provider_id: together provider_model_id: meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 model_type: llm
• metadata: {} model_id: '${env.INFERENCE_MODEL}' provider_id: ollama model_type: llm
• metadata: embedding_dimension: 384 model_id: all-MiniLM-L6-v2 provider_id: ollama provider_model_id: 'all-minilm:latest' model_type: embedding shields:
• shield_id: meta-llama/Llama-Guard-3-8B vector_dbs: [] datasets: [] scoring_fns: [] benchmarks: [] tool_groups:
• toolgroup_id: 'builtin::websearch' provider_id: tavily-search
• toolgroup_id: 'builtin::rag' provider_id: rag-runtime
• toolgroup_id: 'builtin::wolfram_alpha' provider_id: wolfram-alpha server: port: 8321

export TOGETHER_API_KEY="your-api-key" llama stack run run.yaml

定义两个使用不同推理后端的智能体

agent_hosted = Agent( client=client, model="meta-llama/Llama-3.2-3B-Instruct-Turbo", # 由 Together 提供服务 instructions="You are a helpful AI assistant" )

agent_local = Agent( client=client, model="llama3.2:1b", # 由本地 Ollama 提供服务 instructions="You are a helpful AI assistant" )

session_hosted = agent_hosted.create_session("hosted-session") session_local = agent_local.create_session("local-session")