Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions src/aiinfraoptimization/HISTORY.rst
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,10 @@
Release History
===============

1.0.0b2
+++++++
* `az aiinfraoptimization recommendvmsku`: Raise the maximum for `--workload` token budget `prompt-tokens` and `max-output-tokens`.

1.0.0b1
++++++
* Initial release.
Original file line number Diff line number Diff line change
Expand Up @@ -213,17 +213,17 @@ def _build_arguments_schema(cls, *args, **kwargs):
token_budget = cls._args_schema.workload.token_budget
token_budget.max_output_tokens = AAZIntArg(
options=["max-output-tokens"],
help="The maximum number of generated output tokens per inference request. Must be supplied together with `promptTokens`, and the two must total from 1 through 131072.",
help="The maximum number of generated output tokens per inference request. Must be supplied together with `promptTokens`, and the two must total from 1 through 2147483647 without exceeding the model's maximum context length.",
fmt=AAZIntArgFormat(
maximum=131072,
maximum=2147483647,
Comment thread
kkarmakarMS marked this conversation as resolved.
Resolved
Comment thread
kkarmakarMS marked this conversation as resolved.
Resolved
minimum=0,
),
)
token_budget.prompt_tokens = AAZIntArg(
options=["prompt-tokens"],
help="The number of prompt, or input, tokens per inference request. Must be supplied together with `maxOutputTokens`, and the two must total from 1 through 131072.",
help="The number of prompt, or input, tokens per inference request. Must be supplied together with `maxOutputTokens`, and the two must total from 1 through 2147483647 without exceeding the model's maximum context length.",
fmt=AAZIntArgFormat(
maximum=131072,
maximum=2147483647,
minimum=0,
),
)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
interactions:
- request:
body: '{"model": {"headDim": 128, "maxContextLength": 1048576, "numKeyValueHeads":
8, "numLayers": 32, "parameterCount": 8000000000, "weightPrecision": "BF16"},
"targetVmSizes": ["Standard_NC24ads_A100_v4", "Standard_NC40ads_H100_v5"], "workload":
{"maxConcurrentRequestsPerReplica": 1, "tokenBudget": {"maxOutputTokens": 131073,
"promptTokens": 262144}}}'
headers:
Accept:
- application/json
Accept-Encoding:
- gzip, deflate
CommandName:
- aiinfraoptimization recommendvmsku
Connection:
- keep-alive
Content-Length:
- '348'
Content-Type:
- application/json
ParameterSetName:
- --location --model --workload --target-vm-sizes
User-Agent:
- AZURECLI/2.92.0 azsdk-python-core/1.37.0 Python/3.12.10 (Windows-11-10.0.26300-SP0)
method: POST
uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.AIInfraOptimization/locations/eastus2/recommendVmSku?api-version=2026-06-01-preview
response:
body:
string: '{"options":[{"vmSize":"Standard_NC24ads_A100_v4","recommendationRank":1,"readiness":{"status":"VerificationRequired","reason":"Quota
or placement evidence must be verified before attempting deployment."},"topology":{"estimatedNodeCountPerReplica":1,"estimatedTotalNodeCount":1,"accelerator":{"model":"A100","countPerVm":1,"memoryGiBPerAccelerator":80}},"cost":{"currencyCode":"USD","vmHourly":3.673,"deploymentHourly":3.673,"source":"AzureRetail"},"documentationUrl":"https://learn.microsoft.com/en-us/azure/virtual-machines/sizes/gpu-accelerated/nca100v4-series"},{"vmSize":"Standard_NC40ads_H100_v5","recommendationRank":2,"readiness":{"status":"VerificationRequired","reason":"Quota
or placement evidence must be verified before attempting deployment."},"topology":{"estimatedNodeCountPerReplica":1,"estimatedTotalNodeCount":1,"accelerator":{"model":"H100
NVL","countPerVm":1,"memoryGiBPerAccelerator":94}},"cost":{"currencyCode":"USD","vmHourly":6.98,"deploymentHourly":6.98,"source":"AzureRetail"},"documentationUrl":"https://learn.microsoft.com/en-us/azure/virtual-machines/sizes/gpu-accelerated/ncadsh100v5-series"}],"sizing":{"model":{"parameterCount":8000000000,"weightPrecision":"BF16"},"workload":{"maxConcurrentRequestsPerReplica":1,"contextLength":393217},"memory":{"modelWeightsGiB":14.9,"kvCacheGiBPerRequest":48.03,"kvCacheGiBTotal":48.03,"modelAndKvCacheGiB":62.93},"basis":{"parameterCountSource":"Request","modelWeightsCalculation":"EstimatedFromParameterCount","contextLengthSource":"Request"}}}'
headers:
cache-control:
- no-cache
content-length:
- '1515'
content-type:
- application/json; charset=utf-8
date:
- Tue, 17 Nov 2026 06:45:46 GMT
expires:
- '-1'
pragma:
- no-cache
strict-transport-security:
- max-age=31536000; includeSubDomains
x-cache:
- CONFIG_NOCACHE
x-content-type-options:
- nosniff
x-ms-operation-identifier:
- tenantId=72f988bf-86f1-41af-91ab-2d7cd011db47,objectId=9e0b0df5-1f15-45f7-8e9f-e1be2997e32e/japaneast/5308e04f-0715-4b33-95a7-8776f1b9d1d5
x-ms-providerhub-traffic:
- 'True'
x-ms-ratelimit-remaining-subscription-global-writes:
- '11999'
x-ms-ratelimit-remaining-subscription-writes:
- '799'
x-msedge-ref:
- 'Ref A: 0EB4B7219E7B46F7A0C98FD82A7F74A3 Ref B: TYO601100902042 Ref C: 2026-11-17T06:45:45Z'
status:
code: 200
message: OK
version: 1
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,24 @@ def test_aiinfraoptimization_recommendvmsku_boundary_values(self):
self.check("length(options[?cost.source != 'AzureRetail'])", 0),
])

def test_aiinfraoptimization_recommendvmsku_long_context(self):
# Both token fields exceed the former 131072 limit, and the model's context window fits their total.
self.kwargs.update({
'location': 'eastus2',
'model': '{parameter-count:8000000000,weight-precision:BF16,num-layers:32,'
'num-key-value-heads:8,head-dim:128,max-context-length:1048576}',
'workload': '{max-concurrent-requests-per-replica:1,'
'token-budget:{prompt-tokens:262144,max-output-tokens:131073}}',
'target_vm_sizes': '[Standard_NC24ads_A100_v4,Standard_NC40ads_H100_v5]',
})

self.cmd('aiinfraoptimization recommendvmsku --location {location} --model "{model}" '
'--workload "{workload}" --target-vm-sizes "{target_vm_sizes}"',
checks=[
self.check('sizing.workload.contextLength', 393217),
self.greater_than('length(options)', 0),
])

def test_aiinfraoptimization_recommendvmsku_service_error(self):
# The service reads Hugging Face anonymously, so it can't size a gated model without its architecture fields.
self.kwargs.update({
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
# --------------------------------------------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License. See License.txt in the project root for license information.
# --------------------------------------------------------------------------------------------

import json
import unittest
from types import SimpleNamespace
from unittest import mock

from azure.cli.core.aaz._command_ctx import AAZCommandCtx
from azure.cli.core.azclierror import InvalidArgumentValueError
from azure.cli.core.mock import DummyCli
from azure.core import PipelineClient
from azure.core.pipeline.policies import ContentDecodePolicy

from azext_aiinfraoptimization.aaz.latest.aiinfraoptimization import Recommendvmsku


class RecommendvmskuTokenBudgetTest(unittest.TestCase):
# Checks the CLI's own token budget limits with the HTTP call stubbed, so it needs no recording.

@staticmethod
def _send(prompt_tokens, max_output_tokens):
client = PipelineClient(base_url='https://management.azure.com')
session = SimpleNamespace(http_response=SimpleNamespace(status_code=200),
context={ContentDecodePolicy.CONTEXT_NAME: {}})
with mock.patch.object(AAZCommandCtx, 'get_http_client', return_value=client), \
mock.patch.object(client, 'send_request', return_value=session) as send_request:
Recommendvmsku(cli_ctx=DummyCli())({
'subscription': '00000000-0000-0000-0000-000000000000',
'location': 'eastus2',
'workload': {'token_budget': {'prompt_tokens': prompt_tokens,
'max_output_tokens': max_output_tokens}},
})
return json.loads(send_request.call_args.kwargs['request'].body)

def test_accepts_maximum_token_budget(self):
for prompt_tokens, max_output_tokens in ((2147483647, 0), (0, 2147483647)):
with self.subTest(prompt_tokens=prompt_tokens, max_output_tokens=max_output_tokens):
self.assertEqual(self._send(prompt_tokens, max_output_tokens)['workload']['tokenBudget'],
{'promptTokens': prompt_tokens, 'maxOutputTokens': max_output_tokens})

def test_rejects_token_budget_above_maximum(self):
for option, prompt_tokens, max_output_tokens in (('prompt-tokens', 2147483648, 0),
('max-output-tokens', 0, 2147483648)):
with self.subTest(option=option):
with self.assertRaisesRegex(InvalidArgumentValueError,
f"{option}: Invalid format: '2147483648' is greater than 2147483647"):
self._send(prompt_tokens, max_output_tokens)
2 changes: 1 addition & 1 deletion src/aiinfraoptimization/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@


# HISTORY.rst entry.
VERSION = '1.0.0b1'
VERSION = '1.0.0b2'

# The full list of classifiers is available at
# https://pypi.python.org/pypi?%3Aaction=list_classifiers
Expand Down
Loading