diff --git a/src/aiinfraoptimization/HISTORY.rst b/src/aiinfraoptimization/HISTORY.rst index abbff5a61a7..f3d7ee493ab 100644 --- a/src/aiinfraoptimization/HISTORY.rst +++ b/src/aiinfraoptimization/HISTORY.rst @@ -3,6 +3,10 @@ Release History =============== +1.0.0b2 ++++++++ +* `az aiinfraoptimization recommendvmsku`: Raise the maximum for `--workload` token budget `prompt-tokens` and `max-output-tokens`. + 1.0.0b1 ++++++ * Initial release. \ No newline at end of file diff --git a/src/aiinfraoptimization/azext_aiinfraoptimization/aaz/latest/aiinfraoptimization/_recommendvmsku.py b/src/aiinfraoptimization/azext_aiinfraoptimization/aaz/latest/aiinfraoptimization/_recommendvmsku.py index 0dab8ea7d28..a577bde769a 100644 --- a/src/aiinfraoptimization/azext_aiinfraoptimization/aaz/latest/aiinfraoptimization/_recommendvmsku.py +++ b/src/aiinfraoptimization/azext_aiinfraoptimization/aaz/latest/aiinfraoptimization/_recommendvmsku.py @@ -213,17 +213,17 @@ def _build_arguments_schema(cls, *args, **kwargs): token_budget = cls._args_schema.workload.token_budget token_budget.max_output_tokens = AAZIntArg( options=["max-output-tokens"], - help="The maximum number of generated output tokens per inference request. Must be supplied together with `promptTokens`, and the two must total from 1 through 131072.", + help="The maximum number of generated output tokens per inference request. Must be supplied together with `promptTokens`, and the two must total from 1 through 2147483647 without exceeding the model's maximum context length.", fmt=AAZIntArgFormat( - maximum=131072, + maximum=2147483647, minimum=0, ), ) token_budget.prompt_tokens = AAZIntArg( options=["prompt-tokens"], - help="The number of prompt, or input, tokens per inference request. Must be supplied together with `maxOutputTokens`, and the two must total from 1 through 131072.", + help="The number of prompt, or input, tokens per inference request. Must be supplied together with `maxOutputTokens`, and the two must total from 1 through 2147483647 without exceeding the model's maximum context length.", fmt=AAZIntArgFormat( - maximum=131072, + maximum=2147483647, minimum=0, ), ) diff --git a/src/aiinfraoptimization/azext_aiinfraoptimization/tests/latest/recordings/test_aiinfraoptimization_recommendvmsku_long_context.yaml b/src/aiinfraoptimization/azext_aiinfraoptimization/tests/latest/recordings/test_aiinfraoptimization_recommendvmsku_long_context.yaml new file mode 100644 index 00000000000..6c1a9a6b59f --- /dev/null +++ b/src/aiinfraoptimization/azext_aiinfraoptimization/tests/latest/recordings/test_aiinfraoptimization_recommendvmsku_long_context.yaml @@ -0,0 +1,65 @@ +interactions: +- request: + body: '{"model": {"headDim": 128, "maxContextLength": 1048576, "numKeyValueHeads": + 8, "numLayers": 32, "parameterCount": 8000000000, "weightPrecision": "BF16"}, + "targetVmSizes": ["Standard_NC24ads_A100_v4", "Standard_NC40ads_H100_v5"], "workload": + {"maxConcurrentRequestsPerReplica": 1, "tokenBudget": {"maxOutputTokens": 131073, + "promptTokens": 262144}}}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - aiinfraoptimization recommendvmsku + Connection: + - keep-alive + Content-Length: + - '348' + Content-Type: + - application/json + ParameterSetName: + - --location --model --workload --target-vm-sizes + User-Agent: + - AZURECLI/2.92.0 azsdk-python-core/1.37.0 Python/3.12.10 (Windows-11-10.0.26300-SP0) + method: POST + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.AIInfraOptimization/locations/eastus2/recommendVmSku?api-version=2026-06-01-preview + response: + body: + string: '{"options":[{"vmSize":"Standard_NC24ads_A100_v4","recommendationRank":1,"readiness":{"status":"VerificationRequired","reason":"Quota + or placement evidence must be verified before attempting deployment."},"topology":{"estimatedNodeCountPerReplica":1,"estimatedTotalNodeCount":1,"accelerator":{"model":"A100","countPerVm":1,"memoryGiBPerAccelerator":80}},"cost":{"currencyCode":"USD","vmHourly":3.673,"deploymentHourly":3.673,"source":"AzureRetail"},"documentationUrl":"https://learn.microsoft.com/en-us/azure/virtual-machines/sizes/gpu-accelerated/nca100v4-series"},{"vmSize":"Standard_NC40ads_H100_v5","recommendationRank":2,"readiness":{"status":"VerificationRequired","reason":"Quota + or placement evidence must be verified before attempting deployment."},"topology":{"estimatedNodeCountPerReplica":1,"estimatedTotalNodeCount":1,"accelerator":{"model":"H100 + NVL","countPerVm":1,"memoryGiBPerAccelerator":94}},"cost":{"currencyCode":"USD","vmHourly":6.98,"deploymentHourly":6.98,"source":"AzureRetail"},"documentationUrl":"https://learn.microsoft.com/en-us/azure/virtual-machines/sizes/gpu-accelerated/ncadsh100v5-series"}],"sizing":{"model":{"parameterCount":8000000000,"weightPrecision":"BF16"},"workload":{"maxConcurrentRequestsPerReplica":1,"contextLength":393217},"memory":{"modelWeightsGiB":14.9,"kvCacheGiBPerRequest":48.03,"kvCacheGiBTotal":48.03,"modelAndKvCacheGiB":62.93},"basis":{"parameterCountSource":"Request","modelWeightsCalculation":"EstimatedFromParameterCount","contextLengthSource":"Request"}}}' + headers: + cache-control: + - no-cache + content-length: + - '1515' + content-type: + - application/json; charset=utf-8 + date: + - Tue, 17 Nov 2026 06:45:46 GMT + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + x-cache: + - CONFIG_NOCACHE + x-content-type-options: + - nosniff + x-ms-operation-identifier: + - tenantId=72f988bf-86f1-41af-91ab-2d7cd011db47,objectId=9e0b0df5-1f15-45f7-8e9f-e1be2997e32e/japaneast/5308e04f-0715-4b33-95a7-8776f1b9d1d5 + x-ms-providerhub-traffic: + - 'True' + x-ms-ratelimit-remaining-subscription-global-writes: + - '11999' + x-ms-ratelimit-remaining-subscription-writes: + - '799' + x-msedge-ref: + - 'Ref A: 0EB4B7219E7B46F7A0C98FD82A7F74A3 Ref B: TYO601100902042 Ref C: 2026-11-17T06:45:45Z' + status: + code: 200 + message: OK +version: 1 diff --git a/src/aiinfraoptimization/azext_aiinfraoptimization/tests/latest/test_aiinfraoptimization.py b/src/aiinfraoptimization/azext_aiinfraoptimization/tests/latest/test_aiinfraoptimization.py index c8b297ffe92..7e654d7b4e5 100644 --- a/src/aiinfraoptimization/azext_aiinfraoptimization/tests/latest/test_aiinfraoptimization.py +++ b/src/aiinfraoptimization/azext_aiinfraoptimization/tests/latest/test_aiinfraoptimization.py @@ -78,6 +78,24 @@ def test_aiinfraoptimization_recommendvmsku_boundary_values(self): self.check("length(options[?cost.source != 'AzureRetail'])", 0), ]) + def test_aiinfraoptimization_recommendvmsku_long_context(self): + # Both token fields exceed the former 131072 limit, and the model's context window fits their total. + self.kwargs.update({ + 'location': 'eastus2', + 'model': '{parameter-count:8000000000,weight-precision:BF16,num-layers:32,' + 'num-key-value-heads:8,head-dim:128,max-context-length:1048576}', + 'workload': '{max-concurrent-requests-per-replica:1,' + 'token-budget:{prompt-tokens:262144,max-output-tokens:131073}}', + 'target_vm_sizes': '[Standard_NC24ads_A100_v4,Standard_NC40ads_H100_v5]', + }) + + self.cmd('aiinfraoptimization recommendvmsku --location {location} --model "{model}" ' + '--workload "{workload}" --target-vm-sizes "{target_vm_sizes}"', + checks=[ + self.check('sizing.workload.contextLength', 393217), + self.greater_than('length(options)', 0), + ]) + def test_aiinfraoptimization_recommendvmsku_service_error(self): # The service reads Hugging Face anonymously, so it can't size a gated model without its architecture fields. self.kwargs.update({ diff --git a/src/aiinfraoptimization/azext_aiinfraoptimization/tests/latest/test_aiinfraoptimization_unit.py b/src/aiinfraoptimization/azext_aiinfraoptimization/tests/latest/test_aiinfraoptimization_unit.py new file mode 100644 index 00000000000..a4963514896 --- /dev/null +++ b/src/aiinfraoptimization/azext_aiinfraoptimization/tests/latest/test_aiinfraoptimization_unit.py @@ -0,0 +1,50 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import json +import unittest +from types import SimpleNamespace +from unittest import mock + +from azure.cli.core.aaz._command_ctx import AAZCommandCtx +from azure.cli.core.azclierror import InvalidArgumentValueError +from azure.cli.core.mock import DummyCli +from azure.core import PipelineClient +from azure.core.pipeline.policies import ContentDecodePolicy + +from azext_aiinfraoptimization.aaz.latest.aiinfraoptimization import Recommendvmsku + + +class RecommendvmskuTokenBudgetTest(unittest.TestCase): + # Checks the CLI's own token budget limits with the HTTP call stubbed, so it needs no recording. + + @staticmethod + def _send(prompt_tokens, max_output_tokens): + client = PipelineClient(base_url='https://management.azure.com') + session = SimpleNamespace(http_response=SimpleNamespace(status_code=200), + context={ContentDecodePolicy.CONTEXT_NAME: {}}) + with mock.patch.object(AAZCommandCtx, 'get_http_client', return_value=client), \ + mock.patch.object(client, 'send_request', return_value=session) as send_request: + Recommendvmsku(cli_ctx=DummyCli())({ + 'subscription': '00000000-0000-0000-0000-000000000000', + 'location': 'eastus2', + 'workload': {'token_budget': {'prompt_tokens': prompt_tokens, + 'max_output_tokens': max_output_tokens}}, + }) + return json.loads(send_request.call_args.kwargs['request'].body) + + def test_accepts_maximum_token_budget(self): + for prompt_tokens, max_output_tokens in ((2147483647, 0), (0, 2147483647)): + with self.subTest(prompt_tokens=prompt_tokens, max_output_tokens=max_output_tokens): + self.assertEqual(self._send(prompt_tokens, max_output_tokens)['workload']['tokenBudget'], + {'promptTokens': prompt_tokens, 'maxOutputTokens': max_output_tokens}) + + def test_rejects_token_budget_above_maximum(self): + for option, prompt_tokens, max_output_tokens in (('prompt-tokens', 2147483648, 0), + ('max-output-tokens', 0, 2147483648)): + with self.subTest(option=option): + with self.assertRaisesRegex(InvalidArgumentValueError, + f"{option}: Invalid format: '2147483648' is greater than 2147483647"): + self._send(prompt_tokens, max_output_tokens) diff --git a/src/aiinfraoptimization/setup.py b/src/aiinfraoptimization/setup.py index 14d02e8a520..cd5a9ee963c 100644 --- a/src/aiinfraoptimization/setup.py +++ b/src/aiinfraoptimization/setup.py @@ -10,7 +10,7 @@ # HISTORY.rst entry. -VERSION = '1.0.0b1' +VERSION = '1.0.0b2' # The full list of classifiers is available at # https://pypi.python.org/pypi?%3Aaction=list_classifiers