Merge branch 'main' into Tarun/new-env

.github/workflows/assets-release.yaml

@@ -71,14 +71,14 @@ jobs:
         working-directory: ${{ env.main_dir }}
 
       - name: Upload changed assets
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: steps.find-updated-assets.outputs.updated_count > 0
         with:
           name: ${{ env.changed_assets_artifact }}
           path: ${{ runner.temp }}/${{ env.changed_assets_artifact }}
 
       - name: Upload non-testable assets
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: steps.find-updated-assets.outputs.updated_count > 0
         with:
           name: ${{ env.releasable_assets_artifact }}
@@ -102,7 +102,7 @@ jobs:
           exit 1
 
       - name: Upload list of assets
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ env.asset_list_artifact }}
           path: ${{ runner.temp }}/${{ env.asset_list_artifact }}
@@ -179,7 +179,7 @@ jobs:
     steps:
       - name: Download releasable assets
         id: download-artifact
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: ${{ env.releasable_assets_artifact }}
           path: ${{ runner.temp }}/${{ env.releasable_assets_artifact }}
@@ -198,13 +198,13 @@ jobs:
       - name: Download releasable assets
         if: needs.check-release-assets.outputs.has_releasable_assets != ''
         id: download-artifact
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: ${{ env.releasable_assets_artifact }}
           path: ${{ runner.temp }}/${{ env.releasable_assets_artifact }}
 
       - name: Download list of assets
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: ${{ env.asset_list_artifact }}
           path: ${{ runner.temp }}/${{ env.asset_list_artifact }}

.github/workflows/assets-test.yaml

@@ -158,7 +158,7 @@ jobs:
         run: python -u $scripts_test_dir/test_assets.py -i "${{ matrix.asset_config_path }}" -a $asset_config_filename -p $scripts_test_dir/requirements.txt -r $GITHUB_WORKSPACE/$pytest_reports
 
       - name: Upload test results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: ${{ env.pytest_reports }}
@@ -179,7 +179,7 @@ jobs:
     steps:
       - name: Download test results
         id: download-artifact
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: ${{ env.pytest_reports }}
           path: ${{ env.pytest_reports }}

.github/workflows/batch-score-ci.yaml

@@ -71,7 +71,7 @@ jobs:
           RESOURCE_GROUP: ${{ env.resource_group }}
           WORKSPACE_NAME: ${{ env.workspace }}
       - name: Upload test results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: ${{ env.pytest_report_folder }}
@@ -91,7 +91,7 @@ jobs:
       steps:
         - name: Download test results
           id: download-artifact
-          uses: actions/download-artifact@v3
+          uses: actions/download-artifact@v4
           with:
             name: ${{ env.pytest_report_folder }}
             path: ${{ env.pytest_report_folder }}

.github/workflows/batch-score-oss-ci.yaml

@@ -71,7 +71,7 @@ jobs:
           RESOURCE_GROUP: ${{ env.resource_group }}
           WORKSPACE_NAME: ${{ env.workspace }}
       - name: Upload test results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: ${{ env.pytest_report_folder }}
@@ -91,7 +91,7 @@ jobs:
       steps:
         - name: Download test results
           id: download-artifact
-          uses: actions/download-artifact@v3
+          uses: actions/download-artifact@v4
           with:
             name: ${{ env.pytest_report_folder }}
             path: ${{ env.pytest_report_folder }}

.github/workflows/environments-ci.yaml

@@ -89,7 +89,7 @@ jobs:
         run: python -u $scripts_environment_dir/validate_build_logs.py -l '${{ runner.temp }}'/$build_logs_artifact_name
 
       - name: Upload build logs
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: ${{ env.build_logs_artifact_name }}

.github/workflows/model-monitoring-ci.yml

@@ -132,7 +132,7 @@ jobs:
           RESOURCE_GROUP: ${{ env.resource_group }}
           WORKSPACE_NAME: ${{ env.workspace }}
       - name: Upload test results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: ${{ env.pytest_report_folder }}
@@ -152,7 +152,7 @@ jobs:
       steps:
         - name: Download test results
           id: download-artifact
-          uses: actions/download-artifact@v3
+          uses: actions/download-artifact@v4
           with:
             name: ${{ env.pytest_report_folder }}
             path: ${{ env.pytest_report_folder }}

.github/workflows/model-monitoring-gsq-ci.yml

@@ -87,7 +87,7 @@ jobs:
           RESOURCE_GROUP: ${{ env.resource_group }}
           WORKSPACE_NAME: ${{ env.workspace }}
       - name: Upload test results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: ${{ env.pytest_report_folder }}
@@ -107,7 +107,7 @@ jobs:
       steps:
         - name: Download test results
           id: download-artifact
-          uses: actions/download-artifact@v3
+          uses: actions/download-artifact@v4
           with:
             name: ${{ env.pytest_report_folder }}
             path: ${{ env.pytest_report_folder }}

assets/evaluation_on_cloud/environments/evaluations-built-in/context/requirements.txt

@@ -1,4 +1,4 @@
-azure-ai-evaluation=={{latest-pypi-version}}
+azure-ai-evaluation==1.0.0b5
 azureml-mlflow=={{latest-pypi-version}}
 azure-ai-ml=={{latest-pypi-version}}
 

assets/models/system/Phi-4/spec.yaml

@@ -6,6 +6,9 @@ path: ./
 properties:
   inference-min-sku-spec: 24|1|220|64
   inference-recommended-sku: Standard_NC24ads_A100_v4, Standard_NC48ads_A100_v4, Standard_NC96ads_A100_v4, Standard_ND96asr_v4, Standard_ND96amsr_A100_v4
+  finetuning-tasks: chat-completion
+  finetune-min-sku-spec: 24|1|220|64
+  finetune-recommended-sku: Standard_NC24ads_A100_v4, Standard_NC48ads_A100_v4, Standard_NC96ads_A100_v4, Standard_ND96asr_v4, Standard_ND96amsr_A100_v4
   languages: en
   SharedComputeCapacityEnabled: true
 
@@ -43,6 +46,30 @@ tags:
       Standard_ND96asr_v4,
       Standard_ND96amsr_A100_v4
     ]
+  finetune_compute_allow_list:
+    [ 
+      Standard_NC24ads_A100_v4,
+      Standard_NC48ads_A100_v4,
+      Standard_NC96ads_A100_v4,
+      Standard_ND96asr_v4,
+      Standard_ND96amsr_A100_v4
+    ]
+  model_specific_defaults:
+    apply_deepspeed: "true"
+    deepspeed_stage: 3
+    apply_lora: "true"
+    apply_ort: "false"
+    precision: 16
+    ignore_mismatched_sizes: "false"
+    num_train_epochs: 1
+    per_device_train_batch_size: 1
+    per_device_eval_batch_size: 1
+    gradient_accumulation_steps: 1
+    learning_rate: 5e-6
+    lr_scheduler_type: "cosine"
+    logging_strategy: "steps"
+    logging_steps: 10
+    save_total_limit: 1
   benchmark: "quality"
 
-version: 2
+version: 3

assets/models/system/aurora/description.md

@@ -1,21 +1,11 @@
-Aurora: A Foundation Model for the Earth System
-
-Aurora is a machine learning model that can predict atmospheric variables, such as temperature.
+Aurora is a machine learning model that can predict general environmental variables, such as temperature and wind speed.
 It is a _foundation model_, which means that it was first generally trained on a lot of data,
-and then can be adapted to specialised atmospheric forecasting tasks with relatively little data.
+and then can be adapted to specialised environmental forecasting tasks with relatively little data.
 We provide four such specialised versions:
 one for medium-resolution weather prediction,
 one for high-resolution weather prediction,
 one for air pollution prediction,
 and one for ocean wave prediction.
-
-## Resources
-
-* [Aurora Documentation for detailed instruction and examples](https://microsoft.github.io/aurora)
-* [Aurora Academic Paper](https://arxiv.org/abs/2405.13063)
-* [A full-fledged example that runs the model on ERA5](https://microsoft.github.io/aurora/example_era5.html).
-
-## Implementation
-
-_The package currently includes the pretrained model and the fine-tuned version for high-resolution weather forecasting._
-_We are working on the fine-tuned versions for air pollution and ocean wave forecasting, which will be included in due time._
+Currently, this implementation only includes the version for medium-term weather prediction.
+Please see the documentation of the Aurora Foundry Python API linked below for
+precisely which models are available.

assets/models/system/aurora/evaluation.md

@@ -3,4 +3,4 @@ These evaluations not only compare measures of accuracy, such as the root mean s
 but also look at the behaviour in extreme situations, like extreme heat and cold, and rare events, like Storm Ciarán in 2023.
 These evaluations are the main topic of [the paper](https://arxiv.org/pdf/2405.13063).
 
-*Note: The documentation included in this file is for informational purposes only and is not intended to supersede the applicable license terms.*
+*Note: The documentation included here is for informational purposes only and is not intended to supersede the applicable license terms.*

assets/models/system/aurora/notes.md

@@ -1,3 +1,52 @@
+## Resources
+
+* [Documentation of the Aurora Foundry Python API](https://microsoft.github.io/aurora/foundry/intro.html)
+* [A full-fledged example that runs the model on Foundry](https://microsoft.github.io/aurora/foundry/demo_hres_t0.html).
+* [Implementation of the Aurora model](https://github.com/microsoft/aurora)
+* [Documentation of the Aurora implementation](https://microsoft.github.io/aurora/intro.html)
+* [Paper with detailed evaluation](https://arxiv.org/abs/2405.13063)
+
+## Quickstart
+
+First install the model:
+
+```bash
+pip install microsoft-aurora
+```
+
+Then you can make predictions with a Azure Foundry AI endpoint as follows:
+
+```python
+from aurora import Batch
+
+from aurora.foundry import BlobStorageChannel, FoundryClient, submit
+
+
+initial_condition = Batch(...)  # Create initial condition for the model.
+
+for pred in submit(
+    initial_condition,
+    model_name="aurora-0.25-finetuned",
+    num_steps=4,  # Every step predicts six hours ahead.
+    foundry_client=FoundryClient(
+        endpoint="https://endpoint/",
+        token="ENDPOINT_TOKEN",
+    ),
+    # Communication with the endpoint happens via an intermediate blob storage container. You
+    # will need to create one and generate an URL with a SAS token that has both read and write
+    # rights.
+    channel=BlobStorageChannel(
+        "https://storageaccount.blob.core.windows.net/container?<READ_WRITE_SAS_TOKEN>"
+    ),
+):
+    pass  # Do something with `pred`, which is a `Batch`.
+```
+
+## License
+
+This model and the associated model weights are released under the license [CC-BY-NC-SA-4.0](https://spdx.org/licenses/CC-BY-NC-SA-4.0).
+This means that you are free to share and adapt the material for non-commercial purposes, as long as you provide appropriate credit, indicate if changes were made, and distribute your contributions under the same license.
+
 ## Security
 
 See [SECURITY](https://github.com/microsoft/aurora/blob/main/SECURITY.md).
@@ -34,10 +83,3 @@ This project may contain trademarks or logos for projects, products, or services
 Authorized use of Microsoft trademarks or logos is subject to and must follow [Microsoft's Trademark & Brand Guidelines](https://www.microsoft.com/en-us/legal/intellectualproperty/trademarks/usage/general).
 Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship.
 Any use of third-party trademarks or logos are subject to those third-party's policies.
-
-## FAQ
-
-### Why are the fine-tuned versions of Aurora for air quality and ocean wave forecasting missing?
-
-The package currently includes the pretrained model and the fine-tuned version for high-resolution weather forecasting.
-We are working on the fine-tuned versions for air pollution and ocean wave forecasting, which will be included in due time.

assets/models/system/aurora/spec.yaml

@@ -6,22 +6,23 @@ path: ./
 properties:
   languages: EN
   inference-min-sku-spec: 24|1|220|64
-  inference-recommended-sku: Standard_NC24ads_A100_v4, Standard_NC48ads_A100_v4, Standard_NC96ads_A100_v4
+  inference-recommended-sku: Standard_NC24ads_A100_v4
   SharedComputeCapacityEnabled: true
 
 tags:
   task: environmental-forecasting
   Preview: ""
   Featured: ""  
   author: Microsoft
-  license: custom
+  license: cc-by-nc-sa-4.0
   hiddenlayerscanned: "true"
   notes: "notes.md"
   disable-batch: true
   SharedComputeCapacityEnabled: ""
   inference_compute_allow_list:
     - Standard_NC24ads_A100_v4
-    - Standard_NC48ads_A100_v4
-    - Standard_NC96ads_A100_v4
+  sku_to_num_replicas_map:
+    Standard_NC24ads_A100_v4: 1
+    Default: 1
 
-version: 1
+version: 1

assets/models/system/phi-3-medium-128k-instruct/spec.yaml

@@ -68,4 +68,4 @@ tags:
     save_total_limit: 1
     max_seq_length: 4096
   benchmark: "quality"
-version: 6
+version: 7

assets/models/system/phi-3-medium-4k-instruct/spec.yaml

@@ -71,4 +71,4 @@ tags:
     save_total_limit: 1
     max_seq_length: 4096
   benchmark: "quality"
-version: 5
+version: 6

assets/models/system/phi-3-mini-128k-instruct/spec.yaml

@@ -70,4 +70,4 @@ tags:
     logging_steps: 10
     save_total_limit: 1
   benchmark: "quality"
-version: 12
+version: 13

assets/models/system/phi-3-mini-4k-instruct/spec.yaml

@@ -76,4 +76,4 @@ tags:
     logging_steps: 10
     save_total_limit: 1
   benchmark: "quality"
-version: 14
+version: 15

assets/models/system/phi-3-small-128k-instruct/spec.yaml

@@ -64,4 +64,4 @@ tags:
     save_total_limit: 1
     max_seq_length: 4096
   benchmark: "quality"
-version: 4
+version: 5

assets/models/system/phi-3-small-8k-instruct/spec.yaml

@@ -64,4 +64,4 @@ tags:
     save_total_limit: 1
     max_seq_length: 4096
   benchmark: "quality"
-version: 5
+version: 6

assets/models/system/phi-3-vision-128k-instruct/spec.yaml

@@ -23,4 +23,4 @@ tags:
     ]
   inference_supported_envs:
     - vllm
-version: 2
+version: 3

assets/models/system/phi-3.5-mini-128k-instruct/spec.yaml

@@ -71,4 +71,4 @@ tags:
     logging_steps: 10
     save_total_limit: 1
   benchmark: "quality"
-version: 6
+version: 7