diff --git a/tools/azure-rest-api-specs-examples-automation/automation/csv_database.py b/tools/azure-rest-api-specs-examples-automation/automation/csv_database.py index e38fd101852..4d5702e05a4 100644 --- a/tools/azure-rest-api-specs-examples-automation/automation/csv_database.py +++ b/tools/azure-rest-api-specs-examples-automation/automation/csv_database.py @@ -11,9 +11,9 @@ from models import Release -example_repo: str = 'https://github.com/Azure/azure-rest-api-specs-examples' -csvdb_folder: str = 'csvdb' -metadata_branch: str = 'metadata' +example_repo: str = "https://github.com/Azure/azure-rest-api-specs-examples" +csvdb_folder: str = "csvdb" +metadata_branch: str = "metadata" @dataclasses.dataclass(eq=True) @@ -55,76 +55,79 @@ def __init__(self, work_dir: str): self.work_dir = work_dir self.example_metadata_path = path.join(self.work_dir, csvdb_folder) - self.index_file_path = path.join(self.example_metadata_path, 'java-library-example-index.csv') - self.list_file_path = path.join(self.example_metadata_path, 'java-library-example-list.csv') + self.index_file_path = path.join(self.example_metadata_path, "java-library-example-index.csv") + self.list_file_path = path.join(self.example_metadata_path, "java-library-example-list.csv") def checkout(self): # checkout metadata branch from azure-rest-api-specs-examples repo - cmd = ['git', 'clone', - '--quiet', - '--depth', '1', - '--branch', metadata_branch, - example_repo, self.example_metadata_path] - logging.info(f'Checking out repository: {example_repo}, branch {metadata_branch}') - logging.info('Command line: ' + ' '.join(cmd)) + cmd = [ + "git", + "clone", + "--quiet", + "--depth", + "1", + "--branch", + metadata_branch, + example_repo, + self.example_metadata_path, + ] + logging.info(f"Checking out repository: {example_repo}, branch {metadata_branch}") + logging.info("Command line: " + " ".join(cmd)) subprocess.check_call(cmd, cwd=self.work_dir) def load(self): - with open(self.index_file_path, 'r', newline='') as csv_file: - csv_reader = csv.reader(csv_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) + with open(self.index_file_path, "r", newline="") as csv_file: + csv_reader = csv.reader(csv_file, delimiter=",", quotechar='"', quoting=csv.QUOTE_MINIMAL) self.release_db = DatabaseInternal(csv_reader) - with open(self.list_file_path, 'r', newline='') as csv_file: - csv_reader = csv.reader(csv_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) + with open(self.list_file_path, "r", newline="") as csv_file: + csv_reader = csv.reader(csv_file, delimiter=",", quotechar='"', quoting=csv.QUOTE_MINIMAL) self.file_db = DatabaseInternal(csv_reader) def dump(self): - with open(self.index_file_path, 'w', newline='') as csv_file: - csv_writer = csv.writer(csv_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) - csv_writer.writerow(['id', 'name', 'language', 'tag', 'package', 'version', 'date_epoch', 'date']) + with open(self.index_file_path, "w", newline="") as csv_file: + csv_writer = csv.writer(csv_file, delimiter=",", quotechar='"', quoting=csv.QUOTE_MINIMAL) + csv_writer.writerow(["id", "name", "language", "tag", "package", "version", "date_epoch", "date"]) for row in self.release_db.rows: csv_writer.writerow(row) - with open(self.list_file_path, 'w', newline='') as csv_file: - csv_writer = csv.writer(csv_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) - csv_writer.writerow(['id', 'file', 'release_id']) + with open(self.list_file_path, "w", newline="") as csv_file: + csv_writer = csv.writer(csv_file, delimiter=",", quotechar='"', quoting=csv.QUOTE_MINIMAL) + csv_writer.writerow(["id", "file", "release_id"]) for row in self.file_db.rows: csv_writer.writerow(row) def commit(self, tag): if not self.branch: # git checkout new branch - self.date_str = datetime.now().strftime('%Y-%m-%d') - self.branch = f'automation-metadata-{self.date_str}' - cmd = ['git', 'checkout', '-b', self.branch] - logging.info('Command line: ' + ' '.join(cmd)) + self.date_str = datetime.now().strftime("%Y-%m-%d") + self.branch = f"automation-metadata-{self.date_str}" + cmd = ["git", "checkout", "-b", self.branch] + logging.info("Command line: " + " ".join(cmd)) subprocess.check_call(cmd, cwd=self.example_metadata_path) # git add - cmd = ['git', 'add', 'java-library-example-index.csv'] - logging.info('Command line: ' + ' '.join(cmd)) + cmd = ["git", "add", "java-library-example-index.csv"] + logging.info("Command line: " + " ".join(cmd)) subprocess.check_call(cmd, cwd=self.example_metadata_path) - cmd = ['git', 'add', 'java-library-example-list.csv'] - logging.info('Command line: ' + ' '.join(cmd)) + cmd = ["git", "add", "java-library-example-list.csv"] + logging.info("Command line: " + " ".join(cmd)) subprocess.check_call(cmd, cwd=self.example_metadata_path) # git commit - title = f'[Automation] Update metadata on {tag}' - logging.info(f'git commit: {title}') - cmd = ['git', - '-c', 'user.name=azure-sdk', - '-c', 'user.email=azuresdk@microsoft.com', - 'commit', '-m', title] - logging.info('Command line: ' + ' '.join(cmd)) + title = f"[Automation] Update metadata on {tag}" + logging.info(f"git commit: {title}") + cmd = ["git", "-c", "user.name=azure-sdk", "-c", "user.email=azuresdk@microsoft.com", "commit", "-m", title] + logging.info("Command line: " + " ".join(cmd)) subprocess.check_call(cmd, cwd=self.example_metadata_path) def push(self, github_token: str): if self.branch: - title = f'[Automation] Update metadata on {self.date_str}' + title = f"[Automation] Update metadata on {self.date_str}" # git push - remote_uri = 'https://' + github_token + '@' + example_repo[len('https://'):] - cmd = ['git', 'push', remote_uri, self.branch] + remote_uri = "https://" + github_token + "@" + example_repo[len("https://") :] + cmd = ["git", "push", remote_uri, self.branch] # do not print this as it contains token # logging.info('Command line: ' + ' '.join(cmd)) subprocess.check_call(cmd, cwd=self.example_metadata_path) @@ -132,24 +135,25 @@ def push(self, github_token: str): # create github pull request owner = _repository_owner(example_repo) name = _repository_name(example_repo) - head = f'{owner}:{self.branch}' + head = f"{owner}:{self.branch}" repo = GitHubRepository(owner, name, github_token) pull_number = repo.create_pull_request(title, head, metadata_branch) - repo.add_label(pull_number, ['auto-merge']) - logging.info(f'succeeded, pull number {pull_number}') + repo.add_label(pull_number, ["auto-merge"]) + logging.info(f"succeeded, pull number {pull_number}") - def new_release(self, name: str, language: str, tag: str, package: str, version: str, date: datetime, - files: List[str]) -> bool: + def new_release( + self, name: str, language: str, tag: str, package: str, version: str, date: datetime, files: List[str] + ) -> bool: # add a new release and all the example files # return false, if release already exists in DB release_id = self._query_release(name, language) if release_id: - logging.warning(f'Release already exists for {language}#{name}') + logging.warning(f"Release already exists for {language}#{name}") return False date_epoch = int(date.timestamp()) - date_str = datetime.fromtimestamp(date_epoch).strftime('%m/%d/%Y') + date_str = datetime.fromtimestamp(date_epoch).strftime("%m/%d/%Y") release_id = self.release_db.append([name, language, tag, package, version, date_epoch, date_str]) @@ -178,8 +182,8 @@ def _query_release(self, name: str, language: str) -> Union[str, None]: def _repository_owner(repository: str) -> str: - return re.match(r'https://github.com/([^/:]+)/.*', repository).group(1) + return re.match(r"https://github.com/([^/:]+)/.*", repository).group(1) def _repository_name(repository: str) -> str: - return re.match(r'https://github.com/[^/:]+/(.*)', repository).group(1) + return re.match(r"https://github.com/[^/:]+/(.*)", repository).group(1) diff --git a/tools/azure-rest-api-specs-examples-automation/automation/github.py b/tools/azure-rest-api-specs-examples-automation/automation/github.py index d56333e55d5..73976208c28 100644 --- a/tools/azure-rest-api-specs-examples-automation/automation/github.py +++ b/tools/azure-rest-api-specs-examples-automation/automation/github.py @@ -4,7 +4,7 @@ class GitHubRepository: - api_host: str = 'https://api.github.com' + api_host: str = "https://api.github.com" owner: str name: str token: str @@ -15,84 +15,65 @@ def __init__(self, owner: str, name: str, token): self.token = token def create_pull_request(self, title: str, head: str, base: str) -> int: - logging.info(f'Create pull request: {head}') + logging.info(f"Create pull request: {head}") - request_uri = f'{self.api_host}/repos/{self.owner}/{self.name}/pulls' - request_body = { - 'title': title, - 'head': head, - 'base': base - } - pull_request_response = requests.post(request_uri, - json=request_body, - headers=self._headers()) + request_uri = f"{self.api_host}/repos/{self.owner}/{self.name}/pulls" + request_body = {"title": title, "head": head, "base": base} + pull_request_response = requests.post(request_uri, json=request_body, headers=self._headers()) if pull_request_response.status_code == 201: - logging.info('Pull request created') - return pull_request_response.json()['number'] + logging.info("Pull request created") + return pull_request_response.json()["number"] else: - logging.error(f'Request failed: {pull_request_response.status_code}\n{pull_request_response.json()}') + logging.error(f"Request failed: {pull_request_response.status_code}\n{pull_request_response.json()}") pull_request_response.raise_for_status() def list_pull_requests(self) -> List[Dict[str, Any]]: - logging.info(f'List pull requests') + logging.info(f"List pull requests") - request_uri = f'{self.api_host}/repos/{self.owner}/{self.name}/pulls?per_page=100' - pull_request_response = requests.get(request_uri, - headers=self._headers()) + request_uri = f"{self.api_host}/repos/{self.owner}/{self.name}/pulls?per_page=100" + pull_request_response = requests.get(request_uri, headers=self._headers()) if pull_request_response.status_code == 200: - logging.info('Pull request created') + logging.info("Pull request created") return pull_request_response.json() else: - logging.error(f'Request failed: {pull_request_response.status_code}\n{pull_request_response.json()}') + logging.error(f"Request failed: {pull_request_response.status_code}\n{pull_request_response.json()}") return [] def merge_pull_request(self, pull_request: Dict): - title = pull_request['title'] - logging.info(f'Merge pull request: {title}') + title = pull_request["title"] + logging.info(f"Merge pull request: {title}") - pull_number = int(pull_request['number']) + pull_number = int(pull_request["number"]) - request_uri = f'{self.api_host}/repos/{self.owner}/{self.name}/pulls/{pull_number}/merge' - request_body = { - 'commit_title': title, - 'merge_method': 'squash' - } - merge_response = requests.put(request_uri, - json=request_body, - headers=self._headers()) + request_uri = f"{self.api_host}/repos/{self.owner}/{self.name}/pulls/{pull_number}/merge" + request_body = {"commit_title": title, "merge_method": "squash"} + merge_response = requests.put(request_uri, json=request_body, headers=self._headers()) if merge_response.status_code == 200: - logging.info('Pull request merged') + logging.info("Pull request merged") else: - logging.error(f'Request failed: {merge_response.status_code}\n{merge_response.json()}') + logging.error(f"Request failed: {merge_response.status_code}\n{merge_response.json()}") merge_response.raise_for_status() def list_releases(self, per_page: int, page: int = 1) -> List[Dict[str, Any]]: - request_uri = f'{self.api_host}/repos/{self.owner}/{self.name}/releases' - releases_response = requests.get(request_uri, - params={'per_page': per_page, 'page': page}, - headers=self._headers()) + request_uri = f"{self.api_host}/repos/{self.owner}/{self.name}/releases" + releases_response = requests.get( + request_uri, params={"per_page": per_page, "page": page}, headers=self._headers() + ) if releases_response.status_code == 200: return releases_response.json() else: - logging.error(f'Request failed: {releases_response.status_code}\n{releases_response.json()}') + logging.error(f"Request failed: {releases_response.status_code}\n{releases_response.json()}") releases_response.raise_for_status() def add_label(self, pull_number: int, labels: List[str]): - request_uri = f'{self.api_host}/repos/{self.owner}/{self.name}/issues/{pull_number}/labels' - request_body = { - 'labels': labels - } - add_label_response = requests.post(request_uri, - json=request_body, - headers=self._headers()) + request_uri = f"{self.api_host}/repos/{self.owner}/{self.name}/issues/{pull_number}/labels" + request_body = {"labels": labels} + add_label_response = requests.post(request_uri, json=request_body, headers=self._headers()) if add_label_response.status_code == 200: - logging.info('Label added') + logging.info("Label added") else: - logging.error(f'Request failed: {add_label_response.status_code}\n{add_label_response.json()}') + logging.error(f"Request failed: {add_label_response.status_code}\n{add_label_response.json()}") add_label_response.raise_for_status() def _headers(self) -> Dict[str, str]: - return { - 'X-GitHub-Api-Version': '2022-11-28', - 'Authorization': f'token {self.token}' - } + return {"X-GitHub-Api-Version": "2022-11-28", "Authorization": f"token {self.token}"} diff --git a/tools/azure-rest-api-specs-examples-automation/automation/main.py b/tools/azure-rest-api-specs-examples-automation/automation/main.py index 9e009a6e1bd..1a8b3140f44 100644 --- a/tools/azure-rest-api-specs-examples-automation/automation/main.py +++ b/tools/azure-rest-api-specs-examples-automation/automation/main.py @@ -17,7 +17,7 @@ github_token: str -root_path: str = '.' +root_path: str = "." csv_database: CsvDatabase @@ -26,188 +26,188 @@ timeout_secs: float = 45 * 60 * 60 # 45 minutes clean_tmp_dir: bool = True -tmp_folder: str = 'tmp' -tmp_spec_folder: str = 'spec' -tmp_example_folder: str = 'example' -tmp_sdk_folder: str = 'sdk' +tmp_folder: str = "tmp" +tmp_spec_folder: str = "spec" +tmp_example_folder: str = "example" +tmp_sdk_folder: str = "sdk" def load_configuration(command_line: CommandLineConfiguration) -> Configuration: - with open(path.join(root_path, 'automation/configuration.json'), 'r', encoding='utf-8') as f_in: + with open(path.join(root_path, "automation/configuration.json"), "r", encoding="utf-8") as f_in: config = json.load(f_in) now = datetime.now(timezone.utc) - operation_configuration = OperationConfiguration(config['sdkExample']['repository'], - command_line.build_id, - command_line.skip_processed, - command_line.persist_data, - now - timedelta(days=command_line.release_in_days), now) + operation_configuration = OperationConfiguration( + config["sdkExample"]["repository"], + command_line.build_id, + command_line.skip_processed, + command_line.persist_data, + now - timedelta(days=command_line.release_in_days), + now, + ) sdk_configurations = [] - for sdk_config in config['sdkConfigurations']: - script = Script(sdk_config['script']['run']) - release_tag = ReleaseTagConfiguration(sdk_config['releaseTag']['regexMatch'], - sdk_config['releaseTag']['packageRegexGroup'], - sdk_config['releaseTag']['versionRegexGroup']) - ignored_packages = sdk_config['ignoredPackages'] if 'ignoredPackages' in sdk_config else [] - sdk_configuration = SdkConfiguration(sdk_config['name'], - sdk_config['language'], - sdk_config['repository'], - release_tag, script, ignored_packages) + for sdk_config in config["sdkConfigurations"]: + script = Script(sdk_config["script"]["run"]) + release_tag = ReleaseTagConfiguration( + sdk_config["releaseTag"]["regexMatch"], + sdk_config["releaseTag"]["packageRegexGroup"], + sdk_config["releaseTag"]["versionRegexGroup"], + ) + ignored_packages = sdk_config["ignoredPackages"] if "ignoredPackages" in sdk_config else [] + sdk_configuration = SdkConfiguration( + sdk_config["name"], sdk_config["language"], sdk_config["repository"], release_tag, script, ignored_packages + ) sdk_configurations.append(sdk_configuration) return Configuration(operation_configuration, sdk_configurations) def merge_pull_requests(operation: OperationConfiguration): - logging.info('Merge pull requests') + logging.info("Merge pull requests") repo = GitHubRepository(operation.repository_owner, operation.repository_name, github_token) pull_requests = repo.list_pull_requests() for pull_request in pull_requests: - title = pull_request['title'] - if title.startswith('[Automation]'): - if 'labels' in pull_request and any(label['name'] == 'auto-merge' for label in pull_request['labels']): + title = pull_request["title"] + if title.startswith("[Automation]"): + if "labels" in pull_request and any(label["name"] == "auto-merge" for label in pull_request["labels"]): repo.merge_pull_request(pull_request) # wait a few seconds to avoid 409 time.sleep(5) -def process_release(operation: OperationConfiguration, sdk: SdkConfiguration, release: Release, - report: Report): +def process_release(operation: OperationConfiguration, sdk: SdkConfiguration, release: Release, report: Report): # process per release - logging.info(f'Processing release: {release.tag}') + logging.info(f"Processing release: {release.tag}") tmp_root_path = path.join(root_path, tmp_folder) os.makedirs(tmp_root_path, exist_ok=True) - tmp_path = tempfile.mkdtemp(prefix='tmp', dir=tmp_root_path) - logging.info(f'Work directory: {tmp_path}') + tmp_path = tempfile.mkdtemp(prefix="tmp", dir=tmp_root_path) + logging.info(f"Work directory: {tmp_path}") try: example_repo_path = path.join(tmp_path, tmp_example_folder) sdk_repo_path = path.join(tmp_path, tmp_sdk_folder) spec_repo_path = path.join(tmp_root_path, tmp_spec_folder) # checkout azure-rest-api-specs-examples repo - cmd = ['git', 'clone', - '--quiet', - '--depth', '1', - operation.sdk_examples_repository, example_repo_path] - logging.info(f'Checking out repository: {operation.sdk_examples_repository}') - logging.info('Command line: ' + ' '.join(cmd)) + cmd = ["git", "clone", "--quiet", "--depth", "1", operation.sdk_examples_repository, example_repo_path] + logging.info(f"Checking out repository: {operation.sdk_examples_repository}") + logging.info("Command line: " + " ".join(cmd)) subprocess.check_call(cmd, cwd=tmp_path) # checkout sdk repo - cmd = ['git', 'clone', - '-c', 'advice.detachedHead=false', - '--quiet', - '--depth', '1', - '--branch', release.tag, - sdk.repository, sdk_repo_path] - logging.info(f'Checking out repository: {sdk.repository}') - logging.info('Command line: ' + ' '.join(cmd)) + cmd = [ + "git", + "clone", + "-c", + "advice.detachedHead=false", + "--quiet", + "--depth", + "1", + "--branch", + release.tag, + sdk.repository, + sdk_repo_path, + ] + logging.info(f"Checking out repository: {sdk.repository}") + logging.info("Command line: " + " ".join(cmd)) subprocess.check_call(cmd, cwd=tmp_path) # prepare input.json - input_json_path = path.join(tmp_path, 'input.json') - output_json_path = path.join(tmp_path, 'output.json') - with open(input_json_path, 'w', encoding='utf-8') as f_out: + input_json_path = path.join(tmp_path, "input.json") + output_json_path = path.join(tmp_path, "output.json") + with open(input_json_path, "w", encoding="utf-8") as f_out: input_json = { - 'specsPath': spec_repo_path, - 'sdkExamplesPath': example_repo_path, - 'sdkPath': sdk_repo_path, - 'tempPath': tmp_path, - 'release': { - 'tag': release.tag, - 'package': release.package, - 'version': release.version - } + "specsPath": spec_repo_path, + "sdkExamplesPath": example_repo_path, + "sdkPath": sdk_repo_path, + "tempPath": tmp_path, + "release": {"tag": release.tag, "package": release.package, "version": release.version}, } - logging.info(f'Input JSON for worker: {input_json}') + logging.info(f"Input JSON for worker: {input_json}") json.dump(input_json, f_out, indent=2) # run script - logging.info(f'Running worker: {sdk.script.run}') + logging.info(f"Running worker: {sdk.script.run}") start = time.perf_counter() subprocess.check_call([sdk.script.run, input_json_path, output_json_path], cwd=root_path) end = time.perf_counter() - logging.info(f'Worker ran: {str(timedelta(seconds=end-start))}') + logging.info(f"Worker ran: {str(timedelta(seconds=end-start))}") # parse output.json release_name = release.tag succeeded = True files = [] if path.isfile(output_json_path): - with open(output_json_path, 'r', encoding='utf-8') as f_in: + with open(output_json_path, "r", encoding="utf-8") as f_in: output = json.load(f_in) - logging.info(f'Output JSON from worker: {output}') - release_name = output['name'] - succeeded = ('succeeded' == output['status']) - files = output['files'] + logging.info(f"Output JSON from worker: {output}") + release_name = output["name"] + succeeded = "succeeded" == output["status"] + files = output["files"] if not succeeded: - report.statuses[release.tag] = 'failed at worker' - report.aggregated_error.errors.append(RuntimeError(f'Worker failed for release tag: {release.tag}')) + report.statuses[release.tag] = "failed at worker" + report.aggregated_error.errors.append(RuntimeError(f"Worker failed for release tag: {release.tag}")) return # commit and create pull request # check for new examples - cmd = ['git', 'status', '--porcelain'] - logging.info('Command line: ' + ' '.join(cmd)) + cmd = ["git", "status", "--porcelain"] + logging.info("Command line: " + " ".join(cmd)) output = subprocess.check_output(cmd, cwd=example_repo_path) if len(output) == 0: - logging.info(f'No change to repository: {example_repo_path}') - report.statuses[release.tag] = 'succeeded, no change' + logging.info(f"No change to repository: {example_repo_path}") + report.statuses[release.tag] = "succeeded, no change" else: - output_str = str(output, 'utf-8') - logging.info(f'git status:\n{output_str}') + output_str = str(output, "utf-8") + logging.info(f"git status:\n{output_str}") # git add - cmd = ['git', 'add', '--all'] - logging.info('Command line: ' + ' '.join(cmd)) + cmd = ["git", "add", "--all"] + logging.info("Command line: " + " ".join(cmd)) subprocess.check_call(cmd, cwd=example_repo_path) # find added/modified files - cmd = ['git', 'status', '--porcelain'] - logging.info('Command line: ' + ' '.join(cmd)) + cmd = ["git", "status", "--porcelain"] + logging.info("Command line: " + " ".join(cmd)) output = subprocess.check_output(cmd, cwd=example_repo_path) - output_str = str(output, 'utf-8') + output_str = str(output, "utf-8") changed_files = [file.strip()[3:] for file in output_str.splitlines()] # git checkout new branch - branch = f'automation-examples_{sdk.name}_{release.tag}_{operation.build_id}' - cmd = ['git', 'checkout', '-b', branch] - logging.info('Command line: ' + ' '.join(cmd)) + branch = f"automation-examples_{sdk.name}_{release.tag}_{operation.build_id}" + cmd = ["git", "checkout", "-b", branch] + logging.info("Command line: " + " ".join(cmd)) subprocess.check_call(cmd, cwd=example_repo_path) # git commit - title = f'[Automation] Collect examples from {sdk.name}#{release.tag}' - logging.info(f'git commit: {title}') - cmd = ['git', - '-c', 'user.name=azure-sdk', - '-c', 'user.email=azuresdk@microsoft.com', - 'commit', '-m', title] - logging.info('Command line: ' + ' '.join(cmd)) + title = f"[Automation] Collect examples from {sdk.name}#{release.tag}" + logging.info(f"git commit: {title}") + cmd = ["git", "-c", "user.name=azure-sdk", "-c", "user.email=azuresdk@microsoft.com", "commit", "-m", title] + logging.info("Command line: " + " ".join(cmd)) subprocess.check_call(cmd, cwd=example_repo_path) # git push - remote_uri = 'https://' + github_token + '@' + operation.sdk_examples_repository[len('https://'):] - cmd = ['git', 'push', remote_uri, branch] + remote_uri = "https://" + github_token + "@" + operation.sdk_examples_repository[len("https://") :] + cmd = ["git", "push", remote_uri, branch] # do not print this as it contains token # logging.info('Command line: ' + ' '.join(cmd)) subprocess.check_call(cmd, cwd=example_repo_path) try: # create github pull request - head = f'{operation.repository_owner}:{branch}' + head = f"{operation.repository_owner}:{branch}" repo = GitHubRepository(operation.repository_owner, operation.repository_name, github_token) - pull_number = repo.create_pull_request(title, head, 'main') - repo.add_label(pull_number, ['auto-merge']) + pull_number = repo.create_pull_request(title, head, "main") + repo.add_label(pull_number, ["auto-merge"]) except Exception as e: - logging.error(f'Error: {e}') - report.statuses[release.tag] = 'failed to create pull request' + logging.error(f"Error: {e}") + report.statuses[release.tag] = "failed to create pull request" report.aggregated_error.errors.append(e) return @@ -216,16 +216,16 @@ def process_release(operation: OperationConfiguration, sdk: SdkConfiguration, re # commit changes to database commit_database(release_name, sdk.language, release, files) except Exception as e: - logging.error(f'Error: {e}') - report.statuses[release.tag] = 'failed to update database' + logging.error(f"Error: {e}") + report.statuses[release.tag] = "failed to update database" report.aggregated_error.errors.append(e) return - report.statuses[release.tag] = f'succeeded, {len(changed_files)} files changed, pull number {pull_number}' + report.statuses[release.tag] = f"succeeded, {len(changed_files)} files changed, pull number {pull_number}" except subprocess.CalledProcessError as e: - logging.error(f'Call error: {e}') - report.statuses[release.tag] = 'failed to invoke git' + logging.error(f"Call error: {e}") + report.statuses[release.tag] = "failed to invoke git" report.aggregated_error.errors.append(e) finally: if clean_tmp_dir: @@ -242,11 +242,12 @@ def commit_database(release_name: str, language: str, release: Release, changed_ # write to local database and commit to repository # exclude metadata JSON - changed_files = [file for file in changed_files if not file.endswith('.json')] + changed_files = [file for file in changed_files if not file.endswith(".json")] if changed_files: database_succeeded = csv_database.new_release( - release_name, language, release.tag, release.package, release.version, release.date, changed_files) + release_name, language, release.tag, release.package, release.version, release.date, changed_files + ) if database_succeeded: csv_database.dump() csv_database.commit(release_name) @@ -259,7 +260,7 @@ def process_sdk(operation: OperationConfiguration, sdk: SdkConfiguration, report logging.warning(f"Timeout, skip sdk: {sdk.name}") return - logging.info(f'Processing sdk: {sdk.name}') + logging.info(f"Processing sdk: {sdk.name}") count = 0 releases: List[Release] = [] repo = GitHubRepository(sdk.repository_owner, sdk.repository_name, github_token) @@ -272,24 +273,24 @@ def process_sdk(operation: OperationConfiguration, sdk: SdkConfiguration, report break count += len(releases_response_json) for release in releases_response_json: - if not release['draft']: - published_at = datetime.fromisoformat(release['published_at'].replace('Z', '+00:00')) + if not release["draft"]: + published_at = datetime.fromisoformat(release["published_at"].replace("Z", "+00:00")) if operation.date_start < published_at < operation.date_end: - release_tag = release['tag_name'] + release_tag = release["tag_name"] if re.match(sdk.release_tag.regex_match, release_tag): package = re.match(sdk.release_tag.package_regex_group, release_tag).group(1) version = re.match(sdk.release_tag.version_regex_group, release_tag).group(1) release = Release(release_tag, package, version, published_at) releases.append(release) - logging.info(f'Found release tag: {release.tag}') + logging.info(f"Found release tag: {release.tag}") except Exception as e: report.aggregated_error.errors.append(e) break - logging.info(f'Count of all releases: {count}') + logging.info(f"Count of all releases: {count}") releases.sort(key=lambda r: r.date, reverse=True) for release in releases: - logging.info(f'Candidate release tag: {release.tag}, on {release.date.date()}') + logging.info(f"Candidate release tag: {release.tag}, on {release.date.date()}") processed_release_tags = set() if operation.skip_processed: @@ -303,12 +304,12 @@ def process_sdk(operation: OperationConfiguration, sdk: SdkConfiguration, report break if release.tag in processed_release_tags: - logging.info(f'Skip processed tag: {release.tag}') + logging.info(f"Skip processed tag: {release.tag}") processed_release_packages.add(release.package) elif release.package in processed_release_packages: - logging.info(f'Skip processed package: {release.tag}') + logging.info(f"Skip processed package: {release.tag}") elif release.package in sdk.ignored_packages: - logging.info(f'Skip ignored package: {release.tag}') + logging.info(f"Skip ignored package: {release.tag}") else: process_release(operation, sdk, release, report) processed_release_packages.add(release.package) @@ -324,13 +325,10 @@ def process(command_line: CommandLineConfiguration, report: Report): tmp_root_path = path.join(root_path, tmp_folder) os.makedirs(tmp_root_path, exist_ok=True) spec_repo_path = path.join(tmp_root_path, tmp_spec_folder) - spec_repo = 'https://github.com/Azure/azure-rest-api-specs' - cmd = ['git', 'clone', - '--quiet', - '--depth', '1', - spec_repo, spec_repo_path] - logging.info(f'Checking out repository: {spec_repo}') - logging.info('Command line: ' + ' '.join(cmd)) + spec_repo = "https://github.com/Azure/azure-rest-api-specs" + cmd = ["git", "clone", "--quiet", "--depth", "1", spec_repo, spec_repo_path] + logging.info(f"Checking out repository: {spec_repo}") + logging.info("Command line: " + " ".join(cmd)) subprocess.check_call(cmd, cwd=tmp_root_path) # checkout and load database @@ -352,50 +350,70 @@ def main(): global github_token global start_time_secs - logging.basicConfig(level=logging.INFO, - format='%(asctime)s [%(levelname)s] %(message)s', - datefmt='%Y-%m-%d %X') + logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s", datefmt="%Y-%m-%d %X") start_time_secs = time.time() script_path = path.abspath(path.dirname(sys.argv[0])) - root_path = path.abspath(path.join(script_path, '..')) - - parser = argparse.ArgumentParser(description='') - parser.add_argument('--build-id', type=str, required=True, - help='Build ID') - parser.add_argument('--github-token', type=str, required=True, - help='GitHub token') - parser.add_argument('--release-in-days', type=int, required=False, default=3, - help='Process SDK released within given days') - parser.add_argument('--language', type=str, required=False, - help='Process SDK for specific language. Currently supports "java" and "go".') - parser.add_argument('--persist-data', type=str, required=False, default='false', - help='Persist data about release and files to database') - parser.add_argument('--skip-processed', type=str, required=False, default='false', - help='Skip SDK releases that already been processed') - parser.add_argument('--merge-pull-request', type=str, required=False, default='false', - help='Merge GitHub pull request before new processing') + root_path = path.abspath(path.join(script_path, "..")) + + parser = argparse.ArgumentParser(description="") + parser.add_argument("--build-id", type=str, required=True, help="Build ID") + parser.add_argument("--github-token", type=str, required=True, help="GitHub token") + parser.add_argument( + "--release-in-days", type=int, required=False, default=3, help="Process SDK released within given days" + ) + parser.add_argument( + "--language", + type=str, + required=False, + help='Process SDK for specific language. Currently supports "java" and "go".', + ) + parser.add_argument( + "--persist-data", + type=str, + required=False, + default="false", + help="Persist data about release and files to database", + ) + parser.add_argument( + "--skip-processed", + type=str, + required=False, + default="false", + help="Skip SDK releases that already been processed", + ) + parser.add_argument( + "--merge-pull-request", + type=str, + required=False, + default="false", + help="Merge GitHub pull request before new processing", + ) args = parser.parse_args() github_token = args.github_token - command_line_configuration = CommandLineConfiguration(args.build_id, args.release_in_days, args.language, - args.persist_data.lower() == 'true', - args.skip_processed.lower() == 'true', - args.merge_pull_request.lower() == 'true') + command_line_configuration = CommandLineConfiguration( + args.build_id, + args.release_in_days, + args.language, + args.persist_data.lower() == "true", + args.skip_processed.lower() == "true", + args.merge_pull_request.lower() == "true", + ) report = Report({}, AggregatedError([])) process(command_line_configuration, report) if report.statuses: - statuses_str = 'Statuses:' + statuses_str = "Statuses:" for tag, status in report.statuses.items(): - statuses_str += f'\n{tag}: {status}' + statuses_str += f"\n{tag}: {status}" logging.info(statuses_str) if report.aggregated_error.errors: raise RuntimeError(report.aggregated_error.errors) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/tools/azure-rest-api-specs-examples-automation/automation/models.py b/tools/azure-rest-api-specs-examples-automation/automation/models.py index a19b0b763bf..eaa2602d238 100644 --- a/tools/azure-rest-api-specs-examples-automation/automation/models.py +++ b/tools/azure-rest-api-specs-examples-automation/automation/models.py @@ -15,11 +15,11 @@ class OperationConfiguration: @property def repository_owner(self) -> str: - return re.match(r'https://github.com/([^/:]+)/.*', self.sdk_examples_repository).group(1) + return re.match(r"https://github.com/([^/:]+)/.*", self.sdk_examples_repository).group(1) @property def repository_name(self) -> str: - return re.match(r'https://github.com/[^/:]+/(.*)', self.sdk_examples_repository).group(1) + return re.match(r"https://github.com/[^/:]+/(.*)", self.sdk_examples_repository).group(1) @dataclasses.dataclass(eq=True, frozen=True) @@ -45,11 +45,11 @@ class SdkConfiguration: @property def repository_owner(self) -> str: - return re.match(r'https://github.com/([^/:]+)/.*', self.repository).group(1) + return re.match(r"https://github.com/([^/:]+)/.*", self.repository).group(1) @property def repository_name(self) -> str: - return re.match(r'https://github.com/[^/:]+/(.*)', self.repository).group(1) + return re.match(r"https://github.com/[^/:]+/(.*)", self.repository).group(1) @dataclasses.dataclass(eq=True, frozen=True) diff --git a/tools/azure-rest-api-specs-examples-automation/automation/requirements.txt b/tools/azure-rest-api-specs-examples-automation/automation/requirements.txt index f2293605cf1..9ce22693802 100644 --- a/tools/azure-rest-api-specs-examples-automation/automation/requirements.txt +++ b/tools/azure-rest-api-specs-examples-automation/automation/requirements.txt @@ -1 +1,2 @@ requests +yaml diff --git a/tools/azure-rest-api-specs-examples-automation/automation/test_csv_database.py b/tools/azure-rest-api-specs-examples-automation/automation/test_csv_database.py index 36e4a57c6d3..ba5d8b47c34 100644 --- a/tools/azure-rest-api-specs-examples-automation/automation/test_csv_database.py +++ b/tools/azure-rest-api-specs-examples-automation/automation/test_csv_database.py @@ -10,12 +10,12 @@ class TestCsvDatabase(unittest.TestCase): def test(self): - csv_release_content = '''id,name,language,tag,package,version,date_epoch,date + csv_release_content = """id,name,language,tag,package,version,date_epoch,date 1,com.azure.resourcemanager:azure-resourcemanager-confluent:1.0.0-beta.3,java,azure-resourcemanager-confluent_1.0.0-beta.3,azure-resourcemanager-confluent,1.0.0-beta.3,1636608276,11/11/2021 2,com.azure.resourcemanager:azure-resourcemanager-signalr:1.0.0-beta.3,java,azure-resourcemanager-signalr_1.0.0-beta.3,azure-resourcemanager-signalr,1.0.0-beta.3,1636606853,11/11/2021 -''' +""" - csv_file_content = '''id,file,release_id + csv_file_content = """id,file,release_id 1,specification/confluent/resource-manager/Microsoft.Confluent/preview/2021-09-01-preview/examples-java/MarketplaceAgreements_Create.java,1 2,specification/confluent/resource-manager/Microsoft.Confluent/preview/2021-09-01-preview/examples-java/MarketplaceAgreements_List.java,1 3,specification/confluent/resource-manager/Microsoft.Confluent/preview/2021-09-01-preview/examples-java/OrganizationOperations_List.java,1 @@ -47,20 +47,20 @@ def test(self): 29,specification/signalr/resource-manager/Microsoft.SignalRService/stable/2021-10-01/examples-java/SignalR_RegenerateKey.java,2 30,specification/signalr/resource-manager/Microsoft.SignalRService/stable/2021-10-01/examples-java/SignalR_Restart.java,2 31,specification/signalr/resource-manager/Microsoft.SignalRService/stable/2021-10-01/examples-java/SignalR_Update.java,2 -32,specification/signalr/resource-manager/Microsoft.SignalRService/stable/2021-10-01/examples-java/Usages_List.java,2''' +32,specification/signalr/resource-manager/Microsoft.SignalRService/stable/2021-10-01/examples-java/Usages_List.java,2""" - work_dir = path.abspath('.') + work_dir = path.abspath(".") - index_file_path = path.join(work_dir, 'csvdb', 'java-library-example-index.csv') - list_file_path = path.join(work_dir, 'csvdb', 'java-library-example-list.csv') + index_file_path = path.join(work_dir, "csvdb", "java-library-example-index.csv") + list_file_path = path.join(work_dir, "csvdb", "java-library-example-list.csv") - shutil.rmtree(path.join(work_dir, 'csvdb'), ignore_errors=True) - os.mkdir('csvdb') + shutil.rmtree(path.join(work_dir, "csvdb"), ignore_errors=True) + os.mkdir("csvdb") - with open(index_file_path, 'w', newline='') as csv_file: + with open(index_file_path, "w", newline="") as csv_file: csv_file.write(csv_release_content) - with open(list_file_path, 'w', newline='') as csv_file: + with open(list_file_path, "w", newline="") as csv_file: csv_file.write(csv_file_content) test_db = CsvDatabase(work_dir) @@ -75,14 +75,21 @@ def test(self): releases = test_db.query_releases("java") self.assertEqual(2, len(releases)) release1 = releases[0] - self.assertEqual('azure-resourcemanager-confluent_1.0.0-beta.3', release1.tag) - self.assertEqual('azure-resourcemanager-confluent', release1.package) - self.assertEqual('1.0.0-beta.3', release1.version) - - test_db.new_release('com.azure.resourcemanager:azure-resourcemanager-quota:1.0.0-beta.2', 'java', - 'azure-resourcemanager-quota_1.0.0-beta.2', 'azure-resourcemanager-quota', - '1.0.0-beta.2', datetime.fromtimestamp(1636603745), - ['specification/quota/resource-manager/Microsoft.Quota/preview/2021-03-15-preview/examples-java/GetOperations.java']) + self.assertEqual("azure-resourcemanager-confluent_1.0.0-beta.3", release1.tag) + self.assertEqual("azure-resourcemanager-confluent", release1.package) + self.assertEqual("1.0.0-beta.3", release1.version) + + test_db.new_release( + "com.azure.resourcemanager:azure-resourcemanager-quota:1.0.0-beta.2", + "java", + "azure-resourcemanager-quota_1.0.0-beta.2", + "azure-resourcemanager-quota", + "1.0.0-beta.2", + datetime.fromtimestamp(1636603745), + [ + "specification/quota/resource-manager/Microsoft.Quota/preview/2021-03-15-preview/examples-java/GetOperations.java" + ], + ) self.assertEqual(3, len(test_db.release_db.rows)) self.assertEqual(33, len(test_db.file_db.rows)) @@ -95,4 +102,4 @@ def test(self): releases = test_db.query_releases("java") self.assertEqual(3, len(releases)) - shutil.rmtree(path.join(work_dir, 'csvdb'), ignore_errors=True) + shutil.rmtree(path.join(work_dir, "csvdb"), ignore_errors=True) diff --git a/tools/azure-rest-api-specs-examples-automation/directory/examples_dir.py b/tools/azure-rest-api-specs-examples-automation/directory/examples_dir.py new file mode 100644 index 00000000000..ef37271d598 --- /dev/null +++ b/tools/azure-rest-api-specs-examples-automation/directory/examples_dir.py @@ -0,0 +1,50 @@ +from os import path +import yaml +import glob +import re + + +def try_find_resource_manager_example( + specs_path: str, sdk_package_path: str, example_dir: str, example_filename: str +) -> str: + if "/resource-manager/" not in example_dir: + # find the corresponding example file under {specs_path}/specification/{service}/resource-manager + tsp_location_path = path.join(sdk_package_path, "tsp-location.yaml") + if path.exists(tsp_location_path) and path.isfile(tsp_location_path): + # load tsp_dir from tsp-location.yaml + # e.g. tsp_dir = "specification/mongocluster/DocumentDB.MongoCluster.Management" + with open(tsp_location_path, "r") as fin: + yaml_json = yaml.safe_load(fin) + tsp_dir = yaml_json["directory"] + + if tsp_dir: + # find example under directory + # e.g. example_path = "specification/mongocluster/DocumentDB.MongoCluster.Management/examples/2024-03-01-preview/MongoClusters_ListConnectionStrings.json" + example_paths = glob.glob(f"{path.join(specs_path, tsp_dir)}/**/{example_filename}", recursive=True) + + if len(example_paths) > 0: + example_path = example_paths[0] + example_path = path.relpath(example_path, specs_path).replace("\\", "/") + + example_dir = path.dirname(example_path) + + match = re.match(r"specification/([^/]+)/.*/examples/([^/]+)(.*)", example_dir) + if match: + # example: specification/mongocluster/DocumentDB.MongoCluster.Management/examples/2024-03-01-preview + # service: mongocluster + # api_version: 2024-03-01-preview + # additional_path: + + service = match.group(1) + api_version = match.group(2) + additional_path = match.group(3) + + glob_resource_manager_filename = f"specification/{service}/resource-manager/**/{api_version}/examples{additional_path}/{example_filename}" + candidate_resource_manager_filename = glob.glob( + path.join(specs_path, glob_resource_manager_filename), recursive=True + ) + if len(candidate_resource_manager_filename) > 0: + example_path, _ = path.split(candidate_resource_manager_filename[0]) + example_dir = path.relpath(example_path, specs_path).replace("\\", "/") + + return example_dir diff --git a/tools/azure-rest-api-specs-examples-automation/directory/test_examples_dir.py b/tools/azure-rest-api-specs-examples-automation/directory/test_examples_dir.py new file mode 100644 index 00000000000..b0f1049916b --- /dev/null +++ b/tools/azure-rest-api-specs-examples-automation/directory/test_examples_dir.py @@ -0,0 +1,96 @@ +import unittest +import tempfile +from os import path, makedirs + +from examples_dir import try_find_resource_manager_example + + +def create_mock_test_folder() -> tempfile.TemporaryDirectory: + tsp_location_file_content = """directory: specification/mongocluster/DocumentDB.MongoCluster.Management +commit: 7ed015e3dd1b8b1b0e71c9b5e6b6c5ccb8968b3a +repo: Azure/azure-rest-api-specs +additionalDirectories: null +""" + + json_example_file_content = """{ + "operationId": "MongoClusters_ListConnectionStrings", + "title": "List the available connection strings for the Mongo Cluster resource.", + "parameters": { + "subscriptionId": "ffffffff-ffff-ffff-ffff-ffffffffffff", + "resourceGroupName": "TestGroup", + "mongoClusterName": "myMongoCluster", + "api-version": "2024-03-01-preview" + }, + "responses": { + "200": { + "body": { + "connectionStrings": [ + { + "connectionString": "mongodb+srv://:@myMongoCluster.mongocluster.cosmos.azure.com", + "description": "default connection string" + } + ] + } + } + } +} +""" + + tmp_path = path.abspath(".") + tmp_dir = tempfile.TemporaryDirectory(dir=tmp_path) + try: + sdk_path = path.join(tmp_dir.name, "azure-sdk-for-java/sdk/mongocluster/azure-resourcemanager-mongocluster") + makedirs(sdk_path) + with open(path.join(sdk_path, "tsp-location.yaml"), "w+", encoding="utf-8") as file: + file.write(tsp_location_file_content) + + specs_path = path.join( + tmp_dir.name, + "azure-rest-api-specs/specification/mongocluster/DocumentDB.MongoCluster.Management/examples/2024-03-01-preview", + ) + makedirs(specs_path) + with open(path.join(specs_path, "MongoClusters_ListConnectionStrings.json"), "w+", encoding="utf-8") as file: + file.write(json_example_file_content) + + specs_path = path.join( + tmp_dir.name, + "azure-rest-api-specs/specification/mongocluster/resource-manager/Microsoft.DocumentDB/preview/2024-03-01-preview/examples", + ) + makedirs(specs_path) + with open(path.join(specs_path, "MongoClusters_ListConnectionStrings.json"), "w+", encoding="utf-8") as file: + file.write(json_example_file_content) + except Exception as error: + tmp_dir.cleanup() + raise error + + return tmp_dir + + +class TestExamplesDir(unittest.TestCase): + + def test_find_resource_manager_example_typespec(self): + with create_mock_test_folder() as tmp_dir_name: + example_dir = try_find_resource_manager_example( + path.join(tmp_dir_name, "azure-rest-api-specs"), + path.join(tmp_dir_name, "azure-sdk-for-java/sdk/mongocluster/azure-resourcemanager-mongocluster"), + "2024-03-01-preview", + "MongoClusters_ListConnectionStrings.json", + ) + + self.assertEqual( + "specification/mongocluster/resource-manager/Microsoft.DocumentDB/preview/2024-03-01-preview/examples", + example_dir, + ) + + def test_find_resource_manager_example_swagger(self): + example_dir = try_find_resource_manager_example( + "c:/github/azure-rest-api-specs", + "c:/github/azure-sdk-for-java/sdk/mongocluster/azure-resourcemanager-mongocluster", + "specification/mongocluster/resource-manager/Microsoft.DocumentDB/preview/2024-03-01-preview/examples", + "MongoClusters_ListConnectionStrings.json", + ) + + self.assertEqual( + "specification/mongocluster/resource-manager/Microsoft.DocumentDB/preview/2024-03-01-preview/examples", + example_dir, + ) diff --git a/tools/azure-rest-api-specs-examples-automation/dotnet/build.py b/tools/azure-rest-api-specs-examples-automation/dotnet/build.py index 79e790cb1bf..17a0b369706 100644 --- a/tools/azure-rest-api-specs-examples-automation/dotnet/build.py +++ b/tools/azure-rest-api-specs-examples-automation/dotnet/build.py @@ -8,7 +8,7 @@ def check_call(cmd: List[str], work_dir: str): - logging.info('Command line: ' + ' '.join(cmd)) + logging.info("Command line: " + " ".join(cmd)) subprocess.check_call(cmd, cwd=work_dir) @@ -29,26 +29,26 @@ def build(self) -> DotNetBuildResult: # format and validate go files current_example = None try: - logging.info('Initialize project') + logging.info("Initialize project") # project - cmd = ['dotnet', 'new', 'console', '--name', 'example', '--output', '.'] + cmd = ["dotnet", "new", "console", "--name", "example", "--output", "."] check_call(cmd, tmp_dir_name) - cmd = ['dotnet', 'add', 'package', 'Azure.Identity'] + cmd = ["dotnet", "add", "package", "Azure.Identity"] check_call(cmd, tmp_dir_name) # cmd = ['dotnet', 'add', 'package', 'Azure.ResourceManager'] # check_call(cmd, tmp_dir_name) - cmd = ['dotnet', 'add', 'package', self.module, '--version', self.module_version] + cmd = ["dotnet", "add", "package", self.module, "--version", self.module_version] check_call(cmd, tmp_dir_name) - with open(path.join(tmp_dir_name, 'example.csproj'), encoding='utf-8') as f: + with open(path.join(tmp_dir_name, "example.csproj"), encoding="utf-8") as f: content = f.read() - logging.info(f'csproj\n{content}') + logging.info(f"csproj\n{content}") # build per example - filename = 'Program.cs' + filename = "Program.cs" filepath = path.join(tmp_dir_name, filename) file_no = 0 max_file_count = 10 # TODO: for now, only build for first 10 examples @@ -58,19 +58,19 @@ def build(self) -> DotNetBuildResult: if file_no > max_file_count: break - with open(filepath, 'w', encoding='utf-8') as f: + with open(filepath, "w", encoding="utf-8") as f: f.write(example.content) - cmd = ['dotnet', 'clean', '--nologo', '--verbosity', 'quiet'] + cmd = ["dotnet", "clean", "--nologo", "--verbosity", "quiet"] check_call(cmd, tmp_dir_name) - cmd = ['dotnet', 'build', '--no-restore', '--nologo', '--verbosity', 'quiet'] + cmd = ["dotnet", "build", "--no-restore", "--nologo", "--verbosity", "quiet"] check_call(cmd, tmp_dir_name) except subprocess.CalledProcessError as error: - logging.error(f'Call error: {error}') + logging.error(f"Call error: {error}") if current_example: - logging.error(f'Program.cs\n{current_example.content}') + logging.error(f"Program.cs\n{current_example.content}") return DotNetBuildResult(False, []) return DotNetBuildResult(True, self.examples) diff --git a/tools/azure-rest-api-specs-examples-automation/dotnet/main.py b/tools/azure-rest-api-specs-examples-automation/dotnet/main.py index 20a3fd38360..d34e0b6f28d 100644 --- a/tools/azure-rest-api-specs-examples-automation/dotnet/main.py +++ b/tools/azure-rest-api-specs-examples-automation/dotnet/main.py @@ -7,17 +7,28 @@ import logging import dataclasses from typing import List +import importlib.util from models import DotNetExample from build import DotNetBuild -script_path: str = '.' +spec_location = ( + "./directory/examples_dir.py" if path.exists("./directory/examples_dir.py") else "../directory/examples_dir.py" +) +spec = importlib.util.spec_from_file_location("examples_dir", spec_location) +examples_dir = importlib.util.module_from_spec(spec) +spec.loader.exec_module(examples_dir) + + +script_path: str = "." tmp_path: str +specs_path: str +sdk_package_path: str -original_file_key: str = '// Generated from example definition: ' +original_file_key: str = "// Generated from example definition: " -module_relative_path: str = '' +module_relative_path: str = "" @dataclasses.dataclass(eq=True, frozen=True) @@ -64,18 +75,18 @@ def get_dotnet_example_method(lines: List[str], start: int) -> DotNetExampleMeth line = lines[index] if line.strip().startswith(original_file_key): - original_file = line.strip()[len(original_file_key):] + original_file = line.strip()[len(original_file_key) :] # begin of method dotnet_example_method.example_relative_path = original_file dotnet_example_method.line_start = index method_indent = len(line) - len(line.lstrip()) - elif method_indent and line.rstrip() == (' ' * (method_indent - 4) + '}'): + elif method_indent and line.rstrip() == (" " * (method_indent - 4) + "}"): # end of method dotnet_example_method.line_end = index break - dotnet_example_method.content = lines[dotnet_example_method.line_start:dotnet_example_method.line_end] + dotnet_example_method.content = lines[dotnet_example_method.line_start : dotnet_example_method.line_end] return dotnet_example_method @@ -84,21 +95,22 @@ def get_dotnet_using_statements(lines: List[str]) -> List[str]: lines_using_statements = [ # these are some using statements that every sample program should use. "using Azure;\n", - "using Azure.ResourceManager;\n" + "using Azure.ResourceManager;\n", ] for line in lines: - if line.startswith('using '): + if line.startswith("using "): lines_using_statements.append(line) - elif line.startswith('namespace '): + elif line.startswith("namespace "): # remove the prefix first - namespace = line[len('namespace '):].strip() + namespace = line[len("namespace ") :].strip() # remove the '.Samples' suffix if any - if namespace.endswith('.Samples'): - namespace = namespace[:-len('.Samples')] - lines_using_statements.append(f'using {namespace};\n') + if namespace.endswith(".Samples"): + namespace = namespace[: -len(".Samples")] + lines_using_statements.append(f"using {namespace};\n") break return deduplicate_list(lines_using_statements) + def deduplicate_list(list: List[str]) -> List[str]: seen = set() result: List[str] = [] @@ -112,7 +124,7 @@ def deduplicate_list(list: List[str]) -> List[str]: def break_down_aggregated_dotnet_example(lines: List[str]) -> AggregatedDotNetExample: aggregated_dotnet_example = AggregatedDotNetExample([]) aggregated_dotnet_example.class_opening = get_dotnet_using_statements(lines) - aggregated_dotnet_example.class_opening.append('\n') + aggregated_dotnet_example.class_opening.append("\n") dotnet_example_method = get_dotnet_example_method(lines, 0) while dotnet_example_method.is_valid(): @@ -134,7 +146,7 @@ def format_dotnet(lines: List[str]) -> List[str]: last_good_indent = indent - base_indent else: if line.strip(): - line = ' ' * last_good_indent + line + line = " " * last_good_indent + line new_lines.append(line) return new_lines @@ -144,9 +156,9 @@ def process_dotnet_example(filepath: str) -> List[DotNetExample]: # process aggregated DotNet sample to examples filename = path.basename(filepath) - logging.info(f'Processing DotNet aggregated sample: {filename}') + logging.info(f"Processing DotNet aggregated sample: {filename}") - with open(filepath, encoding='utf-8') as f: + with open(filepath, encoding="utf-8") as f: lines = f.readlines() dotnet_examples = [] @@ -154,7 +166,7 @@ def process_dotnet_example(filepath: str) -> List[DotNetExample]: aggregated_dotnet_example = break_down_aggregated_dotnet_example(lines) for dotnet_example_method in aggregated_dotnet_example.methods: if dotnet_example_method.is_valid(): - logging.info(f'Processing DotNet example: {dotnet_example_method.example_relative_path}') + logging.info(f"Processing DotNet example: {dotnet_example_method.example_relative_path}") # re-construct the example class, from example method example_lines = aggregated_dotnet_example.class_opening + format_dotnet(dotnet_example_method.content) @@ -162,12 +174,22 @@ def process_dotnet_example(filepath: str) -> List[DotNetExample]: example_filepath = dotnet_example_method.example_relative_path example_dir, example_filename = path.split(example_filepath) - filename = example_filename.split('.')[0] + try: + example_dir = examples_dir.try_find_resource_manager_example( + specs_path, sdk_package_path, example_dir, example_filename + ) + except NameError: + pass + + filename = example_filename.split(".")[0] # use the examples-dotnet folder for DotNet example - md_dir = (example_dir + '-dotnet') if example_dir.endswith('/examples') \ - else example_dir.replace('/examples/', '/examples-dotnet/') + md_dir = ( + (example_dir + "-dotnet") + if example_dir.endswith("/examples") + else example_dir.replace("/examples/", "/examples-dotnet/") + ) - dotnet_example = DotNetExample(filename, md_dir, ''.join(example_lines)) + dotnet_example = DotNetExample(filename, md_dir, "".join(example_lines)) dotnet_examples.append(dotnet_example) return dotnet_examples @@ -180,132 +202,134 @@ def generate_examples(release: Release, sdk_examples_path: str, dotnet_examples: files = [] for dotnet_example in dotnet_examples: - doc_link = f'https://github.com/Azure/azure-sdk-for-net/blob/{release.tag}/{module_relative_path}/README.md' - - files.extend(write_code_to_file(sdk_examples_path, dotnet_example.target_dir, - dotnet_example.target_filename, '.cs', - dotnet_example.content, doc_link)) + doc_link = f"https://github.com/Azure/azure-sdk-for-net/blob/{release.tag}/{module_relative_path}/README.md" + + files.extend( + write_code_to_file( + sdk_examples_path, + dotnet_example.target_dir, + dotnet_example.target_filename, + ".cs", + dotnet_example.content, + doc_link, + ) + ) return files -def write_code_to_file(sdk_examples_path: str, target_dir: str, filename_root: str, filename_ext: str, - code_content: str, sdk_url: str) -> List[str]: +def write_code_to_file( + sdk_examples_path: str, target_dir: str, filename_root: str, filename_ext: str, code_content: str, sdk_url: str +) -> List[str]: # write code file and metadata file code_filename = filename_root + filename_ext - metadata_filename = filename_root + '.json' + metadata_filename = filename_root + ".json" - metadata_json = {'sdkUrl': sdk_url} + metadata_json = {"sdkUrl": sdk_url} target_dir_path = path.join(sdk_examples_path, target_dir) os.makedirs(target_dir_path, exist_ok=True) code_file_path = path.join(target_dir_path, code_filename) - with open(code_file_path, 'w', encoding='utf-8') as f: + with open(code_file_path, "w", encoding="utf-8") as f: f.write(code_content) - logging.info(f'Code written to file: {code_file_path}') + logging.info(f"Code written to file: {code_file_path}") metadata_file_path = path.join(target_dir_path, metadata_filename) - with open(metadata_file_path, 'w', encoding='utf-8') as f: + with open(metadata_file_path, "w", encoding="utf-8") as f: json.dump(metadata_json, f) - logging.info(f'Metadata written to file: {metadata_file_path}') + logging.info(f"Metadata written to file: {metadata_file_path}") - return [path.join(target_dir, code_filename), - path.join(target_dir, metadata_filename)] + return [path.join(target_dir, code_filename), path.join(target_dir, metadata_filename)] -def create_dotnet_examples(release: Release, - dotnet_module: str, - sdk_examples_path: str, dotnet_examples_path: str) -> (bool, List[str]): +def create_dotnet_examples( + release: Release, dotnet_module: str, sdk_examples_path: str, dotnet_examples_path: str +) -> (bool, List[str]): dotnet_paths = [] for root, dirs, files in os.walk(dotnet_examples_path): for name in files: filepath = path.join(root, name) - if path.splitext(filepath)[1] == '.cs': + if path.splitext(filepath)[1] == ".cs": dotnet_paths.append(filepath) - logging.info(f'Processing SDK examples: {release.package}') + logging.info(f"Processing SDK examples: {release.package}") dotnet_examples = [] for filepath in dotnet_paths: dotnet_examples += process_dotnet_example(filepath) files = [] if dotnet_examples: - dotnet_build = DotNetBuild(tmp_path, dotnet_module.split(',')[0], dotnet_module.split(',')[1], dotnet_examples) + dotnet_build = DotNetBuild(tmp_path, dotnet_module.split(",")[0], dotnet_module.split(",")[1], dotnet_examples) build_result = dotnet_build.build() if build_result.succeeded: files = generate_examples(release, sdk_examples_path, dotnet_examples) else: - logging.error('Build failed') + logging.error("Build failed") return build_result.succeeded, files else: - logging.info('SDK examples not found') + logging.info("SDK examples not found") return True, files def get_module_relative_path(sdk_name: str, sdk_path: str) -> str: global module_relative_path - candidate_sdk_paths = glob.glob(path.join(sdk_path, f'sdk/*/{sdk_name}')) + candidate_sdk_paths = glob.glob(path.join(sdk_path, f"sdk/*/{sdk_name}")) if len(candidate_sdk_paths) > 0: candidate_sdk_paths = [path.relpath(p, sdk_path) for p in candidate_sdk_paths] - logging.info( - f'Use first item of {candidate_sdk_paths} for SDK folder') + logging.info(f"Use first item of {candidate_sdk_paths} for SDK folder") module_relative_path = candidate_sdk_paths[0] else: - raise RuntimeError(f'Source folder not found for SDK {sdk_name}') + raise RuntimeError(f"Source folder not found for SDK {sdk_name}") return module_relative_path def main(): global script_path global tmp_path + global specs_path + global sdk_package_path - logging.basicConfig(level=logging.INFO, - format='%(asctime)s [%(levelname)s] %(message)s', - datefmt='%Y-%m-%d %X') + logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s", datefmt="%Y-%m-%d %X") script_path = path.abspath(path.dirname(sys.argv[0])) parser = argparse.ArgumentParser(description='Requires 2 arguments, path of "input.json" and "output.json".') - parser.add_argument('paths', metavar='path', type=str, nargs=2, - help='path of "input.json" or "output.json"') + parser.add_argument("paths", metavar="path", type=str, nargs=2, help='path of "input.json" or "output.json"') args = parser.parse_args() input_json_path = args.paths[0] output_json_path = args.paths[1] - with open(input_json_path, 'r', encoding='utf-8') as f_in: + with open(input_json_path, "r", encoding="utf-8") as f_in: config = json.load(f_in) - sdk_path = config['sdkPath'] - sdk_examples_path = config['sdkExamplesPath'] - tmp_path = config['tempPath'] + specs_path = config["specsPath"] + sdk_path = config["sdkPath"] + sdk_examples_path = config["sdkExamplesPath"] + tmp_path = config["tempPath"] - release = Release(config['release']['tag'], - config['release']['package'], - config['release']['version']) + release = Release(config["release"]["tag"], config["release"]["package"], config["release"]["version"]) # samples/Generated/Samples module_relative_path_local = get_module_relative_path(release.package, sdk_path) - dotnet_examples_relative_path = path.join(module_relative_path_local, 'samples', 'Generated', 'Samples') + dotnet_examples_relative_path = path.join(module_relative_path_local, "samples", "Generated", "Samples") dotnet_examples_path = path.join(sdk_path, dotnet_examples_relative_path) if not path.exists(dotnet_examples_path): # fallback to tests/Generated/Samples - dotnet_examples_relative_path = path.join(module_relative_path_local, 'tests', 'Generated', 'Samples') + dotnet_examples_relative_path = path.join(module_relative_path_local, "tests", "Generated", "Samples") dotnet_examples_path = path.join(sdk_path, dotnet_examples_relative_path) - dotnet_module = f'{release.package},{release.version}' + sdk_package_path = path.join(sdk_path, module_relative_path_local) + + dotnet_module = f"{release.package},{release.version}" succeeded, files = create_dotnet_examples(release, dotnet_module, sdk_examples_path, dotnet_examples_path) - with open(output_json_path, 'w', encoding='utf-8') as f_out: - output = { - 'status': 'succeeded' if succeeded else 'failed', - 'name': dotnet_module, - 'files': files - } + with open(output_json_path, "w", encoding="utf-8") as f_out: + output = {"status": "succeeded" if succeeded else "failed", "name": dotnet_module, "files": files} json.dump(output, f_out, indent=2) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/tools/azure-rest-api-specs-examples-automation/dotnet/test_build.py b/tools/azure-rest-api-specs-examples-automation/dotnet/test_build.py index b6d9f84a706..83db0e29050 100644 --- a/tools/azure-rest-api-specs-examples-automation/dotnet/test_build.py +++ b/tools/azure-rest-api-specs-examples-automation/dotnet/test_build.py @@ -8,7 +8,7 @@ class TestDotNetBuild(unittest.TestCase): def test_example(self): - code = '''using System; + code = """using System; using System.Threading.Tasks; using System.Xml; using Azure; @@ -38,16 +38,16 @@ def test_example(self): VirtualMachineData resourceData = result.Data; // for demo we just print out the id Console.WriteLine($"Succeeded on id: {resourceData.Id}"); -''' +""" - tmp_path = path.abspath('.') - dotnet_examples = [DotNetExample('code', '', code)] - dotnet_build = DotNetBuild(tmp_path, 'Azure.ResourceManager.Compute', '1.0.1', dotnet_examples) + tmp_path = path.abspath(".") + dotnet_examples = [DotNetExample("code", "", code)] + dotnet_build = DotNetBuild(tmp_path, "Azure.ResourceManager.Compute", "1.0.1", dotnet_examples) result = dotnet_build.build() self.assertTrue(result.succeeded) def test_invalid(self): - code = '''using System; + code = """using System; using System.Threading.Tasks; using System.Xml; using Azure; @@ -64,10 +64,10 @@ def test_invalid(self): string vmName = "myVM"; ResourceIdentifier virtualMachineResourceId = VirtualMachineResource.CreateResourceIdentifier(subscriptionId, resourceGroupName, vmName); VirtualMachineResource virtualMachine = client.GetVirtualMachineResource(virtualMachineResourceId); -''' +""" - tmp_path = path.abspath('.') - dotnet_examples = [DotNetExample('code', '', code)] - dotnet_build = DotNetBuild(tmp_path, 'Azure.ResourceManager.Compute', '1.0.1', dotnet_examples) + tmp_path = path.abspath(".") + dotnet_examples = [DotNetExample("code", "", code)] + dotnet_build = DotNetBuild(tmp_path, "Azure.ResourceManager.Compute", "1.0.1", dotnet_examples) result = dotnet_build.build() self.assertFalse(result.succeeded) diff --git a/tools/azure-rest-api-specs-examples-automation/dotnet/test_main.py b/tools/azure-rest-api-specs-examples-automation/dotnet/test_main.py index 5c940a5f443..0b8f9e973f5 100644 --- a/tools/azure-rest-api-specs-examples-automation/dotnet/test_main.py +++ b/tools/azure-rest-api-specs-examples-automation/dotnet/test_main.py @@ -2,7 +2,7 @@ import parameterized from main import break_down_aggregated_dotnet_example, format_dotnet, get_dotnet_using_statements -file_content = '''// Copyright (c) Microsoft Corporation. All rights reserved. +file_content = """// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. // @@ -323,7 +323,7 @@ } } } -''' +""" class TestMain(unittest.TestCase): @@ -336,12 +336,13 @@ def test_break_down_aggregated_dotnet_example(self): for dotnet_example_method in examples.methods: example_lines = examples.class_opening + format_dotnet(dotnet_example_method.content) - example_content = ''.join(example_lines) + example_content = "".join(example_lines) self.assertIsNotNone(example_content) @parameterized.parameterized.expand( [ - ('''// Copyright (c) Microsoft Corporation. All rights reserved. + ( + """// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. // @@ -358,14 +359,15 @@ def test_break_down_aggregated_dotnet_example(self): namespace Azure.ResourceManager.Compute.Samples { -}''') +}""" + ) ] ) def test_example_usings(self, content: str): lines = content.splitlines(keepends=True) usings = get_dotnet_using_statements(lines) - self.assertIn('using Azure;\n', usings) - self.assertIn('using Azure.Core;\n', usings) - self.assertIn('using Azure.ResourceManager;\n', usings) - self.assertIn('using Azure.ResourceManager.Compute;\n', usings) + self.assertIn("using Azure;\n", usings) + self.assertIn("using Azure.Core;\n", usings) + self.assertIn("using Azure.ResourceManager;\n", usings) + self.assertIn("using Azure.ResourceManager.Compute;\n", usings) diff --git a/tools/azure-rest-api-specs-examples-automation/go/main.py b/tools/azure-rest-api-specs-examples-automation/go/main.py index 53803d37b9e..8c7f40a3100 100644 --- a/tools/azure-rest-api-specs-examples-automation/go/main.py +++ b/tools/azure-rest-api-specs-examples-automation/go/main.py @@ -7,15 +7,26 @@ import logging import dataclasses from typing import List +import importlib.util from models import GoExample, GoVetResult from validate import GoVet -script_path: str = '.' +spec_location = ( + "./directory/examples_dir.py" if path.exists("./directory/examples_dir.py") else "../directory/examples_dir.py" +) +spec = importlib.util.spec_from_file_location("examples_dir", spec_location) +examples_dir = importlib.util.module_from_spec(spec) +spec.loader.exec_module(examples_dir) + + +script_path: str = "." tmp_path: str +specs_path: str +sdk_package_path: str -original_file_key = '// Generated from example definition: ' +original_file_key = "// Generated from example definition: " @dataclasses.dataclass(eq=True, frozen=True) @@ -52,16 +63,16 @@ def is_aggregated_go_example(lines: List[str]) -> bool: def parse_original_file(original_file: str) -> str: - if original_file.startswith('https://'): - spec_main_segment = 'https://github.com/Azure/azure-rest-api-specs/tree/main/' + if original_file.startswith("https://"): + spec_main_segment = "https://github.com/Azure/azure-rest-api-specs/tree/main/" if original_file.startswith(spec_main_segment): - original_file = original_file[len(spec_main_segment):] + original_file = original_file[len(spec_main_segment) :] else: - specification_index = original_file.find('specification/') + specification_index = original_file.find("specification/") if specification_index != -1: original_file = original_file[specification_index:] else: - logging.error(f'Parse relative path from URI {original_file} failed') + logging.error(f"Parse relative path from URI {original_file} failed") original_file = None return original_file @@ -77,13 +88,13 @@ def get_go_example_method(lines: List[str], start: int) -> GoExampleMethodConten line = lines[index] if line.strip().startswith(original_file_key): - original_file = line.strip()[len(original_file_key):] + original_file = line.strip()[len(original_file_key) :] original_file = parse_original_file(original_file) - elif line.startswith('func '): + elif line.startswith("func "): # begin of method go_example_method.example_relative_path = original_file go_example_method.line_start = index - elif line.startswith('}'): + elif line.startswith("}"): # end of method go_example_method.line_end = index + 1 break @@ -92,11 +103,11 @@ def get_go_example_method(lines: List[str], start: int) -> GoExampleMethodConten # backtrace to include comments before the method declaration for index in range(go_example_method.line_start - 1, start - 1, -1): line = lines[index] - if line.strip().startswith('//'): + if line.strip().startswith("//"): go_example_method.line_start = index else: break - go_example_method.content = lines[go_example_method.line_start:go_example_method.line_end] + go_example_method.content = lines[go_example_method.line_start : go_example_method.line_end] return go_example_method @@ -127,7 +138,7 @@ def format_go(lines: List[str]) -> List[str]: new_lines.append(line) else: # start with package - if line.startswith('package '): + if line.startswith("package "): new_lines.append(line) skip_head = False @@ -138,9 +149,9 @@ def process_go_example(filepath: str) -> List[GoExample]: # process aggregated Go sample to examples filename = path.basename(filepath) - logging.info(f'Processing Go aggregated sample: {filename}') + logging.info(f"Processing Go aggregated sample: {filename}") - with open(filepath, encoding='utf-8') as f: + with open(filepath, encoding="utf-8") as f: lines = f.readlines() go_examples = [] @@ -148,7 +159,7 @@ def process_go_example(filepath: str) -> List[GoExample]: aggregated_go_example = break_down_aggregated_go_example(lines) for go_example_method in aggregated_go_example.methods: if go_example_method.is_valid(): - logging.info(f'Processing Go example: {go_example_method.example_relative_path}') + logging.info(f"Processing Go example: {go_example_method.example_relative_path}") # re-construct the example class, from example method example_lines = aggregated_go_example.class_opening + go_example_method.content @@ -156,14 +167,24 @@ def process_go_example(filepath: str) -> List[GoExample]: example_filepath = go_example_method.example_relative_path example_dir, example_filename = path.split(example_filepath) + try: + example_dir = examples_dir.try_find_resource_manager_example( + specs_path, sdk_package_path, example_dir, example_filename + ) + except NameError: + pass + example_lines = format_go(example_lines) - filename = example_filename.split('.')[0] + filename = example_filename.split(".")[0] # use the examples-go folder for Go example - md_dir = (example_dir + '-go') if example_dir.endswith('/examples') \ - else example_dir.replace('/examples/', '/examples-go/') + md_dir = ( + (example_dir + "-go") + if example_dir.endswith("/examples") + else example_dir.replace("/examples/", "/examples-go/") + ) - go_example = GoExample(filename, md_dir, ''.join(example_lines)) + go_example = GoExample(filename, md_dir, "".join(example_lines)) go_examples.append(go_example) return go_examples @@ -174,7 +195,7 @@ def validate_go_examples(go_module: str, go_mod_filepath: str, go_examples: List go_mod = None if path.isfile(go_mod_filepath): - with open(go_mod_filepath, encoding='utf-8') as f: + with open(go_mod_filepath, encoding="utf-8") as f: go_mod = f.read() go_vet = GoVet(tmp_path, go_module, go_mod, go_examples) @@ -188,116 +209,124 @@ def generate_examples(release: Release, sdk_examples_path: str, go_examples: Lis files = [] for go_example in go_examples: - escaped_release_tag = urllib.parse.quote(release.tag, safe='') - doc_link = f'https://github.com/Azure/azure-sdk-for-go/blob/{escaped_release_tag}/' \ - f'{release.package}/README.md' - files.extend(write_code_to_file(sdk_examples_path, go_example.target_dir, go_example.target_filename, '.go', - go_example.content, doc_link)) + escaped_release_tag = urllib.parse.quote(release.tag, safe="") + doc_link = ( + f"https://github.com/Azure/azure-sdk-for-go/blob/{escaped_release_tag}/" f"{release.package}/README.md" + ) + files.extend( + write_code_to_file( + sdk_examples_path, + go_example.target_dir, + go_example.target_filename, + ".go", + go_example.content, + doc_link, + ) + ) return files -def write_code_to_file(sdk_examples_path: str, target_dir: str, filename_root: str, filename_ext: str, - code_content: str, sdk_url: str) -> List[str]: +def write_code_to_file( + sdk_examples_path: str, target_dir: str, filename_root: str, filename_ext: str, code_content: str, sdk_url: str +) -> List[str]: # write code file and metadata file code_filename = filename_root + filename_ext - metadata_filename = filename_root + '.json' + metadata_filename = filename_root + ".json" - metadata_json = {'sdkUrl': sdk_url} + metadata_json = {"sdkUrl": sdk_url} target_dir_path = path.join(sdk_examples_path, target_dir) os.makedirs(target_dir_path, exist_ok=True) code_file_path = path.join(target_dir_path, code_filename) - with open(code_file_path, 'w', encoding='utf-8') as f: + with open(code_file_path, "w", encoding="utf-8") as f: f.write(code_content) - logging.info(f'Code written to file: {code_file_path}') + logging.info(f"Code written to file: {code_file_path}") metadata_file_path = path.join(target_dir_path, metadata_filename) - with open(metadata_file_path, 'w', encoding='utf-8') as f: + with open(metadata_file_path, "w", encoding="utf-8") as f: json.dump(metadata_json, f) - logging.info(f'Metadata written to file: {metadata_file_path}') + logging.info(f"Metadata written to file: {metadata_file_path}") - return [path.join(target_dir, code_filename), - path.join(target_dir, metadata_filename)] + return [path.join(target_dir, code_filename), path.join(target_dir, metadata_filename)] -def create_go_examples(release: Release, - go_module: str, go_mod_filepath: str, - sdk_examples_path: str, go_examples_path: str) -> (bool, List[str]): +def create_go_examples( + release: Release, go_module: str, go_mod_filepath: str, sdk_examples_path: str, go_examples_path: str +) -> (bool, List[str]): go_paths = [] for root, dirs, files in os.walk(go_examples_path): for name in files: filepath = path.join(root, name) - if path.splitext(filepath)[1] == '.go' and filepath.endswith('_test.go'): + if path.splitext(filepath)[1] == ".go" and filepath.endswith("_test.go"): go_paths.append(filepath) - logging.info(f'Processing SDK examples: {release.package}') + logging.info(f"Processing SDK examples: {release.package}") go_examples = [] for filepath in go_paths: go_examples += process_go_example(filepath) files = [] if go_examples: - logging.info('Validating SDK examples') + logging.info("Validating SDK examples") go_vet_result = validate_go_examples(go_module, go_mod_filepath, go_examples) if go_vet_result.succeeded: files = generate_examples(release, sdk_examples_path, go_vet_result.examples) else: - logging.error('Validation failed') + logging.error("Validation failed") return go_vet_result.succeeded, files else: - logging.info('SDK examples not found') + logging.info("SDK examples not found") return True, files def main(): global script_path global tmp_path + global specs_path + global sdk_package_path - logging.basicConfig(level=logging.INFO, - format='%(asctime)s [%(levelname)s] %(message)s', - datefmt='%Y-%m-%d %X') + logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s", datefmt="%Y-%m-%d %X") script_path = path.abspath(path.dirname(sys.argv[0])) parser = argparse.ArgumentParser(description='Requires 2 arguments, path of "input.json" and "output.json".') - parser.add_argument('paths', metavar='path', type=str, nargs=2, - help='path of "input.json" or "output.json"') + parser.add_argument("paths", metavar="path", type=str, nargs=2, help='path of "input.json" or "output.json"') args = parser.parse_args() input_json_path = args.paths[0] output_json_path = args.paths[1] - with open(input_json_path, 'r', encoding='utf-8') as f_in: + with open(input_json_path, "r", encoding="utf-8") as f_in: config = json.load(f_in) - sdk_path = config['sdkPath'] - sdk_examples_path = config['sdkExamplesPath'] - tmp_path = config['tempPath'] + specs_path = config["specsPath"] + sdk_path = config["sdkPath"] + sdk_examples_path = config["sdkExamplesPath"] + tmp_path = config["tempPath"] - release = Release(config['release']['tag'], - config['release']['package'], - config['release']['version']) + release = Release(config["release"]["tag"], config["release"]["package"], config["release"]["version"]) - go_module_major_suffix = '' if release.version.startswith('v0.') or release.version.startswith('v1.')\ + go_module_major_suffix = ( + "" + if release.version.startswith("v0.") or release.version.startswith("v1.") else f'/{release.version.split(".")[0]}' - go_module = f'github.com/Azure/azure-sdk-for-go/{release.package}{go_module_major_suffix}@{release.version}' + ) + go_module = f"github.com/Azure/azure-sdk-for-go/{release.package}{go_module_major_suffix}@{release.version}" go_examples_relative_path = release.package go_examples_path = path.join(sdk_path, go_examples_relative_path) - go_mod_filepath = path.join(sdk_path, release.package, 'go.mod') + go_mod_filepath = path.join(sdk_path, release.package, "go.mod") + + sdk_package_path = path.join(sdk_path, release.package) succeeded, files = create_go_examples(release, go_module, go_mod_filepath, sdk_examples_path, go_examples_path) - with open(output_json_path, 'w', encoding='utf-8') as f_out: - output = { - 'status': 'succeeded' if succeeded else 'failed', - 'name': go_module, - 'files': files - } + with open(output_json_path, "w", encoding="utf-8") as f_out: + output = {"status": "succeeded" if succeeded else "failed", "name": go_module, "files": files} json.dump(output, f_out, indent=2) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/tools/azure-rest-api-specs-examples-automation/go/test_main.py b/tools/azure-rest-api-specs-examples-automation/go/test_main.py index 7c5f6b34d61..1ca88ebae43 100644 --- a/tools/azure-rest-api-specs-examples-automation/go/test_main.py +++ b/tools/azure-rest-api-specs-examples-automation/go/test_main.py @@ -5,15 +5,21 @@ class TestMain(unittest.TestCase): def test_parse_original_file(self): - expected_original_file = 'specification/agrifood/resource-manager/Microsoft.AgFoodPlatform/preview/2020-05-12-preview/examples/FarmBeatsExtensions_List.json' + expected_original_file = "specification/agrifood/resource-manager/Microsoft.AgFoodPlatform/preview/2020-05-12-preview/examples/FarmBeatsExtensions_List.json" - original_file = parse_original_file('https://github.com/Azure/azure-rest-api-specs/tree/main/' + expected_original_file) + original_file = parse_original_file( + "https://github.com/Azure/azure-rest-api-specs/tree/main/" + expected_original_file + ) self.assertEqual(expected_original_file, original_file) - original_file = parse_original_file('https://github.com/Azure/azure-rest-api-specs/tree/some_branch/' + expected_original_file) + original_file = parse_original_file( + "https://github.com/Azure/azure-rest-api-specs/tree/some_branch/" + expected_original_file + ) self.assertEqual(expected_original_file, original_file) - original_file = parse_original_file('https://local/agrifood/resource-manager/Microsoft.AgFoodPlatform/preview/2020-05-12-preview/examples/FarmBeatsExtensions_List.json') + original_file = parse_original_file( + "https://local/agrifood/resource-manager/Microsoft.AgFoodPlatform/preview/2020-05-12-preview/examples/FarmBeatsExtensions_List.json" + ) self.assertIsNone(original_file) original_file = parse_original_file(expected_original_file) diff --git a/tools/azure-rest-api-specs-examples-automation/go/test_validate.py b/tools/azure-rest-api-specs-examples-automation/go/test_validate.py index ab52c5da7fc..85822d1c524 100644 --- a/tools/azure-rest-api-specs-examples-automation/go/test_validate.py +++ b/tools/azure-rest-api-specs-examples-automation/go/test_validate.py @@ -8,21 +8,21 @@ class TestGoVet(unittest.TestCase): def test_example(self): - code = '''package main + code = """package main import "fmt" func main() { fmt.Println("hello world") } -''' +""" - tmp_path = path.abspath('.') - go_examples = [GoExample('code', '', code)] - go_vet = GoVet(tmp_path, 'rsc.io/quote@v1.5.2', '', go_examples) + tmp_path = path.abspath(".") + go_examples = [GoExample("code", "", code)] + go_vet = GoVet(tmp_path, "rsc.io/quote@v1.5.2", "", go_examples) result = go_vet.vet() self.assertTrue(result.succeeded) def test_package_v2(self): - code = r'''package armcompute_test + code = r"""package armcompute_test import ( "context" "log" @@ -52,9 +52,9 @@ def test_package_v2(self): log.Fatalf("failed to pull the result: %v", err) } } -''' +""" - go_mod = r'''module github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v2 + go_mod = r"""module github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v2 go 1.18 @@ -85,11 +85,15 @@ def test_package_v2(self): gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) -''' +""" - tmp_path = path.abspath('.') - go_examples = [GoExample('code', '', code)] - go_vet = GoVet(tmp_path, 'github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v2@v2.0.0', - go_mod, go_examples) + tmp_path = path.abspath(".") + go_examples = [GoExample("code", "", code)] + go_vet = GoVet( + tmp_path, + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v2@v2.0.0", + go_mod, + go_examples, + ) result = go_vet.vet() self.assertTrue(result.succeeded) diff --git a/tools/azure-rest-api-specs-examples-automation/go/validate.py b/tools/azure-rest-api-specs-examples-automation/go/validate.py index 1ec81f12fb7..1c17a349b58 100644 --- a/tools/azure-rest-api-specs-examples-automation/go/validate.py +++ b/tools/azure-rest-api-specs-examples-automation/go/validate.py @@ -9,7 +9,7 @@ def check_call(cmd: List[str], work_dir: str): - logging.info('Command line: ' + ' '.join(cmd)) + logging.info("Command line: " + " ".join(cmd)) subprocess.check_call(cmd, cwd=work_dir) @@ -25,87 +25,87 @@ def __init__(self, tmp_path: str, module: str, go_mod: str, examples: List[GoExa self.module = module self.examples = examples - match = re.search(r'go ([.0-9]*)', go_mod, re.MULTILINE) + match = re.search(r"go ([.0-9]*)", go_mod, re.MULTILINE) if match: self.golang_version = match.group(1) else: - self.golang_version = '1.18' + self.golang_version = "1.18" self.modules = [] - match = re.search(r'github\.com/Azure/azure-sdk-for-go/sdk/azcore (v[.\-\w]*)', go_mod, re.MULTILINE) + match = re.search(r"github\.com/Azure/azure-sdk-for-go/sdk/azcore (v[.\-\w]*)", go_mod, re.MULTILINE) if match: - self.modules.append('github.com/Azure/azure-sdk-for-go/sdk/azcore@' + match.group(1)) - match = re.search(r'github\.com/Azure/azure-sdk-for-go/sdk/azidentity (v[.\-\w]*)', go_mod, re.MULTILINE) + self.modules.append("github.com/Azure/azure-sdk-for-go/sdk/azcore@" + match.group(1)) + match = re.search(r"github\.com/Azure/azure-sdk-for-go/sdk/azidentity (v[.\-\w]*)", go_mod, re.MULTILINE) if match: - self.modules.append('github.com/Azure/azure-sdk-for-go/sdk/azidentity@' + match.group(1)) + self.modules.append("github.com/Azure/azure-sdk-for-go/sdk/azidentity@" + match.group(1)) def vet(self) -> GoVetResult: with tempfile.TemporaryDirectory(dir=self.tmp_path) as tmp_dir_name: # write examples to go files filename_no = 1 for example in self.examples: - filename = 'code' + str(filename_no) + '.go' + filename = "code" + str(filename_no) + ".go" filename_no += 1 filepath = path.join(tmp_dir_name, filename) - with open(filepath, 'w', encoding='utf-8') as f: + with open(filepath, "w", encoding="utf-8") as f: f.write(example.content) # format and validate go files try: - logging.info('Initialize mod') + logging.info("Initialize mod") # mod - cmd = ['go', 'mod', 'init', 'm'] + cmd = ["go", "mod", "init", "m"] check_call(cmd, tmp_dir_name) if self.golang_version: - cmd = ['go', 'mod', 'edit', '-go', self.golang_version] + cmd = ["go", "mod", "edit", "-go", self.golang_version] check_call(cmd, tmp_dir_name) - cmd = ['go', 'mod', 'edit', '-require', self.module] + cmd = ["go", "mod", "edit", "-require", self.module] check_call(cmd, tmp_dir_name) for module in self.modules: - cmd = ['go', 'mod', 'edit', '-require', module] + cmd = ["go", "mod", "edit", "-require", module] check_call(cmd, tmp_dir_name) - cmd = ['go', 'mod', 'tidy'] + cmd = ["go", "mod", "tidy"] check_call(cmd, tmp_dir_name) - with open(path.join(tmp_dir_name, 'go.mod'), encoding='utf-8') as f: + with open(path.join(tmp_dir_name, "go.mod"), encoding="utf-8") as f: content = f.read() - logging.info(f'go.mod\n{content}') + logging.info(f"go.mod\n{content}") - logging.info('Run goimports') + logging.info("Run goimports") # goimports - cmd = ['go', 'install', 'golang.org/x/tools/cmd/goimports@latest'] + cmd = ["go", "install", "golang.org/x/tools/cmd/goimports@latest"] check_call(cmd, tmp_dir_name) - cmd = ['goimports', '-w', '.'] + cmd = ["goimports", "-w", "."] check_call(cmd, tmp_dir_name) - logging.info('Build and vet') + logging.info("Build and vet") # build and vet - cmd = ['go', 'build'] + cmd = ["go", "build"] check_call(cmd, tmp_dir_name) - cmd = ['go', 'vet'] + cmd = ["go", "vet"] check_call(cmd, tmp_dir_name) except subprocess.CalledProcessError as error: - logging.error(f'Call error: {error}') + logging.error(f"Call error: {error}") return GoVetResult(False, []) # read formatted examples from go files formatted_examples = [] filename_no = 1 for example in self.examples: - filename = 'code' + str(filename_no) + '.go' + filename = "code" + str(filename_no) + ".go" filename_no += 1 filepath = path.join(tmp_dir_name, filename) - with open(filepath, encoding='utf-8') as f: + with open(filepath, encoding="utf-8") as f: content = f.read() formatted_examples.append(GoExample(example.target_filename, example.target_dir, content)) diff --git a/tools/azure-rest-api-specs-examples-automation/java/format.py b/tools/azure-rest-api-specs-examples-automation/java/format.py index b03c55d25f0..846942f9250 100644 --- a/tools/azure-rest-api-specs-examples-automation/java/format.py +++ b/tools/azure-rest-api-specs-examples-automation/java/format.py @@ -9,7 +9,7 @@ from modules import JavaExample, JavaFormatResult -OS_WINDOWS = platform.system().lower() == 'windows' +OS_WINDOWS = platform.system().lower() == "windows" class JavaFormat: @@ -22,23 +22,23 @@ def __init__(self, tmp_path: str, maven_path: str): def format(self, examples: List[JavaExample]) -> JavaFormatResult: with tempfile.TemporaryDirectory(dir=self.tmp_path) as tmp_dir_name: - files = ['pom.xml', 'eclipse-format-azure-sdk-for-java.xml'] + files = ["pom.xml", "eclipse-format-azure-sdk-for-java.xml"] for file in files: shutil.copyfile(path.join(self.maven_path, file), path.join(tmp_dir_name, file)) filename_no = 1 for example in examples: - filename = 'Code' + str(filename_no) + '.java' + filename = "Code" + str(filename_no) + ".java" filename_no += 1 filepath = path.join(tmp_dir_name, filename) - with open(filepath, 'w', encoding='utf-8') as f: + with open(filepath, "w", encoding="utf-8") as f: f.write(example.content) - logging.info('Format java code') - cmd = ['mvn' + ('.cmd' if OS_WINDOWS else ''), 'spotless:apply'] - logging.info('Command line: ' + ' '.join(cmd)) + logging.info("Format java code") + cmd = ["mvn" + (".cmd" if OS_WINDOWS else ""), "spotless:apply"] + logging.info("Command line: " + " ".join(cmd)) result = subprocess.run(cmd, cwd=tmp_dir_name) if result.returncode: @@ -48,12 +48,12 @@ def format(self, examples: List[JavaExample]) -> JavaFormatResult: formatted_examples = [] filename_no = 1 for example in examples: - filename = 'Code' + str(filename_no) + '.java' + filename = "Code" + str(filename_no) + ".java" filename_no += 1 filepath = path.join(tmp_dir_name, filename) - with open(filepath, encoding='utf-8') as f: + with open(filepath, encoding="utf-8") as f: content = f.read() formatted_examples.append(JavaExample(example.target_filename, example.target_dir, content)) diff --git a/tools/azure-rest-api-specs-examples-automation/java/main.py b/tools/azure-rest-api-specs-examples-automation/java/main.py index 31d973602b3..8dfb3358f1a 100644 --- a/tools/azure-rest-api-specs-examples-automation/java/main.py +++ b/tools/azure-rest-api-specs-examples-automation/java/main.py @@ -5,22 +5,30 @@ import argparse import logging import dataclasses -import re -import glob from typing import List +import importlib.util from modules import JavaExample, JavaFormatResult from package import MavenPackage from format import JavaFormat -script_path: str = '.' +spec_location = ( + "./directory/examples_dir.py" if path.exists("./directory/examples_dir.py") else "../directory/examples_dir.py" +) +spec = importlib.util.spec_from_file_location("examples_dir", spec_location) +examples_dir = importlib.util.module_from_spec(spec) +spec.loader.exec_module(examples_dir) + + +script_path: str = "." tmp_path: str specs_path: str +sdk_package_path: str -namespace = 'com.azure.resourcemanager' +namespace = "com.azure.resourcemanager" -original_file_key = '* x-ms-original-file:' +original_file_key = "* x-ms-original-file:" @dataclasses.dataclass(eq=True, frozen=True) @@ -49,11 +57,19 @@ class AggregatedJavaExample: class_closing: List[str] = None +def _set_paths(new_specs_path: str, new_sdk_package_path: str): + # for test + global specs_path + global sdk_package_path + specs_path = new_specs_path + sdk_package_path = new_sdk_package_path + + def get_sdk_name_from_package(package: str) -> str: - if package == 'azure-resourcemanager': - return 'resourcemanager' + if package == "azure-resourcemanager": + return "resourcemanager" else: - return package[len('azure-resourcemanager-'):] + return package[len("azure-resourcemanager-") :] def is_aggregated_java_example(lines: List[str]) -> bool: @@ -76,23 +92,23 @@ def get_java_example_method(lines: List[str], start: int) -> JavaExampleMethodCo line = lines[index] if line.strip().startswith(original_file_key): - original_file = line.strip()[len(original_file_key):].strip() + original_file = line.strip()[len(original_file_key) :].strip() # merge rest of the lines peek_index = index + 1 while peek_index < len(lines): peek_line = lines[peek_index] - if peek_line.strip() == '*/': + if peek_line.strip() == "*/": # end of comment block break else: # content of original_file breaks into this line of comment - original_file = original_file + peek_line.strip()[len('*'):].strip() + original_file = original_file + peek_line.strip()[len("*") :].strip() peek_index += 1 - elif line.startswith(' public static void'): + elif line.startswith(" public static void"): # begin of method java_example_method.example_relative_path = original_file java_example_method.line_start = index - elif line.startswith(' }'): + elif line.startswith(" }"): # end of method java_example_method.line_end = index + 1 break @@ -103,12 +119,16 @@ def get_java_example_method(lines: List[str], start: int) -> JavaExampleMethodCo # backtrace to include javadoc and comments before the method declaration for index in range(java_example_method.line_start - 1, start - 1, -1): line = lines[index] - if line.strip().startswith('*') or line.strip().startswith('/*') or line.strip().startswith('*/') \ - or line.strip().startswith('//'): + if ( + line.strip().startswith("*") + or line.strip().startswith("/*") + or line.strip().startswith("*/") + or line.strip().startswith("//") + ): java_example_method.line_start = index else: break - java_example_method.content = lines[java_example_method.line_start:java_example_method.line_end] + java_example_method.content = lines[java_example_method.line_start : java_example_method.line_end] return java_example_method @@ -137,11 +157,11 @@ def format_java(lines: List[str], old_class_name: str, new_class_name: str) -> L for line in lines: if not skip_head: # use new class name - line = line.replace('class ' + old_class_name + ' {', 'class ' + new_class_name + ' {', 1) + line = line.replace("class " + old_class_name + " {", "class " + new_class_name + " {", 1) new_lines.append(line) else: # remove package - if line.startswith('package '): + if line.startswith("package "): skip_head = False return new_lines @@ -151,12 +171,12 @@ def process_java_example(filepath: str) -> List[JavaExample]: # process aggregated Java sample to examples filename = path.basename(filepath) - logging.info(f'Processing Java aggregated sample: {filename}') + logging.info(f"Processing Java aggregated sample: {filename}") - with open(filepath, encoding='utf-8') as f: + with open(filepath, encoding="utf-8") as f: lines = f.readlines() - class_name = filename.split('.')[0] + class_name = filename.split(".")[0] return process_java_example_content(lines, class_name) @@ -166,70 +186,48 @@ def process_java_example_content(lines: List[str], class_name: str) -> List[Java aggregated_java_example = break_down_aggregated_java_example(lines) for java_example_method in aggregated_java_example.methods: if java_example_method.is_valid(): - logging.info(f'Processing java example: {java_example_method.example_relative_path}') + logging.info(f"Processing java example: {java_example_method.example_relative_path}") # re-construct the example class, from example method - example_lines = aggregated_java_example.class_opening + java_example_method.content \ - + aggregated_java_example.class_closing + example_lines = ( + aggregated_java_example.class_opening + + java_example_method.content + + aggregated_java_example.class_closing + ) example_filepath = java_example_method.example_relative_path example_dir, example_filename = path.split(example_filepath) - example_dir = try_find_resource_manager_example(example_dir, example_filename) + try: + example_dir = examples_dir.try_find_resource_manager_example( + specs_path, sdk_package_path, example_dir, example_filename + ) + except NameError: + pass # use Main as class name old_class_name = class_name - new_class_name = 'Main' + new_class_name = "Main" example_lines = format_java(example_lines, old_class_name, new_class_name) - filename = example_filename.split('.')[0] + filename = example_filename.split(".")[0] # use the examples-java folder for Java example - md_dir = (example_dir + '-java') if example_dir.endswith('/examples') \ - else example_dir.replace('/examples/', '/examples-java/') + md_dir = ( + (example_dir + "-java") + if example_dir.endswith("/examples") + else example_dir.replace("/examples/", "/examples-java/") + ) - java_example = JavaExample(filename, md_dir, ''.join(example_lines)) + java_example = JavaExample(filename, md_dir, "".join(example_lines)) java_examples.append(java_example) return java_examples -def set_specs_path(new_specs_path: str): - # for test - global specs_path - specs_path = new_specs_path - - -def try_find_resource_manager_example(example_dir: str, example_filename: str) -> str: - if '/resource-manager/' not in example_dir: - try: - match = re.match(r'specification/([^/]+)/.*/examples/([^/]+)(.*)', example_dir) - if match: - # example: specification/mongocluster/DocumentDB.MongoCluster.Management/examples/2024-03-01-preview - # service: mongocluster - # api_version: 2024-03-01-preview - # additional_path: - - service = match.group(1) - api_version = match.group(2) - additional_path = match.group(3) - - glob_resource_manager_filename = f'specification/{service}/resource-manager/**/{api_version}/examples{additional_path}/{example_filename}' - candidate_resource_manager_filename = glob.glob(path.join(specs_path, glob_resource_manager_filename), - recursive=True) - if len(candidate_resource_manager_filename) > 0: - example_path, _ = path.split(candidate_resource_manager_filename[0]) - example_dir = path.relpath(example_path, specs_path).replace('\\', '/') - except NameError: - # specs_path not defined - pass - - return example_dir - - def validate_java_examples(release: Release, java_examples: List[JavaExample]) -> JavaFormatResult: # batch validate Java examples - java_format = JavaFormat(tmp_path, path.join(script_path, 'javaformat')) + java_format = JavaFormat(tmp_path, path.join(script_path, "javaformat")) java_format_result = java_format.format(java_examples) if java_format_result.succeeded: @@ -246,49 +244,59 @@ def generate_examples(release: Release, sdk_examples_path: str, java_examples: L files = [] for java_example in java_examples: - doc_link = f'https://github.com/Azure/azure-sdk-for-java/blob/{release.tag}/sdk/' \ - f'{release.sdk_name}/{release.package}/README.md' - files.extend(write_code_to_file(sdk_examples_path, java_example.target_dir, java_example.target_filename, - '.java', java_example.content, doc_link)) + doc_link = ( + f"https://github.com/Azure/azure-sdk-for-java/blob/{release.tag}/sdk/" + f"{release.sdk_name}/{release.package}/README.md" + ) + files.extend( + write_code_to_file( + sdk_examples_path, + java_example.target_dir, + java_example.target_filename, + ".java", + java_example.content, + doc_link, + ) + ) return files -def write_code_to_file(sdk_examples_path: str, target_dir: str, filename_root: str, filename_ext: str, - code_content: str, sdk_url: str) -> List[str]: +def write_code_to_file( + sdk_examples_path: str, target_dir: str, filename_root: str, filename_ext: str, code_content: str, sdk_url: str +) -> List[str]: # write code file and metadata file code_filename = filename_root + filename_ext - metadata_filename = filename_root + '.json' + metadata_filename = filename_root + ".json" - metadata_json = {'sdkUrl': sdk_url} + metadata_json = {"sdkUrl": sdk_url} target_dir_path = path.join(sdk_examples_path, target_dir) os.makedirs(target_dir_path, exist_ok=True) code_file_path = path.join(target_dir_path, code_filename) - with open(code_file_path, 'w', encoding='utf-8') as f: + with open(code_file_path, "w", encoding="utf-8") as f: f.write(code_content) - logging.info(f'Code written to file: {code_file_path}') + logging.info(f"Code written to file: {code_file_path}") metadata_file_path = path.join(target_dir_path, metadata_filename) - with open(metadata_file_path, 'w', encoding='utf-8') as f: + with open(metadata_file_path, "w", encoding="utf-8") as f: json.dump(metadata_json, f) - logging.info(f'Metadata written to file: {metadata_file_path}') + logging.info(f"Metadata written to file: {metadata_file_path}") - return [path.join(target_dir, code_filename), - path.join(target_dir, metadata_filename)] + return [path.join(target_dir, code_filename), path.join(target_dir, metadata_filename)] def create_java_examples(release: Release, sdk_examples_path: str, java_examples_path: str) -> (bool, List[str]): - logging.info('Preparing tools and thread pool') + logging.info("Preparing tools and thread pool") - logging.info(f'Processing SDK examples: {release.sdk_name}') + logging.info(f"Processing SDK examples: {release.sdk_name}") java_examples = [] java_paths = [] for root, dirs, files in os.walk(java_examples_path): for name in files: filepath = path.join(root, name) - if path.splitext(filepath)[1] == '.java': + if path.splitext(filepath)[1] == ".java": java_paths.append(filepath) for filepath in java_paths: @@ -296,17 +304,17 @@ def create_java_examples(release: Release, sdk_examples_path: str, java_examples files = [] if java_examples: - logging.info('Validating SDK examples') + logging.info("Validating SDK examples") java_build_result = validate_java_examples(release, java_examples) if java_build_result.succeeded: files = generate_examples(release, sdk_examples_path, java_build_result.examples) else: - logging.error('Validation failed') + logging.error("Validation failed") return java_build_result.succeeded, files else: - logging.info('SDK examples not found') + logging.info("SDK examples not found") return True, files @@ -314,46 +322,48 @@ def main(): global script_path global tmp_path global specs_path + global sdk_package_path - logging.basicConfig(level=logging.INFO, - format='%(asctime)s [%(levelname)s] %(message)s', - datefmt='%Y-%m-%d %X') + logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s", datefmt="%Y-%m-%d %X") script_path = path.abspath(path.dirname(sys.argv[0])) parser = argparse.ArgumentParser(description='Requires 2 arguments, path of "input.json" and "output.json".') - parser.add_argument('paths', metavar='path', type=str, nargs=2, - help='path of "input.json" or "output.json"') + parser.add_argument("paths", metavar="path", type=str, nargs=2, help='path of "input.json" or "output.json"') args = parser.parse_args() input_json_path = args.paths[0] output_json_path = args.paths[1] - with open(input_json_path, 'r', encoding='utf-8') as f_in: + with open(input_json_path, "r", encoding="utf-8") as f_in: config = json.load(f_in) - specs_path = config['specsPath'] - sdk_path = config['sdkPath'] - sdk_examples_path = config['sdkExamplesPath'] - tmp_path = config['tempPath'] + specs_path = config["specsPath"] + sdk_path = config["sdkPath"] + sdk_examples_path = config["sdkExamplesPath"] + tmp_path = config["tempPath"] - release = Release(config['release']['tag'], - config['release']['package'], - config['release']['version'], - get_sdk_name_from_package(config['release']['package'])) + release = Release( + config["release"]["tag"], + config["release"]["package"], + config["release"]["version"], + get_sdk_name_from_package(config["release"]["package"]), + ) - java_examples_relative_path = path.join('sdk', release.sdk_name, release.package, 'src', 'samples') + java_examples_relative_path = path.join("sdk", release.sdk_name, release.package, "src", "samples") java_examples_path = path.join(sdk_path, java_examples_relative_path) + sdk_package_path = path.join(sdk_path, "sdk", release.sdk_name, release.package) + succeeded, files = create_java_examples(release, sdk_examples_path, java_examples_path) - with open(output_json_path, 'w', encoding='utf-8') as f_out: - group = 'com.azure.resourcemanager' + with open(output_json_path, "w", encoding="utf-8") as f_out: + group = "com.azure.resourcemanager" output = { - 'status': 'succeeded' if succeeded else 'failed', - 'name': f'{group}:{release.package}:{release.version}', - 'files': files + "status": "succeeded" if succeeded else "failed", + "name": f"{group}:{release.package}:{release.version}", + "files": files, } json.dump(output, f_out, indent=2) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/tools/azure-rest-api-specs-examples-automation/java/package.py b/tools/azure-rest-api-specs-examples-automation/java/package.py index 9cd50cb9326..5081f99e932 100644 --- a/tools/azure-rest-api-specs-examples-automation/java/package.py +++ b/tools/azure-rest-api-specs-examples-automation/java/package.py @@ -9,11 +9,11 @@ from modules import JavaExample -OS_WINDOWS = platform.system().lower() == 'windows' +OS_WINDOWS = platform.system().lower() == "windows" def replace_class_name(content: str, old_class_name: str, new_class_name: str) -> str: - return content.replace('class ' + old_class_name + ' {', 'class ' + new_class_name + ' {', 1) + return content.replace("class " + old_class_name + " {", "class " + new_class_name + " {", 1) class MavenPackage: @@ -34,29 +34,29 @@ def compile(self, examples: List[JavaExample]) -> bool: filename_no = 1 for example in examples: - class_name = 'Main' + str(filename_no) - code_path = path.join(maven_path, 'src', 'main', 'java', class_name + '.java') + class_name = "Main" + str(filename_no) + code_path = path.join(maven_path, "src", "main", "java", class_name + ".java") filename_no += 1 - content = replace_class_name(example.content, 'Main', class_name) + content = replace_class_name(example.content, "Main", class_name) - with open(code_path, 'w', encoding='utf-8') as f: + with open(code_path, "w", encoding="utf-8") as f: f.write(content) - cmd = ['mvn' + ('.cmd' if OS_WINDOWS else ''), '--no-transfer-progress', 'package'] - logging.info('Run mvn package') - logging.info('Command line: ' + ' '.join(cmd)) + cmd = ["mvn" + (".cmd" if OS_WINDOWS else ""), "--no-transfer-progress", "package"] + logging.info("Run mvn package") + logging.info("Command line: " + " ".join(cmd)) code = subprocess.run(cmd, cwd=maven_path).returncode return code == 0 def __prepare_workspace(self, maven_path: str): # make dir for maven and src/main/java - java_path = path.join(maven_path, 'src', 'main', 'java') + java_path = path.join(maven_path, "src", "main", "java") os.makedirs(java_path, exist_ok=True) # create pom - pom_file_path = path.join(maven_path, 'pom.xml') - pom_str = f''' + pom_file_path = path.join(maven_path, "pom.xml") + pom_str = f""" 4.0.0 com.azure.resourcemanager @@ -90,6 +90,6 @@ def __prepare_workspace(self, maven_path: str): -''' - with open(pom_file_path, 'w', encoding='utf-8') as f: +""" + with open(pom_file_path, "w", encoding="utf-8") as f: f.write(pom_str) diff --git a/tools/azure-rest-api-specs-examples-automation/java/requirements.txt b/tools/azure-rest-api-specs-examples-automation/java/requirements.txt deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tools/azure-rest-api-specs-examples-automation/java/test_format.py b/tools/azure-rest-api-specs-examples-automation/java/test_format.py index 0b116b1a23e..1e5149113ae 100644 --- a/tools/azure-rest-api-specs-examples-automation/java/test_format.py +++ b/tools/azure-rest-api-specs-examples-automation/java/test_format.py @@ -8,13 +8,16 @@ class TestJavaFormat(unittest.TestCase): def test_example(self): - tmp_path = path.abspath('.') - maven_path = path.abspath('./javaformat') + tmp_path = path.abspath(".") + maven_path = path.abspath("./javaformat") java_format = JavaFormat(tmp_path, maven_path) - code = '''class Main {} -''' - result = java_format.format([JavaExample('', '', code)]) + code = """class Main {} +""" + result = java_format.format([JavaExample("", "", code)]) self.assertTrue(result.succeeded) - self.assertEqual('''class Main { + self.assertEqual( + """class Main { } -''', result.examples[0].content) +""", + result.examples[0].content, + ) diff --git a/tools/azure-rest-api-specs-examples-automation/java/test_package.py b/tools/azure-rest-api-specs-examples-automation/java/test_package.py index b746c46a17b..e6da118b8f1 100644 --- a/tools/azure-rest-api-specs-examples-automation/java/test_package.py +++ b/tools/azure-rest-api-specs-examples-automation/java/test_package.py @@ -8,9 +8,9 @@ class TestMavenPackage(unittest.TestCase): def test_correct_example(self): - tmp_path = path.abspath('.') - maven_package = MavenPackage(tmp_path, 'azure-resourcemanager-postgresqlflexibleserver', '1.0.0-beta.3') - code = '''import com.azure.core.util.Context; + tmp_path = path.abspath(".") + maven_package = MavenPackage(tmp_path, "azure-resourcemanager-postgresqlflexibleserver", "1.0.0-beta.3") + code = """import com.azure.core.util.Context; import com.azure.resourcemanager.postgresqlflexibleserver.models.NameAvailabilityRequest; public final class Main { public static void nameAvailability(com.azure.resourcemanager.postgresqlflexibleserver.PostgreSqlManager manager) { @@ -20,15 +20,15 @@ def test_correct_example(self): new NameAvailabilityRequest().withName("name1").withType("Microsoft.DBforPostgreSQL/flexibleServers"), Context.NONE); } -}''' - result = maven_package.compile([JavaExample('', '', code)]) +}""" + result = maven_package.compile([JavaExample("", "", code)]) self.assertTrue(result) def test_incorrect_example(self): - tmp_path = path.abspath('.') - maven_package = MavenPackage(tmp_path, 'azure-resourcemanager-postgresqlflexibleserver', '1.0.0-beta.3') + tmp_path = path.abspath(".") + maven_package = MavenPackage(tmp_path, "azure-resourcemanager-postgresqlflexibleserver", "1.0.0-beta.3") - code1 = '''import com.azure.core.util.Context; + code1 = """import com.azure.core.util.Context; import com.azure.resourcemanager.postgresqlflexibleserver.models.NameAvailabilityRequest; public final class Main { public static void nameAvailability(com.azure.resourcemanager.postgresqlflexibleserver.PostgreSqlManager manager) { @@ -38,10 +38,10 @@ def test_incorrect_example(self): new NameAvailabilityRequest().withName("name1").withType("Microsoft.DBforPostgreSQL/flexibleServers"), Context.NONE); } - }''' + }""" # code missing "import" - code2 = '''public final class Main { + code2 = """public final class Main { public static void nameAvailability(com.azure.resourcemanager.postgresqlflexibleserver.PostgreSqlManager manager) { manager .checkNameAvailabilities() @@ -49,6 +49,6 @@ def test_incorrect_example(self): new NameAvailabilityRequest().withName("name1").withType("Microsoft.DBforPostgreSQL/flexibleServers"), Context.NONE); } -}''' - result = maven_package.compile([JavaExample('', '', code2), JavaExample('', '', code1)]) +}""" + result = maven_package.compile([JavaExample("", "", code2), JavaExample("", "", code1)]) self.assertFalse(result) diff --git a/tools/azure-rest-api-specs-examples-automation/java/test_process.py b/tools/azure-rest-api-specs-examples-automation/java/test_process.py index e61878c510f..58fbab0a7c5 100644 --- a/tools/azure-rest-api-specs-examples-automation/java/test_process.py +++ b/tools/azure-rest-api-specs-examples-automation/java/test_process.py @@ -1,13 +1,18 @@ +import sys import unittest +from os import path from main import process_java_example_content -from main import set_specs_path +from main import _set_paths + +sys.path.append(path.abspath("../directory")) +from test_examples_dir import create_mock_test_folder class TestProcess(unittest.TestCase): def test_process_java_example(self): - java_code = ''' + java_code = """ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. // Code generated by Microsoft (R) AutoRest Code Generator. @@ -126,24 +131,40 @@ def test_process_java_example(self): return map; } } -''' - java_examples = process_java_example_content(java_code.splitlines(keepends=True), - 'DatasetsCreateOrUpdateSamples') +""" + java_examples = process_java_example_content( + java_code.splitlines(keepends=True), "DatasetsCreateOrUpdateSamples" + ) self.assertEqual(2, len(java_examples)) - self.assertEqual('specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/examples-java', java_examples[0].target_dir) - self.assertEqual('Datasets_Create', java_examples[0].target_filename) - self.assertTrue('public final class Main {' in java_examples[0].content) - self.assertTrue('x-ms-original-file: specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/examples/Datasets_Create.json' in java_examples[0].content) - self.assertTrue('public static void datasetsCreate(com.azure.resourcemanager.datafactory.DataFactoryManager manager)' in java_examples[0].content) - - self.assertTrue('public final class Main {' in java_examples[1].content) - self.assertTrue('x-ms-original-file: specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/examples/Datasets_Update.json' in java_examples[1].content) - self.assertTrue('public static void datasetsUpdate(com.azure.resourcemanager.datafactory.DataFactoryManager manager)' in java_examples[1].content) + self.assertEqual( + "specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/examples-java", + java_examples[0].target_dir, + ) + self.assertEqual("Datasets_Create", java_examples[0].target_filename) + self.assertTrue("public final class Main {" in java_examples[0].content) + self.assertTrue( + "x-ms-original-file: specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/examples/Datasets_Create.json" + in java_examples[0].content + ) + self.assertTrue( + "public static void datasetsCreate(com.azure.resourcemanager.datafactory.DataFactoryManager manager)" + in java_examples[0].content + ) + + self.assertTrue("public final class Main {" in java_examples[1].content) + self.assertTrue( + "x-ms-original-file: specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/examples/Datasets_Update.json" + in java_examples[1].content + ) + self.assertTrue( + "public static void datasetsUpdate(com.azure.resourcemanager.datafactory.DataFactoryManager manager)" + in java_examples[1].content + ) def test_process_java_example_original_file_newline(self): - java_code = ''' + java_code = """ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. // Code generated by Microsoft (R) AutoRest Code Generator. @@ -187,18 +208,22 @@ def test_process_java_example_original_file_newline(self): return map; } } -''' - java_examples = process_java_example_content(java_code.splitlines(keepends=True), - 'ClustersCreateOrUpdateSamples') +""" + java_examples = process_java_example_content( + java_code.splitlines(keepends=True), "ClustersCreateOrUpdateSamples" + ) self.assertEqual(1, len(java_examples)) - self.assertTrue('public final class Main {' in java_examples[0].content) - self.assertEqual('specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2020-03-01-preview/examples-java', java_examples[0].target_dir) - self.assertEqual('Cluster_Create', java_examples[0].target_filename) + self.assertTrue("public final class Main {" in java_examples[0].content) + self.assertEqual( + "specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2020-03-01-preview/examples-java", + java_examples[0].target_dir, + ) + self.assertEqual("Cluster_Create", java_examples[0].target_filename) def test_process_java_example_method_signature_newline(self): - java_code = ''' + java_code = """ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. // Code generated by Microsoft (R) AutoRest Code Generator. @@ -241,22 +266,28 @@ def test_process_java_example_method_signature_newline(self): "aaaaaaaaaaaaaaaaaaaaaaaa", "aaaaaaaaaaaaaaaaa", com.azure.core.util.Context.NONE); } } -''' +""" - java_examples = process_java_example_content(java_code.splitlines(keepends=True), - 'VirtualMachineScaleSetVMsStartSamples') + java_examples = process_java_example_content( + java_code.splitlines(keepends=True), "VirtualMachineScaleSetVMsStartSamples" + ) self.assertEqual(2, len(java_examples)) - self.assertEqual('specification/compute/resource-manager/Microsoft.Compute/ComputeRP/stable/2024-03-01/examples-java/virtualMachineScaleSetExamples', java_examples[0].target_dir) - self.assertEqual('VirtualMachineScaleSetVM_Start_MaximumSet_Gen', java_examples[0].target_filename) + self.assertEqual( + "specification/compute/resource-manager/Microsoft.Compute/ComputeRP/stable/2024-03-01/examples-java/virtualMachineScaleSetExamples", + java_examples[0].target_dir, + ) + self.assertEqual("VirtualMachineScaleSetVM_Start_MaximumSet_Gen", java_examples[0].target_filename) - self.assertEqual('specification/compute/resource-manager/Microsoft.Compute/ComputeRP/stable/2024-03-01/examples-java/virtualMachineScaleSetExamples', java_examples[1].target_dir) - self.assertEqual('VirtualMachineScaleSetVM_Start_MinimumSet_Gen', java_examples[1].target_filename) + self.assertEqual( + "specification/compute/resource-manager/Microsoft.Compute/ComputeRP/stable/2024-03-01/examples-java/virtualMachineScaleSetExamples", + java_examples[1].target_dir, + ) + self.assertEqual("VirtualMachineScaleSetVM_Start_MinimumSet_Gen", java_examples[1].target_filename) - @unittest.skip('require call "set_specs_path" with your local azure-rest-api-specs repository') def test_process_java_example_original_file_typespec(self): - java_code = ''' + java_code = """ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. // Code generated by Microsoft (R) TypeSpec Code Generator. @@ -268,8 +299,7 @@ def test_process_java_example_original_file_typespec(self): */ public final class MongoClustersListConnectionStringsSamples { /* - * x-ms-original-file: specification/mongocluster/DocumentDB.MongoCluster.Management/examples/2024-03-01-preview/ - * MongoClusters_ListConnectionStrings.json + * x-ms-original-file: 2024-03-01-preview/MongoClusters_ListConnectionStrings.json */ /** * Sample code: List the available connection strings for the Mongo Cluster resource. @@ -282,12 +312,20 @@ def test_process_java_example_original_file_typespec(self): .listConnectionStringsWithResponse("TestGroup", "myMongoCluster", com.azure.core.util.Context.NONE); } } -''' - - set_specs_path('c:/github/azure-rest-api-specs') - java_examples = process_java_example_content(java_code.splitlines(keepends=True), - 'MongoClustersListConnectionStringsSamples') - - self.assertEqual(1, len(java_examples)) - - self.assertEqual('specification/mongocluster/resource-manager/Microsoft.DocumentDB/preview/2024-03-01-preview/examples-java', java_examples[0].target_dir) +""" + + with create_mock_test_folder() as tmp_dir_name: + _set_paths( + path.join(tmp_dir_name, "azure-rest-api-specs"), + path.join(tmp_dir_name, "azure-sdk-for-java/sdk/mongocluster/azure-resourcemanager-mongocluster"), + ) + java_examples = process_java_example_content( + java_code.splitlines(keepends=True), "MongoClustersListConnectionStringsSamples" + ) + + self.assertEqual(1, len(java_examples)) + + self.assertEqual( + "specification/mongocluster/resource-manager/Microsoft.DocumentDB/preview/2024-03-01-preview/examples-java", + java_examples[0].target_dir, + ) diff --git a/tools/azure-rest-api-specs-examples-automation/js/lint.py b/tools/azure-rest-api-specs-examples-automation/js/lint.py index cefd51dfdc9..1a9bcfbd912 100644 --- a/tools/azure-rest-api-specs-examples-automation/js/lint.py +++ b/tools/azure-rest-api-specs-examples-automation/js/lint.py @@ -9,11 +9,11 @@ from models import JsExample, JsLintResult -OS_WINDOWS = platform.system().lower() == 'windows' +OS_WINDOWS = platform.system().lower() == "windows" def check_call(cmd: List[str], work_dir: str): - logging.info('Command line: ' + ' '.join(cmd)) + logging.info("Command line: " + " ".join(cmd)) subprocess.check_call(cmd, cwd=work_dir) @@ -25,8 +25,9 @@ class JsLint: lint_config_path: str examples: List[JsExample] - def __init__(self, tmp_path: str, module: str, package_json_path: str, lint_config_path: str, - examples: List[JsExample]): + def __init__( + self, tmp_path: str, module: str, package_json_path: str, lint_config_path: str, examples: List[JsExample] + ): self.tmp_path = tmp_path self.module = module self.package_json_path = package_json_path @@ -35,48 +36,48 @@ def __init__(self, tmp_path: str, module: str, package_json_path: str, lint_conf def lint(self) -> JsLintResult: if not path.isfile(self.package_json_path): - logging.error(f'package.json file not found: {self.package_json_path}') + logging.error(f"package.json file not found: {self.package_json_path}") return JsLintResult(False, []) with tempfile.TemporaryDirectory(dir=self.tmp_path) as tmp_dir_name: # write examples to js files filename_no = 1 for example in self.examples: - filename = 'code' + str(filename_no) + '.js' + filename = "code" + str(filename_no) + ".js" filename_no += 1 filepath = path.join(tmp_dir_name, filename) - with open(filepath, 'w', encoding='utf-8') as f: + with open(filepath, "w", encoding="utf-8") as f: f.write(example.content) # lint js files try: # package - logging.info('Initialize package') + logging.info("Initialize package") shutil.copy2(self.package_json_path, tmp_dir_name) # eslint config shutil.copy2(self.lint_config_path, tmp_dir_name) - npm_cmd = 'npm' + ('.cmd' if OS_WINDOWS else '') - npx_cmd = 'npx' + ('.cmd' if OS_WINDOWS else '') + npm_cmd = "npm" + (".cmd" if OS_WINDOWS else "") + npx_cmd = "npx" + (".cmd" if OS_WINDOWS else "") - cmd = [npm_cmd, 'install', self.module, '--save', '--save-exact'] + cmd = [npm_cmd, "install", self.module, "--save", "--save-exact"] check_call(cmd, tmp_dir_name) - cmd = [npm_cmd, 'install', 'eslint@8.57.0', '--save-dev'] + cmd = [npm_cmd, "install", "eslint@8.57.0", "--save-dev"] check_call(cmd, tmp_dir_name) - with open(path.join(tmp_dir_name, 'package.json'), encoding='utf-8') as f: + with open(path.join(tmp_dir_name, "package.json"), encoding="utf-8") as f: content = f.read() - logging.info(f'package.json\n{content}') + logging.info(f"package.json\n{content}") - logging.info('Run eslint') + logging.info("Run eslint") # eslint - cmd = [npx_cmd, 'eslint', '--ext', '.js', '.'] + cmd = [npx_cmd, "eslint", "--ext", ".js", "."] check_call(cmd, tmp_dir_name) except subprocess.CalledProcessError as error: - logging.error(f'Call error: {error}') + logging.error(f"Call error: {error}") return JsLintResult(False, []) return JsLintResult(True, self.examples) diff --git a/tools/azure-rest-api-specs-examples-automation/js/lint_test.py b/tools/azure-rest-api-specs-examples-automation/js/lint_test.py index 101dae85bc7..db33add3cda 100644 --- a/tools/azure-rest-api-specs-examples-automation/js/lint_test.py +++ b/tools/azure-rest-api-specs-examples-automation/js/lint_test.py @@ -8,17 +8,20 @@ class TestJsLint(unittest.TestCase): def test_example(self): - code = '''const { createDefaultHttpClient, createPipelineRequest } = require("@azure/core-rest-pipeline"); + code = """const { createDefaultHttpClient, createPipelineRequest } = require("@azure/core-rest-pipeline"); const httpClient = createDefaultHttpClient(); httpClient.sendRequest(createPipelineRequest("https://httpbin.org/")); -''' +""" - tmp_path = path.abspath('.') - js_examples = [JsExample('code', '', code)] - js_lint = JsLint(tmp_path, '@azure/core-rest-pipeline@1.8.1', - path.join(tmp_path, 'lint', 'package.json'), - path.join(tmp_path, 'lint', '.eslintrc.json'), - js_examples) + tmp_path = path.abspath(".") + js_examples = [JsExample("code", "", code)] + js_lint = JsLint( + tmp_path, + "@azure/core-rest-pipeline@1.8.1", + path.join(tmp_path, "lint", "package.json"), + path.join(tmp_path, "lint", ".eslintrc.json"), + js_examples, + ) result = js_lint.lint() self.assertTrue(result.succeeded) diff --git a/tools/azure-rest-api-specs-examples-automation/js/main.py b/tools/azure-rest-api-specs-examples-automation/js/main.py index feaeb27b76a..4f986007cd4 100644 --- a/tools/azure-rest-api-specs-examples-automation/js/main.py +++ b/tools/azure-rest-api-specs-examples-automation/js/main.py @@ -9,22 +9,33 @@ import dataclasses from typing import List from enum import Enum +import importlib.util from models import JsExample, JsLintResult from lint import JsLint -script_path: str = '.' +spec_location = ( + "./directory/examples_dir.py" if path.exists("./directory/examples_dir.py") else "../directory/examples_dir.py" +) +spec = importlib.util.spec_from_file_location("examples_dir", spec_location) +examples_dir = importlib.util.module_from_spec(spec) +spec.loader.exec_module(examples_dir) + + +script_path: str = "." tmp_path: str +specs_path: str +sdk_package_path: str -original_file_key: str = '* x-ms-original-file: ' +original_file_key: str = "* x-ms-original-file: " -module_relative_path: str = '' +module_relative_path: str = "" class PackageType(Enum): - HLC = '@azure/arm-' - RLC = '@azure-rest/arm-' + HLC = "@azure/arm-" + RLC = "@azure-rest/arm-" @dataclasses.dataclass(eq=True, frozen=True) @@ -71,17 +82,18 @@ def get_js_example_method(lines: List[str], start: int, aggregated_with_main: bo line = lines[index] if line.strip().startswith(original_file_key): - original_file = line.strip()[len(original_file_key):] - elif line.startswith('async function '): + original_file = line.strip()[len(original_file_key) :] + elif line.startswith("async function "): # begin of method js_example_method.example_relative_path = original_file js_example_method.line_start = index - elif '.catch(console.error);' in line \ - or (index > 0 and line.startswith(');') and 'console.error' in lines[index-1]): + elif ".catch(console.error);" in line or ( + index > 0 and line.startswith(");") and "console.error" in lines[index - 1] + ): # end of method js_example_method.line_end = index + 1 break - elif aggregated_with_main and '}' == line.rstrip(): + elif aggregated_with_main and "}" == line.rstrip(): js_example_method.line_end = index + 1 break @@ -98,26 +110,26 @@ def backtrace_comments(js_example_method: JsExampleMethodContent, lines: List[st for index in range(js_example_method.line_start - 1, start - 1, -1): line = lines[index] if block_comment: - if line.strip().startswith('/*'): + if line.strip().startswith("/*"): js_example_method.line_start = index block_comment = False break else: - if line.strip().startswith('//'): + if line.strip().startswith("//"): js_example_method.line_start = index - elif line.strip().startswith('*/'): + elif line.strip().startswith("*/"): js_example_method.line_start = index block_comment = True else: break - js_example_method.content = lines[js_example_method.line_start:js_example_method.line_end] + js_example_method.content = lines[js_example_method.line_start : js_example_method.line_end] def break_down_aggregated_js_example(lines: List[str]) -> AggregatedJsExample: # break down sample Js to multiple examples # check if it is new style with "main()" - aggregated_with_main = len([s for s in lines if 'async function main()' in s]) > 0 + aggregated_with_main = len([s for s in lines if "async function main()" in s]) > 0 aggregated_js_example = AggregatedJsExample([]) js_example_method = get_js_example_method(lines, 0, aggregated_with_main) @@ -132,8 +144,9 @@ def break_down_aggregated_js_example(lines: List[str]) -> AggregatedJsExample: if aggregated_with_main: # remove "dotenv.config()" - aggregated_js_example.class_opening = [s for s in aggregated_js_example.class_opening - if 'require("dotenv").config();' not in s] + aggregated_js_example.class_opening = [ + s for s in aggregated_js_example.class_opening if 'require("dotenv").config();' not in s + ] return aggregated_js_example @@ -149,7 +162,7 @@ def format_js(lines: List[str]) -> List[str]: new_lines.append(line) else: # start with require - if 'require(' in line or 'const {' in line: + if "require(" in line or "const {" in line: new_lines.append(line) skip_head = False @@ -160,9 +173,9 @@ def process_js_example(filepath: str, package_type: PackageType) -> List[JsExamp # process aggregated Js sample to examples filename = path.basename(filepath) - logging.info(f'Processing Js aggregated sample: {filename}') + logging.info(f"Processing Js aggregated sample: {filename}") - with open(filepath, encoding='utf-8') as f: + with open(filepath, encoding="utf-8") as f: lines = f.readlines() example_folder_extension = get_example_folder_extension(package_type) @@ -172,7 +185,7 @@ def process_js_example(filepath: str, package_type: PackageType) -> List[JsExamp aggregated_js_example = break_down_aggregated_js_example(lines) for js_example_method in aggregated_js_example.methods: if js_example_method.is_valid(): - logging.info(f'Processing Js example: {js_example_method.example_relative_path}') + logging.info(f"Processing Js example: {js_example_method.example_relative_path}") # re-construct the example class, from example method example_lines = aggregated_js_example.class_opening + js_example_method.content @@ -180,14 +193,24 @@ def process_js_example(filepath: str, package_type: PackageType) -> List[JsExamp example_filepath = js_example_method.example_relative_path example_dir, example_filename = path.split(example_filepath) + try: + example_dir = examples_dir.try_find_resource_manager_example( + specs_path, sdk_package_path, example_dir, example_filename + ) + except NameError: + pass + example_lines = format_js(example_lines) - filename = example_filename.split('.')[0] + filename = example_filename.split(".")[0] # use the examples-js folder for Js example - md_dir = (example_dir + '-' + example_folder_extension) if example_dir.endswith('/examples') \ - else example_dir.replace('/examples/', f'/examples-{example_folder_extension}/') + md_dir = ( + (example_dir + "-" + example_folder_extension) + if example_dir.endswith("/examples") + else example_dir.replace("/examples/", f"/examples-{example_folder_extension}/") + ) - js_example = JsExample(filename, md_dir, ''.join(example_lines)) + js_example = JsExample(filename, md_dir, "".join(example_lines)) js_examples.append(js_example) return js_examples @@ -198,7 +221,7 @@ def validate_js_examples(js_module: str, package_json_path: str, js_examples: Li global script_path - lint_config_path = path.join(script_path, 'lint', '.eslintrc.json') + lint_config_path = path.join(script_path, "lint", ".eslintrc.json") js_lint = JsLint(tmp_path, js_module, package_json_path, lint_config_path, js_examples) js_lint_result = js_lint.lint() @@ -212,95 +235,103 @@ def generate_examples(release: Release, sdk_examples_path: str, js_examples: Lis files = [] for js_example in js_examples: - escaped_release_tag = urllib.parse.quote(release.tag, safe='') - doc_link = f'https://github.com/Azure/azure-sdk-for-js/blob/{escaped_release_tag}/' \ - f'{module_relative_path}/README.md' - - files.extend(write_code_to_file(sdk_examples_path, js_example.target_dir, js_example.target_filename, '.js', - js_example.content, doc_link)) + escaped_release_tag = urllib.parse.quote(release.tag, safe="") + doc_link = ( + f"https://github.com/Azure/azure-sdk-for-js/blob/{escaped_release_tag}/" f"{module_relative_path}/README.md" + ) + + files.extend( + write_code_to_file( + sdk_examples_path, + js_example.target_dir, + js_example.target_filename, + ".js", + js_example.content, + doc_link, + ) + ) return files -def write_code_to_file(sdk_examples_path: str, target_dir: str, filename_root: str, filename_ext: str, - code_content: str, sdk_url: str) -> List[str]: +def write_code_to_file( + sdk_examples_path: str, target_dir: str, filename_root: str, filename_ext: str, code_content: str, sdk_url: str +) -> List[str]: # write code file and metadata file code_filename = filename_root + filename_ext - metadata_filename = filename_root + '.json' + metadata_filename = filename_root + ".json" - metadata_json = {'sdkUrl': sdk_url} + metadata_json = {"sdkUrl": sdk_url} target_dir_path = path.join(sdk_examples_path, target_dir) os.makedirs(target_dir_path, exist_ok=True) code_file_path = path.join(target_dir_path, code_filename) - with open(code_file_path, 'w', encoding='utf-8') as f: + with open(code_file_path, "w", encoding="utf-8") as f: f.write(code_content) - logging.info(f'Code written to file: {code_file_path}') + logging.info(f"Code written to file: {code_file_path}") metadata_file_path = path.join(target_dir_path, metadata_filename) - with open(metadata_file_path, 'w', encoding='utf-8') as f: + with open(metadata_file_path, "w", encoding="utf-8") as f: json.dump(metadata_json, f) - logging.info(f'Metadata written to file: {metadata_file_path}') + logging.info(f"Metadata written to file: {metadata_file_path}") - return [path.join(target_dir, code_filename), - path.join(target_dir, metadata_filename)] + return [path.join(target_dir, code_filename), path.join(target_dir, metadata_filename)] -def create_js_examples(release: Release, - js_module: str, - sdk_examples_path: str, js_examples_path: str) -> (bool, List[str]): +def create_js_examples( + release: Release, js_module: str, sdk_examples_path: str, js_examples_path: str +) -> (bool, List[str]): js_paths = [] for root, dirs, files in os.walk(js_examples_path): for name in files: filepath = path.join(root, name) - if path.splitext(filepath)[1] == '.js' and filepath.endswith('Sample.js'): + if path.splitext(filepath)[1] == ".js" and filepath.endswith("Sample.js"): js_paths.append(filepath) package_type = get_package_type(release) - logging.info(f'Processing SDK examples: {release.package}') + logging.info(f"Processing SDK examples: {release.package}") js_examples = [] for filepath in js_paths: js_examples += process_js_example(filepath, package_type) files = [] if js_examples: - logging.info('Validating SDK examples') - package_json_path = path.join(js_examples_path, 'package.json') + logging.info("Validating SDK examples") + package_json_path = path.join(js_examples_path, "package.json") js_lint_result = validate_js_examples(js_module, package_json_path, js_examples) if js_lint_result.succeeded: files = generate_examples(release, sdk_examples_path, js_lint_result.examples) else: - logging.error('Validation failed') + logging.error("Validation failed") return js_lint_result.succeeded, files else: - logging.info('SDK examples not found') + logging.info("SDK examples not found") return True, files def get_module_relative_path(sdk_name: str, package_type: PackageType, sdk_path: str) -> str: global module_relative_path - sdk_prefix = 'arm-' - sdk_suffix = '-rest' if package_type is PackageType.RLC else '' - module_relative_path = path.join('sdk', sdk_name, sdk_prefix + sdk_name + sdk_prefix) + sdk_prefix = "arm-" + sdk_suffix = "-rest" if package_type is PackageType.RLC else "" + module_relative_path = path.join("sdk", sdk_name, sdk_prefix + sdk_name + sdk_prefix) if not path.isdir(path.join(sdk_path, module_relative_path)): - candidate_sdk_readmes = glob.glob(path.join(sdk_path, f'sdk/*/{sdk_prefix}{sdk_name}{sdk_suffix}')) + candidate_sdk_readmes = glob.glob(path.join(sdk_path, f"sdk/*/{sdk_prefix}{sdk_name}{sdk_suffix}")) if len(candidate_sdk_readmes) > 0: candidate_sdk_readmes = [path.relpath(p, sdk_path) for p in candidate_sdk_readmes] - logging.info( - f'SDK folder {module_relative_path} not found, use first item of f{candidate_sdk_readmes}') + logging.info(f"SDK folder {module_relative_path} not found, use first item of f{candidate_sdk_readmes}") module_relative_path = candidate_sdk_readmes[0] else: - raise RuntimeError(f'Source folder not found for SDK {sdk_prefix}{sdk_name}{sdk_suffix}') + raise RuntimeError(f"Source folder not found for SDK {sdk_prefix}{sdk_name}{sdk_suffix}") return module_relative_path def get_sample_version(release_version: str) -> str: - version = 'v' + release_version.split('.')[0] - if '-beta' in release_version: - version += '-beta' + version = "v" + release_version.split(".")[0] + if "-beta" in release_version: + version += "-beta" return version @@ -312,60 +343,55 @@ def get_package_type(release: Release) -> PackageType: def get_example_folder_extension(package_type: PackageType) -> str: - return 'js' if package_type is PackageType.HLC else 'js-rlc' + return "js" if package_type is PackageType.HLC else "js-rlc" def main(): global script_path global tmp_path + global specs_path + global sdk_package_path - logging.basicConfig(level=logging.INFO, - format='%(asctime)s [%(levelname)s] %(message)s', - datefmt='%Y-%m-%d %X') + logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s", datefmt="%Y-%m-%d %X") script_path = path.abspath(path.dirname(sys.argv[0])) parser = argparse.ArgumentParser(description='Requires 2 arguments, path of "input.json" and "output.json".') - parser.add_argument('paths', metavar='path', type=str, nargs=2, - help='path of "input.json" or "output.json"') + parser.add_argument("paths", metavar="path", type=str, nargs=2, help='path of "input.json" or "output.json"') args = parser.parse_args() input_json_path = args.paths[0] output_json_path = args.paths[1] - with open(input_json_path, 'r', encoding='utf-8') as f_in: + with open(input_json_path, "r", encoding="utf-8") as f_in: config = json.load(f_in) - sdk_path = config['sdkPath'] - sdk_examples_path = config['sdkExamplesPath'] - tmp_path = config['tempPath'] + specs_path = config["specsPath"] + sdk_path = config["sdkPath"] + sdk_examples_path = config["sdkExamplesPath"] + tmp_path = config["tempPath"] - release = Release(config['release']['tag'], - config['release']['package'], - config['release']['version']) + release = Release(config["release"]["tag"], config["release"]["package"], config["release"]["version"]) package_type = get_package_type(release) if package_type is PackageType.HLC: - sdk_name = release.package[len(PackageType.HLC.value):] + sdk_name = release.package[len(PackageType.HLC.value) :] else: - sdk_name = release.package[len(PackageType.RLC.value):] + sdk_name = release.package[len(PackageType.RLC.value) :] - js_module = f'{release.package}@{release.version}' + js_module = f"{release.package}@{release.version}" sample_version = get_sample_version(release.version) module_relative_path_local = get_module_relative_path(sdk_name, package_type, sdk_path) - js_examples_relative_path = path.join(module_relative_path_local, - 'samples', sample_version, 'javascript') + js_examples_relative_path = path.join(module_relative_path_local, "samples", sample_version, "javascript") js_examples_path = path.join(sdk_path, js_examples_relative_path) + sdk_package_path = path.join(sdk_path, module_relative_path_local) + succeeded, files = create_js_examples(release, js_module, sdk_examples_path, js_examples_path) - with open(output_json_path, 'w', encoding='utf-8') as f_out: - output = { - 'status': 'succeeded' if succeeded else 'failed', - 'name': js_module, - 'files': files - } + with open(output_json_path, "w", encoding="utf-8") as f_out: + output = {"status": "succeeded" if succeeded else "failed", "name": js_module, "files": files} json.dump(output, f_out, indent=2) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/tools/azure-rest-api-specs-examples-automation/js/test_main.py b/tools/azure-rest-api-specs-examples-automation/js/test_main.py index 9c9387f68bd..58fecf8f8b3 100644 --- a/tools/azure-rest-api-specs-examples-automation/js/test_main.py +++ b/tools/azure-rest-api-specs-examples-automation/js/test_main.py @@ -1,16 +1,24 @@ import unittest -from main import get_js_example_method, get_sample_version, get_module_relative_path, \ - break_down_aggregated_js_example, format_js, create_js_examples, Release, PackageType +from main import ( + get_js_example_method, + get_sample_version, + get_module_relative_path, + break_down_aggregated_js_example, + format_js, + create_js_examples, + Release, + PackageType, +) class TestMain(unittest.TestCase): def test_get_sample_version(self): - self.assertEqual('v3', get_sample_version('3.0.0')) - self.assertEqual('v3-beta', get_sample_version('3.0.0-beta.3')) + self.assertEqual("v3", get_sample_version("3.0.0")) + self.assertEqual("v3-beta", get_sample_version("3.0.0-beta.3")) def test_get_js_example_method(self): - code = '''const { StorSimpleManagementClient } = require("@azure/arm-storsimple1200series"); + code = """const { StorSimpleManagementClient } = require("@azure/arm-storsimple1200series"); const { DefaultAzureCredential } = require("@azure/identity"); /** @@ -44,7 +52,7 @@ def test_get_js_example_method(self): } managersUploadRegistrationCertificate().catch(console.error); -''' +""" lines = code.splitlines(keepends=True) @@ -53,7 +61,7 @@ def test_get_js_example_method(self): self.assertIsNotNone(js_example_method.line_end) def test_break_down_aggregated_js_example(self): - code = '''const { StorageManagementClient } = require("@azure/arm-storage"); + code = """const { StorageManagementClient } = require("@azure/arm-storage"); const { DefaultAzureCredential } = require("@azure/identity"); /** @@ -93,22 +101,31 @@ def test_break_down_aggregated_js_example(self): } getContainers().catch(console.error); -''' +""" lines = code.splitlines(keepends=True) aggregated_js_example = break_down_aggregated_js_example(lines) self.assertEqual(2, len(aggregated_js_example.methods)) - self.assertEqual('* This sample demonstrates how to Gets properties of a specified container.', aggregated_js_example.methods[0].content[1].strip()) - self.assertEqual('async function getBlobContainersGetWithAllowProtectedAppendWritesAll() {', aggregated_js_example.methods[0].content[6].strip()) - self.assertEqual('getBlobContainersGetWithAllowProtectedAppendWritesAll().catch(console.error);', aggregated_js_example.methods[0].content[-1].strip()) - - self.assertEqual('async function getContainers() {', aggregated_js_example.methods[1].content[6].strip()) - self.assertEqual('getContainers().catch(console.error);', aggregated_js_example.methods[1].content[-1].strip()) + self.assertEqual( + "* This sample demonstrates how to Gets properties of a specified container.", + aggregated_js_example.methods[0].content[1].strip(), + ) + self.assertEqual( + "async function getBlobContainersGetWithAllowProtectedAppendWritesAll() {", + aggregated_js_example.methods[0].content[6].strip(), + ) + self.assertEqual( + "getBlobContainersGetWithAllowProtectedAppendWritesAll().catch(console.error);", + aggregated_js_example.methods[0].content[-1].strip(), + ) + + self.assertEqual("async function getContainers() {", aggregated_js_example.methods[1].content[6].strip()) + self.assertEqual("getContainers().catch(console.error);", aggregated_js_example.methods[1].content[-1].strip()) def test_break_down_aggregated_js_example_new_style_multiple(self): - code = '''const { SynapseManagementClient } = require("@azure/arm-synapse"); + code = """const { SynapseManagementClient } = require("@azure/arm-synapse"); const { DefaultAzureCredential } = require("@azure/identity"); require("dotenv").config(); @@ -251,17 +268,20 @@ def test_break_down_aggregated_js_example_new_style_multiple(self): } main().catch(console.error); -''' +""" lines = code.splitlines(keepends=True) aggregated_js_example = break_down_aggregated_js_example(lines) self.assertEqual(4, len(aggregated_js_example.methods)) - self.assertEqual('async function createOrUpdateDataMaskingRuleForDefaultMax() {', aggregated_js_example.methods[0].content[6].strip()) + self.assertEqual( + "async function createOrUpdateDataMaskingRuleForDefaultMax() {", + aggregated_js_example.methods[0].content[6].strip(), + ) def test_break_down_aggregated_js_example_new_style_single(self): - code = '''const { SynapseManagementClient } = require("@azure/arm-synapse"); + code = """const { SynapseManagementClient } = require("@azure/arm-synapse"); const { DefaultAzureCredential } = require("@azure/identity"); require("dotenv").config(); @@ -295,7 +315,7 @@ def test_break_down_aggregated_js_example_new_style_single(self): } main().catch(console.error); -''' +""" lines = code.splitlines(keepends=True) @@ -304,15 +324,16 @@ def test_break_down_aggregated_js_example_new_style_single(self): self.assertEqual(3, len(aggregated_js_example.class_opening)) - self.assertEqual('async function listAuditSettingsOfADatabase() {', - aggregated_js_example.methods[0].content[6].strip()) - self.assertEqual('}', aggregated_js_example.methods[0].content[-1].rstrip()) + self.assertEqual( + "async function listAuditSettingsOfADatabase() {", aggregated_js_example.methods[0].content[6].strip() + ) + self.assertEqual("}", aggregated_js_example.methods[0].content[-1].rstrip()) example_lines = aggregated_js_example.class_opening + aggregated_js_example.methods[0].content example_lines = format_js(example_lines) def test_get_js_example_method_new_require_newline(self): - code = '''const { + code = """const { AppComplianceAutomationToolForMicrosoft365, } = require("@azure/arm-appcomplianceautomation"); const { DefaultAzureCredential } = require("@azure/identity"); @@ -338,29 +359,27 @@ def test_get_js_example_method_new_require_newline(self): } main().catch(console.error); -''' +""" lines = code.splitlines(keepends=True) aggregated_js_example = break_down_aggregated_js_example(lines) example_lines = aggregated_js_example.class_opening + aggregated_js_example.methods[0].content example_lines = format_js(example_lines) - self.assertTrue(example_lines[0].startswith('const {')) + self.assertTrue(example_lines[0].startswith("const {")) @unittest.skip def test_get_module_relative_path(self): - sdk_path = 'c:/github/azure-sdk-for-js' - sdk_name = 'mysql-flexible' + sdk_path = "c:/github/azure-sdk-for-js" + sdk_name = "mysql-flexible" module_relative_path = get_module_relative_path(sdk_name, PackageType.HLC, sdk_path) - self.assertEqual('sdk/mysql/azure-mysql-flexible', module_relative_path) + self.assertEqual("sdk/mysql/azure-mysql-flexible", module_relative_path) @unittest.skip def test_create_js_examples(self): - release = Release('@azure/arm-policyinsights_6.0.0-beta.1', - '@azure/arm-policyinsights', - '6.0.0-beta.1') - js_module = f'{release.package}@{release.version}' - sdk_examples_path = 'c:/github/azure-rest-api-specs-examples' - js_examples_path = 'c:/github/azure-sdk-for-js/sdk/policyinsights/arm-policyinsights/samples/v6-beta/javascript' + release = Release("@azure/arm-policyinsights_6.0.0-beta.1", "@azure/arm-policyinsights", "6.0.0-beta.1") + js_module = f"{release.package}@{release.version}" + sdk_examples_path = "c:/github/azure-rest-api-specs-examples" + js_examples_path = "c:/github/azure-sdk-for-js/sdk/policyinsights/arm-policyinsights/samples/v6-beta/javascript" succeeded, files = create_js_examples(release, js_module, sdk_examples_path, js_examples_path) self.assertTrue(succeeded) diff --git a/tools/azure-rest-api-specs-examples-automation/python/main.py b/tools/azure-rest-api-specs-examples-automation/python/main.py index 00f6a19bb39..9ec0f0e5ec6 100644 --- a/tools/azure-rest-api-specs-examples-automation/python/main.py +++ b/tools/azure-rest-api-specs-examples-automation/python/main.py @@ -8,14 +8,25 @@ import logging import dataclasses from typing import List, Optional +import importlib.util -script_path: str = '.' +spec_location = ( + "./directory/examples_dir.py" if path.exists("./directory/examples_dir.py") else "../directory/examples_dir.py" +) +spec = importlib.util.spec_from_file_location("examples_dir", spec_location) +examples_dir = importlib.util.module_from_spec(spec) +spec.loader.exec_module(examples_dir) + + +script_path: str = "." tmp_path: str +specs_path: str +sdk_package_path: str -original_file_key: str = '# x-ms-original-file: ' +original_file_key: str = "# x-ms-original-file: " -module_relative_path: str = '' +module_relative_path: str = "" @dataclasses.dataclass(eq=True, frozen=True) @@ -38,25 +49,36 @@ def parse_python_example(lines: List[str]) -> Optional[PythonExample]: example_relative_path = None for line in lines: if line.strip().startswith(original_file_key): - example_relative_path = line.strip()[len(original_file_key):] + example_relative_path = line.strip()[len(original_file_key) :] break example = None if example_relative_path: example_dir, example_filename = path.split(example_relative_path) - target_dir = (example_dir + '-python') if example_dir.endswith('/examples') \ - else example_dir.replace('/examples/', '/examples-python/') - filename = example_filename.split('.')[0] + + try: + example_dir = examples_dir.try_find_resource_manager_example( + specs_path, sdk_package_path, example_dir, example_filename + ) + except NameError: + pass + + target_dir = ( + (example_dir + "-python") + if example_dir.endswith("/examples") + else example_dir.replace("/examples/", "/examples-python/") + ) + filename = example_filename.split(".")[0] first_line_index = 0 for index, line in enumerate(lines): - if line.strip() and not line.strip().startswith('# '): + if line.strip() and not line.strip().startswith("# "): first_line_index = index break lines = lines[first_line_index:] - example = PythonExample(filename, target_dir, ''.join(lines)) + example = PythonExample(filename, target_dir, "".join(lines)) return example @@ -65,9 +87,9 @@ def process_python_example(filepath: str) -> List[PythonExample]: # process aggregated Python sample to examples filename = path.basename(filepath) - logging.info(f'Processing Python sample: {filename}') + logging.info(f"Processing Python sample: {filename}") - with open(filepath, encoding='utf-8') as f: + with open(filepath, encoding="utf-8") as f: lines = f.readlines() python_examples = [] @@ -85,123 +107,128 @@ def generate_examples(release: Release, sdk_examples_path: str, python_examples: files = [] for python_example in python_examples: - escaped_release_tag = urllib.parse.quote(release.tag, safe='') - doc_link = f'https://github.com/Azure/azure-sdk-for-python/blob/{escaped_release_tag}/' \ - f'{module_relative_path}/README.md' - - files.extend(write_code_to_file(sdk_examples_path, python_example.target_dir, - python_example.target_filename, '.py', python_example.content, doc_link)) + escaped_release_tag = urllib.parse.quote(release.tag, safe="") + doc_link = ( + f"https://github.com/Azure/azure-sdk-for-python/blob/{escaped_release_tag}/" + f"{module_relative_path}/README.md" + ) + + files.extend( + write_code_to_file( + sdk_examples_path, + python_example.target_dir, + python_example.target_filename, + ".py", + python_example.content, + doc_link, + ) + ) return files -def write_code_to_file(sdk_examples_path: str, target_dir: str, filename_root: str, filename_ext: str, - code_content: str, sdk_url: str) -> List[str]: +def write_code_to_file( + sdk_examples_path: str, target_dir: str, filename_root: str, filename_ext: str, code_content: str, sdk_url: str +) -> List[str]: # write code file and metadata file code_filename = filename_root + filename_ext - metadata_filename = filename_root + '.json' + metadata_filename = filename_root + ".json" - metadata_json = {'sdkUrl': sdk_url} + metadata_json = {"sdkUrl": sdk_url} target_dir_path = path.join(sdk_examples_path, target_dir) os.makedirs(target_dir_path, exist_ok=True) code_file_path = path.join(target_dir_path, code_filename) - with open(code_file_path, 'w', encoding='utf-8') as f: + with open(code_file_path, "w", encoding="utf-8") as f: f.write(code_content) - logging.info(f'Code written to file: {code_file_path}') + logging.info(f"Code written to file: {code_file_path}") metadata_file_path = path.join(target_dir_path, metadata_filename) - with open(metadata_file_path, 'w', encoding='utf-8') as f: + with open(metadata_file_path, "w", encoding="utf-8") as f: json.dump(metadata_json, f) - logging.info(f'Metadata written to file: {metadata_file_path}') + logging.info(f"Metadata written to file: {metadata_file_path}") - return [path.join(target_dir, code_filename), - path.join(target_dir, metadata_filename)] + return [path.join(target_dir, code_filename), path.join(target_dir, metadata_filename)] -def create_python_examples(release: Release, - js_module: str, - sdk_examples_path: str, js_examples_path: str) -> (bool, List[str]): +def create_python_examples( + release: Release, js_module: str, sdk_examples_path: str, js_examples_path: str +) -> (bool, List[str]): python_paths = [] for root, dirs, files in os.walk(js_examples_path): for name in files: filepath = path.join(root, name) - if path.splitext(filepath)[1] == '.py': + if path.splitext(filepath)[1] == ".py": python_paths.append(filepath) - logging.info(f'Processing SDK examples: {release.package}') + logging.info(f"Processing SDK examples: {release.package}") python_examples = [] for filepath in python_paths: python_examples += process_python_example(filepath) files = [] if python_examples: - logging.info('Writing SDK examples') + logging.info("Writing SDK examples") files = generate_examples(release, sdk_examples_path, python_examples) return True, files else: - logging.info('SDK examples not found') + logging.info("SDK examples not found") return True, files def get_module_relative_path(sdk_name: str, sdk_path: str) -> str: global module_relative_path - candidate_sdk_paths = glob.glob(path.join(sdk_path, f'sdk/*/{sdk_name}')) + candidate_sdk_paths = glob.glob(path.join(sdk_path, f"sdk/*/{sdk_name}")) if len(candidate_sdk_paths) > 0: candidate_sdk_paths = [path.relpath(p, sdk_path) for p in candidate_sdk_paths] - logging.info( - f'Use first item of {candidate_sdk_paths} for SDK folder') + logging.info(f"Use first item of {candidate_sdk_paths} for SDK folder") module_relative_path = candidate_sdk_paths[0] else: - raise RuntimeError(f'Source folder not found for SDK {sdk_name}') + raise RuntimeError(f"Source folder not found for SDK {sdk_name}") return module_relative_path def main(): global script_path global tmp_path + global specs_path + global sdk_package_path - logging.basicConfig(level=logging.INFO, - format='%(asctime)s [%(levelname)s] %(message)s', - datefmt='%Y-%m-%d %X') + logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s", datefmt="%Y-%m-%d %X") script_path = path.abspath(path.dirname(sys.argv[0])) parser = argparse.ArgumentParser(description='Requires 2 arguments, path of "input.json" and "output.json".') - parser.add_argument('paths', metavar='path', type=str, nargs=2, - help='path of "input.json" or "output.json"') + parser.add_argument("paths", metavar="path", type=str, nargs=2, help='path of "input.json" or "output.json"') args = parser.parse_args() input_json_path = args.paths[0] output_json_path = args.paths[1] - with open(input_json_path, 'r', encoding='utf-8') as f_in: + with open(input_json_path, "r", encoding="utf-8") as f_in: config = json.load(f_in) - sdk_path = config['sdkPath'] - sdk_examples_path = config['sdkExamplesPath'] - tmp_path = config['tempPath'] + specs_path = config["specsPath"] + sdk_path = config["sdkPath"] + sdk_examples_path = config["sdkExamplesPath"] + tmp_path = config["tempPath"] - release = Release(config['release']['tag'], - config['release']['package'], - config['release']['version']) + release = Release(config["release"]["tag"], config["release"]["package"], config["release"]["version"]) - js_module = f'{release.package}@{release.version}' + js_module = f"{release.package}@{release.version}" module_relative_path_local = get_module_relative_path(release.package, sdk_path) - python_examples_relative_path = path.join(module_relative_path_local, 'generated_samples') + python_examples_relative_path = path.join(module_relative_path_local, "generated_samples") python_examples_path = path.join(sdk_path, python_examples_relative_path) + sdk_package_path = path.join(sdk_path, module_relative_path_local) + succeeded, files = create_python_examples(release, js_module, sdk_examples_path, python_examples_path) - with open(output_json_path, 'w', encoding='utf-8') as f_out: - output = { - 'status': 'succeeded' if succeeded else 'failed', - 'name': js_module, - 'files': files - } + with open(output_json_path, "w", encoding="utf-8") as f_out: + output = {"status": "succeeded" if succeeded else "failed", "name": js_module, "files": files} json.dump(output, f_out, indent=2) -if __name__ == '__main__': +if __name__ == "__main__": main()