[Python APIView] tree token parser conversion

packages/python-packages/apistubgentest/apistubgentest/models/__init__.py

@@ -28,7 +28,8 @@
     SomethingWithDecorators,
     SomethingWithInheritedOverloads,
     SomethingWithOverloads,
-    SomethingWithProperties
+    SomethingWithProperties,
+    SomeProtocolDecorator
 )
 from ._dataclasses import (
     DataClassSimple,
@@ -64,5 +65,6 @@
     "SomethingWithDecorators",
     "SomethingWithInheritedOverloads",
     "SomethingWithOverloads",
-    "SomethingWithProperties"
+    "SomethingWithProperties",
+    "SomeProtocolDecorator"
 )

packages/python-packages/apistubgentest/apistubgentest/models/_models.py

@@ -12,7 +12,8 @@
 import datetime
 from enum import Enum, EnumMeta
 import functools
-from typing import Any, overload, Dict, TypedDict, Union, Optional, Generic, TypeVar, NewType, ClassVar
+from typing import Any, overload, Dict, TypedDict, Union, Optional, Generic, TypeVar, NewType, ClassVar, Protocol
+from typing_extensions import runtime_checkable
 
 from ._mixin import MixinWithOverloads
 
@@ -325,3 +326,7 @@ def empty(self) -> bool:
 AliasUnion = NewType('AliasUnion', Union[str, int, bool])
 
 AliasNewType = NewType('AliasNewType', Dict[str, str])
+
+@runtime_checkable
+class SomeProtocolDecorator(Protocol):
+    pass

packages/python-packages/apiview-stub-generator/CHANGELOG.md

@@ -1,5 +1,8 @@
 # Release History
 
+## Version 0.3.13 (Unreleased)
+Update for tree token style parser.
+
 ## Version 0.3.12 (2024-02-27)
 Add support for Cross Language Package ID.
 

packages/python-packages/apiview-stub-generator/README.md

@@ -16,9 +16,9 @@ Following are the steps to create an API review request for python package.
 ### Generate stub File
 `apiview-stub-generator` package is used to generate stub file tokens from either source code repo or from prebuilt wheel package. Following are the steps to generate stub file token.
 
-Install `apiview-stub-generator` package. From the root of the apiview-stub-generator package, run:
+Install `apiview-stub-generator` package on Python <=3.10. From the root of the apiview-stub-generator package, run:
 ```
-pip install . --extra-index-url="https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/"
+(venv310) apiview-stub-generator> pip install . --extra-index-url="https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/"
 ```
 
 Run `apistubgen` command with source repo or whl package as parameter and this will generate a json file with tokens.
@@ -42,6 +42,18 @@ apistubgen --pkg-path C:\git\azure-sdk-for-python\sdk\core\azure-core --out-path
 
 Token file will be created with a naming convention `<package-name>_python.json'
 
+#### ApiStubGen options
+
+The following options are available when running `apistubgen`:
+```
+```
+
+### Running tests
+```
+apiview-stub-generator> pip install . --extra-index-url="https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/"
+apiview-stub-generator> pip install -r dev_requirements.txt
+```
+
 
 ### Upload token file to API review portal
 - Go to ``https://apiview.dev``

packages/python-packages/apiview-stub-generator/apistub/__init__.py

@@ -2,10 +2,13 @@
 
 from ._version import VERSION
 from ._stub_generator import StubGenerator
-from ._token import Token
-from ._token_kind import TokenKind
-from ._apiview import ApiView, Navigation, NavigationTag, Kind
-from ._diagnostic import Diagnostic, DiagnosticLevel
+from ._generated.treestyle.parser.models import (
+    CodeDiagnostic as Diagnostic,
+    CodeDiagnosticLevel as DiagnosticLevel,
+    TokenKind,
+    ReviewToken as Token,
+)
+from ._generated.treestyle.parser.models._patch import ApiView, ReviewLine, ReviewLines
 
 __version__ = VERSION
 
@@ -14,9 +17,8 @@
     "Token",
     "TokenKind",
     "ApiView",
-    "Navigation",
-    "NavigationTag",
-    "Kind",
+    "ReviewLine",
+    "ReviewLines",
     "Diagnostic",
     "DiagnosticLevel",
 ]
@@ -31,8 +33,6 @@ def console_entry_point():
     out_file_path = stub_generator.out_path
     # Generate JSON file name if outpath doesn't have json file name
     if not out_file_path.endswith(".json"):
-        out_file_path = os.path.join(
-            stub_generator.out_path, "{0}_python.json".format(apiview.name)
-        )
+        out_file_path = os.path.join(stub_generator.out_path, "{0}_python.json".format(apiview.package_name))
     with open(out_file_path, "w") as json_file:
         json_file.write(json_tokens)

packages/python-packages/apiview-stub-generator/apistub/__main__.py

@@ -2,4 +2,4 @@
 
 if __name__ == "__main__":
     console_entry_point()
-    exit(0)
+    exit(0)