Skip to content
This repository was archived by the owner on Nov 22, 2022. It is now read-only.

Commit e71c3af

Browse files
zero323zero323
committed
Add dynamic annotations for internal modules
1 parent a66989d commit e71c3af

File tree

10 files changed

+221
-9
lines changed

10 files changed

+221
-9
lines changed

‎doc/api-coverage.rst‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -10,21 +10,21 @@ API Coverage
1010
+------------------------------------------------+---------------------+--------------------+------------+
1111
| `pyspark.broadcast`_ | ✔ | ✔ | Mixed |
1212
+------------------------------------------------+---------------------+--------------------+------------+
13-
| pyspark.cloudpickle | ✘ | ✘ | Internal |
13+
| pyspark.cloudpickle | ✔ | ✘ | Internal |
1414
+------------------------------------------------+---------------------+--------------------+------------+
1515
| `pyspark.conf`_ | ✘ | ✔ | |
1616
+------------------------------------------------+---------------------+--------------------+------------+
1717
| `pyspark.context`_ | ✘ | ✔ | |
1818
+------------------------------------------------+---------------------+--------------------+------------+
19-
| pyspark.daemon | ✘ | ✘ | Internal |
19+
| pyspark.daemon | ✔ | ✘ | Internal |
2020
+------------------------------------------------+---------------------+--------------------+------------+
2121
| `pyspark.files`_ | ✘ | ✔ | |
2222
+------------------------------------------------+---------------------+--------------------+------------+
23-
| pyspark.find\_spark\_home | ✘ | ✘ | Internal |
23+
| pyspark.find\_spark\_home | ✔ | ✘ | Internal |
2424
+------------------------------------------------+---------------------+--------------------+------------+
25-
| pyspark.heapq3 | ✘ | ✘ | Internal |
25+
| pyspark.heapq3 | ✔ | ✘ | Internal |
2626
+------------------------------------------------+---------------------+--------------------+------------+
27-
| pyspark.java\_gateway | ✘ | ✘ | Internal |
27+
| pyspark.java\_gateway | ✔ | ✘ | Internal |
2828
+------------------------------------------------+---------------------+--------------------+------------+
2929
| `pyspark.join`_ | ✘ | ✔ | |
3030
+------------------------------------------------+---------------------+--------------------+------------+
@@ -50,7 +50,7 @@ API Coverage
5050
+------------------------------------------------+---------------------+--------------------+------------+
5151
| `pyspark.ml.param`_ | ✘ | ✔ | |
5252
+------------------------------------------------+---------------------+--------------------+------------+
53-
| pyspark.ml.param.\_shared\_params\_code\_gen | ✘ | ✘ | Internal |
53+
| pyspark.ml.param.\_shared\_params\_code\_gen | ✔ | ✘ | Internal |
5454
+------------------------------------------------+---------------------+--------------------+------------+
5555
| `pyspark.ml.param.shared`_ | ✘ | ✔ | |
5656
+------------------------------------------------+---------------------+--------------------+------------+
@@ -124,9 +124,9 @@ API Coverage
124124
+------------------------------------------------+---------------------+--------------------+------------+
125125
| `pyspark.serializers`_ | ✔ | ✘ | |
126126
+------------------------------------------------+---------------------+--------------------+------------+
127-
| pyspark.shell | ✘ | ✘ | Internal |
127+
| pyspark.shell | ✔ | ✘ | Internal |
128128
+------------------------------------------------+---------------------+--------------------+------------+
129-
| pyspark.shuffle | ✘ | ✘ | Internal |
129+
| pyspark.shuffle | ✔ | ✘ | Internal |
130130
+------------------------------------------------+---------------------+--------------------+------------+
131131
| `pyspark.sql`_ | ✔ | ✘ | |
132132
+------------------------------------------------+---------------------+--------------------+------------+
@@ -212,7 +212,7 @@ API Coverage
212212
+------------------------------------------------+---------------------+--------------------+------------+
213213
| `pyspark.version`_ | ✘ | ✔ | |
214214
+------------------------------------------------+---------------------+--------------------+------------+
215-
| pyspark.worker | ✘ | ✘ | Internal |
215+
| pyspark.worker | ✔ | ✘ | Internal |
216216
+------------------------------------------------+---------------------+--------------------+------------+
217217

218218

‎third_party/3/pyspark/cloudpickle.pyi‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
import pickle
2+
from functools import partial as partial
3+
from pickle import _Pickler as Pickler # type: ignore[attr-defined]
4+
from typing import Any, Optional
5+
6+
DEFAULT_PROTOCOL: Any
7+
string_types: Any
8+
PY3: bool
9+
PY2: bool
10+
11+
def cell_set(cell: Any, value: Any): ...
12+
13+
STORE_GLOBAL: Any
14+
DELETE_GLOBAL: Any
15+
LOAD_GLOBAL: Any
16+
GLOBAL_OPS: Any
17+
HAVE_ARGUMENT: Any
18+
EXTENDED_ARG: Any
19+
20+
def islambda(func: Any): ...
21+
22+
class CloudPickler(Pickler):
23+
dispatch: Any = ...
24+
globals_ref: Any = ...
25+
def __init__(self, file: Any, protocol: Optional[Any] = ...) -> None: ...
26+
def dump(self, obj: Any): ...
27+
def save_memoryview(self, obj: Any) -> None: ...
28+
def save_module(self, obj: Any) -> None: ...
29+
def save_codeobject(self, obj: Any) -> None: ...
30+
def save_function(self, obj: Any, name: Optional[Any] = ...): ...
31+
def save_dynamic_class(self, obj: Any) -> None: ...
32+
def save_function_tuple(self, func: Any) -> None: ...
33+
@classmethod
34+
def extract_code_globals(cls, co: Any): ...
35+
def extract_func_data(self, func: Any): ...
36+
def save_builtin_function(self, obj: Any): ...
37+
def save_global(self, obj: Any, name: Optional[Any] = ..., pack: Any = ...): ...
38+
def save_instancemethod(self, obj: Any) -> None: ...
39+
def save_inst(self, obj: Any) -> None: ...
40+
def save_property(self, obj: Any) -> None: ...
41+
def save_classmethod(self, obj: Any) -> None: ...
42+
def save_itemgetter(self, obj: Any): ...
43+
attrs: Any = ...
44+
index: Any = ...
45+
def save_attrgetter(self, obj: Any): ...
46+
def save_file(self, obj: Any): ...
47+
def save_ellipsis(self, obj: Any) -> None: ...
48+
def save_not_implemented(self, obj: Any) -> None: ...
49+
def save_weakset(self, obj: Any) -> None: ...
50+
def save_logger(self, obj: Any) -> None: ...
51+
def save_root_logger(self, obj: Any) -> None: ...
52+
def save_mappingproxy(self, obj: Any) -> None: ...
53+
def inject_addons(self) -> None: ...
54+
55+
def is_tornado_coroutine(func: Any): ...
56+
def dump(obj: Any, file: Any, protocol: Optional[Any] = ...) -> None: ...
57+
def dumps(obj: Any, protocol: Optional[Any] = ...): ...
58+
load = pickle.load
59+
loads = pickle.loads
60+
61+
def subimport(name: Any): ...
62+
def dynamic_subimport(name: Any, vars: Any): ...
63+
def instance(cls): ...
64+
65+
class _empty_cell_value:
66+
@classmethod
67+
def __reduce__(cls): ...

‎third_party/3/pyspark/daemon.pyi‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
from pyspark.serializers import UTF8Deserializer as UTF8Deserializer, read_int as read_int, write_int as write_int, write_with_length as write_with_length # type: ignore[attr-defined]
2+
from typing import Any
3+
4+
def compute_real_exit_code(exit_code: Any): ...
5+
def worker(sock: Any, authenticated: Any): ...
6+
def manager() -> None: ...

‎third_party/3/pyspark/find_spark_home.pyi‎

Whitespace-only changes.

‎third_party/3/pyspark/heapq3.pyi‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
from typing import Any, Optional
2+
3+
def heappush(heap: Any, item: Any) -> None: ...
4+
def heappop(heap: Any): ...
5+
def heapreplace(heap: Any, item: Any): ...
6+
def heappushpop(heap: Any, item: Any): ...
7+
def heapify(x: Any) -> None: ...
8+
def merge(iterables: Any, key: Optional[Any] = ..., reverse: bool = ...) -> None: ...
9+
def nsmallest(n: Any, iterable: Any, key: Optional[Any] = ...): ...
10+
def nlargest(n: Any, iterable: Any, key: Optional[Any] = ...): ...

‎third_party/3/pyspark/java_gateway.pyi‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
from pyspark.serializers import UTF8Deserializer as UTF8Deserializer, read_int as read_int, write_with_length as write_with_length # type: ignore[attr-defined]
2+
from typing import Any, Optional
3+
4+
xrange = range
5+
6+
def launch_gateway(conf: Optional[Any] = ..., popen_kwargs: Optional[Any] = ...): ...
7+
def local_connect_and_auth(port: Any, auth_secret: Any): ...
8+
def ensure_callback_server_started(gw: Any) -> None: ...

‎third_party/3/pyspark/ml/param/_shared_params_code_gen.pyi‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
header: str

‎third_party/3/pyspark/shell.pyi‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
from pyspark import SparkConf as SparkConf
2+
from pyspark.context import SparkContext as SparkContext
3+
from pyspark.sql import SQLContext as SQLContext, SparkSession as SparkSession
4+
from typing import Any
5+
6+
spark: Any
7+
sc: Any
8+
sql: Any
9+
sqlContext: Any
10+
sqlCtx = sqlContext
11+
code: Any

‎third_party/3/pyspark/shuffle.pyi‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
from pyspark.serializers import AutoBatchedSerializer as AutoBatchedSerializer, BatchedSerializer as BatchedSerializer, CompressedSerializer as CompressedSerializer, FlattenedValuesSerializer as FlattenedValuesSerializer, PickleSerializer as PickleSerializer
2+
from pyspark.util import fail_on_stopiteration as fail_on_stopiteration
3+
from typing import Any, Optional
4+
5+
process: Any
6+
7+
def get_used_memory(): ...
8+
9+
MemoryBytesSpilled: int
10+
DiskBytesSpilled: int
11+
12+
class Aggregator:
13+
createCombiner: Any = ...
14+
mergeValue: Any = ...
15+
mergeCombiners: Any = ...
16+
def __init__(self, createCombiner: Any, mergeValue: Any, mergeCombiners: Any) -> None: ...
17+
18+
class SimpleAggregator(Aggregator):
19+
def __init__(self, combiner: Any): ...
20+
21+
class Merger:
22+
agg: Any = ...
23+
def __init__(self, aggregator: Any) -> None: ...
24+
def mergeValues(self, iterator: Any) -> None: ...
25+
def mergeCombiners(self, iterator: Any) -> None: ...
26+
def items(self) -> None: ...
27+
28+
class ExternalMerger(Merger):
29+
MAX_TOTAL_PARTITIONS: int = ...
30+
memory_limit: Any = ...
31+
serializer: Any = ...
32+
localdirs: Any = ...
33+
partitions: Any = ...
34+
batch: Any = ...
35+
scale: Any = ...
36+
data: Any = ...
37+
pdata: Any = ...
38+
spills: int = ...
39+
def __init__(self, aggregator: Any, memory_limit: int = ..., serializer: Optional[Any] = ..., localdirs: Optional[Any] = ..., scale: int = ..., partitions: int = ..., batch: int = ...) -> None: ...
40+
def mergeValues(self, iterator: Any) -> None: ...
41+
def mergeCombiners(self, iterator: Any, limit: Optional[Any] = ...) -> None: ...
42+
def items(self): ...
43+
44+
class ExternalSorter:
45+
memory_limit: Any = ...
46+
local_dirs: Any = ...
47+
serializer: Any = ...
48+
def __init__(self, memory_limit: Any, serializer: Optional[Any] = ...) -> None: ...
49+
def sorted(self, iterator: Any, key: Optional[Any] = ..., reverse: bool = ...): ...
50+
51+
class ExternalList:
52+
LIMIT: int = ...
53+
values: Any = ...
54+
count: Any = ...
55+
def __init__(self, values: Any) -> None: ...
56+
def __iter__(self) -> Any: ...
57+
def __len__(self): ...
58+
def append(self, value: Any) -> None: ...
59+
def __del__(self) -> None: ...
60+
61+
class ExternalListOfList(ExternalList):
62+
count: Any = ...
63+
def __init__(self, values: Any) -> None: ...
64+
def append(self, value: Any) -> None: ...
65+
def __iter__(self) -> Any: ...
66+
67+
class GroupByKey:
68+
iterator: Any = ...
69+
def __init__(self, iterator: Any) -> None: ...
70+
def __iter__(self) -> Any: ...
71+
72+
class ExternalGroupBy(ExternalMerger):
73+
SORT_KEY_LIMIT: int = ...
74+
def flattened_serializer(self): ...

‎third_party/3/pyspark/worker.pyi‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
from pyspark import shuffle as shuffle
2+
from pyspark.broadcast import Broadcast as Broadcast
3+
from pyspark.files import SparkFiles as SparkFiles
4+
from pyspark.java_gateway import local_connect_and_auth as local_connect_and_auth
5+
from pyspark.rdd import PythonEvalType as PythonEvalType
6+
from pyspark.resourceinformation import ResourceInformation as ResourceInformation
7+
from pyspark.serializers import BatchedSerializer as BatchedSerializer, PickleSerializer as PickleSerializer, SpecialLengths as SpecialLengths, UTF8Deserializer as UTF8Deserializer, read_bool as read_bool, read_int as read_int, read_long as read_long, write_int as write_int, write_long as write_long, write_with_length as write_with_length # type: ignore[attr-defined]
8+
from pyspark.sql.pandas.serializers import ArrowStreamPandasUDFSerializer as ArrowStreamPandasUDFSerializer, CogroupUDFSerializer as CogroupUDFSerializer
9+
from pyspark.sql.pandas.types import to_arrow_type as to_arrow_type
10+
from pyspark.sql.types import StructType as StructType
11+
from pyspark.taskcontext import BarrierTaskContext as BarrierTaskContext, TaskContext as TaskContext
12+
from pyspark.util import fail_on_stopiteration as fail_on_stopiteration
13+
from typing import Any
14+
15+
has_resource_module: bool
16+
basestring = str
17+
pickleSer: Any
18+
utf8_deserializer: Any
19+
20+
def report_times(outfile: Any, boot: Any, init: Any, finish: Any) -> None: ...
21+
def add_path(path: Any) -> None: ...
22+
def read_command(serializer: Any, file: Any): ...
23+
def chain(f: Any, g: Any): ...
24+
def wrap_udf(f: Any, return_type: Any): ...
25+
def wrap_scalar_pandas_udf(f: Any, return_type: Any): ...
26+
def wrap_pandas_iter_udf(f: Any, return_type: Any): ...
27+
def wrap_cogrouped_map_pandas_udf(f: Any, return_type: Any, argspec: Any): ...
28+
def wrap_grouped_map_pandas_udf(f: Any, return_type: Any, argspec: Any): ...
29+
def wrap_grouped_agg_pandas_udf(f: Any, return_type: Any): ...
30+
def wrap_window_agg_pandas_udf(f: Any, return_type: Any, runner_conf: Any, udf_index: Any): ...
31+
def wrap_unbounded_window_agg_pandas_udf(f: Any, return_type: Any): ...
32+
def wrap_bounded_window_agg_pandas_udf(f: Any, return_type: Any): ...
33+
def read_single_udf(pickleSer: Any, infile: Any, eval_type: Any, runner_conf: Any, udf_index: Any): ...
34+
def read_udfs(pickleSer: Any, infile: Any, eval_type: Any): ...
35+
def main(infile: Any, outfile: Any) -> None: ...

0 commit comments

Comments
 (0)