diff --git a/.github/workflows/flake8.yml b/.github/workflows/flake8.yml index 0d82363..7ec5596 100644 --- a/.github/workflows/flake8.yml +++ b/.github/workflows/flake8.yml @@ -24,15 +24,30 @@ jobs: event-type: flake8-event # Additional arguments to pass to flake8, default "." (current directory) args: "--ignore=E121" - - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@master - - uses: actions/setup-python@v2 - with: - python-version: '3.7.7' - - name: Test - uses: onichandame/python-test-action@master - with: - deps_list: 'requirements.txt' + + mypy: + name: mypy + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: '3.7.7' + deps_list: 'requirements.txt' + - name: Run mypy + run: | + pip3 install . + python3 -m mypy . + + Tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - uses: actions/setup-python@v2 + with: + python-version: '3.7.7' + - name: Test + uses: onichandame/python-test-action@master + with: + deps_list: 'requirements.txt' diff --git a/main.py b/main.py index 2d1991c..6be27dd 100644 --- a/main.py +++ b/main.py @@ -1,14 +1,15 @@ from pathlib import Path +from typing import List def ccc(): pass -def Precommit(): +def Precommit(a: List[int]): if True: - print(Path(".") + " 11111" + "22222") + print(str(Path(".")) + " 11111" + "22222") -Precommit() -ccc() \ No newline at end of file +Precommit((1)) +ccc() diff --git a/requirements.txt b/requirements.txt index 039d679..34697f3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,18 @@ +flake8==3.7.9 +codecov==2.0.15 +matplotlib==3.2.1 +coverage==5.0.3 +javalang==0.13.0 +beautifulsoup4==4.8.2 +numpy==1.18.1 +pandas==1.0.0 +mypy==0.770 +networkx==2.4 +cchardet==2.1.6 +lxml==4.5.0 +cached-property==1.2.0 +deprecated==1.2.10 typing-extensions; python_version<'3.8' -tqdm == 4.32.1 \ No newline at end of file +tqdm == 4.32.1 +bs4==0.0.1 +pebble==4.5.3 \ No newline at end of file diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..64d67c8 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,12 @@ +[wheel] +universal = 1 + +[flake8] +ignore = +max-line-length = 120 +max-complexity = 10 + +extensions = ['recommonmark'] + +[mypy] +mypy_path = stubs diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..0a6be87 --- /dev/null +++ b/setup.py @@ -0,0 +1,32 @@ +from setuptools import setup, find_packages +import temp_python + +setup( + name='veniq', + version=temp_python.__version__, + description=temp_python.__doc__.strip(), + long_description='Govno', + url='https://github.com/cqfn/veniq.git', + download_url='https://github.com/cqfn/veniq.git', + author=temp_python.__author__, + author_email='yegor256@gmail.com', + license=temp_python.__licence__, + packages=find_packages(), + entry_points={ + 'console_scripts': [ + 'temp_python = temp_python.__main__:main' + ], + }, + extras_require={}, + install_requires=open('requirements.txt', 'r').readlines(), + tests_require=open('requirements.txt', 'r').readlines(), + classifiers=[ + 'Programming Language :: Python', + 'Environment :: Console', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: MIT License', + 'Topic :: Software Development', + 'Topic :: Utilities' + ], + include_package_data=True, +) diff --git a/stubs/bs4/__init__.pyi b/stubs/bs4/__init__.pyi new file mode 100644 index 0000000..b6fa7e1 --- /dev/null +++ b/stubs/bs4/__init__.pyi @@ -0,0 +1,40 @@ +from .element import Tag +from typing import Any, Optional + +class BeautifulSoup(Tag): + ROOT_TAG_NAME: str = ... + DEFAULT_BUILDER_FEATURES: Any = ... + ASCII_SPACES: str = ... + NO_PARSER_SPECIFIED_WARNING: str = ... + element_classes: Any = ... + builder: Any = ... + is_xml: Any = ... + known_xml: Any = ... + parse_only: Any = ... + markup: Any = ... + def __init__(self, markup: str = ..., features: Optional[Any] = ..., builder: Optional[Any] = ..., parse_only: Optional[Any] = ..., from_encoding: Optional[Any] = ..., exclude_encodings: Optional[Any] = ..., element_classes: Optional[Any] = ..., **kwargs: Any): ... + def __copy__(self): ... + hidden: bool = ... + current_data: Any = ... + currentTag: Any = ... + tagStack: Any = ... + preserve_whitespace_tag_stack: Any = ... + def reset(self) -> None: ... + def new_tag(self, name: Any, namespace: Optional[Any] = ..., nsprefix: Optional[Any] = ..., attrs: Any = ..., sourceline: Optional[Any] = ..., sourcepos: Optional[Any] = ..., **kwattrs: Any): ... + def new_string(self, s: Any, subclass: Optional[Any] = ...): ... + # def insert_before(self, successor: Any) -> None: ... + # def insert_after(self, successor: Any) -> None: ... + def popTag(self): ... + def pushTag(self, tag: Any) -> None: ... + def endData(self, containerClass: Optional[Any] = ...) -> None: ... + def object_was_parsed(self, o: Any, parent: Optional[Any] = ..., most_recent_element: Optional[Any] = ...) -> None: ... + def handle_starttag(self, name: Any, namespace: Any, nsprefix: Any, attrs: Any, sourceline: Optional[Any] = ..., sourcepos: Optional[Any] = ...): ... + def handle_endtag(self, name: Any, nsprefix: Optional[Any] = ...) -> None: ... + def handle_data(self, data: Any) -> None: ... + # def decode(self, pretty_print: bool = ..., eventual_encoding: Any = ..., formatter: str = ...): ... + +class BeautifulStoneSoup(BeautifulSoup): + def __init__(self, *args: Any, **kwargs: Any) -> None: ... + +class StopParsing(Exception): ... +class FeatureNotFound(ValueError): ... diff --git a/stubs/bs4/builder/__init__.pyi b/stubs/bs4/builder/__init__.pyi new file mode 100644 index 0000000..f679f59 --- /dev/null +++ b/stubs/bs4/builder/__init__.pyi @@ -0,0 +1,58 @@ +from typing import Any, Optional + +class TreeBuilderRegistry: + builders_for_feature: Any = ... + builders: Any = ... + def __init__(self) -> None: ... + def register(self, treebuilder_class: Any) -> None: ... + def lookup(self, *features: Any): ... + +class TreeBuilder: + NAME: str = ... + ALTERNATE_NAMES: Any = ... + features: Any = ... + is_xml: bool = ... + picklable: bool = ... + empty_element_tags: Any = ... + DEFAULT_CDATA_LIST_ATTRIBUTES: Any = ... + DEFAULT_PRESERVE_WHITESPACE_TAGS: Any = ... + USE_DEFAULT: Any = ... + TRACKS_LINE_NUMBERS: bool = ... + soup: Any = ... + cdata_list_attributes: Any = ... + preserve_whitespace_tags: Any = ... + store_line_numbers: Any = ... + def __init__(self, multi_valued_attributes: Any = ..., preserve_whitespace_tags: Any = ..., store_line_numbers: Any = ...) -> None: ... + def initialize_soup(self, soup: Any) -> None: ... + def reset(self) -> None: ... + def can_be_empty_element(self, tag_name: Any): ... + def feed(self, markup: Any) -> None: ... + def prepare_markup(self, markup: Any, user_specified_encoding: Optional[Any] = ..., document_declared_encoding: Optional[Any] = ..., exclude_encodings: Optional[Any] = ...) -> None: ... + def test_fragment_to_document(self, fragment: Any): ... + def set_up_substitutions(self, tag: Any): ... + +class SAXTreeBuilder(TreeBuilder): + def feed(self, markup: Any) -> None: ... + def close(self) -> None: ... + def startElement(self, name: Any, attrs: Any) -> None: ... + def endElement(self, name: Any) -> None: ... + def startElementNS(self, nsTuple: Any, nodeName: Any, attrs: Any) -> None: ... + def endElementNS(self, nsTuple: Any, nodeName: Any) -> None: ... + def startPrefixMapping(self, prefix: Any, nodeValue: Any) -> None: ... + def endPrefixMapping(self, prefix: Any) -> None: ... + def characters(self, content: Any) -> None: ... + def startDocument(self) -> None: ... + def endDocument(self) -> None: ... + +class HTMLTreeBuilder(TreeBuilder): + empty_element_tags: Any = ... + block_elements: Any = ... + DEFAULT_CDATA_LIST_ATTRIBUTES: Any = ... + DEFAULT_PRESERVE_WHITESPACE_TAGS: Any = ... + def set_up_substitutions(self, tag: Any): ... + +class ParserRejectedMarkup(Exception): + def __init__(self, message_or_exception: Any) -> None: ... + +# Names in __all__ with no definition: +# HTMLParserTreeBuilder diff --git a/stubs/bs4/builder/_html5lib.pyi b/stubs/bs4/builder/_html5lib.pyi new file mode 100644 index 0000000..5ec17f4 --- /dev/null +++ b/stubs/bs4/builder/_html5lib.pyi @@ -0,0 +1,65 @@ +from bs4.builder import HTMLTreeBuilder +from html5lib.treebuilders import base as treebuilder_base +from typing import Any, Optional + +class HTML5TreeBuilder(HTMLTreeBuilder): + NAME: str = ... + features: Any = ... + TRACKS_LINE_NUMBERS: bool = ... + user_specified_encoding: Any = ... + def prepare_markup(self, markup: Any, user_specified_encoding: Any, document_declared_encoding: Optional[Any] = ..., exclude_encodings: Optional[Any] = ...) -> None: ... + def feed(self, markup: Any) -> None: ... + underlying_builder: Any = ... + def create_treebuilder(self, namespaceHTMLElements: Any): ... + def test_fragment_to_document(self, fragment: Any): ... + +class TreeBuilderForHtml5lib(treebuilder_base.TreeBuilder): + soup: Any = ... + parser: Any = ... + store_line_numbers: Any = ... + def __init__(self, namespaceHTMLElements: Any, soup: Optional[Any] = ..., store_line_numbers: bool = ..., **kwargs: Any) -> None: ... + def documentClass(self): ... + def insertDoctype(self, token: Any) -> None: ... + def elementClass(self, name: Any, namespace: Any): ... + def commentClass(self, data: Any): ... + def fragmentClass(self): ... + def appendChild(self, node: Any) -> None: ... + def getDocument(self): ... + def getFragment(self): ... + def testSerializer(self, element: Any): ... + +class AttrList: + element: Any = ... + attrs: Any = ... + def __init__(self, element: Any) -> None: ... + def __iter__(self) -> Any: ... + def __setitem__(self, name: Any, value: Any) -> None: ... + def items(self): ... + def keys(self): ... + def __len__(self): ... + def __getitem__(self, name: Any): ... + def __contains__(self, name: Any): ... + +class Element(treebuilder_base.Node): + element: Any = ... + soup: Any = ... + namespace: Any = ... + def __init__(self, element: Any, soup: Any, namespace: Any) -> None: ... + def appendChild(self, node: Any) -> None: ... + def getAttributes(self): ... + def setAttributes(self, attributes: Any) -> None: ... + attributes: Any = ... + def insertText(self, data: Any, insertBefore: Optional[Any] = ...) -> None: ... + def insertBefore(self, node: Any, refNode: Any) -> None: ... + def removeChild(self, node: Any) -> None: ... + def reparentChildren(self, new_parent: Any) -> None: ... + def cloneNode(self): ... + def hasContent(self): ... + def getNameTuple(self): ... + nameTuple: Any = ... + +class TextNode(Element): + element: Any = ... + soup: Any = ... + def __init__(self, element: Any, soup: Any) -> None: ... + def cloneNode(self) -> None: ... diff --git a/stubs/bs4/builder/_htmlparser.pyi b/stubs/bs4/builder/_htmlparser.pyi new file mode 100644 index 0000000..3ea458d --- /dev/null +++ b/stubs/bs4/builder/_htmlparser.pyi @@ -0,0 +1,31 @@ +from bs4.builder import HTMLTreeBuilder +from html.parser import HTMLParser +from typing import Any, Optional + +class HTMLParseError(Exception): ... + +class BeautifulSoupHTMLParser(HTMLParser): + already_closed_empty_element: Any = ... + def __init__(self, *args: Any, **kwargs: Any) -> None: ... + def error(self, msg: Any) -> None: ... + def handle_startendtag(self, name: Any, attrs: Any) -> None: ... + def handle_starttag(self, name: Any, attrs: Any, handle_empty_element: bool = ...) -> None: ... + def handle_endtag(self, name: Any, check_already_closed: bool = ...) -> None: ... + def handle_data(self, data: Any) -> None: ... + def handle_charref(self, name: Any) -> None: ... + def handle_entityref(self, name: Any) -> None: ... + def handle_comment(self, data: Any) -> None: ... + def handle_decl(self, data: Any) -> None: ... + def unknown_decl(self, data: Any) -> None: ... + def handle_pi(self, data: Any) -> None: ... + +class HTMLParserTreeBuilder(HTMLTreeBuilder): + is_xml: bool = ... + picklable: bool = ... + NAME: Any = ... + features: Any = ... + TRACKS_LINE_NUMBERS: bool = ... + parser_args: Any = ... + def __init__(self, parser_args: Optional[Any] = ..., parser_kwargs: Optional[Any] = ..., **kwargs: Any) -> None: ... + def prepare_markup(self, markup: Any, user_specified_encoding: Optional[Any] = ..., document_declared_encoding: Optional[Any] = ..., exclude_encodings: Optional[Any] = ...) -> None: ... + def feed(self, markup: Any) -> None: ... diff --git a/stubs/bs4/builder/_lxml.pyi b/stubs/bs4/builder/_lxml.pyi new file mode 100644 index 0000000..c1819dd --- /dev/null +++ b/stubs/bs4/builder/_lxml.pyi @@ -0,0 +1,42 @@ +from bs4.builder import HTMLTreeBuilder, TreeBuilder +from typing import Any, Optional + +class LXMLTreeBuilderForXML(TreeBuilder): + DEFAULT_PARSER_CLASS: Any = ... + is_xml: bool = ... + processing_instruction_class: Any = ... + NAME: str = ... + ALTERNATE_NAMES: Any = ... + features: Any = ... + CHUNK_SIZE: int = ... + DEFAULT_NSMAPS: Any = ... + DEFAULT_NSMAPS_INVERTED: Any = ... + def initialize_soup(self, soup: Any) -> None: ... + def default_parser(self, encoding: Any): ... + def parser_for(self, encoding: Any): ... + empty_element_tags: Any = ... + soup: Any = ... + nsmaps: Any = ... + def __init__(self, parser: Optional[Any] = ..., empty_element_tags: Optional[Any] = ..., **kwargs: Any) -> None: ... + def prepare_markup(self, markup: Any, user_specified_encoding: Optional[Any] = ..., exclude_encodings: Optional[Any] = ..., document_declared_encoding: Optional[Any] = ...) -> None: ... + parser: Any = ... + def feed(self, markup: Any) -> None: ... + def close(self) -> None: ... + def start(self, name: Any, attrs: Any, nsmap: Any = ...) -> None: ... + def end(self, name: Any) -> None: ... + def pi(self, target: Any, data: Any) -> None: ... + def data(self, content: Any) -> None: ... + def doctype(self, name: Any, pubid: Any, system: Any) -> None: ... + def comment(self, content: Any) -> None: ... + def test_fragment_to_document(self, fragment: Any): ... + +class LXMLTreeBuilder(HTMLTreeBuilder, LXMLTreeBuilderForXML): + NAME: Any = ... + ALTERNATE_NAMES: Any = ... + features: Any = ... + is_xml: bool = ... + processing_instruction_class: Any = ... + def default_parser(self, encoding: Any): ... + parser: Any = ... + def feed(self, markup: Any) -> None: ... + def test_fragment_to_document(self, fragment: Any): ... diff --git a/stubs/bs4/dammit.pyi b/stubs/bs4/dammit.pyi new file mode 100644 index 0000000..8edb07a --- /dev/null +++ b/stubs/bs4/dammit.pyi @@ -0,0 +1,64 @@ +from typing import Any, Optional + +chardet_type: Any + +def chardet_dammit(s: Any): ... + +xml_encoding: str +html_meta: str +encoding_res: Any + +class EntitySubstitution: + CHARACTER_TO_HTML_ENTITY: Any = ... + HTML_ENTITY_TO_CHARACTER: Any = ... + CHARACTER_TO_HTML_ENTITY_RE: Any = ... + CHARACTER_TO_XML_ENTITY: Any = ... + BARE_AMPERSAND_OR_BRACKET: Any = ... + AMPERSAND_OR_BRACKET: Any = ... + @classmethod + def quoted_attribute_value(self, value: Any): ... + @classmethod + def substitute_xml(cls, value: Any, make_quoted_attribute: bool = ...): ... + @classmethod + def substitute_xml_containing_entities(cls, value: Any, make_quoted_attribute: bool = ...): ... + @classmethod + def substitute_html(cls, s: Any): ... + +class EncodingDetector: + override_encodings: Any = ... + exclude_encodings: Any = ... + chardet_encoding: Any = ... + is_html: Any = ... + declared_encoding: Any = ... + def __init__(self, markup: Any, override_encodings: Optional[Any] = ..., is_html: bool = ..., exclude_encodings: Optional[Any] = ...) -> None: ... + @property + def encodings(self) -> None: ... + @classmethod + def strip_byte_order_mark(cls, data: Any): ... + @classmethod + def find_declared_encoding(cls, markup: Any, is_html: bool = ..., search_entire_document: bool = ...): ... + +class UnicodeDammit: + CHARSET_ALIASES: Any = ... + ENCODINGS_WITH_SMART_QUOTES: Any = ... + smart_quotes_to: Any = ... + tried_encodings: Any = ... + contains_replacement_characters: bool = ... + is_html: Any = ... + log: Any = ... + detector: Any = ... + markup: Any = ... + unicode_markup: Any = ... + original_encoding: Any = ... + def __init__(self, markup: Any, override_encodings: Any = ..., smart_quotes_to: Optional[Any] = ..., is_html: bool = ..., exclude_encodings: Any = ...) -> None: ... + @property + def declared_html_encoding(self): ... + def find_codec(self, charset: Any): ... + MS_CHARS: Any = ... + MS_CHARS_TO_ASCII: Any = ... + WINDOWS_1252_TO_UTF8: Any = ... + MULTIBYTE_MARKERS_AND_SIZES: Any = ... + FIRST_MULTIBYTE_MARKER: Any = ... + LAST_MULTIBYTE_MARKER: Any = ... + @classmethod + def detwingle(cls, in_bytes: Any, main_encoding: str = ..., embedded_encoding: str = ...): ... diff --git a/stubs/bs4/diagnose.pyi b/stubs/bs4/diagnose.pyi new file mode 100644 index 0000000..9ac0fad --- /dev/null +++ b/stubs/bs4/diagnose.pyi @@ -0,0 +1,25 @@ +from bs4 import BeautifulSoup as BeautifulSoup +from bs4.builder import builder_registry as builder_registry +from html.parser import HTMLParser +from typing import Any + +def diagnose(data: Any) -> None: ... +def lxml_trace(data: Any, html: bool = ..., **kwargs: Any) -> None: ... + +class AnnouncingParser(HTMLParser): + def handle_starttag(self, name: Any, attrs: Any) -> None: ... + def handle_endtag(self, name: Any) -> None: ... + def handle_data(self, data: Any) -> None: ... + def handle_charref(self, name: Any) -> None: ... + def handle_entityref(self, name: Any) -> None: ... + def handle_comment(self, data: Any) -> None: ... + def handle_decl(self, data: Any) -> None: ... + def unknown_decl(self, data: Any) -> None: ... + def handle_pi(self, data: Any) -> None: ... + +def htmlparser_trace(data: Any) -> None: ... +def rword(length: int = ...): ... +def rsentence(length: int = ...): ... +def rdoc(num_elements: int = ...): ... +def benchmark_parsers(num_elements: int = ...) -> None: ... +def profile(num_elements: int = ..., parser: str = ...) -> None: ... diff --git a/stubs/bs4/element.pyi b/stubs/bs4/element.pyi new file mode 100644 index 0000000..c9e6d37 --- /dev/null +++ b/stubs/bs4/element.pyi @@ -0,0 +1,216 @@ +from bs4.formatter import Formatter as Formatter, HTMLFormatter as HTMLFormatter, XMLFormatter as XMLFormatter +from typing import Any, Optional + +DEFAULT_OUTPUT_ENCODING: str +PY3K: Any +nonwhitespace_re: Any +whitespace_re: Any + +class NamespacedAttribute(str): + def __new__(cls, prefix: Any, name: Optional[Any] = ..., namespace: Optional[Any] = ...): ... + +class AttributeValueWithCharsetSubstitution(str): ... + +class CharsetMetaAttributeValue(AttributeValueWithCharsetSubstitution): + def __new__(cls, original_value: Any): ... + +class ContentMetaAttributeValue(AttributeValueWithCharsetSubstitution): + CHARSET_RE: Any = ... + def __new__(cls, original_value: Any): ... + +class PageElement: + parent: Any = ... + previous_element: Any = ... + next_element: Any = ... + next_sibling: Any = ... + previous_sibling: Any = ... + def setup(self, parent: Optional[Any] = ..., previous_element: Optional[Any] = ..., next_element: Optional[Any] = ..., previous_sibling: Optional[Any] = ..., next_sibling: Optional[Any] = ...) -> None: ... + def format_string(self, s: Any, formatter: Any): ... + def formatter_for_name(self, formatter: Any): ... + nextSibling: Any = ... + previousSibling: Any = ... + def replace_with(self, replace_with: Any): ... + replaceWith: Any = ... + def unwrap(self): ... + replace_with_children: Any = ... + replaceWithChildren: Any = ... + def wrap(self, wrap_inside: Any): ... + def extract(self): ... + def insert(self, position: Any, new_child: Any) -> None: ... + def append(self, tag: Any) -> None: ... + def extend(self, tags: Any) -> None: ... + def insert_before(self, *args: Any) -> None: ... + def insert_after(self, *args: Any) -> None: ... + def find_next(self, name: Optional[Any] = ..., attrs: Any = ..., text: Optional[Any] = ..., **kwargs: Any): ... + findNext: Any = ... + def find_all_next(self, name: Optional[Any] = ..., attrs: Any = ..., text: Optional[Any] = ..., limit: Optional[Any] = ..., **kwargs: Any): ... + findAllNext: Any = ... + def find_next_sibling(self, name: Optional[Any] = ..., attrs: Any = ..., text: Optional[Any] = ..., **kwargs: Any): ... + findNextSibling: Any = ... + def find_next_siblings(self, name: Optional[Any] = ..., attrs: Any = ..., text: Optional[Any] = ..., limit: Optional[Any] = ..., **kwargs: Any): ... + findNextSiblings: Any = ... + fetchNextSiblings: Any = ... + def find_previous(self, name: Optional[Any] = ..., attrs: Any = ..., text: Optional[Any] = ..., **kwargs: Any): ... + findPrevious: Any = ... + def find_all_previous(self, name: Optional[Any] = ..., attrs: Any = ..., text: Optional[Any] = ..., limit: Optional[Any] = ..., **kwargs: Any): ... + findAllPrevious: Any = ... + fetchPrevious: Any = ... + def find_previous_sibling(self, name: Optional[Any] = ..., attrs: Any = ..., text: Optional[Any] = ..., **kwargs: Any): ... + findPreviousSibling: Any = ... + def find_previous_siblings(self, name: Optional[Any] = ..., attrs: Any = ..., text: Optional[Any] = ..., limit: Optional[Any] = ..., **kwargs: Any): ... + findPreviousSiblings: Any = ... + fetchPreviousSiblings: Any = ... + def find_parent(self, name: Optional[Any] = ..., attrs: Any = ..., **kwargs: Any): ... + findParent: Any = ... + def find_parents(self, name: Optional[Any] = ..., attrs: Any = ..., limit: Optional[Any] = ..., **kwargs: Any): ... + findParents: Any = ... + fetchParents: Any = ... + @property + def next(self): ... + @property + def previous(self): ... + @property + def next_elements(self) -> None: ... + @property + def next_siblings(self) -> None: ... + @property + def previous_elements(self) -> None: ... + @property + def previous_siblings(self) -> None: ... + @property + def parents(self) -> None: ... + def nextGenerator(self): ... + def nextSiblingGenerator(self): ... + def previousGenerator(self): ... + def previousSiblingGenerator(self): ... + def parentGenerator(self): ... + +class NavigableString(str, PageElement): + PREFIX: str = ... + SUFFIX: str = ... + known_xml: Any = ... + def __new__(cls, value: Any): ... + def __copy__(self): ... + def __getnewargs__(self): ... + def __getattr__(self, attr: Any): ... + def output_ready(self, formatter: str = ...): ... + @property + def name(self) -> None: ... + @name.setter + def name(self, name: Any) -> None: ... + +class PreformattedString(NavigableString): + PREFIX: str = ... + SUFFIX: str = ... + def output_ready(self, formatter: Optional[Any] = ...): ... + +class CData(PreformattedString): + PREFIX: str = ... + SUFFIX: str = ... + +class ProcessingInstruction(PreformattedString): + PREFIX: str = ... + SUFFIX: str = ... + +class XMLProcessingInstruction(ProcessingInstruction): + PREFIX: str = ... + SUFFIX: str = ... + +class Comment(PreformattedString): + PREFIX: str = ... + SUFFIX: str = ... + +class Declaration(PreformattedString): + PREFIX: str = ... + SUFFIX: str = ... + +class Doctype(PreformattedString): + @classmethod + def for_name_and_ids(cls, name: Any, pub_id: Any, system_id: Any): ... + PREFIX: str = ... + SUFFIX: str = ... + +class Tag(PageElement): + parser_class: Any = ... + name: Any = ... + namespace: Any = ... + prefix: Any = ... + sourceline: Any = ... + sourcepos: Any = ... + known_xml: Any = ... + attrs: Any = ... + contents: Any = ... + hidden: bool = ... + can_be_empty_element: Any = ... + cdata_list_attributes: Any = ... + preserve_whitespace_tags: Any = ... + def __init__(self, parser: Optional[Any] = ..., builder: Optional[Any] = ..., name: Optional[Any] = ..., namespace: Optional[Any] = ..., prefix: Optional[Any] = ..., attrs: Optional[Any] = ..., parent: Optional[Any] = ..., previous: Optional[Any] = ..., is_xml: Optional[Any] = ..., sourceline: Optional[Any] = ..., sourcepos: Optional[Any] = ..., can_be_empty_element: Optional[Any] = ..., cdata_list_attributes: Optional[Any] = ..., preserve_whitespace_tags: Optional[Any] = ...) -> None: ... + parserClass: Any = ... + def __copy__(self): ... + @property + def is_empty_element(self): ... + isSelfClosing: Any = ... + @property + def string(self): ... + @string.setter + def string(self, string: Any) -> None: ... + strings: Any = ... + @property + def stripped_strings(self) -> None: ... + def get_text(self, separator: str = ..., strip: bool = ..., types: Any = ...): ... + getText: Any = ... + text: Any = ... + def decompose(self) -> None: ... + def clear(self, decompose: bool = ...) -> None: ... + def smooth(self) -> None: ... + def index(self, element: Any): ... + def get(self, key: Any, default: Optional[Any] = ...): ... + def get_attribute_list(self, key: Any, default: Optional[Any] = ...): ... + def has_attr(self, key: Any): ... + def __hash__(self) -> Any: ... + def __getitem__(self, key: Any): ... + def __iter__(self) -> Any: ... + def __len__(self): ... + def __contains__(self, x: Any): ... + def __bool__(self): ... + def __setitem__(self, key: Any, value: Any) -> None: ... + def __delitem__(self, key: Any) -> None: ... + def __call__(self, *args: Any, **kwargs: Any): ... + def __getattr__(self, tag: Any): ... + def __eq__(self, other: Any) -> Any: ... + def __ne__(self, other: Any) -> Any: ... + def __unicode__(self): ... + def encode(self, encoding: Any = ..., indent_level: Optional[Any] = ..., formatter: str = ..., errors: str = ...): ... + def decode(self, indent_level: Optional[Any] = ..., eventual_encoding: Any = ..., formatter: str = ...): ... + def prettify(self, encoding: Optional[Any] = ..., formatter: str = ...): ... + def decode_contents(self, indent_level: Optional[Any] = ..., eventual_encoding: Any = ..., formatter: str = ...): ... + def encode_contents(self, indent_level: Optional[Any] = ..., encoding: Any = ..., formatter: str = ...): ... + def renderContents(self, encoding: Any = ..., prettyPrint: bool = ..., indentLevel: int = ...): ... + def find(self, name: Optional[Any] = ..., attrs: Any = ..., recursive: bool = ..., text: Optional[Any] = ..., **kwargs: Any): ... + findChild: Any = ... + def find_all(self, name: Optional[Any] = ..., attrs: Any = ..., recursive: bool = ..., text: Optional[Any] = ..., limit: Optional[Any] = ..., **kwargs: Any): ... + findAll: Any = ... + findChildren: Any = ... + @property + def children(self): ... + @property + def descendants(self) -> None: ... + def select_one(self, selector: Any, namespaces: Optional[Any] = ..., **kwargs: Any): ... + def select(self, selector: Any, namespaces: Optional[Any] = ..., limit: Optional[Any] = ..., **kwargs: Any): ... + def childGenerator(self): ... + def recursiveChildGenerator(self): ... + def has_key(self, key: Any): ... + +class SoupStrainer: + name: Any = ... + attrs: Any = ... + text: Any = ... + def __init__(self, name: Optional[Any] = ..., attrs: Any = ..., text: Optional[Any] = ..., **kwargs: Any) -> None: ... + def search_tag(self, markup_name: Optional[Any] = ..., markup_attrs: Any = ...): ... + searchTag: Any = ... + def search(self, markup: Any): ... + +class ResultSet(list): + source: Any = ... + def __init__(self, source: Any, result: Any = ...) -> None: ... + def __getattr__(self, key: Any) -> None: ... diff --git a/stubs/bs4/formatter.pyi b/stubs/bs4/formatter.pyi new file mode 100644 index 0000000..fa1502c --- /dev/null +++ b/stubs/bs4/formatter.pyi @@ -0,0 +1,25 @@ +from bs4.dammit import EntitySubstitution as EntitySubstitution +from typing import Any, Optional + +class Formatter(EntitySubstitution): + XML_FORMATTERS: Any = ... + HTML_FORMATTERS: Any = ... + HTML: str = ... + XML: str = ... + HTML_DEFAULTS: Any = ... + language: Any = ... + entity_substitution: Any = ... + void_element_close_prefix: Any = ... + cdata_containing_tags: Any = ... + def __init__(self, language: Optional[Any] = ..., entity_substitution: Optional[Any] = ..., void_element_close_prefix: str = ..., cdata_containing_tags: Optional[Any] = ...) -> None: ... + def substitute(self, ns: Any): ... + def attribute_value(self, value: Any): ... + def attributes(self, tag: Any): ... + +class HTMLFormatter(Formatter): + REGISTRY: Any = ... + def __init__(self, *args: Any, **kwargs: Any): ... + +class XMLFormatter(Formatter): + REGISTRY: Any = ... + def __init__(self, *args: Any, **kwargs: Any): ... diff --git a/stubs/javalang/__init__.pyi b/stubs/javalang/__init__.pyi new file mode 100644 index 0000000..10eed54 --- /dev/null +++ b/stubs/javalang/__init__.pyi @@ -0,0 +1 @@ +from . import javadoc as javadoc, parse as parse, parser as parser, tokenizer as tokenizer diff --git a/stubs/javalang/ast.pyi b/stubs/javalang/ast.pyi new file mode 100644 index 0000000..7d48d68 --- /dev/null +++ b/stubs/javalang/ast.pyi @@ -0,0 +1,27 @@ +from typing import Any, Iterator, Tuple, TypeVar, Union, Type + +Path = Tuple + +T = TypeVar('T', bound=Node) + + +class MetaNode(type): + def __new__(mcs: Any, name: Any, bases: Any, dict: Any): ... + +class Node(metaclass=MetaNode): + attrs: Any = ... + def __init__(self, **kwargs: Any) -> None: ... + def __equals__(self, other: Any): ... + def __iter__(self) -> Any: ... + # TODO: need proper type, but unfortunatelly it can't be done now + def filter(self, pattern: Union[Type[T], T]) -> Iterator[Tuple[Path, T]]: ... + @property + def children(self): ... + @property + def position(self): ... + @property + def _position(self): ... + +def walk_tree(root: Any) -> None: ... +def dump(ast: Any, file: Any) -> None: ... +def load(file: Any): ... diff --git a/stubs/javalang/javadoc.pyi b/stubs/javalang/javadoc.pyi new file mode 100644 index 0000000..279a40d --- /dev/null +++ b/stubs/javalang/javadoc.pyi @@ -0,0 +1,21 @@ +from typing import Any + +def join(s: Any): ... + +class DocBlock: + description: str = ... + return_doc: Any = ... + params: Any = ... + authors: Any = ... + deprecated: bool = ... + throws: Any = ... + exceptions: Any = ... + tags: Any = ... + def __init__(self) -> None: ... + def add_block(self, name: Any, value: Any) -> None: ... + +blocks_re: Any +leading_space_re: Any +blocks_justify_re: Any + +def parse(raw: Any): ... diff --git a/stubs/javalang/parse.pyi b/stubs/javalang/parse.pyi new file mode 100644 index 0000000..dd1481e --- /dev/null +++ b/stubs/javalang/parse.pyi @@ -0,0 +1,10 @@ +from .parser import Parser as Parser +from .tokenizer import tokenize as tokenize +from typing import Any + +def parse_expression(exp: Any): ... +def parse_member_signature(sig: Any): ... +def parse_constructor_signature(sig: Any): ... +def parse_type(s: Any): ... +def parse_type_signature(sig: Any): ... +def parse(s: Any): ... diff --git a/stubs/javalang/parser.pyi b/stubs/javalang/parser.pyi new file mode 100644 index 0000000..6c285d2 --- /dev/null +++ b/stubs/javalang/parser.pyi @@ -0,0 +1,139 @@ +from . import tree as tree, util as util +from .tokenizer import Annotation as Annotation, BasicType as BasicType, EndOfInput as EndOfInput, Identifier as Identifier, JavaToken as JavaToken, Keyword as Keyword, Literal as Literal, Modifier as Modifier, Operator as Operator +from typing import Any, Optional + +ENABLE_DEBUG_SUPPORT: bool + +def parse_debug(method: Any): ... + +class JavaParserBaseException(Exception): + def __init__(self, message: str = ...) -> None: ... + +class JavaSyntaxError(JavaParserBaseException): + description: Any = ... + at: Any = ... + def __init__(self, description: Any, at: Optional[Any] = ...) -> None: ... + +class JavaParserError(JavaParserBaseException): ... + +class Parser: + operator_precedence: Any = ... + tokens: Any = ... + debug: bool = ... + def __init__(self, tokens: Any) -> None: ... + def set_debug(self, debug: bool = ...) -> None: ... + def parse(self): ... + def illegal(self, description: Any, at: Optional[Any] = ...) -> None: ... + def accept(self, *accepts: Any): ... + def would_accept(self, *accepts: Any): ... + def try_accept(self, *accepts: Any): ... + def build_binary_operation(self, parts: Any, start_level: int = ...): ... + def is_annotation(self, i: int = ...): ... + def is_annotation_declaration(self, i: int = ...): ... + def parse_identifier(self): ... + def parse_qualified_identifier(self): ... + def parse_qualified_identifier_list(self): ... + def parse_compilation_unit(self): ... + def parse_import_declaration(self): ... + def parse_type_declaration(self): ... + def parse_class_or_interface_declaration(self): ... + def parse_normal_class_declaration(self): ... + def parse_enum_declaration(self): ... + def parse_normal_interface_declaration(self): ... + def parse_annotation_type_declaration(self): ... + def parse_type(self): ... + def parse_basic_type(self): ... + def parse_reference_type(self): ... + def parse_type_arguments(self): ... + def parse_type_argument(self): ... + def parse_nonwildcard_type_arguments(self): ... + def parse_type_list(self): ... + def parse_type_arguments_or_diamond(self): ... + def parse_nonwildcard_type_arguments_or_diamond(self): ... + def parse_type_parameters(self): ... + def parse_type_parameter(self): ... + def parse_array_dimension(self): ... + def parse_modifiers(self): ... + def parse_annotations(self): ... + def parse_annotation(self): ... + def parse_annotation_element(self): ... + def parse_element_value_pairs(self): ... + def parse_element_value_pair(self): ... + def parse_element_value(self): ... + def parse_element_value_array_initializer(self): ... + def parse_element_values(self): ... + def parse_class_body(self): ... + def parse_class_body_declaration(self): ... + def parse_member_declaration(self): ... + def parse_method_or_field_declaraction(self): ... + def parse_method_or_field_rest(self): ... + def parse_field_declarators_rest(self): ... + def parse_method_declarator_rest(self): ... + def parse_void_method_declarator_rest(self): ... + def parse_constructor_declarator_rest(self): ... + def parse_generic_method_or_constructor_declaration(self): ... + def parse_interface_body(self): ... + def parse_interface_body_declaration(self): ... + def parse_interface_member_declaration(self): ... + def parse_interface_method_or_field_declaration(self): ... + def parse_interface_method_or_field_rest(self): ... + def parse_constant_declarators_rest(self): ... + def parse_constant_declarator_rest(self): ... + def parse_constant_declarator(self): ... + def parse_interface_method_declarator_rest(self): ... + def parse_void_interface_method_declarator_rest(self): ... + def parse_interface_generic_method_declarator(self): ... + def parse_formal_parameters(self): ... + def parse_variable_modifiers(self): ... + def parse_variable_declators(self): ... + def parse_variable_declarators(self): ... + def parse_variable_declarator(self): ... + def parse_variable_declarator_rest(self): ... + def parse_variable_initializer(self): ... + def parse_array_initializer(self): ... + def parse_block(self): ... + def parse_block_statement(self): ... + def parse_local_variable_declaration_statement(self): ... + def parse_statement(self): ... + def parse_catches(self): ... + def parse_catch_clause(self): ... + def parse_resource_specification(self): ... + def parse_resource(self): ... + def parse_switch_block_statement_groups(self): ... + def parse_switch_block_statement_group(self): ... + def parse_for_control(self): ... + def parse_for_var_control(self): ... + def parse_for_var_control_rest(self): ... + def parse_for_variable_declarator_rest(self): ... + def parse_for_init_or_update(self): ... + def parse_expression(self): ... + def parse_expressionl(self): ... + def parse_expression_2(self): ... + def parse_expression_2_rest(self): ... + def parse_expression_3(self): ... + def parse_method_reference(self): ... + def parse_lambda_expression(self): ... + def parse_lambda_method_body(self): ... + def parse_infix_operator(self): ... + def parse_primary(self): ... + def parse_literal(self): ... + def parse_par_expression(self): ... + def parse_arguments(self): ... + def parse_super_suffix(self): ... + def parse_explicit_generic_invocation_suffix(self): ... + def parse_creator(self): ... + def parse_created_name(self): ... + def parse_class_creator_rest(self): ... + def parse_array_creator_rest(self): ... + def parse_identifier_suffix(self): ... + def parse_explicit_generic_invocation(self): ... + def parse_inner_creator(self): ... + def parse_selector(self): ... + def parse_enum_body(self): ... + def parse_enum_constant(self): ... + def parse_annotation_type_body(self): ... + def parse_annotation_type_element_declarations(self): ... + def parse_annotation_type_element_declaration(self): ... + def parse_annotation_method_or_constant_rest(self): ... + +def parse(tokens: Any, debug: bool = ...): ... diff --git a/stubs/javalang/tokenizer.pyi b/stubs/javalang/tokenizer.pyi new file mode 100644 index 0000000..89475b6 --- /dev/null +++ b/stubs/javalang/tokenizer.pyi @@ -0,0 +1,100 @@ +from collections import namedtuple +from typing import Any, Optional + +class LexerError(Exception): ... + +Position = namedtuple('Position', ['line', 'column']) + +class JavaToken: + value: Any = ... + position: Any = ... + javadoc: Any = ... + def __init__(self, value: Any, position: Optional[Any] = ..., javadoc: Optional[Any] = ...) -> None: ... + def __eq__(self, other: Any) -> Any: ... + +class EndOfInput(JavaToken): ... + +class Keyword(JavaToken): + VALUES: Any = ... + +class Modifier(Keyword): + VALUES: Any = ... + +class BasicType(Keyword): + VALUES: Any = ... + +class Literal(JavaToken): ... +class Integer(Literal): ... +class DecimalInteger(Literal): ... +class OctalInteger(Integer): ... +class BinaryInteger(Integer): ... +class HexInteger(Integer): ... +class FloatingPoint(Literal): ... +class DecimalFloatingPoint(FloatingPoint): ... +class HexFloatingPoint(FloatingPoint): ... + +class Boolean(Literal): + VALUES: Any = ... + +class Character(Literal): ... +class String(Literal): ... +class Null(Literal): ... + +class Separator(JavaToken): + VALUES: Any = ... + +class Operator(JavaToken): + MAX_LEN: int = ... + VALUES: Any = ... + INFIX: Any = ... + PREFIX: Any = ... + POSTFIX: Any = ... + ASSIGNMENT: Any = ... + LAMBDA: Any = ... + METHOD_REFERENCE: Any = ... + def is_infix(self): ... + def is_prefix(self): ... + def is_postfix(self): ... + def is_assignment(self): ... + +class Annotation(JavaToken): ... +class Identifier(JavaToken): ... + +class JavaTokenizer: + IDENT_START_CATEGORIES: Any = ... + IDENT_PART_CATEGORIES: Any = ... + data: Any = ... + ignore_errors: Any = ... + errors: Any = ... + current_line: int = ... + start_of_line: int = ... + operators: Any = ... + whitespace_consumer: Any = ... + javadoc: Any = ... + def __init__(self, data: Any, ignore_errors: bool = ...) -> None: ... + i: int = ... + j: int = ... + def reset(self) -> None: ... + def consume_whitespace(self) -> None: ... + def read_string(self) -> None: ... + def try_operator(self): ... + def read_comment(self): ... + def read_decimal_float_or_integer(self): ... + def read_hex_integer_or_float(self): ... + def read_digits(self, digits: Any) -> None: ... + def read_decimal_integer(self) -> None: ... + def read_hex_integer(self) -> None: ... + def read_bin_integer(self) -> None: ... + def read_octal_integer(self) -> None: ... + def read_integer_or_float(self, c: Any, c_next: Any): ... + def try_separator(self): ... + def decode_data(self): ... + def is_java_identifier_start(self, c: Any): ... + def read_identifier(self): ... + length: Any = ... + def pre_tokenize(self) -> None: ... + def tokenize(self) -> None: ... + def error(self, message: Any, char: Optional[Any] = ...) -> None: ... + +def tokenize(code: Any, ignore_errors: bool = ...): ... +def reformat_tokens(tokens: Any): ... diff --git a/stubs/javalang/tree.pyi b/stubs/javalang/tree.pyi new file mode 100644 index 0000000..726661b --- /dev/null +++ b/stubs/javalang/tree.pyi @@ -0,0 +1,279 @@ +from .ast import Node as Node +from typing import Any, Set, List, Union + +class CompilationUnit(Node): + attrs: Any = ... + imports: Any = ... + +class Import(Node): + attrs: Any = ... + +class Documented(Node): + attrs: Any = ... + +class Declaration(Node): + attrs: Any = ... + modifiers: Set[str] = ... + +class TypeDeclaration(Declaration, Documented): + attrs: Any = ... + @property + def fields(self): ... + @property + def methods(self): ... + @property + def constructors(self): ... + name: str = ... + body: Any = ... + +class PackageDeclaration(Declaration, Documented): + attrs: Any = ... + +class ClassDeclaration(TypeDeclaration): + attrs: Any = ... + implements: List[ReferenceType] = ... + +class EnumDeclaration(TypeDeclaration): + attrs: Any = ... + @property + def fields(self): ... + @property + def methods(self): ... + +class InterfaceDeclaration(TypeDeclaration): + attrs: Any = ... + +class AnnotationDeclaration(TypeDeclaration): + attrs: Any = ... + +class Type(Node): + attrs: Any = ... + name: str = ... + +class BasicType(Type): + attrs: Any = ... + +class ReferenceType(Type): + attrs: Any = ... + +class TypeArgument(Node): + attrs: Any = ... + +class TypeParameter(Node): + attrs: Any = ... + +class Annotation(Node): + attrs: Any = ... + +class ElementValuePair(Node): + attrs: Any = ... + +class ElementArrayValue(Node): + attrs: Any = ... + +class Member(Documented): + attrs: Any = ... + +class MethodDeclaration(Member, Declaration): + attrs: Any = ... + name: str = ... + parameters: List[Union[FormalParameter, InferredFormalParameter]] = ... + return_type: ReferenceType = ... + body: Any = ... + +class FieldDeclaration(Member, Declaration): + attrs: Any = ... + declarators: Any = ... + type: Any = ... + +class ConstructorDeclaration(Declaration, Documented): + attrs: Any = ... + +class ConstantDeclaration(FieldDeclaration): + attrs: Any = ... + +class ArrayInitializer(Node): + attrs: Any = ... + +class VariableDeclaration(Declaration): + attrs: Any = ... + +class LocalVariableDeclaration(VariableDeclaration): + attrs: Any = ... + declarators: Any = ... + type: Any = ... + +class VariableDeclarator(Node): + attrs: Any = ... + name: str = ... + initializer: Any = ... + +class FormalParameter(Declaration): + attrs: Any = ... + type: ReferenceType = ... + name: str = ... + +class InferredFormalParameter(Node): + attrs: Any = ... + name: str = ... + type: Any = ... + +class Statement(Node): + attrs: Any = ... + +class IfStatement(Statement): + attrs: Any = ... + condition: Any = ... + then_statement: Any = ... + else_statement: Any = ... + +class WhileStatement(Statement): + attrs: Any = ... + +class DoStatement(Statement): + attrs: Any = ... + +class ForStatement(Statement): + attrs: Any = ... + +class AssertStatement(Statement): + attrs: Any = ... + condition: Expression = ... + +class BreakStatement(Statement): + attrs: Any = ... + +class ContinueStatement(Statement): + attrs: Any = ... + +class ReturnStatement(Statement): + attrs: Any = ... + +class ThrowStatement(Statement): + attrs: Any = ... + +class SynchronizedStatement(Statement): + attrs: Any = ... + +class TryStatement(Statement): + attrs: Any = ... + +class SwitchStatement(Statement): + attrs: Any = ... + +class BlockStatement(Statement): + attrs: Any = ... + +class StatementExpression(Statement): + attrs: Any = ... + expression: Expression = ... + +class TryResource(Declaration): + attrs: Any = ... + +class CatchClause(Statement): + attrs: Any = ... + +class CatchClauseParameter(Declaration): + attrs: Any = ... + +class SwitchStatementCase(Node): + attrs: Any = ... + +class ForControl(Node): + attrs: Any = ... + +class EnhancedForControl(Node): + attrs: Any = ... + +class Expression(Node): + attrs: Any = ... + +class Assignment(Expression): + attrs: Any = ... + expressionl: Union[This, MemberReference] + +class TernaryExpression(Expression): + attrs: Any = ... + if_true: Expression = ... + if_false: Expression = ... + +class BinaryOperation(Expression): + attrs: Any = ... + operator: str = ... + operandl: Expression = ... + operandr: Expression = ... + +class Cast(Expression): + attrs: Any = ... + +class MethodReference(Expression): + attrs: Any = ... + +class LambdaExpression(Expression): + attrs: Any = ... + +class Primary(Expression): + attrs: Any = ... + +class Literal(Primary): + attrs: Any = ... + value: str = ... + +class This(Primary): + attrs: Any = ... + selectors: Any = ... + +class MemberReference(Primary): + attrs: Any = ... + member: str = ... +class Invocation(Primary): + attrs: Any = ... + +class ExplicitConstructorInvocation(Invocation): + attrs: Any = ... + +class SuperConstructorInvocation(Invocation): + attrs: Any = ... + +class MethodInvocation(Invocation): + attrs: Any = ... + member: str = ... + qualifier: Any = ... + +class SuperMethodInvocation(Invocation): + attrs: Any = ... + member: str = ... + +class SuperMemberReference(Primary): + attrs: Any = ... + +class ArraySelector(Expression): + attrs: Any = ... + +class ClassReference(Primary): + attrs: Any = ... + +class VoidClassReference(ClassReference): + attrs: Any = ... + +class Creator(Primary): + attrs: Any = ... + +class ArrayCreator(Creator): + attrs: Any = ... + +class ClassCreator(Creator): + attrs: Any = ... + +class InnerClassCreator(Creator): + attrs: Any = ... + +class EnumBody(Node): + attrs: Any = ... + +class EnumConstantDeclaration(Declaration, Documented): + attrs: Any = ... + +class AnnotationMethod(Declaration): + attrs: Any = ... diff --git a/stubs/javalang/util.pyi b/stubs/javalang/util.pyi new file mode 100644 index 0000000..8f48789 --- /dev/null +++ b/stubs/javalang/util.pyi @@ -0,0 +1,37 @@ +from typing import Any + +class LookAheadIterator: + iterable: Any = ... + look_ahead: Any = ... + markers: Any = ... + default: Any = ... + value: Any = ... + def __init__(self, iterable: Any) -> None: ... + def __iter__(self) -> Any: ... + def set_default(self, value: Any) -> None: ... + def next(self): ... + def __next__(self): ... + def look(self, i: int = ...): ... + def last(self): ... + def __enter__(self): ... + def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: ... + def push_marker(self) -> None: ... + def pop_marker(self, reset: Any) -> None: ... + +class LookAheadListIterator: + list: Any = ... + marker: int = ... + saved_markers: Any = ... + default: Any = ... + value: Any = ... + def __init__(self, iterable: Any) -> None: ... + def __iter__(self) -> Any: ... + def set_default(self, value: Any) -> None: ... + def next(self): ... + def __next__(self): ... + def look(self, i: int = ...): ... + def last(self): ... + def __enter__(self): ... + def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: ... + def push_marker(self) -> None: ... + def pop_marker(self, reset: Any) -> None: ... diff --git a/temp_python/__init__.py b/temp_python/__init__.py new file mode 100644 index 0000000..ef3bfbe --- /dev/null +++ b/temp_python/__init__.py @@ -0,0 +1,8 @@ +""" +Veniq uses Machine Learning to analyze source code, +find possible refactorings, and suggest those that seem optimal +""" + +__version__ = '0.0.0' +__author__ = 'Me' +__licence__ = 'MIT' diff --git a/temp_python/ast_framework/__init__.py b/temp_python/ast_framework/__init__.py new file mode 100644 index 0000000..efb0d5b --- /dev/null +++ b/temp_python/ast_framework/__init__.py @@ -0,0 +1,10 @@ +from temp_python.ast_framework.ast_node_type import ASTNodeType # noqa: F401 +from temp_python.ast_framework.ast_node import ASTNode # noqa: F401 +from temp_python.ast_framework.ast import AST # noqa: F401 + +# register all standard computed fields from 'computed_fields_catalog' +from temp_python.ast_framework.computed_fields_catalog.standard_fields import ( + register_standard_computed_properties, +) + +register_standard_computed_properties() diff --git a/temp_python/ast_framework/_auxiliary_data.py b/temp_python/ast_framework/_auxiliary_data.py new file mode 100644 index 0000000..d37616b --- /dev/null +++ b/temp_python/ast_framework/_auxiliary_data.py @@ -0,0 +1,419 @@ +# The MIT License (MIT) +# +# Copyright (c) 2020 Aibolit +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from typing import Dict, Set, Type, NamedTuple + +from javalang import tree +from javalang.ast import Node + +from temp_python.ast_framework import ASTNodeType + + +class ASTNodeReference(NamedTuple): + node_index: int + + +javalang_to_ast_node_type: Dict[Type[Node], ASTNodeType] = { + tree.Annotation: ASTNodeType.ANNOTATION, + tree.AnnotationDeclaration: ASTNodeType.ANNOTATION_DECLARATION, + tree.AnnotationMethod: ASTNodeType.ANNOTATION_METHOD, + tree.ArrayCreator: ASTNodeType.ARRAY_CREATOR, + tree.ArrayInitializer: ASTNodeType.ARRAY_INITIALIZER, + tree.ArraySelector: ASTNodeType.ARRAY_SELECTOR, + tree.AssertStatement: ASTNodeType.ASSERT_STATEMENT, + tree.Assignment: ASTNodeType.ASSIGNMENT, + tree.BasicType: ASTNodeType.BASIC_TYPE, + tree.BinaryOperation: ASTNodeType.BINARY_OPERATION, + tree.BlockStatement: ASTNodeType.BLOCK_STATEMENT, + tree.BreakStatement: ASTNodeType.BREAK_STATEMENT, + tree.Cast: ASTNodeType.CAST, + tree.CatchClause: ASTNodeType.CATCH_CLAUSE, + tree.CatchClauseParameter: ASTNodeType.CATCH_CLAUSE_PARAMETER, + tree.ClassCreator: ASTNodeType.CLASS_CREATOR, + tree.ClassDeclaration: ASTNodeType.CLASS_DECLARATION, + tree.ClassReference: ASTNodeType.CLASS_REFERENCE, + tree.CompilationUnit: ASTNodeType.COMPILATION_UNIT, + tree.ConstantDeclaration: ASTNodeType.CONSTANT_DECLARATION, + tree.ConstructorDeclaration: ASTNodeType.CONSTRUCTOR_DECLARATION, + tree.ContinueStatement: ASTNodeType.CONTINUE_STATEMENT, + tree.Creator: ASTNodeType.CREATOR, + tree.Declaration: ASTNodeType.DECLARATION, + tree.Documented: ASTNodeType.DOCUMENTED, + tree.DoStatement: ASTNodeType.DO_STATEMENT, + tree.ElementArrayValue: ASTNodeType.ELEMENT_ARRAY_VALUE, + tree.ElementValuePair: ASTNodeType.ELEMENT_VALUE_PAIR, + tree.EnhancedForControl: ASTNodeType.ENHANCED_FOR_CONTROL, + tree.EnumBody: ASTNodeType.ENUM_BODY, + tree.EnumConstantDeclaration: ASTNodeType.ENUM_CONSTANT_DECLARATION, + tree.EnumDeclaration: ASTNodeType.ENUM_DECLARATION, + tree.ExplicitConstructorInvocation: ASTNodeType.EXPLICIT_CONSTRUCTOR_INVOCATION, + tree.Expression: ASTNodeType.EXPRESSION, + tree.FieldDeclaration: ASTNodeType.FIELD_DECLARATION, + tree.ForControl: ASTNodeType.FOR_CONTROL, + tree.FormalParameter: ASTNodeType.FORMAL_PARAMETER, + tree.ForStatement: ASTNodeType.FOR_STATEMENT, + tree.IfStatement: ASTNodeType.IF_STATEMENT, + tree.Import: ASTNodeType.IMPORT, + tree.InferredFormalParameter: ASTNodeType.INFERRED_FORMAL_PARAMETER, + tree.InnerClassCreator: ASTNodeType.INNER_CLASS_CREATOR, + tree.InterfaceDeclaration: ASTNodeType.INTERFACE_DECLARATION, + tree.Invocation: ASTNodeType.INVOCATION, + tree.LambdaExpression: ASTNodeType.LAMBDA_EXPRESSION, + tree.Literal: ASTNodeType.LITERAL, + tree.LocalVariableDeclaration: ASTNodeType.LOCAL_VARIABLE_DECLARATION, + tree.Member: ASTNodeType.MEMBER, + tree.MemberReference: ASTNodeType.MEMBER_REFERENCE, + tree.MethodDeclaration: ASTNodeType.METHOD_DECLARATION, + tree.MethodInvocation: ASTNodeType.METHOD_INVOCATION, + tree.MethodReference: ASTNodeType.METHOD_REFERENCE, + tree.PackageDeclaration: ASTNodeType.PACKAGE_DECLARATION, + tree.Primary: ASTNodeType.PRIMARY, + tree.ReferenceType: ASTNodeType.REFERENCE_TYPE, + tree.ReturnStatement: ASTNodeType.RETURN_STATEMENT, + tree.Statement: ASTNodeType.STATEMENT, + tree.StatementExpression: ASTNodeType.STATEMENT_EXPRESSION, + tree.SuperConstructorInvocation: ASTNodeType.SUPER_CONSTRUCTOR_INVOCATION, + tree.SuperMemberReference: ASTNodeType.SUPER_MEMBER_REFERENCE, + tree.SuperMethodInvocation: ASTNodeType.SUPER_METHOD_INVOCATION, + tree.SwitchStatement: ASTNodeType.SWITCH_STATEMENT, + tree.SwitchStatementCase: ASTNodeType.SWITCH_STATEMENT_CASE, + tree.SynchronizedStatement: ASTNodeType.SYNCHRONIZED_STATEMENT, + tree.TernaryExpression: ASTNodeType.TERNARY_EXPRESSION, + tree.This: ASTNodeType.THIS, + tree.ThrowStatement: ASTNodeType.THROW_STATEMENT, + tree.TryResource: ASTNodeType.TRY_RESOURCE, + tree.TryStatement: ASTNodeType.TRY_STATEMENT, + tree.Type: ASTNodeType.TYPE, + tree.TypeArgument: ASTNodeType.TYPE_ARGUMENT, + tree.TypeDeclaration: ASTNodeType.TYPE_DECLARATION, + tree.TypeParameter: ASTNodeType.TYPE_PARAMETER, + tree.VariableDeclaration: ASTNodeType.VARIABLE_DECLARATION, + tree.VariableDeclarator: ASTNodeType.VARIABLE_DECLARATOR, + tree.VoidClassReference: ASTNodeType.VOID_CLASS_REFERENCE, + tree.WhileStatement: ASTNodeType.WHILE_STATEMENT, +} + +common_attributes: Set[str] = {'node_type'} + +attributes_by_node_type: Dict[ASTNodeType, Set[str]] = { + ASTNodeType.ANNOTATION_DECLARATION: { + "annotations", + "body", + "documentation", + "modifiers", + "name", + }, + ASTNodeType.ANNOTATION_METHOD: { + "annotations", + "default", + "dimensions", + "modifiers", + "name", + "return_type", + }, + ASTNodeType.ANNOTATION: {"element", "name"}, + ASTNodeType.ARRAY_CREATOR: { + "dimensions", + "initializer", + "postfix_operators", + "prefix_operators", + "qualifier", + "selectors", + "type", + }, + ASTNodeType.ARRAY_INITIALIZER: {"initializers"}, + ASTNodeType.ARRAY_SELECTOR: {"index"}, + ASTNodeType.ASSERT_STATEMENT: {"condition", "label", "value"}, + ASTNodeType.ASSIGNMENT: {"expressionl", "type", "value"}, + ASTNodeType.BASIC_TYPE: {"dimensions", "name"}, + ASTNodeType.BINARY_OPERATION: {"operandl", "operandr", "operator"}, + ASTNodeType.BLOCK_STATEMENT: {"label", "statements"}, + ASTNodeType.BREAK_STATEMENT: {"goto", "label"}, + ASTNodeType.CAST: {"expression", "type"}, + ASTNodeType.CATCH_CLAUSE_PARAMETER: {"annotations", "name", "modifiers", "types"}, + ASTNodeType.CATCH_CLAUSE: {"block", "label", "parameter"}, + ASTNodeType.CLASS_CREATOR: { + "arguments", + "body", + "constructor_type_arguments", + "postfix_operators", + "prefix_operators", + "qualifier", + "selectors", + "type", + }, + ASTNodeType.CLASS_DECLARATION: { + "annotations", + "body", + "documentation", + "extends", + "implements", + "modifiers", + "name", + "type_parameters", + }, + ASTNodeType.CLASS_REFERENCE: { + "postfix_operators", + "prefix_operators", + "qualifier", + "selectors", + "type", + }, + ASTNodeType.COLLECTION: set(), + ASTNodeType.COMPILATION_UNIT: {"imports", "package", "types"}, + ASTNodeType.CONSTANT_DECLARATION: { + "annotations", + "declarators", + "documentation", + "modifiers", + "type", + }, + ASTNodeType.CONSTRUCTOR_DECLARATION: { + "annotations", + "body", + "documentation", + "modifiers", + "name", + "parameters", + "throws", + "type_parameters", + }, + ASTNodeType.CONTINUE_STATEMENT: {"goto", "label"}, + ASTNodeType.CREATOR: { + "postfix_operators", + "prefix_operators", + "qualifier", + "selectors", + "type", + }, + ASTNodeType.DECLARATION: {"annotations", "modifiers"}, + ASTNodeType.DO_STATEMENT: {"body", "condition", "label"}, + ASTNodeType.DOCUMENTED: {"documentation"}, + ASTNodeType.ELEMENT_ARRAY_VALUE: {"values"}, + ASTNodeType.ELEMENT_VALUE_PAIR: {"name", "value"}, + ASTNodeType.ENHANCED_FOR_CONTROL: {"iterable", "var"}, + ASTNodeType.ENUM_BODY: {"constants", "declarations"}, + ASTNodeType.ENUM_CONSTANT_DECLARATION: { + "annotations", + "arguments", + "body", + "documentation", + "modifiers", + "name", + }, + ASTNodeType.ENUM_DECLARATION: { + "annotations", + "body", + "documentation", + "implements", + "modifiers", + "name", + }, + ASTNodeType.EXPLICIT_CONSTRUCTOR_INVOCATION: { + "arguments", + "postfix_operators", + "prefix_operators", + "qualifier", + "selectors", + "type_arguments", + }, + ASTNodeType.EXPRESSION: set(), + ASTNodeType.FIELD_DECLARATION: { + "annotations", + "declarators", + "documentation", + "modifiers", + "type", + }, + ASTNodeType.FOR_CONTROL: {"condition", "init", "update"}, + ASTNodeType.FOR_STATEMENT: {"body", "control", "label"}, + ASTNodeType.FORMAL_PARAMETER: { + "annotations", + "modifiers", + "name", + "type", + "varargs", + }, + ASTNodeType.IF_STATEMENT: { + "condition", + "else_statement", + "label", + "then_statement", + }, + ASTNodeType.IMPORT: {"path", "static", "wildcard"}, + ASTNodeType.INFERRED_FORMAL_PARAMETER: {"name"}, + ASTNodeType.INNER_CLASS_CREATOR: { + "arguments", + "body", + "constructor_type_arguments", + "postfix_operators", + "prefix_operators", + "qualifier", + "selectors", + "type", + }, + ASTNodeType.INTERFACE_DECLARATION: { + "annotations", + "body", + "documentation", + "extends", + "modifiers", + "name", + "type_parameters", + }, + ASTNodeType.INVOCATION: { + "arguments", + "postfix_operators", + "prefix_operators", + "qualifier", + "selectors", + "type_arguments", + }, + ASTNodeType.LAMBDA_EXPRESSION: {"body", "parameters"}, + ASTNodeType.LITERAL: { + "postfix_operators", + "prefix_operators", + "qualifier", + "selectors", + "value", + }, + ASTNodeType.LOCAL_VARIABLE_DECLARATION: { + "annotations", + "declarators", + "modifiers", + "type", + }, + ASTNodeType.MEMBER_REFERENCE: { + "member", + "postfix_operators", + "prefix_operators", + "qualifier", + "selectors", + }, + ASTNodeType.MEMBER: {"documentation"}, + ASTNodeType.METHOD_DECLARATION: { + "annotations", + "body", + "documentation", + "modifiers", + "name", + "parameters", + "return_type", + "throws", + "type_parameters", + }, + ASTNodeType.METHOD_INVOCATION: { + "arguments", + "member", + "postfix_operators", + "prefix_operators", + "qualifier", + "selectors", + "type_arguments", + }, + ASTNodeType.METHOD_REFERENCE: {"expression", "method", "type_arguments"}, + ASTNodeType.PACKAGE_DECLARATION: { + "annotations", + "documentation", + "modifiers", + "name", + }, + ASTNodeType.PRIMARY: { + "postfix_operators", + "prefix_operators", + "qualifier", + "selectors", + }, + ASTNodeType.REFERENCE_TYPE: {"arguments", "dimensions", "name", "sub_type"}, + ASTNodeType.RETURN_STATEMENT: {"expression", "label"}, + ASTNodeType.STATEMENT_EXPRESSION: {"expression", "label"}, + ASTNodeType.STATEMENT: {"label"}, + ASTNodeType.STRING: {"string"}, + ASTNodeType.SUPER_CONSTRUCTOR_INVOCATION: { + "arguments", + "postfix_operators", + "prefix_operators", + "qualifier", + "selectors", + "type_arguments", + }, + ASTNodeType.SUPER_MEMBER_REFERENCE: { + "member", + "postfix_operators", + "prefix_operators", + "qualifier", + "selectors", + }, + ASTNodeType.SUPER_METHOD_INVOCATION: { + "arguments", + "member", + "postfix_operators", + "prefix_operators", + "qualifier", + "selectors", + "type_arguments", + }, + ASTNodeType.SWITCH_STATEMENT_CASE: {"case", "statements"}, + ASTNodeType.SWITCH_STATEMENT: {"cases", "expression", "label"}, + ASTNodeType.SYNCHRONIZED_STATEMENT: {"block", "label", "lock"}, + ASTNodeType.TERNARY_EXPRESSION: {"condition", "if_false", "if_true"}, + ASTNodeType.THIS: { + "postfix_operators", + "prefix_operators", + "qualifier", + "selectors", + }, + ASTNodeType.THROW_STATEMENT: {"expression", "label"}, + ASTNodeType.TRY_RESOURCE: {"annotations", "name", "modifiers", "type", "value"}, + ASTNodeType.TRY_STATEMENT: { + "block", + "catches", + "finally_block", + "label", + "resources", + }, + ASTNodeType.TYPE_ARGUMENT: {"pattern_type", "type"}, + ASTNodeType.TYPE_DECLARATION: { + "annotations", + "body", + "documentation", + "modifiers", + "name", + }, + ASTNodeType.TYPE_PARAMETER: {"extends", "name"}, + ASTNodeType.TYPE: {"dimensions", "name"}, + ASTNodeType.VARIABLE_DECLARATION: { + "annotations", + "declarators", + "modifiers", + "type", + }, + ASTNodeType.VARIABLE_DECLARATOR: {"dimensions", "initializer", "name"}, + ASTNodeType.VOID_CLASS_REFERENCE: { + "postfix_operators", + "prefix_operators", + "qualifier", + "selectors", + "type", + }, + ASTNodeType.WHILE_STATEMENT: {"body", "condition", "label"}, +} diff --git a/temp_python/ast_framework/ast.py b/temp_python/ast_framework/ast.py new file mode 100644 index 0000000..280e8ca --- /dev/null +++ b/temp_python/ast_framework/ast.py @@ -0,0 +1,383 @@ +# The MIT License (MIT) +# +# Copyright (c) 2020 Aibolit +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from collections import namedtuple +from itertools import islice, repeat, chain + +from deprecated import deprecated # type: ignore +from javalang.tree import Node +from networkx import DiGraph, dfs_labeled_edges, dfs_preorder_nodes # type: ignore +from typing import Union, Any, Callable, Set, List, \ + Iterator, Tuple, Dict, cast, Optional + +from temp_python.ast_framework.ast_node_type import ASTNodeType +from temp_python.ast_framework._auxiliary_data import \ + javalang_to_ast_node_type, attributes_by_node_type, ASTNodeReference +from temp_python.ast_framework.ast_node import ASTNode + +MethodInvocationParams = namedtuple('MethodInvocationParams', ['object_name', 'method_name']) + +MemberReferenceParams = namedtuple('MemberReferenceParams', ('object_name', 'member_name', 'unary_operator')) + +BinaryOperationParams = namedtuple('BinaryOperationParams', ('operation', 'left_side', 'right_side')) + +TraverseCallback = Callable[[ASTNode], None] + + +class AST: + def __init__(self, networkx_tree: DiGraph, root: int): + self.tree = networkx_tree + self.root = root + + @staticmethod + def build_from_javalang(javalang_ast_root: Node) -> 'AST': + tree = DiGraph() + javalang_node_to_index_map: Dict[Node, int] = {} + root = AST._add_subtree_from_javalang_node(tree, javalang_ast_root, + javalang_node_to_index_map) + AST._replace_javalang_nodes_in_attributes(tree, javalang_node_to_index_map) + return AST(tree, root) + + def __str__(self) -> str: + printed_graph = '' + depth = 0 + for _, destination, edge_type in dfs_labeled_edges(self.tree, self.root): + if edge_type == 'forward': + printed_graph += '| ' * depth + node_type = self.tree.nodes[destination]['node_type'] + printed_graph += str(node_type) + ': ' + if node_type == ASTNodeType.STRING: + printed_graph += self.tree.nodes[destination]['string'] + ', ' + printed_graph += f'node index = {destination}' + node_line = self.tree.nodes[destination]['line'] + if node_line is not None: + printed_graph += f', line = {node_line}' + printed_graph += '\n' + depth += 1 + elif edge_type == 'reverse': + depth -= 1 + return printed_graph + + def get_root(self) -> ASTNode: + return ASTNode(self.tree, self.root) + + def __iter__(self) -> Iterator[ASTNode]: + for node_index in self.tree.nodes: + yield ASTNode(self.tree, node_index) + + def get_subtrees(self, *root_type: ASTNodeType) -> Iterator['AST']: + ''' + Yields subtrees with given type of the root. + If such subtrees are one including the other, only the larger one is + going to be in resulted sequence. + ''' + is_inside_subtree = False + current_subtree_root = -1 # all node indexes are positive + subtree: List[int] = [] + for _, destination, edge_type in dfs_labeled_edges(self.tree, self.root): + if edge_type == 'forward': + if is_inside_subtree: + subtree.append(destination) + elif self.tree.nodes[destination]['node_type'] in root_type: + subtree.append(destination) + is_inside_subtree = True + current_subtree_root = destination + elif edge_type == 'reverse' and destination == current_subtree_root: + is_inside_subtree = False + yield AST(self.tree.subgraph(subtree), current_subtree_root) + subtree = [] + current_subtree_root = -1 + + def get_subtree(self, node: ASTNode) -> 'AST': + subtree_nodes_indexes = dfs_preorder_nodes(self.tree, node.node_index) + subtree = self.tree.subgraph(subtree_nodes_indexes) + return AST(subtree, node.node_index) + + def traverse( + self, + on_node_entering: TraverseCallback, + on_node_leaving: TraverseCallback = lambda node: None, + source_node: Optional[ASTNode] = None, + undirected=False + ): + traverse_graph = self.tree.to_undirected(as_view=True) if undirected else self.tree + if source_node is None: + source_node = self.get_root() + + for _, destination, edge_type in dfs_labeled_edges(traverse_graph, source_node.node_index): + if edge_type == "forward": + on_node_entering(ASTNode(self.tree, destination)) + elif edge_type == "reverse": + on_node_leaving(ASTNode(self.tree, destination)) + + @deprecated(reason='Use ASTNode functionality instead.') + def children_with_type(self, node: int, child_type: ASTNodeType) -> Iterator[int]: + ''' + Yields children of node with given type. + ''' + for child in self.tree.succ[node]: + if self.tree.nodes[child]['node_type'] == child_type: + yield child + + @deprecated(reason='Use ASTNode functionality instead.') + def list_all_children_with_type(self, node: int, child_type: ASTNodeType) -> List[int]: + list_node: List[int] = [] + for child in self.tree.succ[node]: + list_node = list_node + self.list_all_children_with_type(child, child_type) + if self.tree.nodes[child]['node_type'] == child_type: + list_node.append(child) + return sorted(list_node) + + @deprecated(reason='Use ASTNode functionality instead.') + def all_children_with_type(self, node: int, child_type: ASTNodeType) -> Iterator[int]: + ''' + Yields all children of node with given type. + ''' + for child in self.list_all_children_with_type(node, child_type): + yield child + + @deprecated(reason='Use ASTNode functionality instead.') + def get_first_n_children_with_type(self, node: int, child_type: ASTNodeType, quantity: int) -> List[int]: + ''' + Returns first quantity of children of node with type child_type. + Resulted list is padded with None to length quantity. + ''' + children_with_type = (child for child in self.tree.succ[node] if self.get_type(child) == child_type) + children_with_type_padded = chain(children_with_type, repeat(None)) + return list(islice(children_with_type_padded, 0, quantity)) + + @deprecated(reason='Use ASTNode functionality instead.') + def get_binary_operation_name(self, node: int) -> str: + assert(self.get_type(node) == ASTNodeType.BINARY_OPERATION) + name_node, = islice(self.children_with_type(node, ASTNodeType.STRING), 1) + return self.get_attr(name_node, 'string') + + @deprecated(reason='Use ASTNode functionality instead.') + def get_line_number_from_children(self, node: int) -> int: + for child in self.tree.succ[node]: + cur_line = self.get_attr(child, 'line') + if cur_line is not None: + return cur_line + return 0 + + @deprecated(reason='Use get_proxy_nodes instead.') + def get_nodes(self, type: Union[ASTNodeType, None] = None) -> Iterator[int]: + for node in self.tree.nodes: + if type is None or self.tree.nodes[node]['node_type'] == type: + yield node + + def get_proxy_nodes(self, *types: ASTNodeType) -> Iterator[ASTNode]: + for node in self.tree.nodes: + if len(types) == 0 or self.tree.nodes[node]['node_type'] in types: + yield ASTNode(self.tree, node) + + @deprecated(reason='Use ASTNode functionality instead.') + def get_attr(self, node: int, attr_name: str, default_value: Any = None) -> Any: + return self.tree.nodes[node].get(attr_name, default_value) + + @deprecated(reason='Use ASTNode functionality instead.') + def get_type(self, node: int) -> ASTNodeType: + return self.get_attr(node, 'node_type') + + @deprecated(reason='Use ASTNode functionality instead.') + def get_method_invocation_params(self, invocation_node: int) -> MethodInvocationParams: + assert(self.get_type(invocation_node) == ASTNodeType.METHOD_INVOCATION) + # first two STRING nodes represent object and method names + children = list(self.children_with_type(invocation_node, ASTNodeType.STRING)) + if len(children) == 1: + return MethodInvocationParams('', self.get_attr(children[0], 'string')) + + return MethodInvocationParams(self.get_attr(children[0], 'string'), + self.get_attr(children[1], 'string')) + + @deprecated(reason='Use ASTNode functionality instead.') + def get_member_reference_params(self, member_reference_node: int) -> MemberReferenceParams: + assert(self.get_type(member_reference_node) == ASTNodeType.MEMBER_REFERENCE) + params = [self.get_attr(child, 'string') for child in + self.children_with_type(member_reference_node, ASTNodeType.STRING)] + + member_reference_params: MemberReferenceParams + if len(params) == 1: + member_reference_params = MemberReferenceParams(object_name='', member_name=params[0], + unary_operator='') + elif len(params) == 2: + member_reference_params = MemberReferenceParams(object_name=params[0], member_name=params[1], + unary_operator='') + elif len(params) == 3: + member_reference_params = MemberReferenceParams(unary_operator=params[0], object_name=params[1], + member_name=params[2]) + else: + raise ValueError('Node has 0 or more then 3 children with type "STRING": ' + str(params)) + + return member_reference_params + + @deprecated(reason='Use ASTNode functionality instead.') + def get_binary_operation_params(self, binary_operation_node: int) -> BinaryOperationParams: + assert(self.get_type(binary_operation_node) == ASTNodeType.BINARY_OPERATION) + operation_node, left_side_node, right_side_node = self.tree.succ[binary_operation_node] + return BinaryOperationParams(self.get_attr(operation_node, 'string'), left_side_node, right_side_node) + + @staticmethod + def _add_subtree_from_javalang_node(tree: DiGraph, javalang_node: Union[Node, Set[Any], str], + javalang_node_to_index_map: Dict[Node, int]) -> int: + node_index, node_type = AST._add_javalang_node(tree, javalang_node) + if node_index != AST._UNKNOWN_NODE_TYPE and \ + node_type not in {ASTNodeType.COLLECTION, ASTNodeType.STRING}: + javalang_standard_node = cast(Node, javalang_node) + javalang_node_to_index_map[javalang_standard_node] = node_index + AST._add_javalang_children(tree, javalang_standard_node.children, node_index, + javalang_node_to_index_map) + return node_index + + @staticmethod + def _add_javalang_children(tree: DiGraph, children: List[Any], parent_index: int, + javalang_node_to_index_map: Dict[Node, int]) -> None: + for child in children: + if isinstance(child, list): + AST._add_javalang_children(tree, child, parent_index, javalang_node_to_index_map) + else: + child_index = AST._add_subtree_from_javalang_node(tree, child, javalang_node_to_index_map) + if child_index != AST._UNKNOWN_NODE_TYPE: + tree.add_edge(parent_index, child_index) + + @staticmethod + def _add_javalang_node(tree: DiGraph, javalang_node: Union[Node, Set[Any], str]) -> Tuple[int, ASTNodeType]: + node_index = AST._UNKNOWN_NODE_TYPE + node_type = ASTNodeType.UNKNOWN + if isinstance(javalang_node, Node): + node_index, node_type = AST._add_javalang_standard_node(tree, javalang_node) + elif isinstance(javalang_node, set): + node_index = AST._add_javalang_collection_node(tree, javalang_node) + node_type = ASTNodeType.COLLECTION + elif isinstance(javalang_node, str): + node_index = AST._add_javalang_string_node(tree, javalang_node) + node_type = ASTNodeType.STRING + + return node_index, node_type + + @staticmethod + def _add_javalang_standard_node(tree: DiGraph, javalang_node: Node) -> Tuple[int, ASTNodeType]: + node_index = len(tree) + 1 + node_type = javalang_to_ast_node_type[type(javalang_node)] + + attr_names = attributes_by_node_type[node_type] + attributes = {attr_name: getattr(javalang_node, attr_name) for attr_name in attr_names} + + attributes['node_type'] = node_type + attributes['line'] = javalang_node.position.line if javalang_node.position is not None else None + + AST._post_process_javalang_attributes(tree, node_type, attributes) + + tree.add_node(node_index, **attributes) + return node_index, node_type + + @staticmethod + def _post_process_javalang_attributes(tree: DiGraph, node_type: ASTNodeType, attributes: Dict[str, Any]) -> None: + """ + Replace some attributes with more appropriate values for convenient work + """ + + if node_type == ASTNodeType.METHOD_DECLARATION and attributes["body"] is None: + attributes["body"] = [] + + if node_type == ASTNodeType.LAMBDA_EXPRESSION and isinstance(attributes["body"], Node): + attributes["body"] = [attributes["body"]] + + if node_type in {ASTNodeType.METHOD_INVOCATION, ASTNodeType.MEMBER_REFERENCE} and \ + attributes["qualifier"] == "": + attributes["qualifier"] = None + + @staticmethod + def _add_javalang_collection_node(tree: DiGraph, collection_node: Set[Any]) -> int: + node_index = len(tree) + 1 + tree.add_node(node_index, node_type=ASTNodeType.COLLECTION, line=None) + # we expect only strings in collection + # we add them here as children + for item in collection_node: + if type(item) == str: + string_node_index = AST._add_javalang_string_node(tree, item) + tree.add_edge(node_index, string_node_index) + elif item is not None: + raise ValueError('Unexpected javalang AST node type {} inside \ + "COLLECTION" node'.format(type(item))) + return node_index + + @staticmethod + def _add_javalang_string_node(tree: DiGraph, string_node: str) -> int: + node_index = len(tree) + 1 + tree.add_node(node_index, node_type=ASTNodeType.STRING, string=string_node, line=None) + return node_index + + @staticmethod + def _replace_javalang_nodes_in_attributes(tree: DiGraph, + javalang_node_to_index_map: Dict[Node, int]) -> None: + ''' + All javalang nodes found in networkx nodes attributes are replaced + with references to according networkx nodes. + Supported attributes types: + - just javalang Node + - list of javalang Nodes and other such lists (with any depth) + ''' + for node, attributes in tree.nodes.items(): + for attribute_name in attributes: + attribute_value = attributes[attribute_name] + if isinstance(attribute_value, Node): + node_reference = AST._create_reference_to_node(attribute_value, + javalang_node_to_index_map) + tree.add_node(node, **{attribute_name: node_reference}) + elif isinstance(attribute_value, list): + node_references = \ + AST._replace_javalang_nodes_in_list(attribute_value, + javalang_node_to_index_map) + tree.add_node(node, **{attribute_name: node_references}) + + @staticmethod + def _replace_javalang_nodes_in_list(javalang_nodes_list: List[Any], + javalang_node_to_index_map: Dict[Node, int]) -> List[Any]: + ''' + javalang_nodes_list: list of javalang Nodes or other such lists (with any depth) + All javalang nodes are replaces with according references + NOTICE: Any is used, because mypy does not support recurrent type definitions + ''' + node_references_list: List[Any] = [] + for item in javalang_nodes_list: + if isinstance(item, Node): + node_references_list.append( + AST._create_reference_to_node(item, javalang_node_to_index_map)) + elif isinstance(item, list): + node_references_list.append( + AST._replace_javalang_nodes_in_list(item, javalang_node_to_index_map)) + elif isinstance(item, (int, str)) or item is None: + node_references_list.append(item) + else: + raise RuntimeError('Cannot parse "Javalang" attribute:\n' + f'{item}\n' + 'Expected: Node, list of Nodes, integer or string') + + return node_references_list + + @staticmethod + def _create_reference_to_node(javalang_node: Node, + javalang_node_to_index_map: Dict[Node, int]) -> ASTNodeReference: + return ASTNodeReference(javalang_node_to_index_map[javalang_node]) + + _UNKNOWN_NODE_TYPE = -1 diff --git a/temp_python/ast_framework/ast_node.py b/temp_python/ast_framework/ast_node.py new file mode 100644 index 0000000..c9adb68 --- /dev/null +++ b/temp_python/ast_framework/ast_node.py @@ -0,0 +1,184 @@ +# The MIT License (MIT) +# +# Copyright (c) 2020 Aibolit +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from typing import Any, List, Iterator, Optional + +from networkx import DiGraph, dfs_preorder_nodes # type: ignore +from cached_property import cached_property # type: ignore + +from temp_python.ast_framework._auxiliary_data import ( + common_attributes, + attributes_by_node_type, + ASTNodeReference, +) +from temp_python.ast_framework import ASTNodeType +from temp_python.ast_framework.computed_fields_registry import computed_fields_registry + + +class ASTNode: + def __init__(self, graph: DiGraph, node_index: int): + self._graph = graph + self._node_index = node_index + + @property + def children(self) -> Iterator["ASTNode"]: + for child_index in self._graph.succ[self._node_index]: + yield ASTNode(self._graph, child_index) + + @property + def node_index(self) -> int: + return self._node_index + + @cached_property + def line(self) -> int: + line = self._get_line(self._node_index) + if line is not None: + return line + + children_lines: List[int] = [ + self._get_line(child_index) # type: ignore # all Nones filtered out in list comprehension + for child_index in dfs_preorder_nodes(self._graph, self._node_index) + if self._get_line(child_index) is not None + ] + + # try to find source code line information from nodes reachable from self + if children_lines: + return min(children_lines) + + # try to find source code line information from parents up to the root + parent_index = self._get_parent(self._node_index) + while line is None and parent_index is not None: + line = self._get_line(parent_index) + parent_index = self._get_parent(parent_index) + + if line is not None: + return line + + raise RuntimeError( + f"Failed to retrieve source code line information for {repr(self)} node. " + "All nodes in a path from root to it and all nodes, reachable from it, " + "does not have any source code line information either." + ) + + def __getattr__(self, attribute_name: str): + node_type = self._get_type(self._node_index) + javalang_fields = attributes_by_node_type[node_type] + computed_fields = computed_fields_registry.get_fields(node_type) + if attribute_name not in common_attributes and \ + attribute_name not in javalang_fields and \ + attribute_name not in computed_fields: + raise AttributeError( + "Failed to retrieve property. " + f"'{node_type}' node does not have '{attribute_name}' attribute." + ) + + if attribute_name in computed_fields: + attribute = computed_fields[attribute_name](self) + else: + attribute = self._graph.nodes[self._node_index][attribute_name] + + # common_attributes and javalang_fields may contain ASTNodeReference + # which needs to be replaces with actual ASTNode for convince API + # computed_fields are using javalang_fields and there is no ASTNodeReference + if attribute_name not in computed_fields: + if isinstance(attribute, ASTNodeReference): + attribute = self._create_node_from_reference(attribute) + elif isinstance(attribute, list): + attribute = self._replace_references_with_nodes(attribute) + return attribute + + def __dir__(self) -> List[str]: + node_type = self._get_type(self._node_index) + return ASTNode._public_fixed_interface + \ + list(common_attributes) + \ + list(attributes_by_node_type[node_type]) + \ + list(computed_fields_registry.get_fields(node_type).keys()) + + def __str__(self) -> str: + text_representation = f"node index: {self._node_index}" + node_type = self._get_type(self._node_index) + for attribute_name in sorted( + common_attributes | attributes_by_node_type[node_type] + ): + attribute_value = self.__getattr__(attribute_name) + + if isinstance(attribute_value, ASTNode): + attribute_representation = repr(attribute_value) + elif isinstance(attribute_value, str) and "\n" in attribute_value: + attribute_representation = "\n\t" + attribute_value.replace( + "\n", "\n\t" + ) + else: + attribute_representation = str(attribute_value) + + text_representation += f"\n{attribute_name}: {attribute_representation}" + + return text_representation + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, ASTNode): + raise NotImplementedError( + f"ASTNode support comparission only with themselves, but {type(other)} was provided." + ) + return self._graph == other._graph and self._node_index == other._node_index + + def __hash__(self): + return hash(self._node_index) + + def _replace_references_with_nodes( + self, list_with_references: List[Any] + ) -> List[Any]: + list_with_nodes: List[Any] = [] + for item in list_with_references: + if isinstance(item, ASTNodeReference): + list_with_nodes.append(self._create_node_from_reference(item)) + elif isinstance(item, list): + list_with_nodes.append(self._replace_references_with_nodes(item)) + elif isinstance(item, (int, str)) or item is None: + list_with_nodes.append(item) + else: + raise RuntimeError( + "Failed parsing attribute.\n" + f"An {item} with {type(item)} was found.\n" + "Expected: int, str, ASNodeReference of list of them." + ) + + return list_with_nodes + + def _create_node_from_reference(self, reference: ASTNodeReference) -> "ASTNode": + return ASTNode(self._graph, reference.node_index) + + def _get_type(self, node_index: int) -> ASTNodeType: + return self._graph.nodes[node_index]["node_type"] + + def _get_line(self, node_index: int) -> Optional[int]: + return self._graph.nodes[node_index]["line"] + + def _get_parent(self, node_index: int) -> Optional[int]: + # there is maximum one parent in a tree + return next(self._graph.predecessors(node_index), None) + + # names of methods and properties, which is not generated dynamically + _public_fixed_interface = ["children", "node_index", "line"] diff --git a/temp_python/ast_framework/ast_node_type.py b/temp_python/ast_framework/ast_node_type.py new file mode 100644 index 0000000..1cc24e7 --- /dev/null +++ b/temp_python/ast_framework/ast_node_type.py @@ -0,0 +1,109 @@ +# The MIT License (MIT) +# +# Copyright (c) 2020 Aibolit +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from enum import Enum, auto + + +class ASTNodeType(Enum): + ANNOTATION = auto() + ANNOTATION_DECLARATION = auto() + ANNOTATION_METHOD = auto() + ARRAY_CREATOR = auto() + ARRAY_INITIALIZER = auto() + ARRAY_SELECTOR = auto() + ASSERT_STATEMENT = auto() + ASSIGNMENT = auto() + BASIC_TYPE = auto() + BINARY_OPERATION = auto() + BLOCK_STATEMENT = auto() + BREAK_STATEMENT = auto() + CAST = auto() + CATCH_CLAUSE = auto() + CATCH_CLAUSE_PARAMETER = auto() + CLASS_CREATOR = auto() + CLASS_DECLARATION = auto() + CLASS_REFERENCE = auto() + COLLECTION = auto() # Custom type, represent set (as a node) in AST + COMPILATION_UNIT = auto() + CONSTANT_DECLARATION = auto() + CONSTRUCTOR_DECLARATION = auto() + CONTINUE_STATEMENT = auto() + CREATOR = auto() + DECLARATION = auto() + DO_STATEMENT = auto() + DOCUMENTED = auto() + ELEMENT_ARRAY_VALUE = auto() + ELEMENT_VALUE_PAIR = auto() + ENHANCED_FOR_CONTROL = auto() + ENUM_BODY = auto() + ENUM_CONSTANT_DECLARATION = auto() + ENUM_DECLARATION = auto() + EXPLICIT_CONSTRUCTOR_INVOCATION = auto() + EXPRESSION = auto() + FIELD_DECLARATION = auto() + FOR_CONTROL = auto() + FOR_STATEMENT = auto() + FORMAL_PARAMETER = auto() + IF_STATEMENT = auto() + IMPORT = auto() + INFERRED_FORMAL_PARAMETER = auto() + INNER_CLASS_CREATOR = auto() + INTERFACE_DECLARATION = auto() + INVOCATION = auto() + LAMBDA_EXPRESSION = auto() + LITERAL = auto() + LOCAL_VARIABLE_DECLARATION = auto() + MEMBER = auto() + MEMBER_REFERENCE = auto() + METHOD_DECLARATION = auto() + METHOD_INVOCATION = auto() + METHOD_REFERENCE = auto() + PACKAGE_DECLARATION = auto() + PRIMARY = auto() + REFERENCE_TYPE = auto() + RETURN_STATEMENT = auto() + STATEMENT = auto() + STATEMENT_EXPRESSION = auto() + STRING = auto() # Custom type, represent just string in AST + SUPER_CONSTRUCTOR_INVOCATION = auto() + SUPER_MEMBER_REFERENCE = auto() + SUPER_METHOD_INVOCATION = auto() + SWITCH_STATEMENT = auto() + SWITCH_STATEMENT_CASE = auto() + SYNCHRONIZED_STATEMENT = auto() + TERNARY_EXPRESSION = auto() + THIS = auto() + THROW_STATEMENT = auto() + TRY_RESOURCE = auto() + TRY_STATEMENT = auto() + TYPE = auto() + TYPE_ARGUMENT = auto() + TYPE_DECLARATION = auto() + TYPE_PARAMETER = auto() + UNKNOWN = auto() # Custom type, used only in parsing javalang AST + VARIABLE_DECLARATION = auto() + VARIABLE_DECLARATOR = auto() + VOID_CLASS_REFERENCE = auto() + WHILE_STATEMENT = auto() + + def __str__(self) -> str: + return self.name.replace('_', ' ').capitalize() diff --git a/temp_python/ast_framework/computed_fields_catalog/__init__.py b/temp_python/ast_framework/computed_fields_catalog/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/temp_python/ast_framework/computed_fields_catalog/chained_fields.py b/temp_python/ast_framework/computed_fields_catalog/chained_fields.py new file mode 100644 index 0000000..ffde7f7 --- /dev/null +++ b/temp_python/ast_framework/computed_fields_catalog/chained_fields.py @@ -0,0 +1,62 @@ +# The MIT License (MIT) +# +# Copyright (c) 2020 Aibolit +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from typing import Union, Any, Callable +from itertools import chain + +from temp_python.ast_framework import ASTNode + + +def chain_field_getter_factory(*steps: Union[str, int]) -> Callable[[ASTNode], Any]: + """ + A chained field is a field of some other field and so on. + For example name of class field is retrieved like `field_decl.declarators[0].name` using javalang fields. + To automate this chain fields can be used. + Chain field is specified by sequence of strings (field names) and integers (list indexes). + If a field is list of ASTNodes and next step in chain is string + then we get that field from every node in list. + If we get list of lists, it gets flatten. + List with single items is unwrapped. + """ + + def get_chain_field(node: ASTNode) -> Any: + field = node + for step in steps: + if isinstance(field, list) and \ + isinstance(step, str) and \ + all(isinstance(item, ASTNode) for item in field): + # get attribute from all elements from a list + field = [getattr(item, step) for item in field] + + if all(isinstance(item, list) for item in field): # flattening list + field = list(chain.from_iterable(field)) + + elif isinstance(field, list) and isinstance(step, int): + field = field[step] + elif isinstance(field, ASTNode) and isinstance(step, str): + field = getattr(field, step) + else: + raise RuntimeError(f"Failed to apply step {step} to field {field}.") + + return field + + return get_chain_field diff --git a/temp_python/ast_framework/computed_fields_catalog/nodes_filter.py b/temp_python/ast_framework/computed_fields_catalog/nodes_filter.py new file mode 100644 index 0000000..94e4275 --- /dev/null +++ b/temp_python/ast_framework/computed_fields_catalog/nodes_filter.py @@ -0,0 +1,48 @@ +# The MIT License (MIT) +# +# Copyright (c) 2020 Aibolit +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from typing import Callable, Iterator + +from temp_python.ast_framework import ASTNode, ASTNodeType + + +def nodes_filter_factory( + base_field_name: str, *node_types: ASTNodeType +) -> Callable[[ASTNode], Iterator[ASTNode]]: + """ + Create filter, which takes 'body_field_name' field of incoming node, + checks if it list of ASTNode, and return it filtered by node_type. + """ + + def filter(base_node: ASTNode) -> Iterator[ASTNode]: + base_field = getattr(base_node, base_field_name) + if isinstance(base_field, list): + for node in base_field: + if isinstance(node, ASTNode) and node.node_type in node_types: + yield node + else: + raise RuntimeError( + f"Failed computing ASTNode field based on {base_field_name} field. " + f"Expected list, but got {base_field} of type {type(base_field)}." + ) + + return filter diff --git a/temp_python/ast_framework/computed_fields_catalog/standard_fields.py b/temp_python/ast_framework/computed_fields_catalog/standard_fields.py new file mode 100644 index 0000000..4cc80ea --- /dev/null +++ b/temp_python/ast_framework/computed_fields_catalog/standard_fields.py @@ -0,0 +1,81 @@ +# The MIT License (MIT) +# +# Copyright (c) 2020 Aibolit +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from temp_python.ast_framework.computed_fields_registry import computed_fields_registry +from temp_python.ast_framework import ASTNodeType + +from .nodes_filter import nodes_filter_factory +from .chained_fields import chain_field_getter_factory + + +def register_standard_computed_properties() -> None: + _register_standard_nodes_filters() + _register_standard_chain_fields() + + +def _register_standard_nodes_filters() -> None: + computed_fields_registry.register( + nodes_filter_factory("body", ASTNodeType.CONSTRUCTOR_DECLARATION), + "constructors", + ASTNodeType.CLASS_DECLARATION, + ASTNodeType.INTERFACE_DECLARATION, + ASTNodeType.ANNOTATION_DECLARATION, + ) + + computed_fields_registry.register( + nodes_filter_factory("body", ASTNodeType.METHOD_DECLARATION), + "methods", + ASTNodeType.CLASS_DECLARATION, + ASTNodeType.INTERFACE_DECLARATION, + ASTNodeType.ANNOTATION_DECLARATION, + ) + + computed_fields_registry.register( + nodes_filter_factory("body", ASTNodeType.FIELD_DECLARATION), + "fields", + ASTNodeType.CLASS_DECLARATION, + ASTNodeType.INTERFACE_DECLARATION, + ASTNodeType.ANNOTATION_DECLARATION, + ) + + computed_fields_registry.register( + nodes_filter_factory("declarations", ASTNodeType.METHOD_DECLARATION), + "methods", + ASTNodeType.ENUM_DECLARATION, + ) + + computed_fields_registry.register( + nodes_filter_factory("declarations", ASTNodeType.FIELD_DECLARATION), + "fields", + ASTNodeType.ENUM_DECLARATION, + ) + + +def _register_standard_chain_fields() -> None: + computed_fields_registry.register( + chain_field_getter_factory("declarators", "name"), + "names", + ASTNodeType.CONSTANT_DECLARATION, + ASTNodeType.FIELD_DECLARATION, + ASTNodeType.LOCAL_VARIABLE_DECLARATION, + ASTNodeType.VARIABLE_DECLARATION, + ) diff --git a/temp_python/ast_framework/computed_fields_registry.py b/temp_python/ast_framework/computed_fields_registry.py new file mode 100644 index 0000000..76d1871 --- /dev/null +++ b/temp_python/ast_framework/computed_fields_registry.py @@ -0,0 +1,69 @@ +# The MIT License (MIT) +# +# Copyright (c) 2020 Aibolit +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from collections import defaultdict +import sys +from typing import Dict, Callable, Any, TYPE_CHECKING + +if TYPE_CHECKING: + from temp_python.ast_framework import ASTNode, ASTNodeType # noqa: F401 + + +class _ComputedFieldsRegistry: + def __init__(self) -> None: + RegistryType = Dict["ASTNodeType", Dict[str, Callable[["ASTNode"], Any]]] + self._registry: RegistryType = defaultdict(dict) + + def register( + self, + compute_field: Callable[["ASTNode"], Any], + name: str, + *node_types: "ASTNodeType", + ) -> None: + for node_type in node_types: + computed_fields = self._registry[node_type] + if name in computed_fields and \ + not _ComputedFieldsRegistry._is_in_interactive_shell(): + raise RuntimeError( + f"Registry already has computed field " + f"named '{name}' for node type {node_type}." + ) + + computed_fields[name] = compute_field + + def get_fields( + self, node_type: "ASTNodeType" + ) -> Dict[str, Callable[["ASTNode"], Any]]: + return self._registry[node_type] + + def clear(self) -> None: + self._registry = defaultdict(dict) + + @staticmethod + def _is_in_interactive_shell() -> bool: + """ + Taken from comments to this answer https://stackoverflow.com/a/6879085/2129920 + """ + return bool(getattr(sys, "ps1", sys.flags.interactive)) + + +computed_fields_registry = _ComputedFieldsRegistry() diff --git a/temp_python/ast_framework/java_class.py b/temp_python/ast_framework/java_class.py new file mode 100644 index 0000000..0b8eacd --- /dev/null +++ b/temp_python/ast_framework/java_class.py @@ -0,0 +1,74 @@ +# The MIT License (MIT) +# +# Copyright (c) 2020 Aibolit +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +from cached_property import cached_property # type: ignore +from deprecated import deprecated # type: ignore + +from typing import Dict, Set, TYPE_CHECKING +from networkx import DiGraph # type: ignore + +from temp_python.ast_framework import AST, ASTNodeType +from temp_python.ast_framework.java_class_method import JavaClassMethod +from temp_python.ast_framework.java_class_field import JavaClassField + +if TYPE_CHECKING: + from temp_python.ast_framework.java_package import JavaPackage + + +@deprecated("This functionality must be transmitted to ASTNode") +class JavaClass(AST): + def __init__(self, tree: DiGraph, root: int, java_package: 'JavaPackage'): + self.tree = tree + self.root = root + self._java_package = java_package + + @cached_property + def name(self) -> str: + try: + class_name = next(self.children_with_type(self.root, ASTNodeType.STRING)) + return self.tree.nodes[class_name]['string'] + except StopIteration: + raise ValueError("Provided AST does not has 'STRING' node type right under the root") + + @property + def package(self) -> 'JavaPackage': + return self._java_package + + @cached_property + def methods(self) -> Dict[str, Set[JavaClassMethod]]: + methods: Dict[str, Set[JavaClassMethod]] = {} + for method_ast in self.get_subtrees(ASTNodeType.METHOD_DECLARATION): + method = JavaClassMethod(method_ast.tree, method_ast.root, self) + if method.name in methods: + methods[method.name].add(method) + else: + methods[method.name] = {method} + return methods + + @cached_property + def fields(self) -> Dict[str, JavaClassField]: + fields: Dict[str, JavaClassField] = {} + for field_ast in self.get_subtrees(ASTNodeType.FIELD_DECLARATION): + field = JavaClassField(field_ast.tree, field_ast.root, self) + fields[field.name] = field + return fields diff --git a/temp_python/ast_framework/java_class_decomposition.py b/temp_python/ast_framework/java_class_decomposition.py new file mode 100644 index 0000000..70c9ab4 --- /dev/null +++ b/temp_python/ast_framework/java_class_decomposition.py @@ -0,0 +1,212 @@ +# The MIT License (MIT) +# +# Copyright (c) 2020 Aibolit +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from typing import List, Dict, Set, Iterator, Any + +from networkx import DiGraph, strongly_connected_components, weakly_connected_components # type: ignore + +from temp_python.ast_framework import AST, ASTNodeType + +from temp_python.patterns.classic_setter.classic_setter import ClassicSetter as setter +from temp_python.patterns.classic_getter.classic_getter import ClassicGetter as getter + + +def find_patterns(tree: AST, patterns: List[Any]) -> Set[str]: + """ + Searches all setters in a component + :param patterns: list of patterns to check + :param tree: ast tree + :return: list of method name which are setters + """ + + patterns_method_names: Set[str] = set() + for method_declaration in tree.get_root().methods: + method_ast = tree.get_subtree(method_declaration) + for pattern in patterns: + if is_ast_pattern(method_ast, pattern): + patterns_method_names.add(method_declaration.name) + + return patterns_method_names + + +def is_ast_pattern(class_ast: AST, Pattern) -> bool: + """ + Checks whether ast is some pattern + :param Pattern: pattern class + :param class_ast: ast tree + :return: True if it is setter, otherwise - False + """ + return len(Pattern().value(class_ast)) > 0 + + +def decompose_java_class( + class_ast: AST, + strength: str, + ignore_setters=False, + ignore_getters=False) -> List[AST]: + """ + Splits java_class fields and methods by their usage and + construct for each case an AST with only those fields and methods kept. + :param class_ast: component + :param ignore_getters: should ignore getters + :param ignore_setters: should ignore setters + :param strength: controls splitting criteria. Use "strong" or "weak" + for splitting fields and methods by strong and weak connectivity. + """ + + usage_graph = _create_usage_graph(class_ast) + + components: Iterator[Set[int]] + if strength == "strong": + components = strongly_connected_components(usage_graph) + elif strength == "weak": + components = weakly_connected_components(usage_graph) + else: + raise ValueError( + f"'strength' argument must be either 'strong' or 'weak', but '{strength}' was provided." + ) + + class_parts: List[AST] = [] + patterns_to_ignore: List[Any] = [] + if ignore_getters: + patterns_to_ignore.append(lambda: getter()) + if ignore_setters: + patterns_to_ignore.append(lambda: setter()) + + if ignore_setters or ignore_getters: + prohibited_function_names = find_patterns(class_ast, patterns_to_ignore) + + for component in components: + + field_names = { + usage_graph.nodes[node]["name"] + for node in component + if usage_graph.nodes[node]["type"] == "field" + } + method_names = { + usage_graph.nodes[node]["name"] + for node in component + if usage_graph.nodes[node]["type"] == "method" + } + + if ignore_setters or ignore_getters: + method_names = method_names.difference(prohibited_function_names) + + filtered = _filter_class_methods_and_fields(class_ast, field_names, method_names) + + class_parts.append( + filtered + ) + + return class_parts + + +def _create_usage_graph(class_ast: AST) -> DiGraph: + usage_graph = DiGraph() + fields_ids: Dict[str, int] = {} + methods_ids: Dict[str, int] = {} + + class_declaration = class_ast.get_root() + + for field_declaration in class_declaration.fields: + # several fields can be declared at one line + for field_name in field_declaration.names: + fields_ids[field_name] = len(fields_ids) + usage_graph.add_node(fields_ids[field_name], type="field", name=field_name) + + for method_declaration in class_declaration.methods: + method_name = method_declaration.name + + # overloaded methods considered as single node in usage_graph + if method_name not in methods_ids: + methods_ids[method_name] = len(fields_ids) + 1 + len(methods_ids) + usage_graph.add_node( + methods_ids[method_name], type="method", name=method_name + ) + + for method_declaration in class_declaration.methods: + method_ast = class_ast.get_subtree(method_declaration) + + for invoked_method_name in _find_local_method_invocations(method_ast): + if invoked_method_name in methods_ids: + usage_graph.add_edge( + methods_ids[method_declaration.name], + methods_ids[invoked_method_name], + ) + + for used_field_name in _find_fields_usage(method_ast): + if used_field_name in fields_ids: + usage_graph.add_edge( + methods_ids[method_declaration.name], fields_ids[used_field_name] + ) + + return usage_graph + + +def _find_local_method_invocations(method_ast: AST) -> Set[str]: + invoked_methods: Set[str] = set() + for method_invocation in method_ast.get_proxy_nodes(ASTNodeType.METHOD_INVOCATION): + if method_invocation.qualifier is None: + invoked_methods.add(method_invocation.member) + + return invoked_methods + + +def _find_fields_usage(method_ast: AST) -> Set[str]: + local_variables: Set[str] = set() + for variable_declaration in method_ast.get_proxy_nodes( + ASTNodeType.LOCAL_VARIABLE_DECLARATION + ): + local_variables.update(variable_declaration.names) + + method_declaration = method_ast.get_root() + for parameter in method_declaration.parameters: + local_variables.add(parameter.name) + + used_fields: Set[str] = set() + for member_reference in method_ast.get_proxy_nodes(ASTNodeType.MEMBER_REFERENCE): + if member_reference.qualifier is None and \ + member_reference.member not in local_variables: + used_fields.add(member_reference.member) + + return used_fields + + +def _filter_class_methods_and_fields( + class_ast: AST, + allowed_fields_names: Set[str], + allowed_methods_names: Set[str] +) -> AST: + class_declaration = class_ast.get_root() + allowed_nodes = {class_declaration.node_index} + + for field_declaration in class_declaration.fields: + if len(allowed_fields_names & set(field_declaration.names)) != 0: + field_ast = class_ast.get_subtree(field_declaration) + allowed_nodes.update(node.node_index for node in field_ast) + + for method_declaration in class_declaration.methods: + if method_declaration.name in allowed_methods_names: + method_ast = class_ast.get_subtree(method_declaration) + allowed_nodes.update(node.node_index for node in method_ast) + + return AST(class_ast.tree.subgraph(allowed_nodes), class_declaration.node_index) diff --git a/temp_python/ast_framework/java_class_field.py b/temp_python/ast_framework/java_class_field.py new file mode 100644 index 0000000..676bf79 --- /dev/null +++ b/temp_python/ast_framework/java_class_field.py @@ -0,0 +1,53 @@ +# The MIT License (MIT) +# +# Copyright (c) 2020 Aibolit +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from cached_property import cached_property # type: ignore +from deprecated import deprecated # type: ignore + +from typing import TYPE_CHECKING +from networkx import DiGraph # type: ignore + +from temp_python.ast_framework import AST, ASTNodeType + +if TYPE_CHECKING: + from temp_python.ast_framework.java_class import JavaClass + + +@deprecated("This functionality must be transmitted to ASTNode") +class JavaClassField(AST): + def __init__(self, tree: DiGraph, root: int, java_class: 'JavaClass'): + self.tree = tree + self.root = root + self._java_class = java_class + + @cached_property + def name(self) -> str: + try: + field_declarator = next(self.children_with_type(self.root, ASTNodeType.VARIABLE_DECLARATOR)) + field_name = next(self.children_with_type(field_declarator, ASTNodeType.STRING)) + return self.tree.nodes[field_name]['string'] + except StopIteration: + raise ValueError("Provided AST does not has 'STRING' node type right under the root") + + @property + def java_class(self) -> 'JavaClass': + return self._java_class diff --git a/temp_python/ast_framework/java_class_method.py b/temp_python/ast_framework/java_class_method.py new file mode 100644 index 0000000..49e3b6e --- /dev/null +++ b/temp_python/ast_framework/java_class_method.py @@ -0,0 +1,91 @@ +# The MIT License (MIT) +# +# Copyright (c) 2020 Aibolit +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from cached_property import cached_property # type: ignore +from typing import Dict, Set, TYPE_CHECKING +from networkx import DiGraph, dfs_tree # type: ignore +from deprecated import deprecated # type: ignore + +from temp_python.utils.cfg_builder import build_cfg +from temp_python.ast_framework import AST, ASTNodeType +from temp_python.ast_framework.java_class_field import JavaClassField + +if TYPE_CHECKING: + from temp_python.ast_framework.java_class import JavaClass + + +@deprecated("This functionality must be transmitted to ASTNode") +class JavaClassMethod(AST): + def __init__(self, tree: DiGraph, root: int, java_class: 'JavaClass'): + self.tree = tree + self.root = root + self._java_class = java_class + + @cached_property + def name(self) -> str: + try: + method_name = next(self.children_with_type(self.root, ASTNodeType.STRING)) + return self.tree.nodes[method_name]['string'] + except StopIteration: + raise ValueError("Provided AST does not has 'STRING' node type right under the root") + + @property + def java_class(self) -> 'JavaClass': + return self._java_class + + @cached_property + def parameters(self) -> Dict[str, AST]: + parameters: Dict[str, AST] = {} + for parameter_node in self.children_with_type(self.root, ASTNodeType.FORMAL_PARAMETER): + parameter_name_node = next(iter(self.children_with_type(parameter_node, ASTNodeType.STRING))) + parameter_name = self.get_attr(parameter_name_node, 'string') + parameters[parameter_name] = AST(dfs_tree(self.tree, parameter_node), parameter_node) + + return parameters + + @cached_property + def used_methods(self) -> Dict[str, Set['JavaClassMethod']]: + method_invocation_nodes = self.get_nodes(ASTNodeType.METHOD_INVOCATION) + used_method_invocation_params = (self.get_method_invocation_params(node) for node + in method_invocation_nodes) + used_local_method_invocation_params = (params for params in used_method_invocation_params + if len(params.object_name) == 0) + used_local_method_names = {params.method_name for params in used_local_method_invocation_params} + + return {method_name: self.java_class.methods[method_name] for method_name in self.java_class.methods + if method_name in used_local_method_names} + + @cached_property + def used_fields(self) -> Dict[str, JavaClassField]: + used_member_reference_params = (self.get_member_reference_params(node) for node in + self.get_nodes(ASTNodeType.MEMBER_REFERENCE)) + used_local_member_reference_names = {params.member_name for params in used_member_reference_params + if params.object_name == ''} + # Local member references may lead to method parameters instead of class fields if they have same names + used_local_member_reference_names -= self.parameters.keys() + class_fields = self.java_class.fields + return {field_name: class_fields[field_name] for field_name in used_local_member_reference_names} + + @cached_property + def cfg(self) -> DiGraph: + '''Make Control Flow Graph representation of this method''' + return build_cfg(self) diff --git a/temp_python/ast_framework/java_package.py b/temp_python/ast_framework/java_package.py new file mode 100644 index 0000000..85c63db --- /dev/null +++ b/temp_python/ast_framework/java_package.py @@ -0,0 +1,56 @@ +# The MIT License (MIT) +# +# Copyright (c) 2020 Aibolit +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from cached_property import cached_property # type: ignore +from deprecated import deprecated # type: ignore + +from typing import Dict + +from temp_python.utils.ast_builder import build_ast +from temp_python.ast_framework import AST, ASTNodeType +from temp_python.ast_framework.java_class import JavaClass + + +@deprecated("This functionality must be transmitted to ASTNode") +class JavaPackage(AST): + def __init__(self, filename: str): + ast = AST.build_from_javalang(build_ast(filename)) + super().__init__(ast.tree, ast.root) + + @cached_property + def name(self) -> str: + try: + package_declaration = next(self.children_with_type(self.root, ASTNodeType.PACKAGE_DECLARATION)) + package_name = next(self.children_with_type(package_declaration, ASTNodeType.STRING)) + return self.tree.nodes[package_name]['string'] + except StopIteration: + pass + + return '.' # default package name + + @cached_property + def java_classes(self) -> Dict[str, JavaClass]: + classes: Dict[str, JavaClass] = {} + for class_ast in self.get_subtrees(ASTNodeType.CLASS_DECLARATION): + java_class = JavaClass(class_ast.tree, class_ast.root, self) + classes[java_class.name] = java_class + return classes diff --git a/temp_python/ast_framework/scope.py b/temp_python/ast_framework/scope.py new file mode 100644 index 0000000..6db8e5f --- /dev/null +++ b/temp_python/ast_framework/scope.py @@ -0,0 +1,97 @@ +# The MIT License (MIT) +# +# Copyright (c) 2020 Aibolit +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from typing import List, Iterator, Optional + +from networkx import DiGraph # type: ignore + +from .ast import AST +from .ast_node import ASTNode +from .ast_node_type import ASTNodeType +from .scope_extractors import extract_scopes + + +class Scope: + def __init__(self, scope_tree: DiGraph, scope_id: int): + self._scope_tree = scope_tree + self._scope_id = scope_id + + @staticmethod + def build_from_method_ast(method_ast: AST) -> "Scope": + method_declaration = method_ast.get_root() + assert ( + method_declaration.node_type == ASTNodeType.METHOD_DECLARATION + ), "Building scopes tree supported only for method declaration." + + scope_tree = DiGraph() + root_scopes_id = Scope._create_scopes_from_node(method_declaration, method_ast, scope_tree) + + assert len(root_scopes_id) == 1, "Method declaration must produce a single scope." + return Scope(scope_tree, root_scopes_id[0]) + + @property + def parent_scope(self) -> Optional["Scope"]: + parent_scope_id: Optional[int] = next(self._scope_tree.predecessors(self._scope_id), None) + + if parent_scope_id is None: + return None + + return Scope(self._scope_tree, parent_scope_id) + + @property + def nested_scopes(self) -> Iterator["Scope"]: + for nested_scope_id in self._scope_tree.successors(self._scope_id): + yield Scope(self._scope_tree, nested_scope_id) + + @property + def statements(self) -> List[ASTNode]: + return self._scope_tree.nodes[self._scope_id]["statements"] + + @property + def parent_node(self) -> ASTNode: + return self._scope_tree.nodes[self._scope_id]["parent_node"] + + @property + def parameters(self) -> List[ASTNode]: + return self._scope_tree.nodes[self._scope_id]["parameters"] + + @staticmethod + def _create_scopes_from_node(node: ASTNode, method_ast: AST, scope_tree: DiGraph) -> List[int]: + scopes = extract_scopes(node, method_ast) + + new_scopes_ids: List[int] = [] + for scope in scopes: + new_scope_id = len(scope_tree) + new_scopes_ids.append(new_scope_id) + scope_tree.add_node( + new_scope_id, + statements=scope.statements, + parent_node=scope.parent_node, + parameters=scope.parameters, + ) + + for statement in scope.statements: + nested_scopes_ids = Scope._create_scopes_from_node(statement, method_ast, scope_tree) + for nested_scope_id in nested_scopes_ids: + scope_tree.add_edge(new_scope_id, nested_scope_id) + + return new_scopes_ids diff --git a/temp_python/ast_framework/scope_extractors.py b/temp_python/ast_framework/scope_extractors.py new file mode 100644 index 0000000..bdc190f --- /dev/null +++ b/temp_python/ast_framework/scope_extractors.py @@ -0,0 +1,239 @@ +# The MIT License (MIT) +# +# Copyright (c) 2020 Aibolit +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from typing import Dict, List, Callable, NamedTuple +from itertools import chain + +from .ast import AST +from .ast_node import ASTNode +from .ast_node_type import ASTNodeType + + +class ScopeAttributes(NamedTuple): + statements: List[ASTNode] + parent_node: ASTNode + parameters: List[ASTNode] = [] + + +def extract_scopes(node: ASTNode, ast: AST) -> List[ScopeAttributes]: + try: + return _scope_extractors_by_node_type[node.node_type](node, ast) + except KeyError: + return [] + + +def _extract_scopes_from_assert(assert_node: ASTNode, method_ast: AST) -> List[ScopeAttributes]: + assert assert_node.node_type == ASTNodeType.ASSERT_STATEMENT + + expression_ast = method_ast.get_subtree(assert_node.condition) + return _find_scopes_in_expressions(expression_ast) + + +def _extract_scopes_from_block(block_node: ASTNode, _) -> List[ScopeAttributes]: + assert block_node.node_type == ASTNodeType.BLOCK_STATEMENT + + return [ScopeAttributes(statements=block_node.statements, parent_node=block_node)] + + +def _extract_scopes_from_expression_statement( + expression_statement: ASTNode, method_ast: AST +) -> List[ScopeAttributes]: + assert expression_statement.node_type in { + ASTNodeType.STATEMENT_EXPRESSION, + ASTNodeType.RETURN_STATEMENT, + ASTNodeType.THROW_STATEMENT, + } + + expression_ast = method_ast.get_subtree(expression_statement.expression) + return _find_scopes_in_expressions(expression_ast) + + +def _extract_scopes_from_for_cycle(for_cycle: ASTNode, method_ast: AST) -> List[ScopeAttributes]: + assert for_cycle.node_type == ASTNodeType.FOR_STATEMENT + + control_ast = method_ast.get_subtree(for_cycle.control) + scopes = _find_scopes_in_expressions(control_ast) + scopes.append( + ScopeAttributes(statements=_get_block_statements_list(for_cycle.body), parent_node=for_cycle) + ) + + return scopes + + +def _extract_scopes_from_if_statement(if_node: ASTNode, method_ast: AST) -> List[ScopeAttributes]: + assert if_node.node_type == ASTNodeType.IF_STATEMENT + + condition_ast = method_ast.get_subtree(if_node.condition) + scopes = _find_scopes_in_expressions(condition_ast) + scopes.append( + ScopeAttributes(statements=_get_block_statements_list(if_node.then_statement), parent_node=if_node) + ) + + while if_node.else_statement is not None and if_node.else_statement.node_type == ASTNodeType.IF_STATEMENT: + if_node = if_node.else_statement + condition_ast = method_ast.get_subtree(if_node.condition) + scopes.extend(_find_scopes_in_expressions(condition_ast)) + scopes.append( + ScopeAttributes( + statements=_get_block_statements_list(if_node.then_statement), parent_node=if_node + ) + ) + + if if_node.else_statement is not None: + scopes.append( + ScopeAttributes( + statements=_get_block_statements_list(if_node.else_statement), parent_node=if_node + ) + ) + + return scopes + + +def _extract_scopes_from_variable_declaration( + variable_declaration: ASTNode, method_ast: AST +) -> List[ScopeAttributes]: + assert variable_declaration.node_type == ASTNodeType.LOCAL_VARIABLE_DECLARATION + + return list( + chain.from_iterable( + _find_scopes_in_expressions(method_ast.get_subtree(declarator.initializer)) + for declarator in variable_declaration.declarators + ) + ) + + +def _extract_scopes_from_method(method_declaration: ASTNode, _) -> List[ScopeAttributes]: + assert method_declaration.node_type == ASTNodeType.METHOD_DECLARATION + + return [ScopeAttributes(statements=method_declaration.body, parent_node=method_declaration)] + + +def _extract_scopes_from_switch_statement( + switch_statement: ASTNode, method_ast: AST +) -> List[ScopeAttributes]: + assert switch_statement.node_type == ASTNodeType.SWITCH_STATEMENT + + expression_ast = method_ast.get_subtree(switch_statement.expression) + scopes = _find_scopes_in_expressions(expression_ast) + + # all case statements belong to one scope + # thats why cases are not surrounded with curly braces + scopes.append( + ScopeAttributes( + statements=list( + chain.from_iterable(case_node.statements for case_node in switch_statement.cases) + ), + parent_node=switch_statement, + ) + ) + + return scopes + + +def _extract_scopes_from_synchronized(synchronized_block: ASTNode, method_ast: AST) -> List[ScopeAttributes]: + assert synchronized_block.node_type == ASTNodeType.SYNCHRONIZED_STATEMENT + + lock_ast = method_ast.get_subtree(synchronized_block.lock) + scopes = _find_scopes_in_expressions(lock_ast) + scopes.append(ScopeAttributes(statements=synchronized_block.block, parent_node=synchronized_block)) + + return scopes + + +def _extract_scopes_from_try_statement(try_node: ASTNode, method_ast: AST) -> List[ScopeAttributes]: + assert try_node.node_type == ASTNodeType.TRY_STATEMENT + + scopes: List[ScopeAttributes] = [] + + for resource in try_node.resources: + initializer_ast = method_ast.get_subtree(resource.value) + scopes.extend(_find_scopes_in_expressions(initializer_ast)) + + scopes.append(ScopeAttributes(statements=try_node.block, parent_node=try_node)) + + for catch in try_node.catches: + scopes.append(ScopeAttributes(statements=catch.block, parent_node=try_node)) + + if try_node.finally_block is not None: + scopes.append(ScopeAttributes(statements=try_node.finally_block, parent_node=try_node)) + + return scopes + + +def _extract_scopes_from_while_cycle(while_cycle: ASTNode, method_ast: AST) -> List[ScopeAttributes]: + assert while_cycle.node_type in {ASTNodeType.DO_STATEMENT, ASTNodeType.WHILE_STATEMENT} + + condition_ast = method_ast.get_subtree(while_cycle.condition) + scopes = _find_scopes_in_expressions(condition_ast) + scopes.append( + ScopeAttributes(statements=_get_block_statements_list(while_cycle.body), parent_node=while_cycle) + ) + + return scopes + + +def _find_scopes_in_expressions(expression_ast: AST) -> List[ScopeAttributes]: + """ + Finds top level lambda expressions and returns their bodies. + Each found nested scope represented by a list of its statements. List of such list is returned. + TODO: Add support for others scopes can be found in expressions like anonymous classes. + """ + + nested_scopes_statements: List[ScopeAttributes] = [] + for nested_scope in expression_ast.get_subtrees(ASTNodeType.LAMBDA_EXPRESSION): + lambda_declaration = nested_scope.get_root() + nested_scopes_statements.append( + ScopeAttributes( + statements=lambda_declaration.body, + parent_node=lambda_declaration, + parameters=lambda_declaration.parameters, + ) + ) + + return nested_scopes_statements + + +def _get_block_statements_list(node: ASTNode) -> List[ASTNode]: + if node.node_type == ASTNodeType.BLOCK_STATEMENT: + return node.statements + + # there may be a single statement without curly braces + # for consistency we wrap it in a list + return [node] + + +_scope_extractors_by_node_type: Dict[ASTNodeType, Callable[[ASTNode, AST], List[ScopeAttributes]]] = { + ASTNodeType.ASSERT_STATEMENT: _extract_scopes_from_assert, + ASTNodeType.BLOCK_STATEMENT: _extract_scopes_from_block, + ASTNodeType.DO_STATEMENT: _extract_scopes_from_while_cycle, + ASTNodeType.FOR_STATEMENT: _extract_scopes_from_for_cycle, + ASTNodeType.IF_STATEMENT: _extract_scopes_from_if_statement, + ASTNodeType.LOCAL_VARIABLE_DECLARATION: _extract_scopes_from_variable_declaration, + ASTNodeType.METHOD_DECLARATION: _extract_scopes_from_method, + ASTNodeType.RETURN_STATEMENT: _extract_scopes_from_expression_statement, + ASTNodeType.STATEMENT_EXPRESSION: _extract_scopes_from_expression_statement, + ASTNodeType.SWITCH_STATEMENT: _extract_scopes_from_switch_statement, + ASTNodeType.SYNCHRONIZED_STATEMENT: _extract_scopes_from_synchronized, + ASTNodeType.THROW_STATEMENT: _extract_scopes_from_expression_statement, + ASTNodeType.TRY_STATEMENT: _extract_scopes_from_try_statement, + ASTNodeType.WHILE_STATEMENT: _extract_scopes_from_while_cycle, +} diff --git a/temp_python/patterns/classic_getter/__init__.py b/temp_python/patterns/classic_getter/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/temp_python/patterns/classic_getter/classic_getter.py b/temp_python/patterns/classic_getter/classic_getter.py new file mode 100644 index 0000000..a2d254c --- /dev/null +++ b/temp_python/patterns/classic_getter/classic_getter.py @@ -0,0 +1,35 @@ +from typing import List +from temp_python.ast_framework import ASTNodeType, AST +from temp_python.ast_framework.ast_node import ASTNode + + +class ClassicGetter: + ''' + The method's name starts with get. Return statement is the only statement, + excepting asserts. + ''' + + def _check_body_nodes(self, check_setter_body: List[ASTNode]) -> bool: + ''' + Check whether nodes are agree with the following types + (in self.suitable_nodes) or not. + ''' + for node in check_setter_body: + has_expression_value = hasattr(node, "expression") + if not has_expression_value: + return False + + is_return = node.node_type == ASTNodeType.RETURN_STATEMENT + is_expression_memeber_ref = node.expression.node_type == ASTNodeType.MEMBER_REFERENCE + if is_return and is_expression_memeber_ref: + return True + + return False + + def value(self, ast: AST) -> List[int]: + lines: List[int] = [] + for node in ast.get_proxy_nodes(ASTNodeType.METHOD_DECLARATION): + method_name = node.name + if method_name.startswith('get') and self._check_body_nodes(node.body): + lines.append(node.line) + return sorted(lines) diff --git a/temp_python/patterns/classic_setter/__init__.py b/temp_python/patterns/classic_setter/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/temp_python/patterns/classic_setter/classic_setter.py b/temp_python/patterns/classic_setter/classic_setter.py new file mode 100644 index 0000000..3d5e57b --- /dev/null +++ b/temp_python/patterns/classic_setter/classic_setter.py @@ -0,0 +1,33 @@ +from typing import List +from temp_python.ast_framework import ASTNodeType, AST +from temp_python.ast_framework.ast_node import ASTNode + + +class ClassicSetter: + ''' + The method's name starts with set. There are attributes + assigning in the method. Also, asserts are ignored. + ''' + suitable_nodes: List[ASTNodeType] = [ + ASTNodeType.ASSERT_STATEMENT, + ASTNodeType.STATEMENT_EXPRESSION, + ] + + def _check_body_nodes(self, check_setter_body: List[ASTNode]) -> bool: + ''' + Check whether nodes are agree with the following types + (in self.suitable_nodes) or not. + ''' + for node in check_setter_body: + if node.node_type not in self.suitable_nodes: + return False + return True + + def value(self, ast: AST) -> List[int]: + lines: List[int] = [] + for node in ast.get_proxy_nodes(ASTNodeType.METHOD_DECLARATION): + method_name = node.name + if node.return_type is None and method_name.startswith('set') and \ + self._check_body_nodes(node.body): + lines.append(node.line) + return sorted(lines) diff --git a/temp_python/utils/__init__.py b/temp_python/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/temp_python/utils/ast_builder.py b/temp_python/utils/ast_builder.py new file mode 100644 index 0000000..b449675 --- /dev/null +++ b/temp_python/utils/ast_builder.py @@ -0,0 +1,8 @@ +from javalang.parse import parse +from javalang.tree import CompilationUnit + +from temp_python.utils.encoding_detector import read_text_with_autodetected_encoding + + +def build_ast(filename: str) -> CompilationUnit: + return parse(read_text_with_autodetected_encoding(filename)) diff --git a/temp_python/utils/cfg_builder.py b/temp_python/utils/cfg_builder.py new file mode 100644 index 0000000..36aaeed --- /dev/null +++ b/temp_python/utils/cfg_builder.py @@ -0,0 +1,59 @@ +# The MIT License (MIT) +# +# Copyright (c) 2020 Aibolit +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from networkx import DiGraph, disjoint_union # type: ignore +from temp_python.ast_framework import AST, ASTNodeType +from typing import Tuple + + +NODE_TYPES = [ + ASTNodeType.ASSIGNMENT, + ASTNodeType.RETURN_STATEMENT +] + + +def build_cfg(tree: AST) -> DiGraph: + '''Create Control Flow Graph''' + g = DiGraph() + g.add_node(0) + for node in tree: + if node.node_type not in NODE_TYPES: + continue + _g = _mk_cfg_graph(node.node_type) + g = _compose_two_graphs(g, _g) + return g + + +def _mk_cfg_graph(node: ASTNodeType) -> Tuple[DiGraph, int]: + '''Takes in Javalang statement and returns corresponding CFG''' + g = DiGraph() + g.add_node(0) + return g + + +def _compose_two_graphs(g1: DiGraph, g2: DiGraph) -> DiGraph: + '''Compose two graphs by creating the edge between last of the fist graph and fist of the second. + We assume that node in the each graph G has order from 0 to len(G)-1 + ''' + g = disjoint_union(g1, g2) + g.add_edge(len(g1) - 1, len(g1)) + return g diff --git a/temp_python/utils/cohesiongraph.py b/temp_python/utils/cohesiongraph.py new file mode 100644 index 0000000..f60e304 --- /dev/null +++ b/temp_python/utils/cohesiongraph.py @@ -0,0 +1,233 @@ +# The MIT License (MIT) +# +# Copyright (c) 2020 Aibolit +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import networkx as nx # type: ignore +from networkx import Graph +from temp_python.utils.filter import Filters + +from typing import List, Tuple, Union, TypeVar +from javalang.tree import ClassDeclaration, MethodDeclaration, \ + MemberReference, FieldDeclaration, MethodInvocation, This, Node, LocalVariableDeclaration + + +FldExh = Tuple[str, Tuple[str, str]] +MthExh = Tuple[str, Tuple[Tuple[str, str], ...]] +HasMember = Union[MemberReference, MethodInvocation] +HasSelector = Union[MemberReference, MethodInvocation, This] +T = TypeVar('T', bound=Node) +S = TypeVar('S', HasSelector, HasMember) +ThisNodes = Tuple[tuple, This] +SelMemNodes = Tuple[tuple, S] +RefNodes = Tuple[tuple, MemberReference] +InvNodes = Tuple[tuple, MethodInvocation] +LocalNodes = Tuple[tuple, LocalVariableDeclaration] +MthNodes = Tuple[tuple, MethodDeclaration] +Nodes = Tuple[tuple, T] +EdgeNode = Union[MthExh, FldExh] + + +class CohesionGraph: + + filtrate = Filters() + + def value(self, tree: Node) -> Graph: + + G: Graph = nx.Graph() + + for class_path, class_node in tree.filter(ClassDeclaration): + + # Extract all methods and fields + # from ClassDeclaration node + + field_nodes: List[Nodes] = \ + self.filtrate.filter_node_lvl(class_node, FieldDeclaration) + full_field_exhaust: List[FldExh] = \ + list(self.filtrate.exhaust_field(field_node) for path, field_node in field_nodes) + clear_field_exhaust: List[FldExh] = \ + self.filtrate.clean_for_repetitions(full_field_exhaust) + method_nodes: List[MthNodes] = \ + self.filtrate.filter_node_lvl(class_node, MethodDeclaration) + method_nodes_filtered: List[MthNodes] = \ + self.filtrate.filter_getters_setters(method_nodes) + full_method_exhaust: List[MthExh] = \ + list(self.filtrate.exhaust_method(method_node) for path, method_node in method_nodes_filtered) + clear_method_exhaust: List[MthExh] = \ + self.filtrate.clean_for_repetitions(full_method_exhaust) + + for method in clear_method_exhaust: # Add all MethodDeclarations to graph + G.add_node(method[0] + str(hash(method[1]))) + for field in clear_field_exhaust: # Add all FieldDeclarations to graph + G.add_node(field[0] + str(hash(field[1]))) + + for method_path, method_node in method_nodes_filtered: + # Find and compare all existing + # MemberReferences, MethodInvocations, + # This statements, LocalVariableDeclarations + # to themselves and objects added to graph G + reference_nodes: List[RefNodes] = \ + self.filtrate.filter_node_lvl(method_node, MemberReference) + invocation_nodes: List[InvNodes] = \ + self.filtrate.filter_node_lvl(method_node, MethodInvocation) + this_nodes: List[ThisNodes] = \ + self.filtrate.filter_node_lvl(method_node, This) + local_nodes: List[LocalNodes] = \ + self.filtrate.filter_node_lvl(method_node, LocalVariableDeclaration) + local_exhaust: List[FldExh] = \ + list(self.filtrate.exhaust_field(local_node) for path, local_node in local_nodes) + method_exhaust: MthExh = self.filtrate.exhaust_method(method_node) + + self.add_references_to_graph(G, reference_nodes, local_exhaust, clear_field_exhaust, method_exhaust) + self.add_this_to_graph(G, this_nodes, clear_field_exhaust, method_exhaust) + self.add_invocations_to_graph( + G, invocation_nodes, clear_method_exhaust, method_exhaust, local_exhaust, clear_field_exhaust) + break # Stop after first class + return G + + # ------------------------------------------------ + # Funcs for adding edges to graph + + def add_invocations_to_graph( + self, + G: Graph, + invocation_nodes: List[SelMemNodes], + full_method_exhaust: List[MthExh], + method_exhaust: MthExh, + local_exhaust: List[FldExh], + full_field_exhaust: List[FldExh] + ) -> None: + """Adds nodes to graph G + + Gets list of invocation names as input and + compares them to existing list of exhausted "MethodDeclarations" + After successful comparison calls "add_vertices_edges" + Adding nodes and edges between. + """ + for invocation_path, invocation_node in invocation_nodes: + if isinstance(invocation_node.selectors, list): # Check for inv being first in whole statement + for method in full_method_exhaust: + if invocation_node.member == method[0]: + self.add_vertices_edges(G, 'invocation', method_exhaust, method) + inv_arguments = self.filtrate.get_arguments(invocation_node) + inv_fields: List[str] = inv_arguments[1] + inv_funcs: List[str] = inv_arguments[0] + if len(inv_fields) > 0: + self.add_invocation_fields(G, inv_fields, local_exhaust, full_field_exhaust, method_exhaust) + if len(inv_funcs) > 0: + self.add_invocation_funcs(G, inv_funcs, full_method_exhaust, method_exhaust) + + def add_invocation_fields( + self, + G: Graph, + inv_fields: List[str], + local_exhaust: List[FldExh], + full_field_exhaust: List[FldExh], + method_exhaust: MthExh + ) -> None: + """Adds nodes to graph G + + Gets a list of invocation field attributes as input and + compares them to existing list of exhausted "FieldDeclarations" + After successful comparison calls "add_vertices_edges" + Adding nodes and edges between. + """ + for inv_argument in inv_fields: + if inv_argument not in [x[0] for x in local_exhaust]: + for field in full_field_exhaust: + if inv_argument == field[0]: + self.add_vertices_edges(G, 'reference', method_exhaust, field) + + def add_invocation_funcs( + self, + G: Graph, + inv_funcs: List[str], + full_method_exhaust: List[MthExh], + method_exhaust: MthExh + ) -> None: + """Adds nodes to graph G + + Gets a list of invocation method attributes as input and + compares them to existing list of exhausted "MethodDeclarations" + After successful comparison calls "add_vertices_edges" + Adding nodes and edges between. + """ + # ToDo: make a func for a return type check + for inv_argument in inv_funcs: + for method in full_method_exhaust: + if inv_argument == method[0]: + self.add_vertices_edges(G, 'invocation', method_exhaust, method) + + def add_references_to_graph( + self, + G: Graph, + reference_nodes: List[SelMemNodes], + local_exhaust: List[FldExh], + full_field_exhaust: List[FldExh], + method_exhaust: MthExh + ) -> None: + """Adds nodes to graph G + + Gets list of "MemberReferences" nodes as input and + compares them to existing list of exhausted FieldDeclarations. + After successful comparison calls "add_vertices_edges" + Adding nodes and edges between. + """ + for reference_path, reference_node in reference_nodes: + if isinstance(reference_node.selectors, list): # Check for node being "alone" + if reference_node.member not in [x[0] for x in local_exhaust] and \ + reference_node.member not in [x for x, _ in method_exhaust[1]]: + for field in full_field_exhaust: + if reference_node.member == field[0]: + self.add_vertices_edges(G, 'reference', method_exhaust, field) + + def add_this_to_graph( + self, + G: Graph, + this_nodes: List[ThisNodes], + full_field_exhaust: List[FldExh], + method_exhaust: MthExh + ) -> None: + """Adds nodes to graph G + + Gets list of "This" nodes as input and + compares them to existing list of exhausted "FieldDeclarations" + After successful comparison calls "add_vertices_edges" + Adding nodes and edges between. + """ + for this_path, this_node in this_nodes: + for field in full_field_exhaust: + if len(this_node.selectors) == 1 and isinstance(this_node.selectors[0], MemberReference): + if this_node.selectors[0].member in field: + self.add_vertices_edges(G, 'reference', method_exhaust, field) + + def add_vertices_edges(self, G, edge_type: str, first_node: EdgeNode, second_node: EdgeNode) -> None: + """Adds nodes to graph G + + Gets two objects as input and + adds nodes and an edge between. + If nodes already exist: + creates an edge between + """ + + G.add_node(first_node[0] + str(hash(first_node[1]))) + G.add_node((second_node[0]) + str(hash(second_node[1]))) + G.add_edge(first_node[0] + str(hash(first_node[1])), + (second_node[0]) + str(hash(second_node[1])), type=edge_type) diff --git a/temp_python/utils/encoding_detector.py b/temp_python/utils/encoding_detector.py new file mode 100644 index 0000000..8646d5e --- /dev/null +++ b/temp_python/utils/encoding_detector.py @@ -0,0 +1,21 @@ +from cchardet import detect # type: ignore + + +def detect_encoding_of_file(filename: str): + with open(filename, 'rb') as target_file: + return detect_encoding_of_data(target_file.read()) + + +def detect_encoding_of_data(data: bytes): + return detect(data)['encoding'] + + +def read_text_with_autodetected_encoding(filename: str): + with open(filename, 'rb') as target_file: + data = target_file.read() + + if not data: + return '' # In case of empty file, return empty string + + encoding = detect_encoding_of_data(data) or 'utf-8' + return data.decode(encoding) diff --git a/temp_python/utils/filter.py b/temp_python/utils/filter.py new file mode 100644 index 0000000..bb8d4f9 --- /dev/null +++ b/temp_python/utils/filter.py @@ -0,0 +1,122 @@ +from typing import List, Tuple, Any, Union, TypeVar, Type +from javalang.tree import ClassDeclaration, InterfaceDeclaration, MethodDeclaration, \ + MemberReference, FieldDeclaration, MethodInvocation, This, Node, LocalVariableDeclaration + +FldExh = Tuple[str, Tuple[str, str]] +MthExh = Tuple[str, Tuple[Tuple[str, str], ...]] +HasMember = Union[MemberReference, MethodInvocation] +HasSelector = Union[MemberReference, MethodInvocation, This] +T = TypeVar('T', bound=Node) +S = TypeVar('S', HasSelector, HasMember) +MthNodes = Tuple[tuple, MethodDeclaration] +Nodes = Tuple[tuple, T] +AnyField = Union[FieldDeclaration, LocalVariableDeclaration] + + +class Filters: + + def __init__(self): + pass + + def filter_node_lvl(self, node: Node, javalang_class: Type[Node]) -> List[Nodes]: + """Filters nodes by desired javalang class. + + Gets node(node) of any javalang.Tree type and filters it by + desired type(javalang_class). + Returns a generator with (path, node) inside. + """ + temp_list = [] + for filtered_path, filtered_node in node.filter(javalang_class): + if self.get_class_depth(filtered_path) == 1: + temp_list.append((filtered_path, filtered_node)) + return temp_list + + @staticmethod + def filter_getters_setters(method_node_list: List[MthNodes]) -> List[MthNodes]: + """Filters nodes by name. + + Gets list of nodes of "MethodDeclaration" type and filters it by + name, so that no methods with name starting with "set" or "get" + go to return list. + Returns a generator with (path, node) inside. + """ + + # ToDo: implement get/set detection with .body + temp_list = [] + for path, node in method_node_list: + if node.name.startswith(('get', 'set')): + pass + else: + temp_list.append((path, node)) + return temp_list + + @staticmethod + def get_class_depth(path: tuple) -> int: + """Returns an int displaying level of given node's nesting level. + + Gets a node of any javalang.tree type and calculates it's nesting level + Returns an int. + """ + + class_level = 0 + for step in path: + if isinstance(step, (ClassDeclaration, InterfaceDeclaration, MethodDeclaration)): + class_level += 1 + return class_level + + @staticmethod + def exhaust_method(method_node: MethodDeclaration) -> MthExh: + """ Exhausts name and input vars, types for given MethodDeclaration node. + + Returns a tuple containing name and all parameters + that given method gets as an input. + """ + + parameter_list = [] + name: str = method_node.name + for parameter in method_node.parameters: + parameter_list.append((parameter.name, parameter.type.name)) + parameter_tuple: Tuple[Tuple[str, str], ...] = tuple(parameter_list) + return name, parameter_tuple + + @staticmethod + def exhaust_field(field_node: AnyField) -> FldExh: + """ Exhausts name and type for given FieldDeclaration or LocalVariableDeclaration node. + + Returns a tuple containing name and type of field. + """ + + # ToDo: get rid of'type' in parameter_tuple + name = field_node.declarators[0].name + try: + parameter_tuple: Tuple[str, str] = ('type', field_node.type.name) + except AttributeError: + return "", ("", "") + return name, parameter_tuple + + @staticmethod + def get_arguments(invocation_node: MethodInvocation) -> Tuple[List[str], List[str]]: + """Gets arguments passed to given MethodInvocation node. + + Returns two tuples containing all arguments and methods passed + to given MethodInvocation node. + """ + + list_of_funcs = [] + list_of_fields = [] + for argument in invocation_node.arguments: # type: ignore + if isinstance(argument, MethodInvocation): + list_of_funcs.append(argument.member) + elif isinstance(argument, MemberReference): + list_of_fields.append(argument.member) + return list_of_funcs, list_of_fields + + @staticmethod + def clean_for_repetitions(list_of_exhaust: List[Any]) -> List[Any]: + """Gets any list and removes all repetitions. + + Returns list with no repetitive objects. + """ + + list_of_exhaust = list(dict.fromkeys(list_of_exhaust)) + return list_of_exhaust diff --git a/temp_python/utils/java_parser.py b/temp_python/utils/java_parser.py new file mode 100644 index 0000000..3f65efe --- /dev/null +++ b/temp_python/utils/java_parser.py @@ -0,0 +1,167 @@ +# The MIT License (MIT) +# +# Copyright (c) 2020 Aibolit +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from typing import List, Type + +import javalang + +from temp_python.utils.lines import Lines + + +# mapping between javalang node class names and Java keywords +NODE_KEYWORD_MAP = { + 'SuperMethodInvocation': 'super', + 'WhileStatement': 'while', + 'ForStatement': 'for', + 'TryStatement': 'try', + 'CatchClause': 'catch', + 'SynchronizedStatement': 'synchronized' +} + +# list of nodes creating new scope +NEW_SCOPE_NODES = [ + javalang.tree.MethodDeclaration, + javalang.tree.IfStatement, + javalang.tree.ForStatement, + javalang.tree.SwitchStatement, + javalang.tree.TryStatement, + javalang.tree.DoStatement, + javalang.tree.WhileStatement, + javalang.tree.BlockStatement, + javalang.tree.CatchClause, + javalang.tree.SynchronizedStatement +] + + +class ASTNode: + def __init__(self, line, method_line, node, scope): + self.line = line # node line number in the file + self.method_line = method_line # line number where parent method declared + self.node = node # javalang AST node object + self.scope = scope # ID of scope this node belongs + + +class JavalangImproved: + """ + Thi class flattens AST to provie easier interface. + Deprecated: flatening the AST leads to lose of information and bugs. + All patterns using it should start traversing the tree manually. + """ + + def __init__(self, filename: str): + tree, lines = Lines(filename).value() + self.tree = tree + self.lines = lines + + def __find_keyword(self, lines, keyword, start): + ''' + Args: + lines (List[str]): List of lines from parsed source code file + keyword (str): keyword to find + start (int): Line number to start search + ''' + for i in range(start - 1, len(lines)): + if keyword in lines[i]: + return i + 1 + return -1 + + def __fix_line_number_if_possible(self, node: javalang.ast.Node, line_n): + ''' + Try to figure out the "true" line number of AST node in the source file + ''' + node_class_name = node.__class__.__name__ + if node_class_name not in NODE_KEYWORD_MAP: + return line_n + line = self.__find_keyword( + self.lines, + NODE_KEYWORD_MAP[node_class_name], + line_n + ) + if line == -1: + return line_n + return line + + def __tree_to_nodes( + self, + tree: javalang.ast.Node, + line=1, + parent_method_line=None, + scope=0 + ) -> List[ASTNode]: + ''' + Return AST nodes with line numbers sorted by line number + + Args: + tree (javalang.ast.Node): AST node + line (int): Supposed line number of processed AST node in the file + parent_method_line (int): Nearest line number of the method this node located + scope (int): ID of scope processed AST node located + ''' + + if hasattr(tree, 'position') and tree.position: + line = tree.position.line + + line = self.__fix_line_number_if_possible(tree, line) + if type(tree) in NEW_SCOPE_NODES: + scope += 1 + + if type(tree) in [javalang.tree.MethodDeclaration, + javalang.tree.ConstructorDeclaration, + javalang.tree.LambdaExpression]: + parent_method_line = line + + res: List[ASTNode] = [] + for child in tree.children: + nodes_arr = child if isinstance(child, list) else [child] + for node in nodes_arr: + if not hasattr(node, 'children'): + continue + left_siblings_line = res[-1].line if len(res) > 0 else line + res += self.__tree_to_nodes( + node, + left_siblings_line, + parent_method_line, + scope + ) + + return [ASTNode(line, parent_method_line, tree, scope)] + res + + def tree_to_nodes(self) -> List[ASTNode]: + '''Return AST nodes as list with line numbers sorted by line number''' + nodes = self.__tree_to_nodes(self.tree) + return sorted(nodes, key=lambda v: v.line) + + def filter(self, ntypes: List[Type[javalang.tree.Node]]) -> List[ASTNode]: + nodes = self.tree_to_nodes() + return list( + filter(lambda v: type(v.node) in ntypes, nodes) + ) + + def get_empty_lines(self) -> List[int]: + '''Figure out lines that are either empty or multiline statements''' + lines_with_nodes = self.get_non_empty_lines() + max_line = max(lines_with_nodes) + return list(set(range(1, max_line + 1)).difference(lines_with_nodes)) + + def get_non_empty_lines(self) -> List[int]: + '''Figure out file lines that contains statements''' + return list(map(lambda v: v.line, self.tree_to_nodes())) diff --git a/temp_python/utils/lines.py b/temp_python/utils/lines.py new file mode 100644 index 0000000..d375b3d --- /dev/null +++ b/temp_python/utils/lines.py @@ -0,0 +1,23 @@ +from javalang.ast import Node +from javalang.tree import CompilationUnit +from typing import List, Tuple + +from javalang.parse import parse + +from temp_python.utils.encoding_detector import read_text_with_autodetected_encoding + + +class Lines: + """ + Return the lines for some AST + Deprecated: This class is used ony once by JavalangImproved + and does not provide any complex functionality, so should be removed. + """ + def __init__(self, filename: str): + source_code = read_text_with_autodetected_encoding(filename) + + self._lines = source_code.splitlines(keepends=True) + self._tree: CompilationUnit = parse(source_code) + + def value(self) -> Tuple[Node, List[str]]: + return self._tree, self._lines diff --git a/temp_python/utils/scope_status.py b/temp_python/utils/scope_status.py new file mode 100644 index 0000000..b07044b --- /dev/null +++ b/temp_python/utils/scope_status.py @@ -0,0 +1,70 @@ +# The MIT License (MIT) +# +# Copyright (c) 2020 Aibolit +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from enum import Enum, auto + +from typing import List, Set + + +class ScopeStatusFlags(Enum): + ONLY_VARIABLE_DECLARATIONS_PRESENT = auto() + INSIDE_VARIABLE_DECLARATION_SUBTREE = auto() + INSIDE_CALLING_SUPER_CLASS_CONSTRUCTOR_SUBTREE = auto() + INSIDE_ANNOTATION_SUBTREE = auto() + + +class ScopeStatus: + _default_scope_status: Set[ScopeStatusFlags] = \ + {ScopeStatusFlags.ONLY_VARIABLE_DECLARATIONS_PRESENT} + + def __init__(self): + # Copy _default_scope_status to prevent its modification + self._scope_stack: List[Set[ScopeStatusFlags]] = \ + [ScopeStatus._default_scope_status.copy()] + + def get_status(self): + try: + return self._scope_stack[-1] + except IndexError: + raise RuntimeError("No scopes registered.") + + def add_flag(self, flag): + try: + self._scope_stack[-1].add(flag) + except IndexError: + raise RuntimeError("No scopes registered.") + + def remove_flag(self, flag): + try: + self._scope_stack[-1].discard(flag) + except IndexError: + raise RuntimeError("No scopes registered.") + + def enter_new_scope(self, new_scope_status=_default_scope_status): + # Copy new_scope_status to prevent its modification + self._scope_stack.append(new_scope_status.copy()) + + def leave_current_scope(self): + try: + self._scope_stack.pop() + except IndexError: + raise RuntimeError("No scopes registered.") diff --git a/temp_python/utils/utils.py b/temp_python/utils/utils.py new file mode 100644 index 0000000..c90e9ed --- /dev/null +++ b/temp_python/utils/utils.py @@ -0,0 +1,17 @@ +import re + + +class RemoveComments: + + def __init__(self): + pass + + @staticmethod + def remove_comments(string): + # remove all occurrences streamed comments (/*COMMENT */) from string + string = re.sub(re.compile(r"/\*.*?\*/", re.DOTALL), "", + string) + # remove all occurrence single-line comments (//COMMENT\n ) from string + string = re.sub(re.compile(r"//.*?\n"), "", + string) + return string