From 81c704576a732f2d694c0a995c870a2fae839110 Mon Sep 17 00:00:00 2001 From: shmyga Date: Mon, 20 Mar 2023 00:14:42 +0300 Subject: [PATCH] refactor: add pipe module --- project_manager/default_preset.toml | 4 +- project_manager/pipe/__init__.py | 3 + project_manager/pipe/core.py | 83 +++++++++++++++++++++++++ project_manager/pipe/type/__init__.py | 0 project_manager/pipe/type/equal.py | 16 +++++ project_manager/pipe/type/file.py | 37 +++++++++++ project_manager/pipe/type/get.py | 21 +++++++ project_manager/pipe/type/not_.py | 13 ++++ project_manager/pipe/type/search.py | 20 ++++++ project_manager/pipe/util.py | 13 ++++ project_manager/preset/entry.py | 12 ++-- project_manager/preset/model.py | 9 +-- project_manager/preset/value.py | 88 --------------------------- 13 files changed, 219 insertions(+), 100 deletions(-) create mode 100644 project_manager/pipe/__init__.py create mode 100644 project_manager/pipe/core.py create mode 100644 project_manager/pipe/type/__init__.py create mode 100644 project_manager/pipe/type/equal.py create mode 100644 project_manager/pipe/type/file.py create mode 100644 project_manager/pipe/type/get.py create mode 100644 project_manager/pipe/type/not_.py create mode 100644 project_manager/pipe/type/search.py create mode 100644 project_manager/pipe/util.py delete mode 100644 project_manager/preset/value.py diff --git a/project_manager/default_preset.toml b/project_manager/default_preset.toml index 11fc131..7de7f26 100644 --- a/project_manager/default_preset.toml +++ b/project_manager/default_preset.toml @@ -43,11 +43,11 @@ filename = "package.json" [presets.nodejs.tasks.setup.npm] conditions = ["file(package-lock.json)"] -commands = ["npm install-clean"] +commands = ["npm install-clean --legacy-peer-deps"] [presets.nodejs.tasks.setup.npm-legacy] conditions = ["file(package-lock.json) | not"] -commands = ["npm install"] +commands = ["npm install --legacy-peer-deps"] [presets.nodejs.tasks.test.jest] conditions = ["file(package.json) | get(devDependencies.jest)"] diff --git a/project_manager/pipe/__init__.py b/project_manager/pipe/__init__.py new file mode 100644 index 0000000..1cd4f2c --- /dev/null +++ b/project_manager/pipe/__init__.py @@ -0,0 +1,3 @@ +from .core import PIPE_REGISTRY + +PIPE_REGISTRY.register_module("project_manager.pipe.type") diff --git a/project_manager/pipe/core.py b/project_manager/pipe/core.py new file mode 100644 index 0000000..9713619 --- /dev/null +++ b/project_manager/pipe/core.py @@ -0,0 +1,83 @@ +import importlib +import re +from pathlib import Path +from typing import Any, Generic, List, Optional, Type, TypeVar + +__all__ = ("Pipe", "PipeRegistry", "PipeParser", "PipeRunner", "PIPE_REGISTRY") + +I = TypeVar("I") +O = TypeVar("O") + + +class Pipe(Generic[I, O]): + type: str + + def apply(self, value: I, basepath: Path) -> Optional[O]: + raise NotImplementedError + + +class PipeRegistry: + def __init__(self): + self._pipes = {} + + def register(self, pipe_type: Type[Pipe]): + self._pipes[pipe_type.type] = pipe_type + + def register_module(self, module_name: str): + module = importlib.import_module(module_name) + modulepath = Path(module.__file__).parent + for item in modulepath.iterdir(): + if item.suffix == ".py" and not item.name.startswith("__"): + sub_module = importlib.import_module(f"{module_name}.{item.stem}") + for v in sub_module.__dict__.values(): + if isinstance(v, type): + for baseclass in v.__bases__: + if baseclass == Pipe: + self.register(v) + + def get(self, pipe_type: str) -> Type[Pipe]: + return self._pipes[pipe_type] + + +class PipeParser: + _PIPE_PATTERN = re.compile("^(\\w+)\\((.*)\\)") + + def parse(self, pipeline: str) -> List[Pipe]: + chain = [item.strip() for item in pipeline.split("|")] + result = [] + for part in chain: + matched = self._PIPE_PATTERN.match(part) + if matched: + pipe_name = matched.group(1) + pipe_args = [item.strip() for item in matched.group(2).split(",")] + else: + pipe_name = part.strip() + pipe_args = [] + pipe_type = PIPE_REGISTRY.get(pipe_name) + result.append(pipe_type(*pipe_args)) + return result + + +class PipeRunner: + def __init__(self, basepath: Path, parser: Optional[PipeParser] = None): + self._basepath = basepath + self._parser = parser or PipeParser() + + def run(self, pipes: List[Pipe]) -> Any: + result = None + for pipe in pipes: + result = pipe.apply(result, self._basepath) + return result + + def resolve_value(self, pipeline: str) -> Any: + pipe = self._parser.parse(pipeline) + return self.run(pipe) + + def check_conditions(self, conditions: List[str]) -> bool: + for condition in conditions: + if not self.resolve_value(condition): + return False + return True + + +PIPE_REGISTRY = PipeRegistry() diff --git a/project_manager/pipe/type/__init__.py b/project_manager/pipe/type/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/project_manager/pipe/type/equal.py b/project_manager/pipe/type/equal.py new file mode 100644 index 0000000..3336f70 --- /dev/null +++ b/project_manager/pipe/type/equal.py @@ -0,0 +1,16 @@ +from pathlib import Path +from typing import Any + +from ..core import Pipe + +__all__ = ("EqualPipe",) + + +class EqualPipe(Pipe[Any, bool]): + type = "equal" + + def __init__(self, value: Any): + self._value = value + + def apply(self, value: Any, basepath: Path) -> bool: + return self._value == value diff --git a/project_manager/pipe/type/file.py b/project_manager/pipe/type/file.py new file mode 100644 index 0000000..c0ba867 --- /dev/null +++ b/project_manager/pipe/type/file.py @@ -0,0 +1,37 @@ +from pathlib import Path +from typing import Any, Optional + +from ..core import Pipe +from ..util import reflect + +__all__ = ("FilePipe",) + + +class FilePipe(Pipe[None, Any]): + type = "file" + _LOADERS = { + ".toml": "tomli:load", + ".json": "json:load", + } + + def __init__(self, filename: Optional[str], loader: Optional[str] = None): + self._filename = filename + self._loader = loader + + def _resolve_loader(self, filepath: Path): + loader = self._loader or self._LOADERS.get(filepath.suffix) + return reflect(loader) if loader else None + + def apply(self, value: Optional[str], basepath: Path) -> Optional[Any]: + filepath = Path(value or self._filename) # TODO: raise none filepath exception + if not filepath.is_absolute(): + filepath = basepath / filepath + if not filepath.exists(): + return None + loader = self._resolve_loader(filepath) + with open(filepath, "rb") as fd: + if loader: + result = loader(fd) + else: + result = fd.read() + return result diff --git a/project_manager/pipe/type/get.py b/project_manager/pipe/type/get.py new file mode 100644 index 0000000..8ab9402 --- /dev/null +++ b/project_manager/pipe/type/get.py @@ -0,0 +1,21 @@ +from pathlib import Path +from typing import Any, Optional + +from ..core import Pipe + +__all__ = ("GetPipe",) + + +class GetPipe(Pipe[dict, Any]): + type = "get" + + def __init__(self, keypath: str): + self._keypath = keypath + + def apply(self, value: dict, basepath: Path) -> Optional[Any]: + path = self._keypath.split(".") + result = value.get(path[0]) + for key in path[1:]: + if result: + result = result.get(key) + return result diff --git a/project_manager/pipe/type/not_.py b/project_manager/pipe/type/not_.py new file mode 100644 index 0000000..8bcfcc9 --- /dev/null +++ b/project_manager/pipe/type/not_.py @@ -0,0 +1,13 @@ +from pathlib import Path +from typing import Any + +from ..core import Pipe + +__all__ = ("NotPipe",) + + +class NotPipe(Pipe[Any, bool]): + type = "not" + + def apply(self, value: Any, basepath: Path) -> bool: + return not value diff --git a/project_manager/pipe/type/search.py b/project_manager/pipe/type/search.py new file mode 100644 index 0000000..756e646 --- /dev/null +++ b/project_manager/pipe/type/search.py @@ -0,0 +1,20 @@ +import re +from pathlib import Path +from typing import Optional + +from ..core import Pipe + +__all__ = ("SearchPipe",) + + +class SearchPipe(Pipe[str, str]): + type = "search" + + def __init__(self, patter: str, position: str = "1"): + self._patter = patter + self._position = int(position) + + def apply(self, value: str, basepath: Path) -> Optional[str]: + matched = re.compile(self._patter).search(value) + if matched: + return matched.group(self._position) diff --git a/project_manager/pipe/util.py b/project_manager/pipe/util.py new file mode 100644 index 0000000..a4dd461 --- /dev/null +++ b/project_manager/pipe/util.py @@ -0,0 +1,13 @@ +import importlib +from typing import Any + +__all__ = ("reflect",) + + +def reflect(value: str) -> Any: + if ":" in value: + module_name, item_name = value.split(":") + else: + module_name, item_name = "builtins", value + module = importlib.import_module(module_name) + return getattr(module, item_name) diff --git a/project_manager/preset/entry.py b/project_manager/preset/entry.py index 14b5e5e..d5f52cd 100644 --- a/project_manager/preset/entry.py +++ b/project_manager/preset/entry.py @@ -1,30 +1,34 @@ +from cmath import pi from collections import defaultdict from pathlib import Path from typing import Optional from ..entry import Entry, EntryDetector +from ..pipe.core import PipeParser, PipeRunner from .util import PresetUtil -from .value import ValueResolver __all__ = ("PresetEntryDetector",) class PresetEntryDetector(EntryDetector): + _PIPE_PARSER = PipeParser() + def __init__(self, filename: Path): self._preset = PresetUtil.resolve_preset(filename) + self._pipeparser = PipeParser() def detect_entry(self, basepath: Path) -> Optional["Entry"]: + piperunner = PipeRunner(basepath) tasks = defaultdict(list) for preset_name, preset in self._preset.presets.items(): filename = basepath / preset.filename if filename.exists(): - value_resolver = ValueResolver(basepath) for task_name, task_pack in preset.tasks.items(): for task_item_name, task in task_pack.items(): - if value_resolver.check_conditions(task.conditions): + if piperunner.check_conditions(task.conditions): variables = ( { - k: value_resolver.resolve_value(v) + k: piperunner.resolve_value(v) for k, v in task.variables.items() } if task.variables diff --git a/project_manager/preset/model.py b/project_manager/preset/model.py index 7b35019..e7a327d 100644 --- a/project_manager/preset/model.py +++ b/project_manager/preset/model.py @@ -2,15 +2,12 @@ from typing import Dict, List, Optional from pydantic import BaseModel -__all__ = ("PipeStr", "TaskPreset", "FilePreset", "Preset") - - -PipeStr = str +__all__ = ("TaskPreset", "FilePreset", "Preset") class TaskPreset(BaseModel): - conditions: List[PipeStr] - variables: Optional[Dict[str, PipeStr]] + conditions: List[str] + variables: Optional[Dict[str, str]] commands: List[str] diff --git a/project_manager/preset/value.py b/project_manager/preset/value.py deleted file mode 100644 index df36ac7..0000000 --- a/project_manager/preset/value.py +++ /dev/null @@ -1,88 +0,0 @@ -import importlib -import re -from pathlib import Path -from typing import Any, List, Optional - -from .model import PipeStr - -__all__ = ("ValueResolver",) - - -class ValueResolver: - PIPE_PATTERN = re.compile("^(\\w+)\\((.*)\\)") - FILE_LOADERS = { - ".toml": "tomli:load", - ".json": "json:load", - } - - def __init__(self, basepath: Path): - self._basepath = basepath - - def _reflect(self, value: str) -> Any: - if ":" in value: - module_name, item_name = value.split(":") - else: - module_name, item_name = "builtins", value - module = importlib.import_module(module_name) - return getattr(module, item_name) - - def pipe_file(self, pipe, filename: str, loader: Optional[str] = None): - filepath = Path(filename) - if not filepath.is_absolute(): - filepath = self._basepath / filepath - if loader is None: - loader = self.FILE_LOADERS.get(Path(filename).suffix) - if not filepath.exists(): - return None - with open(filepath, "rb") as fd: - if loader: - loader_fun = self._reflect(loader) - data = loader_fun(fd) - else: - data = fd.read() - return data - - def pipe_get(self, pipe, keypath: str): - path = keypath.split(".") - value = pipe.get(path[0]) - for key in path[1:]: - if value: - value = value.get(key) - return value - - def pipe_search(self, pipe, pattern: str, position_arg: str = "1"): - matched = re.compile(pattern).search(pipe) - position = int(position_arg) - if matched: - return matched.group(position) - - def pipe_equal(self, pipe, value: Any): - return pipe == value - - def pipe_convert(self, pipe, converter: str): - converter_fun = self._reflect(converter) - return converter_fun(pipe) - - def pipe_not(self, pipe): - return not pipe - - def resolve_value(self, operator: str) -> Any: - chain = [item.strip() for item in operator.split("|")] - pipe = None - for part in chain: - matched = self.PIPE_PATTERN.match(part) - if matched: - pipe_name = matched.group(1) - pipe_args = [item.strip() for item in matched.group(2).split(",")] - else: - pipe_name = part.strip() - pipe_args = [] - pipe_func = getattr(self, f"pipe_{pipe_name}") - pipe = pipe_func(pipe, *pipe_args) - return pipe - - def check_conditions(self, conditions: List[PipeStr]) -> bool: - for condition in conditions: - if not self.resolve_value(condition): - return False - return True