diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 011fcd0..298b87b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,4 +1,4 @@ -name: Test Suite +name: MetaCall Examples CI on: workflow_dispatch: @@ -7,15 +7,12 @@ on: tags: - 'v*.*.*' branches: - - master + - main jobs: - build: - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, windows-latest] # TODO: macos-latest - runs-on: ${{ matrix.os }} + LinuxUbuntuRun: + name: Linux - Ubuntu Run + runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v4 @@ -23,30 +20,66 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.12' + + - name: Install MetaCall + run: | + sudo apt update + curl -sL https://raw.githubusercontent.com/metacall/install/master/install.sh | sh - - name: Install MetaCall (Linux) - if: matrix.os == 'ubuntu-latest' - run: curl -sL https://raw.githubusercontent.com/metacall/install/master/install.sh | sh + - name: Install Dependencies + run: | + pip install -r requirements.txt + + - name: Run Tests Suits + run: | + pip install -r requirements.txt + find test-suites -type f -name "*.yaml" -exec python ./testing.py -f {} -V \; - - name: Install MetaCall (Windows) - if: matrix.os == 'windows-latest' - run: powershell -NoProfile -ExecutionPolicy unrestricted -Command "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; &([scriptblock]::Create((Invoke-WebRequest -UseBasicParsing 'https://raw.githubusercontent.com/metacall/install/master/install.ps1')))" + # WindowsRun: + # name: Windows Run + # runs-on: windows-latest + # steps: + # - name: Checkout code + # uses: actions/checkout@v4 - # TODO: - # - name: Install MetaCall (MacOS) - # if: matrix.os == 'macos-latest' - # run: curl -sL https://raw.githubusercontent.com/metacall/install/master/install.sh | sh + # - name: Set up Python + # uses: actions/setup-python@v5 + # with: + # python-version: '3.12' + + # - name: Install MetaCall + # shell: pwsh + # run: | + # powershell -NoProfile -ExecutionPolicy unrestricted -Command "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; &([scriptblock]::Create((Invoke-WebRequest -UseBasicParsing 'https://raw.githubusercontent.com/metacall/install/master/install.ps1')))" + + # - name: Run Test Suits + # shell: pwsh + # run: | + # pip install pyyaml + # Get-ChildItem -Path test-suits -Filter *.yaml | ForEach-Object { python ./metacall-test.py -f $_.FullName -V } + # Clear-Host + # for /f %f in (failed-test-cases.txt) do type %f && exit /b 1 - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt + # MacOSRun: + # name: MacOS Run + # runs-on: macos-latest + # steps: + # - name: Checkout code + # uses: actions/checkout@v4 - - name: Run tests - shell: bash - run: | - for test in ./suites/*.yaml; do - echo "Running $test" - python3 ./testing.py -f $test -V - done + # - name: Set up Python + # uses: actions/setup-python@v5 + # with: + # python-version: '3.12' + + # - name: Install MetaCall + # run: | + # curl -sL https://raw.githubusercontent.com/metacall/install/master/install.sh | sh + + # - name: Run Tests Suits + # run: | + # pip install pyyaml + # find test-suits -type f -name "*.yaml" -exec python ./metacall-test.py -f {} -V \; + # clear + # for file in failed-test-cases.txt; do [ -s "$file" ] && cat failed-test-cases.txt && exit 1; done diff --git a/README.md b/README.md index 5f834f0..ab5a5b0 100644 --- a/README.md +++ b/README.md @@ -12,10 +12,10 @@ code-files: test-cases: - name: Check the password is generated in the correct length command: call getRandomPassword(12) - expected-stdout: '\"[\w\W]{12}\"' + expected-pattern: '\"[\w\W]{12}\"' - name: Check the password is generated in the correct length command: call getRandomPassword() - expected-stdout: 'missing 1 required positional argument' + expected-pattern: 'missing 1 required positional argument' ``` ## Arguments @@ -31,5 +31,5 @@ options: ``` Example: ```bash -./testing.py -f suites/test-time-app-web.yaml -V +python3 ./testing.py -f test-suites/test-time-app-web.yaml -V ``` diff --git a/TODO.md b/TODO.md index 08ac58f..5e574d1 100644 --- a/TODO.md +++ b/TODO.md @@ -1,11 +1,11 @@ # ToDos -- [ ] Create a stable version that runs existing tests. -- [ ] If any test fail, it should exit with error code 1. Right now tests are failing and it is passing with exit code 0. +- [x] Create a stable version that runs existing tests. +- [x] If any test fail, it should exit with error code 1. Right now tests are failing and it is passing with exit code 0. - [ ] Support FaaS. - [ ] Update/fix test cases. -- [x] Run the pipeline on Linux. -- [x] Run the pipeline on Windows. +- [ ] Run the pipeline on Linux. +- [ ] Run the pipeline on Windows. - [ ] Run the pipeline on MacOS. - [ ] Link the pipeline with the distributables and install script. - [ ] Update the Readme. diff --git a/suites/test-auth-function-mesh.yaml b/test-auth-function-mesh.yaml similarity index 76% rename from suites/test-auth-function-mesh.yaml rename to test-auth-function-mesh.yaml index 8cbd7d6..593e816 100644 --- a/suites/test-auth-function-mesh.yaml +++ b/test-auth-function-mesh.yaml @@ -6,10 +6,10 @@ code-files: test-cases: - name: Check the encrypt function command: call encrypt("asd") - expected-stdout: 'eyJhbGciOiJIUzI1NiJ9.YXNk.QNa-p8QpuHcVUDMN_Ih4x4vidWp31365GM4zrSr3t0s' + expected-pattern: 'eyJhbGciOiJIUzI1NiJ9.YXNk.QNa-p8QpuHcVUDMN_Ih4x4vidWp31365GM4zrSr3t0s' - name: Check the decrypt function command: call decrypt("eyJhbGciOiJIUzI1NiJ9.YXNk.QNa-p8QpuHcVUDMN_Ih4x4vidWp31365GM4zrSr3t0s") - expected-stdout: 'asd' + expected-pattern: 'asd' diff --git a/suites/test-auth-middleware.yaml b/test-auth-middleware.yaml similarity index 82% rename from suites/test-auth-middleware.yaml rename to test-auth-middleware.yaml index fa98946..e2c8095 100644 --- a/suites/test-auth-middleware.yaml +++ b/test-auth-middleware.yaml @@ -6,13 +6,13 @@ code-files: test-cases: - name: Check the sum function command: call sum(3,4) - expected-stdout: '7' + expected-pattern: '7' - name: Check the signin function command: call signin("Mostafa","1557") - expected-stdout: '.*' + expected-pattern: '.*' - name: Check the reverse function command: call reverse("abcdefg") - expected-stdout: '.*' + expected-pattern: "gfedcba" diff --git a/suites/test-random-password-generator.yaml b/test-suites/test-random-password-generator.yaml similarity index 72% rename from suites/test-random-password-generator.yaml rename to test-suites/test-random-password-generator.yaml index 23a5345..9ffc5d4 100644 --- a/suites/test-random-password-generator.yaml +++ b/test-suites/test-random-password-generator.yaml @@ -7,9 +7,9 @@ code-files: test-cases: - name: Check the password is generated in the correct length command: call getRandomPassword(12) - expected-stdout: '\"[\w\W]{12}\"' - - name: Check the password is generated in the correct length + expected-pattern: '[\S]{12}' + - name: Check for missing argument command: call getRandomPassword() - expected-stdout: 'missing 1 required positional argument' + expected-pattern: 'missing 1 required positional argument' \ No newline at end of file diff --git a/suites/test-string-manipulation.yaml b/test-suites/test-string-manipulation.yaml similarity index 75% rename from suites/test-string-manipulation.yaml rename to test-suites/test-string-manipulation.yaml index f12326d..618bc22 100644 --- a/suites/test-string-manipulation.yaml +++ b/test-suites/test-string-manipulation.yaml @@ -6,13 +6,13 @@ code-files: test-cases: - name: Check the longest_repetition function command: call longest_repetition("aaa") - expected-stdout: "[ 'a', 3 ]" + expected-pattern: "[ 'a', 3 ]" - name: Check the longest_repetition function with a different string command: call longest_repetition("bbbaaabaaaa") - expected-stdout: "[ 'a', 4 ]" + expected-pattern: "[ 'a', 4 ]" - name: Check the longest_repetition function with an empty string command: call longest_repetition("") - expected-stdout: '\[null,0\]' + expected-pattern: "NodeJS Loader could not convert the value of type 'Invalid' to N-API" diff --git a/suites/test-time-app-web.yaml b/test-suites/test-time-app-web.yaml similarity index 69% rename from suites/test-time-app-web.yaml rename to test-suites/test-time-app-web.yaml index 9bc57d6..0a6eb6d 100644 --- a/suites/test-time-app-web.yaml +++ b/test-suites/test-time-app-web.yaml @@ -5,13 +5,13 @@ code-files: test-cases: - name: Check the time is generated in the correct format command: call time() - expected-stdout: '\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}' + expected-pattern: '\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}' - name: Check calling the time function with a parameter command: call time(1) - expected-stdout: 'takes 0 positional arguments but 1 was given' + expected-pattern: 'takes 0 positional arguments but 1 was given' - name: Check index.html is fully returned command: call index() - expected-stdout: '[\w\W]*' + expected-pattern: '[\w\W]*' diff --git a/testing.py b/testing.py index 6f0e558..3033347 100644 --- a/testing.py +++ b/testing.py @@ -18,13 +18,17 @@ def main(): test_suite_file_name = args.file + + test_suites_extractor = TestSuitesExtractor(test_suite_file_name) project_name, repo_url, test_suites = test_suites_extractor.extract_test_suites() + logger.info(f"Project: {project_name}") + repo_manager = RepoManager(repo_url) repo_manager.clone_repo_if_not_exist() - test_runner = TestRunner("composite") + test_runner = TestRunner(["cli"]) test_runner.run_tests(project_name, test_suites) if __name__ == "__main__": diff --git a/testing/logger.py b/testing/logger.py index f8df373..8a0f492 100644 --- a/testing/logger.py +++ b/testing/logger.py @@ -15,6 +15,7 @@ def __init__(self): else: Logger._instance = self self.logger = logging.getLogger("CLI_Tool") + self.level = "INFO" # default level handler = logging.StreamHandler() formatter = logging.Formatter('%(levelname)s - %(message)s') handler.setFormatter(formatter) @@ -29,8 +30,12 @@ def set_level(self, level): "ERROR": logging.ERROR, "CRITICAL": logging.CRITICAL } + self.level = level.upper() self.logger.setLevel(level_map.get(level.upper(), logging.INFO)) + def get_level(self): + return self.level + def debug(self, msg): self.logger.debug(msg) diff --git a/testing/repo_manager.py b/testing/repo_manager.py index 38dccee..792f595 100644 --- a/testing/repo_manager.py +++ b/testing/repo_manager.py @@ -11,14 +11,15 @@ def clone_repo_if_not_exist(self): try: repo_name = self.repo_url.split('/')[-1].split('.')[0] if os.path.isdir(repo_name): - self.logger.debug("Repo is already cloned!") + self.logger.warning("Repo is already cloned!") else: process = subprocess.Popen(['git', 'clone', self.repo_url], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) _, stderr = process.communicate() stderr = stderr.decode('utf-8') - if "Fatal" in stderr or "fatal" in stderr or "error" in stderr or "Error" in stderr or "ERROR" in stderr: - raise Exception(stderr) + error_keywords = ["Fatal", "fatal", "error", "Error", "ERROR"] + if any(keyword in stderr for keyword in error_keywords): + raise ValueError(stderr) self.logger.debug("Repo is cloned successfully!") - except Exception as e: + except ValueError as e: self.logger.error(f"Error: {e}") exit() diff --git a/testing/runner/cli_interface.py b/testing/runner/cli_interface.py index 719da8a..916bfa1 100644 --- a/testing/runner/cli_interface.py +++ b/testing/runner/cli_interface.py @@ -5,6 +5,8 @@ class CLIInterface(RunnerInterface): def __init__(self): self.logger = Logger.get_instance() + def get_name(self): + return "cli" def get_runtime_tag(self, file_name): file_extension = file_name.split('.')[-1] @@ -21,23 +23,23 @@ def get_runtime_tag(self, file_name): else: raise ValueError("Error: file extension not supported!") - def run_test_command(self, filename, file_path, test_case_command): + def run_test_command(self, file_path, test_case_command): + file_name = file_path.split('/')[-1] try: process = subprocess.Popen(['metacall'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - self.logger.debug("Metacall CLI started...") - - commands = ['load ' + ' ' + self.get_runtime_tag(filename) + ' ' + file_path, test_case_command, 'exit'] + commands = ['load ' + ' ' + self.get_runtime_tag(file_name) + ' ' + file_path, test_case_command, 'exit'] commands = '\n'.join(commands) + '\n' # join the commands with a newline character - + process.stdin.write(f"{commands}".encode('utf-8')) process.stdin.flush() stdout, _ = process.communicate() + out_str = stdout.decode('utf-8').strip().split('λ') - out_str = out_str[2] - return out_str + + return out_str[2] - except Exception as e: + except subprocess.CalledProcessError as e: self.logger.error(f"Error: {e}") - return None + return "" diff --git a/testing/runner/composite_runner_interface.py b/testing/runner/composite_runner_interface.py deleted file mode 100644 index a2e79ae..0000000 --- a/testing/runner/composite_runner_interface.py +++ /dev/null @@ -1,15 +0,0 @@ -from testing.runner.runner_interface import RunnerInterface - -class CompositeRunnerInterface(RunnerInterface): - def __init__(self): - self.runners = [] - - def add_runner(self, runner): - self.runners.append(runner) - - def run_test_command(self, filename, file_path, test_case_command): - results = {} - for runner in self.runners: - runner_name = type(runner).__name__ - results[runner_name] = runner.run_test_command(filename, file_path, test_case_command) - return results diff --git a/testing/runner/faas_interface.py b/testing/runner/faas_interface.py index 688feba..4adce55 100644 --- a/testing/runner/faas_interface.py +++ b/testing/runner/faas_interface.py @@ -1,7 +1,13 @@ from testing.runner.runner_interface import RunnerInterface class FaaSInterface(RunnerInterface): - def run_test_command(self, filename, file_path, test_case_command): + def __init__(self): + pass + + def get_name(self): + return "faas" + + def run_test_command(self, file_path, test_case_command): # Implement the FaaS call here # For now, return a placeholder string return "FaaS output placeholder" diff --git a/testing/runner/interface_factory.py b/testing/runner/interface_factory.py index 884edd7..d993d50 100644 --- a/testing/runner/interface_factory.py +++ b/testing/runner/interface_factory.py @@ -1,6 +1,5 @@ from testing.runner.cli_interface import CLIInterface from testing.runner.faas_interface import FaaSInterface -from testing.runner.composite_runner_interface import CompositeRunnerInterface class InterfaceFactory: @staticmethod @@ -9,10 +8,5 @@ def get_interface(interface_type): return CLIInterface() elif interface_type == "faas": return FaaSInterface() - elif interface_type == "composite": - composite = CompositeRunnerInterface() - composite.add_runner(CLIInterface()) - composite.add_runner(FaaSInterface()) - return composite else: raise ValueError(f"Unknown interface type: {interface_type}") diff --git a/testing/runner/runner_interface.py b/testing/runner/runner_interface.py index 9922ebc..be435ca 100644 --- a/testing/runner/runner_interface.py +++ b/testing/runner/runner_interface.py @@ -2,5 +2,5 @@ class RunnerInterface(ABC): @abstractmethod - def run_test_command(self, filename, file_path, test_case_command): + def run_test_command(self, file_path, test_case_command): pass diff --git a/testing/test_runner.py b/testing/test_runner.py index 239bf12..0eb1ee8 100644 --- a/testing/test_runner.py +++ b/testing/test_runner.py @@ -1,37 +1,70 @@ import re +import unittest from testing.runner.interface_factory import InterfaceFactory from testing.logger import Logger -class TestRunner: - def __init__(self, interface_type): - self.interface = InterfaceFactory.get_interface(interface_type) - self.logger = Logger.get_instance() +class TestCaseGenerator: + @staticmethod + def create_test_method(interface, test_case_name, test_case_command, test_case_expected_stdout): + def test_method(self): + out_str = interface.run_test_command(self.file_path, test_case_command) + passed = self.check_match(out_str, test_case_expected_stdout) + self.assertTrue(passed, f"{interface.get_name()}_{test_case_name} - Expected: {test_case_expected_stdout}, Actual: {out_str}") + return test_method - def run_tests(self, project_name, test_suites): - self.logger.info(f"{project_name}\n================================") - for file_path, test_cases in test_suites: - file_name = file_path.split('/')[-1] + @staticmethod + def check_match(actual, expected_pattern): + actual_cleaned = re.sub(r'\s+', ' ', actual).strip() + expected_pattern_cleaned = re.sub(r'\s+', ' ', expected_pattern).strip() + return bool(re.search(expected_pattern_cleaned, actual_cleaned, re.DOTALL)) - self.logger.info(f"{file_name}\n=============") - for test_case_order, test_case in enumerate(test_cases): - test_case_name, test_case_command, test_case_expected_stdout = test_case - self.logger.debug(f"{test_case_order}. Test Case Command: {test_case_command}") +class DynamicTestSuiteFactory: + def __init__(self, logger, interfaces): + self.logger = logger + self.interfaces = interfaces - results = self.interface.run_test_command(file_name, file_path, test_case_command) + def create_test_suite(self, file_path, test_cases): + logger = self.logger - all_passed = True - for runner_name, out_str in results.items(): + class DynamicTestSuite(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.file_path = file_path + cls.logger = logger - passed = self.check_output(out_str, test_case_expected_stdout) - self.logger.info(f"{runner_name} - Test Case: {test_case_name} {'PASSED' if passed else 'FAILED'}") - if not passed: - all_passed = False - self.logger.debug(f"{runner_name} - Expected: {test_case_expected_stdout}") - self.logger.debug(f"{runner_name} - Actual: {out_str}") + @staticmethod + def check_match(actual, expected_pattern): + return TestCaseGenerator.check_match(actual, expected_pattern) - self.logger.info(f"Overall Test Case: {test_case_name} {'PASSED' if all_passed else 'FAILED'}") + for test_case in test_cases: + test_case_name, test_case_command, test_case_expected_stdout = test_case + for interface in self.interfaces: + test_method = TestCaseGenerator.create_test_method(interface, test_case_name, test_case_command, test_case_expected_stdout) + test_method_name = f'testCase_{interface.get_name()}_{test_case_name}' + setattr(DynamicTestSuite, test_method_name, test_method) - def check_output(self, actual, expected_pattern): - actual_cleaned = re.sub(r'\s+', ' ', actual).strip() - expected_pattern_cleaned = re.sub(r'\s+', ' ', expected_pattern).strip() - return bool(re.search(expected_pattern_cleaned, actual_cleaned, re.DOTALL)) + return DynamicTestSuite + +class TestRunner: + def __init__(self, interface_types): + self.logger = Logger.get_instance() + self.interfaces = [InterfaceFactory.get_interface(interface_type) for interface_type in interface_types] + self.test_suite_factory = DynamicTestSuiteFactory(self.logger, self.interfaces) + self.test_verbosity = 2 if self.logger.get_level() == "DEBUG" else 1 + + def create_project_test_suites(self, test_suites): + test_loader = unittest.TestLoader() + master_suite = unittest.TestSuite() + + for file_path, test_cases in test_suites: + test_suite = self.test_suite_factory.create_test_suite(file_path, test_cases) + master_suite.addTests(test_loader.loadTestsFromTestCase(test_suite)) + return master_suite + + def run_tests(self, project_name, test_suites): + master_suite = self.create_project_test_suites(test_suites) + runner = unittest.TextTestRunner(verbosity=self.test_verbosity) + result = runner.run(master_suite) + if not result.wasSuccessful(): + self.logger.error("Some tests failed!") + exit(1) diff --git a/testing/test_suites_extractor.py b/testing/test_suites_extractor.py index bd4d1a7..54e41e2 100644 --- a/testing/test_suites_extractor.py +++ b/testing/test_suites_extractor.py @@ -20,7 +20,7 @@ def extract_test_suites(self): code_files = data['code-files'] test_suites = [] for code_file in code_files: - test_cases = [(test_case['name'], test_case['command'], test_case['expected-stdout']) for test_case in code_file['test-cases']] + test_cases = [(test_case['name'], test_case['command'], test_case['expected-pattern']) for test_case in code_file['test-cases']] test_suites.append((code_file['path'], test_cases)) except KeyError as e: self.logger.error(f"Error: parsing yaml file, missing key:{e}")