diff --git a/.gitignore b/.gitignore index 2f372208..94962f5c 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ /support/default_validator/default_validator /support/interactive/interactive build/ +*_images/ diff --git a/examples/different/input_validators/different.ctd b/examples/different/input_validators/different.ctd deleted file mode 100644 index 66d8e60d..00000000 --- a/examples/different/input_validators/different.ctd +++ /dev/null @@ -1,6 +0,0 @@ -SET(cases = 0) -WHILE(!ISEOF) - INT(0, 10^15) SPACE INT(0, 10^15) NEWLINE - SET(cases = cases + 1) -END -ASSERT(1 <= cases && cases <= 40) diff --git a/examples/different/output_visualizer/outputvisualizer.py b/examples/different/output_visualizer/outputvisualizer.py new file mode 100644 index 00000000..dfc799b8 --- /dev/null +++ b/examples/different/output_visualizer/outputvisualizer.py @@ -0,0 +1,35 @@ +import sys +import os +from PIL import Image, ImageDraw, ImageFont + +def main(): + if not len(sys.argv) == 3: + print("Usage: output_visualizer.py ") + sys.exit(1) + + with open (sys.argv[1], 'r') as f: + submission_output = f.read() + feedback_dir = sys.argv[2] + + # Create an image with a white background + width, height = 400, 200 + image = Image.new('RGB', (width, height), color=(255, 255, 255)) + draw = ImageDraw.Draw(image) + + # Use default font + font = ImageFont.load_default() + + # Construct text to display (limit length for simplicity) + text = f"Output:\n{submission_output[:100]}" + + # Draw the text onto the image + draw.multiline_text((10, 10), text, fill=(0, 0, 0), font=font) + + # Save the image to feedback directory + outfile = os.path.join(feedback_dir, "visualizer_output.png") + image.save(outfile) + print(f"Image saved to {outfile}") + sys.exit(0) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/problemtools/verifyproblem.py b/problemtools/verifyproblem.py index cd708c4e..1aeb945e 100644 --- a/problemtools/verifyproblem.py +++ b/problemtools/verifyproblem.py @@ -11,6 +11,7 @@ import hashlib import collections import os +from pathlib import Path import signal import re import shutil @@ -99,6 +100,7 @@ def __init__(self, args: argparse.Namespace, executor: ThreadPoolExecutor|None) self.data_filter: Pattern[str] = args.data_filter self.submission_filter: Pattern[str] = args.submission_filter self.fixed_timelim: int|None = args.fixed_timelim + self.save_output_visualizer_images: bool = args.save_visualizer self.executor = executor self._background_work: list[concurrent.futures.Future[object]] = [] @@ -322,8 +324,16 @@ def run_submission(self, sub, runner: Runner, context: Context) -> Result: def run_submission_real(self, sub, context: Context, timelim: int, timelim_low: int, timelim_high: int) -> Result: # This may be called off-main thread. + + feedbackdir = os.path.join(self._problem.tmpdir, f"feedback-{self.counter}") + # The problem directory persists long enough that reuse is a problem + if os.path.exists(feedbackdir): + shutil.rmtree(feedbackdir) + os.makedirs(feedbackdir) + + outfile = "" # TODO: what should we do for interactive/multipass? if self._problem.get(ProblemTestCases)['is_interactive']: - res_high = self._problem.classes[OutputValidators.PART_NAME].validate_interactive(self, sub, timelim_high, self._problem.classes[Submissions.PART_NAME]) + res_high = self._problem.classes[OutputValidators.PART_NAME].validate_interactive(self, sub, timelim_high, self._problem.classes[Submissions.PART_NAME], feedbackdir) else: outfile = os.path.join(self._problem.tmpdir, f'output-{self.counter}') errfile = os.path.join(self._problem.tmpdir, f'error-{self.counter}') @@ -341,7 +351,7 @@ def run_submission_real(self, sub, context: Context, timelim: int, timelim_low: info = None res_high = SubmissionResult('RTE', additional_info=info) else: - res_high = self._problem.classes[OutputValidators.PART_NAME].validate(self, outfile) + res_high = self._problem.classes[OutputValidators.PART_NAME].validate(self, outfile, feedbackdir) res_high.runtime = runtime if res_high.runtime <= timelim_low: @@ -365,6 +375,16 @@ def run_submission_real(self, sub, context: Context, timelim: int, timelim_low: res.set_ac_runtime() res_low.set_ac_runtime() res_high.set_ac_runtime() + + visualizer = self._problem.classes.get(OutputVisualizer.PART_NAME) + if visualizer.visualizer_exists(): + output_path = Path(f"{self._problem.shortname}_images") / Path(sub.path).name / Path(self.infile).stem + if outfile: + visualizer.visualize(outfile, feedbackdir, output_path, context) + else: + with tempfile.NamedTemporaryFile() as f: + visualizer.visualize(f.name, feedbackdir, output_path, context) + return (res, res_low, res_high) def _init_result_for_testcase(self, res: SubmissionResult) -> SubmissionResult: @@ -1368,7 +1388,15 @@ def _actual_validators(self) -> list: return [val for val in vals if val is not None] - def validate_interactive(self, testcase: TestCase, submission, timelim: int, errorhandler: Submissions) -> SubmissionResult: + def validate_interactive(self, testcase: TestCase, submission, timelim: int, errorhandler: Submissions, feedbackdir: str) -> SubmissionResult: + """ + Validate a submission against all output validators. + + Parameters: + testcase: The test case we are validating. + submission: The submission to validate. + feedback_dir_path: Path to feedback directory. If None, a temporary directory will be created and cleaned up. + """ # This may be called off-main thread. interactive_output_re = r'\d+ \d+\.\d+ \d+ \d+\.\d+ (validator|submission)' res = SubmissionResult('JE') @@ -1383,13 +1411,23 @@ def validate_interactive(self, testcase: TestCase, submission, timelim: int, err val_timelim = self.problem.get(ProblemConfig)['limits']['validation_time'] val_memlim = self.problem.get(ProblemConfig)['limits']['validation_memory'] + first_validator = True for val in self._actual_validators(): if val.compile()[0]: - feedbackdir = tempfile.mkdtemp(prefix='feedback', dir=self.problem.tmpdir) + # Subtle point: if we're running multipass, we must ensure feedback dir is not reset + # If we're running multipass, there exists exactly one output validator + # Only deleting on second iteration is still fine with legacy and multiple output validators + if not first_validator: + feedback_path = Path(feedbackdir) + shutil.rmtree(feedback_path) + feedback_path.mkdir() + first_validator = False + validator_args[2] = feedbackdir + os.sep f = tempfile.NamedTemporaryFile(delete=False) interactive_out = f.name f.close() + i_status, _ = interactive.run(outfile=interactive_out, args=initargs + val.get_runcmd(memlim=val_memlim) + validator_args + [';'] + submission_args, work_dir=submission.path) if is_RTE(i_status): @@ -1425,25 +1463,48 @@ def validate_interactive(self, testcase: TestCase, submission, timelim: int, err res.validator_first = (first == 'validator') os.unlink(interactive_out) - shutil.rmtree(feedbackdir) if res.verdict != 'AC': + res.from_validator = True return res # TODO: check that all output validators give same result return res - def validate(self, testcase: TestCase, submission_output: str) -> SubmissionResult: + def validate(self, testcase: TestCase, submission_output: str, feedback_dir_path: str|None = None) -> SubmissionResult: + """ + Run all output validators on the given test case and submission output. + + Parameters: + testcase: The test case we are validating. + submission_output: Path to out file of submission. + feedback_dir_path: Path to feedback directory. If None, a temporary directory will be created and cleaned up. + """ res = SubmissionResult('JE') + res.from_validator = True val_timelim = self.problem.get(ProblemConfig)['limits']['validation_time'] val_memlim = self.problem.get(ProblemConfig)['limits']['validation_memory'] flags = self.problem.get(ProblemConfig)['validator_flags'].split() + testcase.testcasegroup.config['output_validator_flags'].split() + + first_validator = True for val in self._actual_validators(): if val.compile()[0]: - feedbackdir = tempfile.mkdtemp(prefix='feedback', dir=self.problem.tmpdir) + # Subtle point: if we're running multipass, we must ensure feedback dir is not reset + # If we're running multipass, there exists exactly one output validator + # Only deleting on second iteration is still fine with legacy and multiple output validators + if not first_validator: + feedback_path = Path(feedbackdir) + shutil.rmtree(feedback_path) + feedback_path.mkdir() + first_validator = False + + if feedback_dir_path: + feedbackdir = feedback_dir_path + else: + feedbackdir = tempfile.mkdtemp(prefix='feedback', dir=self.problem.tmpdir) validator_output = tempfile.mkdtemp(prefix='checker_out', dir=self.problem.tmpdir) outfile = validator_output + "/out.txt" errfile = validator_output + "/err.txt" - status, runtime = val.run(submission_output, + status, runtime = val.run(infile=submission_output, args=[testcase.infile, testcase.ansfile, feedbackdir] + flags, timelim=val_timelim, memlim=val_memlim, outfile=outfile, errfile=errfile) @@ -1460,8 +1521,10 @@ def validate(self, testcase: TestCase, submission_output: str) -> SubmissionResu except IOError as e: self.info("Failed to read validator output: %s", e) res = self._parse_validator_results(val, status, feedbackdir, testcase) - shutil.rmtree(feedbackdir) shutil.rmtree(validator_output) + if feedback_dir_path is None: + shutil.rmtree(feedbackdir) + res.from_validator = True if res.verdict != 'AC': return res @@ -1469,6 +1532,129 @@ def validate(self, testcase: TestCase, submission_output: str) -> SubmissionResu return res +class OutputVisualizer(ProblemPart): + """Handles output visualizers. Runs an output visualizer for each + testcase on each submission. Always runs visualizer to sanity check + output, saves images to disk if flag is passed. + + Example of created file structure when run on different: + different_images + ├── different.c + │ ├── 01 + │ │ └── visualizer_output.png + │ ├── 02_extreme_cases + │ │ └── visualizer_output.png + │ └── 1 + │ └── visualizer_output.png + └── different.cc + ├── 01 + │ └── visualizer_output.png + ├── 02_extreme_cases + │ └── visualizer_output.png + └── 1 + └── visualizer_output.png + """ + PART_NAME = 'output_visualizer' + + ALLOWED_FILE_EXTENSIONS = [".png", ".jpg", ".jpeg", ".svg"] + + def setup(self): + self._visualizer = run.find_programs(os.path.join(self.problem.probdir,'output_visualizer'), + work_dir=self.problem.tmpdir, + language_config=self.problem.language_config) + + self.count = 0 + self._has_precompiled = False + + def __str__(self) -> str: + return 'output visualizer' + + @staticmethod + def setup_dependencies(): + return [OutputValidators] + + # Does an early compilatilation of the visualizer + def start_background_work(self, context: Context) -> None: #kan + if not self._has_precompiled: + context.submit_background_work(lambda v: v.compile(), self._visualizer) + self._has_precompiled = True + + def check(self, context: Context) -> bool: + if len(self._visualizer) != 1: + self.warning(f'Wrong amount of visualizers. \nExcpected: 1\nActual: {len(self._visualizer)}') + + # Checks if a file's extension is allowed, and if so validates its header + def check_is_valid_image(self, file) -> bool: + simple_header_formats = [ + ('.png', b'\x89PNG\r\n\x1a\n'), # PNG file header + ('jpg', b'\xff\xd8\xff\xe0\x10\x00JF') # JPEG and JPG file header + ] + simple_header_formats.append(('jpeg', simple_header_formats[1][1])) + # If the file is not an svg it then reads in the first 8 bytes and checks + # that its header / magic number is as expected + if any(file.suffix == end for end, _ in simple_header_formats): + header = [header for end, header in simple_header_formats if file.suffix == end][0] + with open(file, "rb") as f: + file_signature = f.read(8) + if not file_signature.startswith(header): + self.warning(f"File {file} has incorrect file header") + return False + return True + elif file.endswith('.svg'): + try: + #TODO make this more robust + # Reads the XML declaration and first 256 characters, as bool: + return bool(self._visualizer) + + def visualize(self, result_file: str, feedback_dir: str, output_dir: Path, context: Context): + """Run visualizer on result_file, and optionally save it to disk""" + generated_image_paths = [] + + if not self.visualizer_exists(): + return + visualizer = self._visualizer[0] + + #Tries to run the visualzier and raises a warning if failed + try: + status, runtime = visualizer.run(args=[result_file,feedback_dir]) + if status != 0: + self.warning(f'The output visualizer crashed, status: {status}') + except Exception as e: + self.warning(f'Error running output visualizer: {e}') + + any_invalid_headers = False + + for file in Path(feedback_dir).iterdir(): + if file.is_file() and file.suffix in self.ALLOWED_FILE_EXTENSIONS: + if self.check_is_valid_image(file): + generated_image_paths.append(file) + else: + any_invalid_headers = True + + if context.save_output_visualizer_images: + for image in generated_image_paths: + output_dir.mkdir(parents=True, exist_ok=True) + shutil.copy(image, output_dir / Path(image).name) + + #Raises a warning if the file signature is wrong or the list is empty + if not generated_image_paths: + if any_invalid_headers: + self.warning("The visualizer did not generate images with proper file headers") + else: + self.warning("The visualizer did not generate any images") + + class Runner: def __init__(self, problem: Problem, sub, context: Context, timelim: int, timelim_low: int, timelim_high: int) -> None: self._problem = problem @@ -1735,9 +1921,11 @@ def check(self, context: Context) -> bool: 'graders': [Graders], 'data': [ProblemTestCases], 'submissions': [Submissions], + 'visualizers': [OutputVisualizer] #TODO for testing }, '2023-07': { # TODO: Add all the parts 'statement': [ProblemStatement2023_07, Attachments], + 'visualizers': [OutputVisualizer] } } @@ -1841,6 +2029,7 @@ def check(self, args: argparse.Namespace) -> tuple[int, int]: self.msg(f'Checking {part}') for item in self.part_mapping[part]: self.classes[item.PART_NAME].check(context) + #TODO Dirsystem with multipass except VerifyError: pass finally: @@ -1905,6 +2094,9 @@ def argparser_basic_arguments(parser: argparse.ArgumentParser) -> None: default='automatic', choices=list(PROBLEM_FORMATS.keys()) + ['automatic'], help='which problem format should the package be interpreted as, or "automatic" if it should be figured out from problem.yaml') + parser.add_argument('-sv', '--save_visualizer', + action='store_true', + help="Save visualizer outputs to disk") def argparser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description='Validate a problem package in the Kattis problem format.')