| import pdb | |
| import subprocess | |
| import re | |
| # Output file | |
| output_file = "pass_rate_output.txt" | |
| # Clearing the output file before appending new content | |
| with open(output_file, "w") as file: | |
| file.write("") | |
| # List of input paths | |
| input_path_lists = [ | |
| "test/zero_shot/wild_test/generation/lean4_random_15k_all/2/1/", | |
| ] | |
| def get_output(input_string, k): | |
| pattern = r"zero_shot/(\w+)/(.+?)/(\w+)" | |
| match = re.search(pattern, input_string) | |
| if match: | |
| part1 = match.group(1) | |
| part2 = match.group(3) + f"pass{k}.jsonl" | |
| result = "/".join([part1, part2]) | |
| print(result) | |
| else: | |
| print("No match found.") | |
| assert True | |
| return result | |
| # List of input paths | |
| input_path_lists = [ | |
| # "../auto-info/generate_result/zero_shot/gsm8k_train/generation/lean4_random_15k_all/2/10/", | |
| # "../auto-info/generate_result/zero_shot/math_train/generation/lean4_random_15k_all/2/10/", | |
| # "../auto-info/generate_result/zero_shot/wild_test/generation/lean4_rft/1/1", | |
| # "../auto-info/generate_result/zero_shot/wild_test/generation/lean4_rft/2/1", | |
| # "../auto-info/generate_result/zero_shot/wild_test/generation/lean4_rft/3/1", | |
| # "../auto-info/generate_result/zero_shot/wild_test/generation/lean4_verifier/1/1", | |
| # "../auto-info/generate_result/zero_shot/wild_test/generation/lean4_verifier/2/1", | |
| # "../auto-info/generate_result/zero_shot/wild_test/generation/lean4_verifier/3/1", | |
| # "../auto-info/generate_result/zero_shot/wild_test/generation/lean4_verifier_rft/1/1", | |
| # "../auto-info/generate_result/zero_shot/wild_test/generation/lean4_verifier_rft/2/1", | |
| # "../auto-info/generate_result/zero_shot/wild_test/generation/lean4_verifier_rft/3/1", | |
| # "/opt/tiger/auto-info/generate_result/zero_shot/lean4_basic_test/generation/lean4_rft/1/1/", | |
| # "/opt/tiger/auto-info/generate_result/zero_shot/lean4_basic_test/generation/lean4_rft/2/1/", | |
| # "/opt/tiger/auto-info/generate_result/zero_shot/lean4_basic_test/generation/lean4_rft/3/1/", | |
| # "/opt/tiger/auto-info/generate_result/zero_shot/lean4_random_test/generation/lean4_rft/1/1/", | |
| # "/opt/tiger/auto-info/generate_result/zero_shot/lean4_random_test/generation/lean4_rft/2/1/", | |
| # "/opt/tiger/auto-info/generate_result/zero_shot/lean4_random_test/generation/lean4_rft/3/1/", | |
| # "/opt/tiger/auto-info/generate_result/zero_shot/lean4_basic_test/generation/lean4_verifier/1/1/", | |
| # "/opt/tiger/auto-info/generate_result/zero_shot/lean4_basic_test/generation/lean4_verifier/2/1/", | |
| # "/opt/tiger/auto-info/generate_result/zero_shot/lean4_basic_test/generation/lean4_verifier/3/1/", | |
| # "/opt/tiger/auto-info/generate_result/zero_shot/lean4_random_test/generation/lean4_verifier/1/1/", | |
| # "/opt/tiger/auto-info/generate_result/zero_shot/lean4_random_test/generation/lean4_verifier/2/1/", | |
| # "/opt/tiger/auto-info/generate_result/zero_shot/lean4_random_test/generation/lean4_verifier/3/1/", | |
| # "/opt/tiger/auto-info/generate_result/zero_shot/lean4_basic_test/generation/lean4_verifier_rft/1/1/", | |
| # "/opt/tiger/auto-info/generate_result/zero_shot/lean4_basic_test/generation/lean4_verifier_rft/2/1/", | |
| # "/opt/tiger/auto-info/generate_result/zero_shot/lean4_basic_test/generation/lean4_verifier_rft/3/1/", | |
| # "/opt/tiger/auto-info/generate_result/zero_shot/lean4_random_test/generation/lean4_verifier_rft/1/1/", | |
| # "/opt/tiger/auto-info/generate_result/zero_shot/lean4_random_test/generation/lean4_verifier_rft/2/1/", | |
| # "/opt/tiger/auto-info/generate_result/zero_shot/lean4_random_test/generation/lean4_verifier_rft/3/1/", | |
| # "/opt/tiger/auto-info/generate_result/zero_shot/wild_test/generation/lean4_rft/1/1/", | |
| # "/opt/tiger/auto-info/generate_result/zero_shot/wild_test/generation/lean4_rft/2/1/", | |
| # "/opt/tiger/auto-info/generate_result/zero_shot/wild_test/generation/lean4_rft/3/1/", | |
| # "/opt/tiger/auto-info/generate_result/zero_shot/wild_test/generation/lean4_verifier/1/1/", | |
| # "/opt/tiger/auto-info/generate_result/zero_shot/wild_test/generation/lean4_verifier/2/1/", | |
| # "/opt/tiger/auto-info/generate_result/zero_shot/wild_test/generation/lean4_verifier/3/1/", | |
| # "/opt/tiger/auto-info/generate_result/zero_shot/wild_test/generation/lean4_verifier_rft/1/1/", | |
| # "/opt/tiger/auto-info/generate_result/zero_shot/wild_test/generation/lean4_verifier_rft/2/1/", | |
| # "/opt/tiger/auto-info/generate_result/zero_shot/wild_test/generation/lean4_verifier_rft/3/1/", | |
| # "/opt/tiger/auto-info/generate_result/zero_shot/lean4_15k_train/generation/lean4_random_15k_all/2/20/", | |
| # "/opt/tiger/auto-info/generate_result/zero_shot/lean4_basic_test/generation/lean4_random_15k_all/2/5/", | |
| # "/opt/tiger/auto-info/generate_result/zero_shot/lean4_random_test/generation/lean4_random_15k_all/2/5/", | |
| # "/opt/tiger/mariana/auto-info/generate_result/zero_shot/lean4_random_test/generation/lean4_random_5k/2/1/", | |
| # "test/zero_shot/lean4_random_test/generation/lean4_random_15k_all/3/1/", | |
| # "/opt/tiger/mariana/auto-info/generate_result/zero_shot/lean4_basic_test/generation/lean4_random_15k_all/2/1/", | |
| # "/opt/tiger/mariana/auto-info/generate_result/zero_shot/lean4_random_test/generation/lean4_random_15k_all/2/1/", | |
| # "test/zero_shot/lean4_random_test/generation/lean4_random_15k_all/3/1/", | |
| # "test/zero_shot/lean4_basic_test/generation/lean4_random_15k_all/3/1/", | |
| # "/opt/tiger/auto-info/generate_result/zero_shot/lean4_basic_test/generation/lean4_random_15k_all_mathrft/1/1/", | |
| # "/opt/tiger/auto-info/generate_result/zero_shot/lean4_random_test/generation/lean4_random_15k_all_mathrft/1/1/", | |
| # "/opt/tiger/auto-info/generate_result/zero_shot/wild_test/generation/lean4_random_15k_all_mathrft/1/1/", | |
| # "/opt/tiger/auto-info/generate_result/zero_shot/lean4_basic_test/generation/lean4_random_15k_all_mathrft/2/1/", | |
| # "/opt/tiger/auto-info/generate_result/zero_shot/lean4_random_test/generation/lean4_random_15k_all_mathrft/2/1/", | |
| # "/opt/tiger/auto-info/generate_result/zero_shot/wild_test/generation/lean4_random_15k_all_mathrft/2/1/", | |
| # "/opt/tiger/auto-info/generate_result/zero_shot/lean4_basic_test/generation/lean4_random_15k_all_mathrft/3/1/", | |
| # "/opt/tiger/auto-info/generate_result/zero_shot/lean4_random_test/generation/lean4_random_15k_all_mathrft/3/1/", | |
| # "/opt/tiger/auto-info/generate_result/zero_shot/wild_test/generation/lean4_random_15k_all_mathrft/3/1/", | |
| # "/opt/tiger/auto-info/generate_result/zero_shot/gsm8k_train/generation/lean4_random_15k_all_mathrft/2/10/", | |
| # "/opt/tiger/auto-info/generate_result/zero_shot/math_train/generation/lean4_random_15k_all_mathrft/2/10/", | |
| # "/opt/tiger/auto-info/generate_result/zero_shot/lean4_15k_train/generation/lean4_random_15k_all_mathrft/2/10/", | |
| # Add more input paths as needed | |
| ] | |
| # Iterate through the input paths and run the command | |
| for input_path in input_path_lists: | |
| k = 5 | |
| if "wild_test" in input_path or "gsm8k_train" in input_path or "math_train" in input_path: | |
| print(f"wild") | |
| print(f"Running for input path: {input_path}", file=open(output_file, "a")) | |
| command = f"python3 pass_rate_notlean_test.py --input_path {input_path} --output_path {get_output(input_path,k)} --k {k}" | |
| subprocess.run(command, shell=True, stdout=open(output_file, "a"), stderr=subprocess.STDOUT) | |
| print("\n\n",file=open(output_file, "a")) | |
| else: | |
| print(f"lean") | |
| print(f"Running for input path: {input_path}", file=open(output_file, "a")) | |
| command = f"python3 pass_rate_new_test.py --input_path {input_path} --output_path {get_output(input_path, k)} --k {k}" | |
| subprocess.run(command, shell=True, stdout=open(output_file, "a"), stderr=subprocess.STDOUT) | |
| print("\n\n",file=open(output_file, "a")) | |