[Solved] Auto-scheduler cannot find a valid schedule

Hi all,

I am learning about TVM’s auto-scheduler these days. But when I ran the tutorial code given in Optimizing Operators with Auto-scheduling — tvm 0.9.dev182+ge718f5a8a documentation, it simply said the auto-scheduler cannot find a valid schedule:

RuntimeError: Cannot find any valid schedule for ["matmul_add", 1024, 1024, 1024, "float32"] in file matmul.json

I’d appreciate any help!

Hi!

Since the tuning results are saved in matmul.json in the same directory of the tutorial file, as specified in the tutorial, could you check the content of matmul.json :thinking: ?

For example, after executing the tutorial python file, my matmul.json is

{"i": [["[\"matmul_add\", 1024, 1024, 1024, \"float32\"]", "llvm -keys=cpu -link-params=0", [8, 64, 64, 0, 0, 0, 0, 0], "", 1, []], [[], [["SP", 2, 0, 1024, [1, 256, 2], 1], ["SP", 2, 4, 1024, [2, 1, 1], 1], ["SP", 2, 8, 1024, [32], 1], ["RE", 2, [0, 4, 1, 5, 8, 2, 6, 9, 3, 7]], ["FU", 2, [0, 1]], ["AN", 2, 0, 3], ["AN", 4, 0, 3], ["PR", 2, 0, "auto_unroll_max_step$16"], ["AN", 2, 8, 2]]]], "r": [[0.150646], 0, 1.18064, 1640007387], "v": "v0.6"}
{"i": [["[\"matmul_add\", 1024, 1024, 1024, \"float32\"]", "llvm -keys=cpu -link-params=0", [8, 64, 64, 0, 0, 0, 0, 0], "", 1, []], [[], [["SP", 2, 0, 1024, [1, 16, 64], 1], ["SP", 2, 4, 1024, [8, 2, 2], 1], ["SP", 2, 8, 1024, [64], 1], ["RE", 2, [0, 4, 1, 5, 8, 2, 6, 9, 3, 7]], ["CR", 4], ["FU", 2, [0, 1]], ["AN", 2, 0, 3], ["AN", 4, 0, 3], ["PR", 2, 0, "auto_unroll_max_step$64"], ["AN", 2, 8, 2]]]], "r": [[0.137297], 0, 1.44352, 1640007388], "v": "v0.6"}
{"i": [["[\"matmul_add\", 1024, 1024, 1024, \"float32\"]", "llvm -keys=cpu -link-params=0", [8, 64, 64, 0, 0, 0, 0, 0], "", 1, []], [[], [["SP", 2, 0, 1024, [4, 32, 8], 1], ["SP", 2, 4, 1024, [1, 8, 64], 1], ["SP", 2, 8, 1024, [16], 1], ["RE", 2, [0, 4, 1, 5, 8, 2, 6, 9, 3, 7]], ["FSP", 4, 0, 0, 1], ["FSP", 4, 2, 1, 1], ["RE", 4, [0, 2, 1, 3]], ["CA", 2, 4, 1], ["FU", 4, [0, 1]], ["AN", 4, 0, 3], ["PR", 2, 0, "auto_unroll_max_step$16"]]]], "r": [[0.131515], 0, 1.6067, 1640007389], "v": "v0.6"}
{"i": [["[\"matmul_add\", 1024, 1024, 1024, \"float32\"]", "llvm -keys=cpu -link-params=0", [8, 64, 64, 0, 0, 0, 0, 0], "", 1, []], [[], [["SP", 2, 0, 1024, [64, 8, 1], 1], ["SP", 2, 4, 1024, [1, 8, 1], 1], ["SP", 2, 8, 1024, [4], 1], ["RE", 2, [0, 4, 1, 5, 8, 2, 6, 9, 3, 7]], ["FSP", 4, 0, 0, 2], ["FSP", 4, 3, 1, 2], ["RE", 4, [0, 3, 1, 4, 2, 5]], ["CA", 2, 4, 3], ["FU", 4, [0, 1, 2]], ["AN", 4, 0, 3], ["PR", 2, 0, "auto_unroll_max_step$16"]]]], "r": [[0.0552857], 0, 0.882808, 1640007389], "v": "v0.6"}
{"i": [["[\"matmul_add\", 1024, 1024, 1024, \"float32\"]", "llvm -keys=cpu -link-params=0", [8, 64, 64, 0, 0, 0, 0, 0], "", 1, []], [[], [["SP", 2, 0, 1024, [16, 2, 4], 1], ["SP", 2, 4, 1024, [1, 8, 1], 1], ["SP", 2, 8, 1024, [64], 1], ["RE", 2, [0, 4, 1, 5, 8, 2, 6, 9, 3, 7]], ["CR", 4], ["FU", 2, [0, 1]], ["AN", 2, 0, 3], ["AN", 4, 0, 3], ["PR", 2, 0, "auto_unroll_max_step$64"], ["AN", 2, 8, 2]]]], "r": [[0.0857], 0, 1.13702, 1640007390], "v": "v0.6"}
{"i": [["[\"matmul_add\", 1024, 1024, 1024, \"float32\"]", "llvm -keys=cpu -link-params=0", [8, 64, 64, 0, 0, 0, 0, 0], "", 1, []], [[], [["SP", 2, 0, 1024, [64, 4, 4], 1], ["SP", 2, 4, 1024, [2, 2, 2], 1], ["SP", 2, 8, 1024, [2], 1], ["RE", 2, [0, 4, 1, 5, 8, 2, 6, 9, 3, 7]], ["FSP", 4, 0, 0, 1], ["FSP", 4, 2, 1, 1], ["RE", 4, [0, 2, 1, 3]], ["CA", 2, 4, 1], ["FU", 4, [0, 1]], ["AN", 4, 0, 3], ["PR", 2, 0, "auto_unroll_max_step$64"], ["AN", 2, 9, 2], ["AN", 4, 2, 2]]]], "r": [[0.0608416], 0, 1.02185, 1640007390], "v": "v0.6"}
{"i": [["[\"matmul_add\", 1024, 1024, 1024, \"float32\"]", "llvm -keys=cpu -link-params=0", [8, 64, 64, 0, 0, 0, 0, 0], "", 1, []], [[], [["SP", 2, 0, 1024, [16, 2, 16], 1], ["SP", 2, 4, 1024, [2, 16, 1], 1], ["SP", 2, 8, 1024, [2], 1], ["RE", 2, [0, 4, 1, 5, 8, 2, 6, 9, 3, 7]], ["FSP", 4, 0, 0, 2], ["FSP", 4, 3, 1, 2], ["RE", 4, [0, 3, 1, 4, 2, 5]], ["CA", 2, 4, 3], ["FU", 4, [0, 1, 2]], ["AN", 4, 0, 3], ["PR", 2, 0, "auto_unroll_max_step$64"], ["AN", 2, 9, 2]]]], "r": [[0.0336179], 0, 1.11559, 1640007391], "v": "v0.6"}
{"i": [["[\"matmul_add\", 1024, 1024, 1024, \"float32\"]", "llvm -keys=cpu -link-params=0", [8, 64, 64, 0, 0, 0, 0, 0], "", 1, []], [[], [["SP", 2, 0, 1024, [128, 4, 2], 1], ["SP", 2, 4, 1024, [1, 256, 1], 1], ["SP", 2, 8, 1024, [64], 1], ["RE", 2, [0, 4, 1, 5, 8, 2, 6, 9, 3, 7]], ["FSP", 4, 0, 0, 1], ["FSP", 4, 2, 1, 1], ["RE", 4, [0, 2, 1, 3]], ["CA", 2, 4, 1], ["FU", 4, [0, 1]], ["AN", 4, 0, 3], ["PR", 2, 0, "auto_unroll_max_step$0"], ["AN", 2, 9, 2]]]], "r": [[0.113403], 0, 1.0202, 1640007392], "v": "v0.6"}
{"i": [["[\"matmul_add\", 1024, 1024, 1024, \"float32\"]", "llvm -keys=cpu -link-params=0", [8, 64, 64, 0, 0, 0, 0, 0], "", 1, []], [[], [["SP", 2, 0, 1024, [32, 4, 2], 1], ["SP", 2, 4, 1024, [1024, 1, 1], 1], ["SP", 2, 8, 1024, [1], 1], ["RE", 2, [0, 4, 1, 5, 8, 2, 6, 9, 3, 7]], ["CR", 4], ["FU", 2, [0, 1, 2, 3]], ["AN", 2, 0, 3], ["AN", 4, 0, 3], ["PR", 2, 0, "auto_unroll_max_step$512"], ["AN", 2, 6, 2]]]], "r": [[0.0891908], 0, 0.77039, 1640007393], "v": "v0.6"}
{"i": [["[\"matmul_add\", 1024, 1024, 1024, \"float32\"]", "llvm -keys=cpu -link-params=0", [8, 64, 64, 0, 0, 0, 0, 0], "", 1, []], [[], [["SP", 2, 0, 1024, [4, 8, 2], 1], ["SP", 2, 4, 1024, [1, 8, 2], 1], ["SP", 2, 8, 1024, [2], 1], ["RE", 2, [0, 4, 1, 5, 8, 2, 6, 9, 3, 7]], ["FSP", 4, 0, 0, 2], ["FSP", 4, 3, 1, 2], ["RE", 4, [0, 3, 1, 4, 2, 5]], ["CA", 2, 4, 3], ["FU", 4, [0, 1]], ["AN", 4, 0, 3], ["PR", 2, 0, "auto_unroll_max_step$64"], ["AN", 2, 9, 2], ["AN", 4, 4, 2]]]], "r": [[0.0504146], 0, 0.910845, 1640007393], "v": "v0.6"}
{"i": [["[\"matmul_add\", 1024, 1024, 1024, \"float32\"]", "llvm -keys=cpu -link-params=0", [8, 64, 64, 0, 0, 0, 0, 0], "", 1, []], [[], [["SP", 2, 0, 1024, [256, 1, 4], 1], ["SP", 2, 4, 1024, [32, 1, 32], 1], ["SP", 2, 8, 1024, [32], 1], ["RE", 2, [0, 4, 1, 5, 8, 2, 6, 9, 3, 7]], ["FSP", 4, 0, 0, 2], ["FSP", 4, 3, 1, 2], ["RE", 4, [0, 3, 1, 4, 2, 5]], ["CA", 2, 4, 3], ["FU", 4, [0, 1, 2]], ["AN", 4, 0, 3], ["PR", 2, 0, "auto_unroll_max_step$64"]]]], "r": [[0.0266626], 0, 1.08314, 1640007424], "v": "v0.6"}
{"i": [["[\"matmul_add\", 1024, 1024, 1024, \"float32\"]", "llvm -keys=cpu -link-params=0", [8, 64, 64, 0, 0, 0, 0, 0], "", 1, []], [[], [["SP", 2, 0, 1024, [128, 1, 8], 1], ["SP", 2, 4, 1024, [2, 64, 4], 1], ["SP", 2, 8, 1024, [32], 1], ["RE", 2, [0, 4, 1, 5, 8, 2, 6, 9, 3, 7]], ["FSP", 4, 0, 0, 1], ["FSP", 4, 2, 1, 1], ["RE", 4, [0, 2, 1, 3]], ["CA", 2, 4, 1], ["FU", 4, [0, 1]], ["AN", 4, 0, 3], ["PR", 2, 0, "auto_unroll_max_step$512"], ["AN", 2, 9, 2]]]], "r": [[0.0833978], 0, 1.82687, 1640007424], "v": "v0.6"}
{"i": [["[\"matmul_add\", 1024, 1024, 1024, \"float32\"]", "llvm -keys=cpu -link-params=0", [8, 64, 64, 0, 0, 0, 0, 0], "", 1, []], [[], [["SP", 2, 0, 1024, [64, 16, 1], 1], ["SP", 2, 4, 1024, [1, 1, 4], 1], ["SP", 2, 8, 1024, [32], 1], ["RE", 2, [0, 4, 1, 5, 8, 2, 6, 9, 3, 7]], ["FSP", 4, 0, 0, 1], ["FSP", 4, 2, 1, 1], ["RE", 4, [0, 2, 1, 3]], ["CA", 2, 4, 1], ["FU", 4, [0, 1]], ["AN", 4, 0, 3], ["PR", 2, 0, "auto_unroll_max_step$64"], ["AN", 2, 9, 2], ["AN", 4, 2, 2]]]], "r": [[0.0313274], 0, 0.856808, 1640007425], "v": "v0.6"}
{"i": [["[\"matmul_add\", 1024, 1024, 1024, \"float32\"]", "llvm -keys=cpu -link-params=0", [8, 64, 64, 0, 0, 0, 0, 0], "", 1, []], [[], [["SP", 2, 0, 1024, [4, 1, 2], 1], ["SP", 2, 4, 1024, [1, 2, 32], 1], ["SP", 2, 8, 1024, [1], 1], ["RE", 2, [0, 4, 1, 5, 8, 2, 6, 9, 3, 7]], ["FSP", 4, 0, 0, 1], ["FSP", 4, 2, 1, 1], ["RE", 4, [0, 2, 1, 3]], ["CA", 2, 4, 1], ["FU", 4, [0, 1]], ["AN", 4, 0, 3], ["PR", 2, 0, "auto_unroll_max_step$512"]]]], "r": [[0.0298633], 0, 1.12206, 1640007425], "v": "v0.6"}
{"i": [["[\"matmul_add\", 1024, 1024, 1024, \"float32\"]", "llvm -keys=cpu -link-params=0", [8, 64, 64, 0, 0, 0, 0, 0], "", 1, []], [[], [["SP", 2, 0, 1024, [4, 8, 8], 1], ["SP", 2, 4, 1024, [4, 1, 1], 1], ["SP", 2, 8, 1024, [8], 1], ["RE", 2, [0, 4, 1, 5, 8, 2, 6, 9, 3, 7]], ["FSP", 4, 0, 0, 1], ["FSP", 4, 2, 1, 1], ["RE", 4, [0, 2, 1, 3]], ["CA", 2, 4, 1], ["FU", 4, [0, 1]], ["AN", 4, 0, 3], ["PR", 2, 0, "auto_unroll_max_step$64"], ["AN", 2, 9, 2], ["AN", 4, 2, 2]]]], "r": [[0.137484], 0, 1.41401, 1640007426], "v": "v0.6"}

And this doesn’t run into any unexpected error :slight_smile: .

Thanks for your reply. Mine is:

{"i": [["[\"matmul_add\", 1024, 1024, 1024, \"float32\"]", "llvm -keys=cpu -link-params=0", [8, 64, 64, 0, 0, 0, 0, 0], "", 1, []], [[], [["SP", 2, 0, 1024, [16, 1, 2], 1], ["SP", 2, 4, 1024, [2, 1, 4], 1], ["SP", 2, 8, 1024, [64], 1], ["RE", 2, [0, 4, 1, 5, 8, 2, 6, 9, 3, 7]], ["FSP", 4, 0, 0, 1], ["FSP", 4, 2, 1, 1], ["RE", 4, [0, 2, 1, 3]], ["CA", 2, 4, 1], ["FU", 4, [0, 1]], ["AN", 4, 0, 3], ["PR", 2, 0, "auto_unroll_max_step$0"], ["AN", 2, 9, 2], ["AN", 4, 2, 2]]]], "r": [[1e+10], 2, 15, 1640012843], "v": "v0.6"}
{"i": [["[\"matmul_add\", 1024, 1024, 1024, \"float32\"]", "llvm -keys=cpu -link-params=0", [8, 64, 64, 0, 0, 0, 0, 0], "", 1, []], [[], [["SP", 2, 0, 1024, [1, 128, 4], 1], ["SP", 2, 4, 1024, [64, 16, 1], 1], ["SP", 2, 8, 1024, [8], 1], ["RE", 2, [0, 4, 1, 5, 8, 2, 6, 9, 3, 7]], ["FU", 2, [0, 1, 2, 3]], ["AN", 2, 0, 3], ["AN", 4, 0, 3], ["PR", 2, 0, "auto_unroll_max_step$16"], ["AN", 2, 6, 2]]]], "r": [[1e+10], 2, 15, 1640012843], "v": "v0.6"}
{"i": [["[\"matmul_add\", 1024, 1024, 1024, \"float32\"]", "llvm -keys=cpu -link-params=0", [8, 64, 64, 0, 0, 0, 0, 0], "", 1, []], [[], [["SP", 2, 0, 1024, [1, 32, 4], 1], ["SP", 2, 4, 1024, [1, 32, 32], 1], ["SP", 2, 8, 1024, [32], 1], ["RE", 2, [0, 4, 1, 5, 8, 2, 6, 9, 3, 7]], ["FSP", 4, 0, 0, 1], ["FSP", 4, 2, 1, 1], ["RE", 4, [0, 2, 1, 3]], ["CA", 2, 4, 1], ["FU", 4, [0, 1]], ["AN", 4, 0, 3], ["PR", 2, 0, "auto_unroll_max_step$0"], ["AN", 2, 9, 2]]]], "r": [[1e+10], 2, 15, 1640012843], "v": "v0.6"}
{"i": [["[\"matmul_add\", 1024, 1024, 1024, \"float32\"]", "llvm -keys=cpu -link-params=0", [8, 64, 64, 0, 0, 0, 0, 0], "", 1, []], [[], [["SP", 2, 0, 1024, [8, 16, 1], 1], ["SP", 2, 4, 1024, [32, 16, 1], 1], ["SP", 2, 8, 1024, [4], 1], ["RE", 2, [0, 4, 1, 5, 8, 2, 6, 9, 3, 7]], ["FSP", 4, 0, 0, 1], ["FSP", 4, 2, 1, 1], ["RE", 4, [0, 2, 1, 3]], ["CA", 2, 4, 1], ["FU", 4, [0, 1]], ["AN", 4, 0, 3], ["PR", 2, 0, "auto_unroll_max_step$0"], ["AN", 2, 9, 2]]]], "r": [[1e+10], 2, 15, 1640012843], "v": "v0.6"}
{"i": [["[\"matmul_add\", 1024, 1024, 1024, \"float32\"]", "llvm -keys=cpu -link-params=0", [8, 64, 64, 0, 0, 0, 0, 0], "", 1, []], [[], [["SP", 2, 0, 1024, [2, 4, 16], 1], ["SP", 2, 4, 1024, [1, 16, 16], 1], ["SP", 2, 8, 1024, [1], 1], ["RE", 2, [0, 4, 1, 5, 8, 2, 6, 9, 3, 7]], ["FSP", 4, 0, 0, 2], ["FSP", 4, 3, 1, 2], ["RE", 4, [0, 3, 1, 4, 2, 5]], ["CA", 2, 4, 3], ["FU", 4, [0, 1, 2, 3]], ["AN", 4, 0, 3], ["PR", 2, 0, "auto_unroll_max_step$0"], ["AN", 2, 9, 2]]]], "r": [[1e+10], 2, 15, 1640012843], "v": "v0.6"}
{"i": [["[\"matmul_add\", 1024, 1024, 1024, \"float32\"]", "llvm -keys=cpu -link-params=0", [8, 64, 64, 0, 0, 0, 0, 0], "", 1, []], [[], [["SP", 2, 0, 1024, [8, 4, 2], 1], ["SP", 2, 4, 1024, [4, 32, 4], 1], ["SP", 2, 8, 1024, [8], 1], ["RE", 2, [0, 4, 1, 5, 8, 2, 6, 9, 3, 7]], ["FSP", 4, 0, 0, 2], ["FSP", 4, 3, 1, 2], ["RE", 4, [0, 3, 1, 4, 2, 5]], ["CA", 2, 4, 3], ["FU", 4, [0, 1, 2]], ["AN", 4, 0, 3], ["PR", 2, 0, "auto_unroll_max_step$0"], ["AN", 2, 9, 2]]]], "r": [[1e+10], 2, 15, 1640012843], "v": "v0.6"}
{"i": [["[\"matmul_add\", 1024, 1024, 1024, \"float32\"]", "llvm -keys=cpu -link-params=0", [8, 64, 64, 0, 0, 0, 0, 0], "", 1, []], [[], [["SP", 2, 0, 1024, [2, 2, 4], 1], ["SP", 2, 4, 1024, [32, 8, 2], 1], ["SP", 2, 8, 1024, [32], 1], ["RE", 2, [0, 4, 1, 5, 8, 2, 6, 9, 3, 7]], ["FSP", 4, 0, 0, 2], ["FSP", 4, 3, 1, 2], ["RE", 4, [0, 3, 1, 4, 2, 5]], ["CA", 2, 4, 3], ["FU", 4, [0, 1, 2]], ["AN", 4, 0, 3], ["PR", 2, 0, "auto_unroll_max_step$64"], ["AN", 2, 9, 2], ["AN", 4, 3, 2]]]], "r": [[1e+10], 2, 15, 1640012843], "v": "v0.6"}
{"i": [["[\"matmul_add\", 1024, 1024, 1024, \"float32\"]", "llvm -keys=cpu -link-params=0", [8, 64, 64, 0, 0, 0, 0, 0], "", 1, []], [[], [["SP", 2, 0, 1024, [1, 1, 16], 1], ["SP", 2, 4, 1024, [256, 2, 1], 1], ["SP", 2, 8, 1024, [4], 1], ["RE", 2, [0, 4, 1, 5, 8, 2, 6, 9, 3, 7]], ["FSP", 4, 0, 0, 1], ["FSP", 4, 2, 1, 1], ["RE", 4, [0, 2, 1, 3]], ["CA", 2, 4, 1], ["FU", 4, [0, 1]], ["AN", 4, 0, 3], ["PR", 2, 0, "auto_unroll_max_step$512"], ["AN", 2, 9, 2]]]], "r": [[1e+10], 2, 15, 1640012843], "v": "v0.6"}
{"i": [["[\"matmul_add\", 1024, 1024, 1024, \"float32\"]", "llvm -keys=cpu -link-params=0", [8, 64, 64, 0, 0, 0, 0, 0], "", 1, []], [[], [["SP", 2, 0, 1024, [512, 1, 1], 1], ["SP", 2, 4, 1024, [2, 4, 2], 1], ["SP", 2, 8, 1024, [16], 1], ["RE", 2, [0, 4, 1, 5, 8, 2, 6, 9, 3, 7]], ["FU", 2, [0, 1, 2]], ["AN", 2, 0, 3], ["AN", 4, 0, 3], ["PR", 2, 0, "auto_unroll_max_step$16"], ["AN", 2, 7, 2]]]], "r": [[1e+10], 2, 15, 1640012843], "v": "v0.6"}
{"i": [["[\"matmul_add\", 1024, 1024, 1024, \"float32\"]", "llvm -keys=cpu -link-params=0", [8, 64, 64, 0, 0, 0, 0, 0], "", 1, []], [[], [["SP", 2, 0, 1024, [16, 2, 4], 1], ["SP", 2, 4, 1024, [2, 2, 2], 1], ["SP", 2, 8, 1024, [4], 1], ["RE", 2, [0, 4, 1, 5, 8, 2, 6, 9, 3, 7]], ["FSP", 4, 0, 0, 1], ["FSP", 4, 2, 1, 1], ["RE", 4, [0, 2, 1, 3]], ["CA", 2, 4, 1], ["FU", 4, [0, 1]], ["AN", 4, 0, 3], ["PR", 2, 0, "auto_unroll_max_step$64"], ["AN", 2, 9, 2], ["AN", 4, 2, 2]]]], "r": [[1e+10], 2, 15, 1640012843], "v": "v0.6"}

BTW, I am using the latest tvm (commit hash: 5cb5c5bda7976666da9c9f862654cbbfabadc69f).

You should let it tune for more trials or enable verbose mode to let it dump the failure log for better investigation.

Meanwhile, from your tuning log, the error code seems COMPILE_HOST = 2 # Errors happen when compiling code on host (e.g., tvm.build), so you may want to check your environment.

1 Like

I enabled the verbose mode and checked my environment. The problem turned out to be the missing cloudpickle package. Now it works as expected!

hello, i enconter the same problem, and i am wondering how you enable the verbose mode, thanks please

Hi, sorry that I forgot the details, but this API doc, especially the verbose parameter, might be helpful.