From f8ca1b3524a53a05072723d01b99b1b8ffba28a7 Mon Sep 17 00:00:00 2001 From: Frank Plowman Date: Mon, 8 Jan 2024 13:05:12 +0000 Subject: [PATCH 1/2] threads.py: Add `--csv` option for CSV summary --- tools/threads.py | 74 +++++++++++++++++++++++++++++------------------- 1 file changed, 45 insertions(+), 29 deletions(-) diff --git a/tools/threads.py b/tools/threads.py index ac8a471..653f3de 100755 --- a/tools/threads.py +++ b/tools/threads.py @@ -20,11 +20,27 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from dataclasses import dataclass +from itertools import product import statistics import subprocess from utils.runner import * from utils.perfapp import * +@dataclass +class ThreadTestRun: + decoder: str + sequence: str + threads: int + asm: bool + fps: float + + def readable(self): + return f"{self.sequence}, {self.decoder} {'asm' if self.asm else 'no asm'} {self.threads} threads: {self.fps} fps" + + def csv(self): + return f"{self.decoder},{self.sequence},{self.threads},{self.asm},{self.fps}" + class ThreadRunner(TestRunner): __summary = {} __app = None @@ -35,10 +51,12 @@ def run(self): for f in files: self.__test(f) + print() self.__print_summary() def add_args(self, parser): parser.add_argument("--vvdec-path", type=str) + parser.add_argument("--csv", action="store_true") def __get_app(self): if self.args.vvdec_path: @@ -47,37 +65,35 @@ def __get_app(self): def __test(self, input): fn = os.path.basename(input) - self.__summary[fn] = [[], []] - for i in [1, 0]: - fps = self.__summary[fn][i] - for j in [16, 8, 4, 2, 1]: - self.__app.set_asm(i) - self.__app.set_threads(j) - cmd = self.__app.get_cmd(input) - print(cmd) - try: - o = subprocess.run(cmd.split(), capture_output=True, timeout=5 * 60) - if o.returncode: - raise Exception(o.stderr) - o = self.__app.get_fps(o) - print("fps = ", o) - fps.append(o) - except Exception as e: - print(e) - raise + self.runs = [] + + for asm, threads in product([True, False], [1, 2, 4, 8, 16]): + self.__app.set_asm(asm) + self.__app.set_threads(threads) + cmd = self.__app.get_cmd(input) + print(cmd) + try: + o = subprocess.run(cmd.split(), capture_output=True, timeout=5 * 60) + if o.returncode: + raise Exception(o.stderr) + o = self.__app.get_fps(o) + print("fps = ", o) + except Exception as e: + print(e) + raise + + run = ThreadTestRun("vvdec" if self.args.vvdec_path else "ffvvc", fn, threads, asm, o) + self.runs.append(run) def __print_summary(self): - for k,v in self.__summary.items(): - print_summary(k, "no asm", v[0]) - print_summary(k, "with asm", v[1]) - pass - -def print_summary(fn, msg, a): - s = fn + ", " + msg + " fps : {" - for fps in a: - s += " " + str(fps) + "," - s += " }" - print(s) + if self.args.csv: + print("Decoder,Sequence,Threads,SIMD,FPS") + + for run in self.runs: + if not self.args.csv: + print(run.readable()) + else: + print(run.csv()) if __name__ == "__main__": t = ThreadRunner() From be68c72a8b06370be865a960a3c738d76ce56e5c Mon Sep 17 00:00:00 2001 From: Frank Plowman Date: Wed, 24 Jan 2024 13:21:53 +0000 Subject: [PATCH 2/2] Add OpenVVC PerfApp --- tools/perf.py | 3 +++ tools/threads.py | 7 +++++-- tools/utils/perfapp.py | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/tools/perf.py b/tools/perf.py index c1e8f92..b7404ee 100755 --- a/tools/perf.py +++ b/tools/perf.py @@ -38,9 +38,12 @@ def run(self): self.__print_summary() def add_args(self, parser): + parser.add_argument("--openvvc-path", type=str) parser.add_argument("--vvdec-path", type=str) def __get_app(self): + if self.args.openvvc_path: + return OpenVVCApp(self.args.openvvc_path) if self.args.vvdec_path: return VVDecApp(self.args.vvdec_path) return FFmpegApp(self.args.ffmpeg_path) diff --git a/tools/threads.py b/tools/threads.py index 653f3de..b763847 100755 --- a/tools/threads.py +++ b/tools/threads.py @@ -55,10 +55,13 @@ def run(self): self.__print_summary() def add_args(self, parser): + parser.add_argument("--openvvc-path", type=str) parser.add_argument("--vvdec-path", type=str) parser.add_argument("--csv", action="store_true") def __get_app(self): + if self.args.openvvc_path: + return OpenVVCApp(self.args.openvvc_path) if self.args.vvdec_path: return VVDecApp(self.args.vvdec_path) return FFmpegApp(self.args.ffmpeg_path) @@ -67,7 +70,7 @@ def __test(self, input): fn = os.path.basename(input) self.runs = [] - for asm, threads in product([True, False], [1, 2, 4, 8, 16]): + for asm, threads in product([True, False], range(1, 12 + 1)): self.__app.set_asm(asm) self.__app.set_threads(threads) cmd = self.__app.get_cmd(input) @@ -82,7 +85,7 @@ def __test(self, input): print(e) raise - run = ThreadTestRun("vvdec" if self.args.vvdec_path else "ffvvc", fn, threads, asm, o) + run = ThreadTestRun(self.__app.get_name(), fn, threads, asm, o) self.runs.append(run) def __print_summary(self): diff --git a/tools/utils/perfapp.py b/tools/utils/perfapp.py index 4c9b481..618ea89 100644 --- a/tools/utils/perfapp.py +++ b/tools/utils/perfapp.py @@ -43,6 +43,8 @@ def get_fps(self, o): o = re.findall(r'fps=.*?q',o.stderr.decode())[-1] o = float(o.replace("fps=", "").replace("q", "").strip()) return o + def get_name(self): + return "ffmpeg" class VVDecApp(PerfApp): def __init__(self, path): @@ -58,3 +60,34 @@ def get_fps(self, o): o = re.findall(r'@ .*?fps',o.stdout.decode())[0] o = float(o.replace("fps", "").replace("@", "").strip()) return o + def get_name(self): + return "vvdec" + +class OpenVVCApp(PerfApp): + def __init__(self, path): + super().__init__(self) + self.__path = path + pass + def get_cmd(self, input): + extra_pre = "time " + extra_post = " -t " + str(self._threads) if self._threads else " " + extra_post += " -e " + str(self._threads) if self._threads else " " + extra_post += " -i " + input + extra_post += " -o /dev/null" + cmd = extra_pre + self.__path + extra_post + return cmd + def get_fps(self, o): + if match := re.search(r"Decoded ([0-9]+) pictures", o.stderr.decode()): + frame_count = int(match.group(1)) + else: + raise Exception("No frame count found") + if match := re.search(r"([0-9]+):([0-9]{2}).([0-9]{2})elapsed", o.stderr.decode()): + rtime_m = int(match.group(1)) + rtime_s = int(match.group(2)) + rtime_cs = int(match.group(3)) + rtime = rtime_m * 60 + rtime_s + rtime_cs / 100 + else: + raise Exception("No time found") + return frame_count / rtime + def get_name(self): + return "openvvc"